1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 static int 809 mlx5_flow_info_get(struct rte_eth_dev *dev, 810 struct rte_flow_port_info *port_info, 811 struct rte_flow_queue_info *queue_info, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_port_configure(struct rte_eth_dev *dev, 815 const struct rte_flow_port_attr *port_attr, 816 uint16_t nb_queue, 817 const struct rte_flow_queue_attr *queue_attr[], 818 struct rte_flow_error *err); 819 820 static struct rte_flow_pattern_template * 821 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 822 const struct rte_flow_pattern_template_attr *attr, 823 const struct rte_flow_item items[], 824 struct rte_flow_error *error); 825 826 static int 827 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 828 struct rte_flow_pattern_template *template, 829 struct rte_flow_error *error); 830 static struct rte_flow_actions_template * 831 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 832 const struct rte_flow_actions_template_attr *attr, 833 const struct rte_flow_action actions[], 834 const struct rte_flow_action masks[], 835 struct rte_flow_error *error); 836 static int 837 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 838 struct rte_flow_actions_template *template, 839 struct rte_flow_error *error); 840 841 static struct rte_flow_template_table * 842 mlx5_flow_table_create(struct rte_eth_dev *dev, 843 const struct rte_flow_template_table_attr *attr, 844 struct rte_flow_pattern_template *item_templates[], 845 uint8_t nb_item_templates, 846 struct rte_flow_actions_template *action_templates[], 847 uint8_t nb_action_templates, 848 struct rte_flow_error *error); 849 static int 850 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 851 struct rte_flow_template_table *table, 852 struct rte_flow_error *error); 853 static struct rte_flow * 854 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 855 uint32_t queue, 856 const struct rte_flow_op_attr *attr, 857 struct rte_flow_template_table *table, 858 const struct rte_flow_item items[], 859 uint8_t pattern_template_index, 860 const struct rte_flow_action actions[], 861 uint8_t action_template_index, 862 void *user_data, 863 struct rte_flow_error *error); 864 static int 865 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 866 uint32_t queue, 867 const struct rte_flow_op_attr *attr, 868 struct rte_flow *flow, 869 void *user_data, 870 struct rte_flow_error *error); 871 static int 872 mlx5_flow_pull(struct rte_eth_dev *dev, 873 uint32_t queue, 874 struct rte_flow_op_result res[], 875 uint16_t n_res, 876 struct rte_flow_error *error); 877 static int 878 mlx5_flow_push(struct rte_eth_dev *dev, 879 uint32_t queue, 880 struct rte_flow_error *error); 881 882 static struct rte_flow_action_handle * 883 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 884 const struct rte_flow_op_attr *attr, 885 const struct rte_flow_indir_action_conf *conf, 886 const struct rte_flow_action *action, 887 void *user_data, 888 struct rte_flow_error *error); 889 890 static int 891 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 892 const struct rte_flow_op_attr *attr, 893 struct rte_flow_action_handle *handle, 894 const void *update, 895 void *user_data, 896 struct rte_flow_error *error); 897 898 static int 899 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 900 const struct rte_flow_op_attr *attr, 901 struct rte_flow_action_handle *handle, 902 void *user_data, 903 struct rte_flow_error *error); 904 905 static const struct rte_flow_ops mlx5_flow_ops = { 906 .validate = mlx5_flow_validate, 907 .create = mlx5_flow_create, 908 .destroy = mlx5_flow_destroy, 909 .flush = mlx5_flow_flush, 910 .isolate = mlx5_flow_isolate, 911 .query = mlx5_flow_query, 912 .dev_dump = mlx5_flow_dev_dump, 913 .get_aged_flows = mlx5_flow_get_aged_flows, 914 .action_handle_create = mlx5_action_handle_create, 915 .action_handle_destroy = mlx5_action_handle_destroy, 916 .action_handle_update = mlx5_action_handle_update, 917 .action_handle_query = mlx5_action_handle_query, 918 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 919 .tunnel_match = mlx5_flow_tunnel_match, 920 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 921 .tunnel_item_release = mlx5_flow_tunnel_item_release, 922 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 923 .flex_item_create = mlx5_flow_flex_item_create, 924 .flex_item_release = mlx5_flow_flex_item_release, 925 .info_get = mlx5_flow_info_get, 926 .configure = mlx5_flow_port_configure, 927 .pattern_template_create = mlx5_flow_pattern_template_create, 928 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 929 .actions_template_create = mlx5_flow_actions_template_create, 930 .actions_template_destroy = mlx5_flow_actions_template_destroy, 931 .template_table_create = mlx5_flow_table_create, 932 .template_table_destroy = mlx5_flow_table_destroy, 933 .async_create = mlx5_flow_async_flow_create, 934 .async_destroy = mlx5_flow_async_flow_destroy, 935 .pull = mlx5_flow_pull, 936 .push = mlx5_flow_push, 937 .async_action_handle_create = mlx5_flow_async_action_handle_create, 938 .async_action_handle_update = mlx5_flow_async_action_handle_update, 939 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 940 }; 941 942 /* Tunnel information. */ 943 struct mlx5_flow_tunnel_info { 944 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 945 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 946 }; 947 948 static struct mlx5_flow_tunnel_info tunnels_info[] = { 949 { 950 .tunnel = MLX5_FLOW_LAYER_VXLAN, 951 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 952 }, 953 { 954 .tunnel = MLX5_FLOW_LAYER_GENEVE, 955 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 956 }, 957 { 958 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 959 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 960 }, 961 { 962 .tunnel = MLX5_FLOW_LAYER_GRE, 963 .ptype = RTE_PTYPE_TUNNEL_GRE, 964 }, 965 { 966 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 967 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 968 }, 969 { 970 .tunnel = MLX5_FLOW_LAYER_MPLS, 971 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 972 }, 973 { 974 .tunnel = MLX5_FLOW_LAYER_NVGRE, 975 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 976 }, 977 { 978 .tunnel = MLX5_FLOW_LAYER_IPIP, 979 .ptype = RTE_PTYPE_TUNNEL_IP, 980 }, 981 { 982 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 983 .ptype = RTE_PTYPE_TUNNEL_IP, 984 }, 985 { 986 .tunnel = MLX5_FLOW_LAYER_GTP, 987 .ptype = RTE_PTYPE_TUNNEL_GTPU, 988 }, 989 }; 990 991 992 993 /** 994 * Translate tag ID to register. 995 * 996 * @param[in] dev 997 * Pointer to the Ethernet device structure. 998 * @param[in] feature 999 * The feature that request the register. 1000 * @param[in] id 1001 * The request register ID. 1002 * @param[out] error 1003 * Error description in case of any. 1004 * 1005 * @return 1006 * The request register on success, a negative errno 1007 * value otherwise and rte_errno is set. 1008 */ 1009 int 1010 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1011 enum mlx5_feature_name feature, 1012 uint32_t id, 1013 struct rte_flow_error *error) 1014 { 1015 struct mlx5_priv *priv = dev->data->dev_private; 1016 struct mlx5_sh_config *config = &priv->sh->config; 1017 enum modify_reg start_reg; 1018 bool skip_mtr_reg = false; 1019 1020 switch (feature) { 1021 case MLX5_HAIRPIN_RX: 1022 return REG_B; 1023 case MLX5_HAIRPIN_TX: 1024 return REG_A; 1025 case MLX5_METADATA_RX: 1026 switch (config->dv_xmeta_en) { 1027 case MLX5_XMETA_MODE_LEGACY: 1028 return REG_B; 1029 case MLX5_XMETA_MODE_META16: 1030 return REG_C_0; 1031 case MLX5_XMETA_MODE_META32: 1032 return REG_C_1; 1033 } 1034 break; 1035 case MLX5_METADATA_TX: 1036 return REG_A; 1037 case MLX5_METADATA_FDB: 1038 switch (config->dv_xmeta_en) { 1039 case MLX5_XMETA_MODE_LEGACY: 1040 return REG_NON; 1041 case MLX5_XMETA_MODE_META16: 1042 return REG_C_0; 1043 case MLX5_XMETA_MODE_META32: 1044 return REG_C_1; 1045 } 1046 break; 1047 case MLX5_FLOW_MARK: 1048 switch (config->dv_xmeta_en) { 1049 case MLX5_XMETA_MODE_LEGACY: 1050 return REG_NON; 1051 case MLX5_XMETA_MODE_META16: 1052 return REG_C_1; 1053 case MLX5_XMETA_MODE_META32: 1054 return REG_C_0; 1055 } 1056 break; 1057 case MLX5_MTR_ID: 1058 /* 1059 * If meter color and meter id share one register, flow match 1060 * should use the meter color register for match. 1061 */ 1062 if (priv->mtr_reg_share) 1063 return priv->mtr_color_reg; 1064 else 1065 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1066 REG_C_3; 1067 case MLX5_MTR_COLOR: 1068 case MLX5_ASO_FLOW_HIT: 1069 case MLX5_ASO_CONNTRACK: 1070 case MLX5_SAMPLE_ID: 1071 /* All features use the same REG_C. */ 1072 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1073 return priv->mtr_color_reg; 1074 case MLX5_COPY_MARK: 1075 /* 1076 * Metadata COPY_MARK register using is in meter suffix sub 1077 * flow while with meter. It's safe to share the same register. 1078 */ 1079 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1080 case MLX5_APP_TAG: 1081 /* 1082 * If meter is enable, it will engage the register for color 1083 * match and flow match. If meter color match is not using the 1084 * REG_C_2, need to skip the REG_C_x be used by meter color 1085 * match. 1086 * If meter is disable, free to use all available registers. 1087 */ 1088 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1089 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1090 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1091 if (id > (uint32_t)(REG_C_7 - start_reg)) 1092 return rte_flow_error_set(error, EINVAL, 1093 RTE_FLOW_ERROR_TYPE_ITEM, 1094 NULL, "invalid tag id"); 1095 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1096 return rte_flow_error_set(error, ENOTSUP, 1097 RTE_FLOW_ERROR_TYPE_ITEM, 1098 NULL, "unsupported tag id"); 1099 /* 1100 * This case means meter is using the REG_C_x great than 2. 1101 * Take care not to conflict with meter color REG_C_x. 1102 * If the available index REG_C_y >= REG_C_x, skip the 1103 * color register. 1104 */ 1105 if (skip_mtr_reg && priv->sh->flow_mreg_c 1106 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1107 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1108 return rte_flow_error_set(error, EINVAL, 1109 RTE_FLOW_ERROR_TYPE_ITEM, 1110 NULL, "invalid tag id"); 1111 if (priv->sh->flow_mreg_c 1112 [id + 1 + start_reg - REG_C_0] != REG_NON) 1113 return priv->sh->flow_mreg_c 1114 [id + 1 + start_reg - REG_C_0]; 1115 return rte_flow_error_set(error, ENOTSUP, 1116 RTE_FLOW_ERROR_TYPE_ITEM, 1117 NULL, "unsupported tag id"); 1118 } 1119 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1120 } 1121 MLX5_ASSERT(false); 1122 return rte_flow_error_set(error, EINVAL, 1123 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1124 NULL, "invalid feature name"); 1125 } 1126 1127 /** 1128 * Check extensive flow metadata register support. 1129 * 1130 * @param dev 1131 * Pointer to rte_eth_dev structure. 1132 * 1133 * @return 1134 * True if device supports extensive flow metadata register, otherwise false. 1135 */ 1136 bool 1137 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1138 { 1139 struct mlx5_priv *priv = dev->data->dev_private; 1140 1141 /* 1142 * Having available reg_c can be regarded inclusively as supporting 1143 * extensive flow metadata register, which could mean, 1144 * - metadata register copy action by modify header. 1145 * - 16 modify header actions is supported. 1146 * - reg_c's are preserved across different domain (FDB and NIC) on 1147 * packet loopback by flow lookup miss. 1148 */ 1149 return priv->sh->flow_mreg_c[2] != REG_NON; 1150 } 1151 1152 /** 1153 * Get the lowest priority. 1154 * 1155 * @param[in] dev 1156 * Pointer to the Ethernet device structure. 1157 * @param[in] attributes 1158 * Pointer to device flow rule attributes. 1159 * 1160 * @return 1161 * The value of lowest priority of flow. 1162 */ 1163 uint32_t 1164 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1165 const struct rte_flow_attr *attr) 1166 { 1167 struct mlx5_priv *priv = dev->data->dev_private; 1168 1169 if (!attr->group && !attr->transfer) 1170 return priv->sh->flow_max_priority - 2; 1171 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1172 } 1173 1174 /** 1175 * Calculate matcher priority of the flow. 1176 * 1177 * @param[in] dev 1178 * Pointer to the Ethernet device structure. 1179 * @param[in] attr 1180 * Pointer to device flow rule attributes. 1181 * @param[in] subpriority 1182 * The priority based on the items. 1183 * @param[in] external 1184 * Flow is user flow. 1185 * @return 1186 * The matcher priority of the flow. 1187 */ 1188 uint16_t 1189 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1190 const struct rte_flow_attr *attr, 1191 uint32_t subpriority, bool external) 1192 { 1193 uint16_t priority = (uint16_t)attr->priority; 1194 struct mlx5_priv *priv = dev->data->dev_private; 1195 1196 if (!attr->group && !attr->transfer) { 1197 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1198 priority = priv->sh->flow_max_priority - 1; 1199 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1200 } else if (!external && attr->transfer && attr->group == 0 && 1201 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1202 return (priv->sh->flow_max_priority - 1) * 3; 1203 } 1204 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1205 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1206 return priority * 3 + subpriority; 1207 } 1208 1209 /** 1210 * Verify the @p item specifications (spec, last, mask) are compatible with the 1211 * NIC capabilities. 1212 * 1213 * @param[in] item 1214 * Item specification. 1215 * @param[in] mask 1216 * @p item->mask or flow default bit-masks. 1217 * @param[in] nic_mask 1218 * Bit-masks covering supported fields by the NIC to compare with user mask. 1219 * @param[in] size 1220 * Bit-masks size in bytes. 1221 * @param[in] range_accepted 1222 * True if range of values is accepted for specific fields, false otherwise. 1223 * @param[out] error 1224 * Pointer to error structure. 1225 * 1226 * @return 1227 * 0 on success, a negative errno value otherwise and rte_errno is set. 1228 */ 1229 int 1230 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1231 const uint8_t *mask, 1232 const uint8_t *nic_mask, 1233 unsigned int size, 1234 bool range_accepted, 1235 struct rte_flow_error *error) 1236 { 1237 unsigned int i; 1238 1239 MLX5_ASSERT(nic_mask); 1240 for (i = 0; i < size; ++i) 1241 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1242 return rte_flow_error_set(error, ENOTSUP, 1243 RTE_FLOW_ERROR_TYPE_ITEM, 1244 item, 1245 "mask enables non supported" 1246 " bits"); 1247 if (!item->spec && (item->mask || item->last)) 1248 return rte_flow_error_set(error, EINVAL, 1249 RTE_FLOW_ERROR_TYPE_ITEM, item, 1250 "mask/last without a spec is not" 1251 " supported"); 1252 if (item->spec && item->last && !range_accepted) { 1253 uint8_t spec[size]; 1254 uint8_t last[size]; 1255 unsigned int i; 1256 int ret; 1257 1258 for (i = 0; i < size; ++i) { 1259 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1260 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1261 } 1262 ret = memcmp(spec, last, size); 1263 if (ret != 0) 1264 return rte_flow_error_set(error, EINVAL, 1265 RTE_FLOW_ERROR_TYPE_ITEM, 1266 item, 1267 "range is not valid"); 1268 } 1269 return 0; 1270 } 1271 1272 /** 1273 * Adjust the hash fields according to the @p flow information. 1274 * 1275 * @param[in] dev_flow. 1276 * Pointer to the mlx5_flow. 1277 * @param[in] tunnel 1278 * 1 when the hash field is for a tunnel item. 1279 * @param[in] layer_types 1280 * RTE_ETH_RSS_* types. 1281 * @param[in] hash_fields 1282 * Item hash fields. 1283 * 1284 * @return 1285 * The hash fields that should be used. 1286 */ 1287 uint64_t 1288 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1289 int tunnel __rte_unused, uint64_t layer_types, 1290 uint64_t hash_fields) 1291 { 1292 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1293 int rss_request_inner = rss_desc->level >= 2; 1294 1295 /* Check RSS hash level for tunnel. */ 1296 if (tunnel && rss_request_inner) 1297 hash_fields |= IBV_RX_HASH_INNER; 1298 else if (tunnel || rss_request_inner) 1299 return 0; 1300 #endif 1301 /* Check if requested layer matches RSS hash fields. */ 1302 if (!(rss_desc->types & layer_types)) 1303 return 0; 1304 return hash_fields; 1305 } 1306 1307 /** 1308 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1309 * if several tunnel rules are used on this queue, the tunnel ptype will be 1310 * cleared. 1311 * 1312 * @param rxq_ctrl 1313 * Rx queue to update. 1314 */ 1315 static void 1316 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1317 { 1318 unsigned int i; 1319 uint32_t tunnel_ptype = 0; 1320 1321 /* Look up for the ptype to use. */ 1322 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1323 if (!rxq_ctrl->flow_tunnels_n[i]) 1324 continue; 1325 if (!tunnel_ptype) { 1326 tunnel_ptype = tunnels_info[i].ptype; 1327 } else { 1328 tunnel_ptype = 0; 1329 break; 1330 } 1331 } 1332 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1333 } 1334 1335 /** 1336 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1337 * flow. 1338 * 1339 * @param[in] dev 1340 * Pointer to the Ethernet device structure. 1341 * @param[in] dev_handle 1342 * Pointer to device flow handle structure. 1343 */ 1344 void 1345 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1346 struct mlx5_flow_handle *dev_handle) 1347 { 1348 struct mlx5_priv *priv = dev->data->dev_private; 1349 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1350 struct mlx5_ind_table_obj *ind_tbl = NULL; 1351 unsigned int i; 1352 1353 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1354 struct mlx5_hrxq *hrxq; 1355 1356 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1357 dev_handle->rix_hrxq); 1358 if (hrxq) 1359 ind_tbl = hrxq->ind_table; 1360 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1361 struct mlx5_shared_action_rss *shared_rss; 1362 1363 shared_rss = mlx5_ipool_get 1364 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1365 dev_handle->rix_srss); 1366 if (shared_rss) 1367 ind_tbl = shared_rss->ind_tbl; 1368 } 1369 if (!ind_tbl) 1370 return; 1371 for (i = 0; i != ind_tbl->queues_n; ++i) { 1372 int idx = ind_tbl->queues[i]; 1373 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1374 1375 MLX5_ASSERT(rxq_ctrl != NULL); 1376 if (rxq_ctrl == NULL) 1377 continue; 1378 /* 1379 * To support metadata register copy on Tx loopback, 1380 * this must be always enabled (metadata may arive 1381 * from other port - not from local flows only. 1382 */ 1383 if (tunnel) { 1384 unsigned int j; 1385 1386 /* Increase the counter matching the flow. */ 1387 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1388 if ((tunnels_info[j].tunnel & 1389 dev_handle->layers) == 1390 tunnels_info[j].tunnel) { 1391 rxq_ctrl->flow_tunnels_n[j]++; 1392 break; 1393 } 1394 } 1395 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1396 } 1397 } 1398 } 1399 1400 static void 1401 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1402 { 1403 struct mlx5_priv *priv = dev->data->dev_private; 1404 struct mlx5_rxq_ctrl *rxq_ctrl; 1405 1406 if (priv->mark_enabled) 1407 return; 1408 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1409 rxq_ctrl->rxq.mark = 1; 1410 } 1411 priv->mark_enabled = 1; 1412 } 1413 1414 /** 1415 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1416 * 1417 * @param[in] dev 1418 * Pointer to the Ethernet device structure. 1419 * @param[in] flow 1420 * Pointer to flow structure. 1421 */ 1422 static void 1423 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1424 { 1425 struct mlx5_priv *priv = dev->data->dev_private; 1426 uint32_t handle_idx; 1427 struct mlx5_flow_handle *dev_handle; 1428 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1429 1430 MLX5_ASSERT(wks); 1431 if (wks->mark) 1432 flow_rxq_mark_flag_set(dev); 1433 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1434 handle_idx, dev_handle, next) 1435 flow_drv_rxq_flags_set(dev, dev_handle); 1436 } 1437 1438 /** 1439 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1440 * device flow if no other flow uses it with the same kind of request. 1441 * 1442 * @param dev 1443 * Pointer to Ethernet device. 1444 * @param[in] dev_handle 1445 * Pointer to the device flow handle structure. 1446 */ 1447 static void 1448 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1449 struct mlx5_flow_handle *dev_handle) 1450 { 1451 struct mlx5_priv *priv = dev->data->dev_private; 1452 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1453 struct mlx5_ind_table_obj *ind_tbl = NULL; 1454 unsigned int i; 1455 1456 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1457 struct mlx5_hrxq *hrxq; 1458 1459 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1460 dev_handle->rix_hrxq); 1461 if (hrxq) 1462 ind_tbl = hrxq->ind_table; 1463 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1464 struct mlx5_shared_action_rss *shared_rss; 1465 1466 shared_rss = mlx5_ipool_get 1467 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1468 dev_handle->rix_srss); 1469 if (shared_rss) 1470 ind_tbl = shared_rss->ind_tbl; 1471 } 1472 if (!ind_tbl) 1473 return; 1474 MLX5_ASSERT(dev->data->dev_started); 1475 for (i = 0; i != ind_tbl->queues_n; ++i) { 1476 int idx = ind_tbl->queues[i]; 1477 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1478 1479 MLX5_ASSERT(rxq_ctrl != NULL); 1480 if (rxq_ctrl == NULL) 1481 continue; 1482 if (tunnel) { 1483 unsigned int j; 1484 1485 /* Decrease the counter matching the flow. */ 1486 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1487 if ((tunnels_info[j].tunnel & 1488 dev_handle->layers) == 1489 tunnels_info[j].tunnel) { 1490 rxq_ctrl->flow_tunnels_n[j]--; 1491 break; 1492 } 1493 } 1494 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1495 } 1496 } 1497 } 1498 1499 /** 1500 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1501 * @p flow if no other flow uses it with the same kind of request. 1502 * 1503 * @param dev 1504 * Pointer to Ethernet device. 1505 * @param[in] flow 1506 * Pointer to the flow. 1507 */ 1508 static void 1509 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1510 { 1511 struct mlx5_priv *priv = dev->data->dev_private; 1512 uint32_t handle_idx; 1513 struct mlx5_flow_handle *dev_handle; 1514 1515 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1516 handle_idx, dev_handle, next) 1517 flow_drv_rxq_flags_trim(dev, dev_handle); 1518 } 1519 1520 /** 1521 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1522 * 1523 * @param dev 1524 * Pointer to Ethernet device. 1525 */ 1526 static void 1527 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1528 { 1529 struct mlx5_priv *priv = dev->data->dev_private; 1530 unsigned int i; 1531 1532 for (i = 0; i != priv->rxqs_n; ++i) { 1533 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1534 unsigned int j; 1535 1536 if (rxq == NULL || rxq->ctrl == NULL) 1537 continue; 1538 rxq->ctrl->rxq.mark = 0; 1539 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1540 rxq->ctrl->flow_tunnels_n[j] = 0; 1541 rxq->ctrl->rxq.tunnel = 0; 1542 } 1543 priv->mark_enabled = 0; 1544 } 1545 1546 /** 1547 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1548 * 1549 * @param[in] dev 1550 * Pointer to the Ethernet device structure. 1551 */ 1552 void 1553 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1554 { 1555 struct mlx5_priv *priv = dev->data->dev_private; 1556 unsigned int i; 1557 1558 for (i = 0; i != priv->rxqs_n; ++i) { 1559 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1560 struct mlx5_rxq_data *data; 1561 1562 if (rxq == NULL || rxq->ctrl == NULL) 1563 continue; 1564 data = &rxq->ctrl->rxq; 1565 if (!rte_flow_dynf_metadata_avail()) { 1566 data->dynf_meta = 0; 1567 data->flow_meta_mask = 0; 1568 data->flow_meta_offset = -1; 1569 data->flow_meta_port_mask = 0; 1570 } else { 1571 data->dynf_meta = 1; 1572 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1573 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1574 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1575 } 1576 } 1577 } 1578 1579 /* 1580 * return a pointer to the desired action in the list of actions. 1581 * 1582 * @param[in] actions 1583 * The list of actions to search the action in. 1584 * @param[in] action 1585 * The action to find. 1586 * 1587 * @return 1588 * Pointer to the action in the list, if found. NULL otherwise. 1589 */ 1590 const struct rte_flow_action * 1591 mlx5_flow_find_action(const struct rte_flow_action *actions, 1592 enum rte_flow_action_type action) 1593 { 1594 if (actions == NULL) 1595 return NULL; 1596 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1597 if (actions->type == action) 1598 return actions; 1599 return NULL; 1600 } 1601 1602 /* 1603 * Validate the flag action. 1604 * 1605 * @param[in] action_flags 1606 * Bit-fields that holds the actions detected until now. 1607 * @param[in] attr 1608 * Attributes of flow that includes this action. 1609 * @param[out] error 1610 * Pointer to error structure. 1611 * 1612 * @return 1613 * 0 on success, a negative errno value otherwise and rte_errno is set. 1614 */ 1615 int 1616 mlx5_flow_validate_action_flag(uint64_t action_flags, 1617 const struct rte_flow_attr *attr, 1618 struct rte_flow_error *error) 1619 { 1620 if (action_flags & MLX5_FLOW_ACTION_MARK) 1621 return rte_flow_error_set(error, EINVAL, 1622 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1623 "can't mark and flag in same flow"); 1624 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1625 return rte_flow_error_set(error, EINVAL, 1626 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1627 "can't have 2 flag" 1628 " actions in same flow"); 1629 if (attr->egress) 1630 return rte_flow_error_set(error, ENOTSUP, 1631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1632 "flag action not supported for " 1633 "egress"); 1634 return 0; 1635 } 1636 1637 /* 1638 * Validate the mark action. 1639 * 1640 * @param[in] action 1641 * Pointer to the queue action. 1642 * @param[in] action_flags 1643 * Bit-fields that holds the actions detected until now. 1644 * @param[in] attr 1645 * Attributes of flow that includes this action. 1646 * @param[out] error 1647 * Pointer to error structure. 1648 * 1649 * @return 1650 * 0 on success, a negative errno value otherwise and rte_errno is set. 1651 */ 1652 int 1653 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1654 uint64_t action_flags, 1655 const struct rte_flow_attr *attr, 1656 struct rte_flow_error *error) 1657 { 1658 const struct rte_flow_action_mark *mark = action->conf; 1659 1660 if (!mark) 1661 return rte_flow_error_set(error, EINVAL, 1662 RTE_FLOW_ERROR_TYPE_ACTION, 1663 action, 1664 "configuration cannot be null"); 1665 if (mark->id >= MLX5_FLOW_MARK_MAX) 1666 return rte_flow_error_set(error, EINVAL, 1667 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1668 &mark->id, 1669 "mark id must in 0 <= id < " 1670 RTE_STR(MLX5_FLOW_MARK_MAX)); 1671 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1672 return rte_flow_error_set(error, EINVAL, 1673 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1674 "can't flag and mark in same flow"); 1675 if (action_flags & MLX5_FLOW_ACTION_MARK) 1676 return rte_flow_error_set(error, EINVAL, 1677 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1678 "can't have 2 mark actions in same" 1679 " flow"); 1680 if (attr->egress) 1681 return rte_flow_error_set(error, ENOTSUP, 1682 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1683 "mark action not supported for " 1684 "egress"); 1685 return 0; 1686 } 1687 1688 /* 1689 * Validate the drop action. 1690 * 1691 * @param[in] action_flags 1692 * Bit-fields that holds the actions detected until now. 1693 * @param[in] attr 1694 * Attributes of flow that includes this action. 1695 * @param[out] error 1696 * Pointer to error structure. 1697 * 1698 * @return 1699 * 0 on success, a negative errno value otherwise and rte_errno is set. 1700 */ 1701 int 1702 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1703 const struct rte_flow_attr *attr, 1704 struct rte_flow_error *error) 1705 { 1706 if (attr->egress) 1707 return rte_flow_error_set(error, ENOTSUP, 1708 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1709 "drop action not supported for " 1710 "egress"); 1711 return 0; 1712 } 1713 1714 /* 1715 * Validate the queue action. 1716 * 1717 * @param[in] action 1718 * Pointer to the queue action. 1719 * @param[in] action_flags 1720 * Bit-fields that holds the actions detected until now. 1721 * @param[in] dev 1722 * Pointer to the Ethernet device structure. 1723 * @param[in] attr 1724 * Attributes of flow that includes this action. 1725 * @param[out] error 1726 * Pointer to error structure. 1727 * 1728 * @return 1729 * 0 on success, a negative errno value otherwise and rte_errno is set. 1730 */ 1731 int 1732 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1733 uint64_t action_flags, 1734 struct rte_eth_dev *dev, 1735 const struct rte_flow_attr *attr, 1736 struct rte_flow_error *error) 1737 { 1738 struct mlx5_priv *priv = dev->data->dev_private; 1739 const struct rte_flow_action_queue *queue = action->conf; 1740 1741 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1742 return rte_flow_error_set(error, EINVAL, 1743 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1744 "can't have 2 fate actions in" 1745 " same flow"); 1746 if (attr->egress) 1747 return rte_flow_error_set(error, ENOTSUP, 1748 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1749 "queue action not supported for egress."); 1750 if (mlx5_is_external_rxq(dev, queue->index)) 1751 return 0; 1752 if (!priv->rxqs_n) 1753 return rte_flow_error_set(error, EINVAL, 1754 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1755 NULL, "No Rx queues configured"); 1756 if (queue->index >= priv->rxqs_n) 1757 return rte_flow_error_set(error, EINVAL, 1758 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1759 &queue->index, 1760 "queue index out of range"); 1761 if (mlx5_rxq_get(dev, queue->index) == NULL) 1762 return rte_flow_error_set(error, EINVAL, 1763 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1764 &queue->index, 1765 "queue is not configured"); 1766 return 0; 1767 } 1768 1769 /** 1770 * Validate queue numbers for device RSS. 1771 * 1772 * @param[in] dev 1773 * Configured device. 1774 * @param[in] queues 1775 * Array of queue numbers. 1776 * @param[in] queues_n 1777 * Size of the @p queues array. 1778 * @param[out] error 1779 * On error, filled with a textual error description. 1780 * @param[out] queue_idx 1781 * On error, filled with an offending queue index in @p queues array. 1782 * 1783 * @return 1784 * 0 on success, a negative errno code on error. 1785 */ 1786 static int 1787 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1788 const uint16_t *queues, uint32_t queues_n, 1789 const char **error, uint32_t *queue_idx) 1790 { 1791 const struct mlx5_priv *priv = dev->data->dev_private; 1792 bool is_hairpin = false; 1793 bool is_ext_rss = false; 1794 uint32_t i; 1795 1796 for (i = 0; i != queues_n; ++i) { 1797 struct mlx5_rxq_ctrl *rxq_ctrl; 1798 1799 if (mlx5_is_external_rxq(dev, queues[0])) { 1800 is_ext_rss = true; 1801 continue; 1802 } 1803 if (is_ext_rss) { 1804 *error = "Combining external and regular RSS queues is not supported"; 1805 *queue_idx = i; 1806 return -ENOTSUP; 1807 } 1808 if (queues[i] >= priv->rxqs_n) { 1809 *error = "queue index out of range"; 1810 *queue_idx = i; 1811 return -EINVAL; 1812 } 1813 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 1814 if (rxq_ctrl == NULL) { 1815 *error = "queue is not configured"; 1816 *queue_idx = i; 1817 return -EINVAL; 1818 } 1819 if (i == 0 && rxq_ctrl->is_hairpin) 1820 is_hairpin = true; 1821 if (is_hairpin != rxq_ctrl->is_hairpin) { 1822 *error = "combining hairpin and regular RSS queues is not supported"; 1823 *queue_idx = i; 1824 return -ENOTSUP; 1825 } 1826 } 1827 return 0; 1828 } 1829 1830 /* 1831 * Validate the rss action. 1832 * 1833 * @param[in] dev 1834 * Pointer to the Ethernet device structure. 1835 * @param[in] action 1836 * Pointer to the queue action. 1837 * @param[out] error 1838 * Pointer to error structure. 1839 * 1840 * @return 1841 * 0 on success, a negative errno value otherwise and rte_errno is set. 1842 */ 1843 int 1844 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1845 const struct rte_flow_action *action, 1846 struct rte_flow_error *error) 1847 { 1848 struct mlx5_priv *priv = dev->data->dev_private; 1849 const struct rte_flow_action_rss *rss = action->conf; 1850 int ret; 1851 const char *message; 1852 uint32_t queue_idx; 1853 1854 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1855 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1856 return rte_flow_error_set(error, ENOTSUP, 1857 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1858 &rss->func, 1859 "RSS hash function not supported"); 1860 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1861 if (rss->level > 2) 1862 #else 1863 if (rss->level > 1) 1864 #endif 1865 return rte_flow_error_set(error, ENOTSUP, 1866 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1867 &rss->level, 1868 "tunnel RSS is not supported"); 1869 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1870 if (rss->key_len == 0 && rss->key != NULL) 1871 return rte_flow_error_set(error, ENOTSUP, 1872 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1873 &rss->key_len, 1874 "RSS hash key length 0"); 1875 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1876 return rte_flow_error_set(error, ENOTSUP, 1877 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1878 &rss->key_len, 1879 "RSS hash key too small"); 1880 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1881 return rte_flow_error_set(error, ENOTSUP, 1882 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1883 &rss->key_len, 1884 "RSS hash key too large"); 1885 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1886 return rte_flow_error_set(error, ENOTSUP, 1887 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1888 &rss->queue_num, 1889 "number of queues too large"); 1890 if (rss->types & MLX5_RSS_HF_MASK) 1891 return rte_flow_error_set(error, ENOTSUP, 1892 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1893 &rss->types, 1894 "some RSS protocols are not" 1895 " supported"); 1896 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1897 !(rss->types & RTE_ETH_RSS_IP)) 1898 return rte_flow_error_set(error, EINVAL, 1899 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1900 "L3 partial RSS requested but L3 RSS" 1901 " type not specified"); 1902 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1903 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1904 return rte_flow_error_set(error, EINVAL, 1905 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1906 "L4 partial RSS requested but L4 RSS" 1907 " type not specified"); 1908 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 1909 return rte_flow_error_set(error, EINVAL, 1910 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1911 NULL, "No Rx queues configured"); 1912 if (!rss->queue_num) 1913 return rte_flow_error_set(error, EINVAL, 1914 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1915 NULL, "No queues configured"); 1916 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1917 &message, &queue_idx); 1918 if (ret != 0) { 1919 return rte_flow_error_set(error, -ret, 1920 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1921 &rss->queue[queue_idx], message); 1922 } 1923 return 0; 1924 } 1925 1926 /* 1927 * Validate the rss action. 1928 * 1929 * @param[in] action 1930 * Pointer to the queue action. 1931 * @param[in] action_flags 1932 * Bit-fields that holds the actions detected until now. 1933 * @param[in] dev 1934 * Pointer to the Ethernet device structure. 1935 * @param[in] attr 1936 * Attributes of flow that includes this action. 1937 * @param[in] item_flags 1938 * Items that were detected. 1939 * @param[out] error 1940 * Pointer to error structure. 1941 * 1942 * @return 1943 * 0 on success, a negative errno value otherwise and rte_errno is set. 1944 */ 1945 int 1946 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1947 uint64_t action_flags, 1948 struct rte_eth_dev *dev, 1949 const struct rte_flow_attr *attr, 1950 uint64_t item_flags, 1951 struct rte_flow_error *error) 1952 { 1953 const struct rte_flow_action_rss *rss = action->conf; 1954 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1955 int ret; 1956 1957 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1958 return rte_flow_error_set(error, EINVAL, 1959 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1960 "can't have 2 fate actions" 1961 " in same flow"); 1962 ret = mlx5_validate_action_rss(dev, action, error); 1963 if (ret) 1964 return ret; 1965 if (attr->egress) 1966 return rte_flow_error_set(error, ENOTSUP, 1967 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1968 "rss action not supported for " 1969 "egress"); 1970 if (rss->level > 1 && !tunnel) 1971 return rte_flow_error_set(error, EINVAL, 1972 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1973 "inner RSS is not supported for " 1974 "non-tunnel flows"); 1975 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1976 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1979 "RSS on eCPRI is not supported now"); 1980 } 1981 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1982 !(item_flags & 1983 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1984 rss->level > 1) 1985 return rte_flow_error_set(error, EINVAL, 1986 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1987 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1988 return 0; 1989 } 1990 1991 /* 1992 * Validate the default miss action. 1993 * 1994 * @param[in] action_flags 1995 * Bit-fields that holds the actions detected until now. 1996 * @param[out] error 1997 * Pointer to error structure. 1998 * 1999 * @return 2000 * 0 on success, a negative errno value otherwise and rte_errno is set. 2001 */ 2002 int 2003 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2004 const struct rte_flow_attr *attr, 2005 struct rte_flow_error *error) 2006 { 2007 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2008 return rte_flow_error_set(error, EINVAL, 2009 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2010 "can't have 2 fate actions in" 2011 " same flow"); 2012 if (attr->egress) 2013 return rte_flow_error_set(error, ENOTSUP, 2014 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2015 "default miss action not supported " 2016 "for egress"); 2017 if (attr->group) 2018 return rte_flow_error_set(error, ENOTSUP, 2019 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2020 "only group 0 is supported"); 2021 if (attr->transfer) 2022 return rte_flow_error_set(error, ENOTSUP, 2023 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2024 NULL, "transfer is not supported"); 2025 return 0; 2026 } 2027 2028 /* 2029 * Validate the count action. 2030 * 2031 * @param[in] dev 2032 * Pointer to the Ethernet device structure. 2033 * @param[in] attr 2034 * Attributes of flow that includes this action. 2035 * @param[out] error 2036 * Pointer to error structure. 2037 * 2038 * @return 2039 * 0 on success, a negative errno value otherwise and rte_errno is set. 2040 */ 2041 int 2042 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2043 const struct rte_flow_attr *attr, 2044 struct rte_flow_error *error) 2045 { 2046 if (attr->egress) 2047 return rte_flow_error_set(error, ENOTSUP, 2048 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2049 "count action not supported for " 2050 "egress"); 2051 return 0; 2052 } 2053 2054 /* 2055 * Validate the ASO CT action. 2056 * 2057 * @param[in] dev 2058 * Pointer to the Ethernet device structure. 2059 * @param[in] conntrack 2060 * Pointer to the CT action profile. 2061 * @param[out] error 2062 * Pointer to error structure. 2063 * 2064 * @return 2065 * 0 on success, a negative errno value otherwise and rte_errno is set. 2066 */ 2067 int 2068 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2069 const struct rte_flow_action_conntrack *conntrack, 2070 struct rte_flow_error *error) 2071 { 2072 RTE_SET_USED(dev); 2073 2074 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2075 return rte_flow_error_set(error, EINVAL, 2076 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2077 "Invalid CT state"); 2078 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2079 return rte_flow_error_set(error, EINVAL, 2080 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2081 "Invalid last TCP packet flag"); 2082 return 0; 2083 } 2084 2085 /** 2086 * Verify the @p attributes will be correctly understood by the NIC and store 2087 * them in the @p flow if everything is correct. 2088 * 2089 * @param[in] dev 2090 * Pointer to the Ethernet device structure. 2091 * @param[in] attributes 2092 * Pointer to flow attributes 2093 * @param[out] error 2094 * Pointer to error structure. 2095 * 2096 * @return 2097 * 0 on success, a negative errno value otherwise and rte_errno is set. 2098 */ 2099 int 2100 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2101 const struct rte_flow_attr *attributes, 2102 struct rte_flow_error *error) 2103 { 2104 struct mlx5_priv *priv = dev->data->dev_private; 2105 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2106 2107 if (attributes->group) 2108 return rte_flow_error_set(error, ENOTSUP, 2109 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2110 NULL, "groups is not supported"); 2111 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2112 attributes->priority >= priority_max) 2113 return rte_flow_error_set(error, ENOTSUP, 2114 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2115 NULL, "priority out of range"); 2116 if (attributes->egress) 2117 return rte_flow_error_set(error, ENOTSUP, 2118 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2119 "egress is not supported"); 2120 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2121 return rte_flow_error_set(error, ENOTSUP, 2122 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2123 NULL, "transfer is not supported"); 2124 if (!attributes->ingress) 2125 return rte_flow_error_set(error, EINVAL, 2126 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2127 NULL, 2128 "ingress attribute is mandatory"); 2129 return 0; 2130 } 2131 2132 /** 2133 * Validate ICMP6 item. 2134 * 2135 * @param[in] item 2136 * Item specification. 2137 * @param[in] item_flags 2138 * Bit-fields that holds the items detected until now. 2139 * @param[in] ext_vlan_sup 2140 * Whether extended VLAN features are supported or not. 2141 * @param[out] error 2142 * Pointer to error structure. 2143 * 2144 * @return 2145 * 0 on success, a negative errno value otherwise and rte_errno is set. 2146 */ 2147 int 2148 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2149 uint64_t item_flags, 2150 uint8_t target_protocol, 2151 struct rte_flow_error *error) 2152 { 2153 const struct rte_flow_item_icmp6 *mask = item->mask; 2154 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2155 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2156 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2157 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2158 MLX5_FLOW_LAYER_OUTER_L4; 2159 int ret; 2160 2161 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2162 return rte_flow_error_set(error, EINVAL, 2163 RTE_FLOW_ERROR_TYPE_ITEM, item, 2164 "protocol filtering not compatible" 2165 " with ICMP6 layer"); 2166 if (!(item_flags & l3m)) 2167 return rte_flow_error_set(error, EINVAL, 2168 RTE_FLOW_ERROR_TYPE_ITEM, item, 2169 "IPv6 is mandatory to filter on" 2170 " ICMP6"); 2171 if (item_flags & l4m) 2172 return rte_flow_error_set(error, EINVAL, 2173 RTE_FLOW_ERROR_TYPE_ITEM, item, 2174 "multiple L4 layers not supported"); 2175 if (!mask) 2176 mask = &rte_flow_item_icmp6_mask; 2177 ret = mlx5_flow_item_acceptable 2178 (item, (const uint8_t *)mask, 2179 (const uint8_t *)&rte_flow_item_icmp6_mask, 2180 sizeof(struct rte_flow_item_icmp6), 2181 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2182 if (ret < 0) 2183 return ret; 2184 return 0; 2185 } 2186 2187 /** 2188 * Validate ICMP item. 2189 * 2190 * @param[in] item 2191 * Item specification. 2192 * @param[in] item_flags 2193 * Bit-fields that holds the items detected until now. 2194 * @param[out] error 2195 * Pointer to error structure. 2196 * 2197 * @return 2198 * 0 on success, a negative errno value otherwise and rte_errno is set. 2199 */ 2200 int 2201 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2202 uint64_t item_flags, 2203 uint8_t target_protocol, 2204 struct rte_flow_error *error) 2205 { 2206 const struct rte_flow_item_icmp *mask = item->mask; 2207 const struct rte_flow_item_icmp nic_mask = { 2208 .hdr.icmp_type = 0xff, 2209 .hdr.icmp_code = 0xff, 2210 .hdr.icmp_ident = RTE_BE16(0xffff), 2211 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2212 }; 2213 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2214 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2215 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2216 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2217 MLX5_FLOW_LAYER_OUTER_L4; 2218 int ret; 2219 2220 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2221 return rte_flow_error_set(error, EINVAL, 2222 RTE_FLOW_ERROR_TYPE_ITEM, item, 2223 "protocol filtering not compatible" 2224 " with ICMP layer"); 2225 if (!(item_flags & l3m)) 2226 return rte_flow_error_set(error, EINVAL, 2227 RTE_FLOW_ERROR_TYPE_ITEM, item, 2228 "IPv4 is mandatory to filter" 2229 " on ICMP"); 2230 if (item_flags & l4m) 2231 return rte_flow_error_set(error, EINVAL, 2232 RTE_FLOW_ERROR_TYPE_ITEM, item, 2233 "multiple L4 layers not supported"); 2234 if (!mask) 2235 mask = &nic_mask; 2236 ret = mlx5_flow_item_acceptable 2237 (item, (const uint8_t *)mask, 2238 (const uint8_t *)&nic_mask, 2239 sizeof(struct rte_flow_item_icmp), 2240 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2241 if (ret < 0) 2242 return ret; 2243 return 0; 2244 } 2245 2246 /** 2247 * Validate Ethernet item. 2248 * 2249 * @param[in] item 2250 * Item specification. 2251 * @param[in] item_flags 2252 * Bit-fields that holds the items detected until now. 2253 * @param[out] error 2254 * Pointer to error structure. 2255 * 2256 * @return 2257 * 0 on success, a negative errno value otherwise and rte_errno is set. 2258 */ 2259 int 2260 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2261 uint64_t item_flags, bool ext_vlan_sup, 2262 struct rte_flow_error *error) 2263 { 2264 const struct rte_flow_item_eth *mask = item->mask; 2265 const struct rte_flow_item_eth nic_mask = { 2266 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2267 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2268 .type = RTE_BE16(0xffff), 2269 .has_vlan = ext_vlan_sup ? 1 : 0, 2270 }; 2271 int ret; 2272 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2273 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2274 MLX5_FLOW_LAYER_OUTER_L2; 2275 2276 if (item_flags & ethm) 2277 return rte_flow_error_set(error, ENOTSUP, 2278 RTE_FLOW_ERROR_TYPE_ITEM, item, 2279 "multiple L2 layers not supported"); 2280 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2281 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2282 return rte_flow_error_set(error, EINVAL, 2283 RTE_FLOW_ERROR_TYPE_ITEM, item, 2284 "L2 layer should not follow " 2285 "L3 layers"); 2286 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2287 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2288 return rte_flow_error_set(error, EINVAL, 2289 RTE_FLOW_ERROR_TYPE_ITEM, item, 2290 "L2 layer should not follow VLAN"); 2291 if (item_flags & MLX5_FLOW_LAYER_GTP) 2292 return rte_flow_error_set(error, EINVAL, 2293 RTE_FLOW_ERROR_TYPE_ITEM, item, 2294 "L2 layer should not follow GTP"); 2295 if (!mask) 2296 mask = &rte_flow_item_eth_mask; 2297 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2298 (const uint8_t *)&nic_mask, 2299 sizeof(struct rte_flow_item_eth), 2300 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2301 return ret; 2302 } 2303 2304 /** 2305 * Validate VLAN item. 2306 * 2307 * @param[in] item 2308 * Item specification. 2309 * @param[in] item_flags 2310 * Bit-fields that holds the items detected until now. 2311 * @param[in] dev 2312 * Ethernet device flow is being created on. 2313 * @param[out] error 2314 * Pointer to error structure. 2315 * 2316 * @return 2317 * 0 on success, a negative errno value otherwise and rte_errno is set. 2318 */ 2319 int 2320 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2321 uint64_t item_flags, 2322 struct rte_eth_dev *dev, 2323 struct rte_flow_error *error) 2324 { 2325 const struct rte_flow_item_vlan *spec = item->spec; 2326 const struct rte_flow_item_vlan *mask = item->mask; 2327 const struct rte_flow_item_vlan nic_mask = { 2328 .tci = RTE_BE16(UINT16_MAX), 2329 .inner_type = RTE_BE16(UINT16_MAX), 2330 }; 2331 uint16_t vlan_tag = 0; 2332 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2333 int ret; 2334 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2335 MLX5_FLOW_LAYER_INNER_L4) : 2336 (MLX5_FLOW_LAYER_OUTER_L3 | 2337 MLX5_FLOW_LAYER_OUTER_L4); 2338 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2339 MLX5_FLOW_LAYER_OUTER_VLAN; 2340 2341 if (item_flags & vlanm) 2342 return rte_flow_error_set(error, EINVAL, 2343 RTE_FLOW_ERROR_TYPE_ITEM, item, 2344 "multiple VLAN layers not supported"); 2345 else if ((item_flags & l34m) != 0) 2346 return rte_flow_error_set(error, EINVAL, 2347 RTE_FLOW_ERROR_TYPE_ITEM, item, 2348 "VLAN cannot follow L3/L4 layer"); 2349 if (!mask) 2350 mask = &rte_flow_item_vlan_mask; 2351 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2352 (const uint8_t *)&nic_mask, 2353 sizeof(struct rte_flow_item_vlan), 2354 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2355 if (ret) 2356 return ret; 2357 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2358 struct mlx5_priv *priv = dev->data->dev_private; 2359 2360 if (priv->vmwa_context) { 2361 /* 2362 * Non-NULL context means we have a virtual machine 2363 * and SR-IOV enabled, we have to create VLAN interface 2364 * to make hypervisor to setup E-Switch vport 2365 * context correctly. We avoid creating the multiple 2366 * VLAN interfaces, so we cannot support VLAN tag mask. 2367 */ 2368 return rte_flow_error_set(error, EINVAL, 2369 RTE_FLOW_ERROR_TYPE_ITEM, 2370 item, 2371 "VLAN tag mask is not" 2372 " supported in virtual" 2373 " environment"); 2374 } 2375 } 2376 if (spec) { 2377 vlan_tag = spec->tci; 2378 vlan_tag &= mask->tci; 2379 } 2380 /* 2381 * From verbs perspective an empty VLAN is equivalent 2382 * to a packet without VLAN layer. 2383 */ 2384 if (!vlan_tag) 2385 return rte_flow_error_set(error, EINVAL, 2386 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2387 item->spec, 2388 "VLAN cannot be empty"); 2389 return 0; 2390 } 2391 2392 /** 2393 * Validate IPV4 item. 2394 * 2395 * @param[in] item 2396 * Item specification. 2397 * @param[in] item_flags 2398 * Bit-fields that holds the items detected until now. 2399 * @param[in] last_item 2400 * Previous validated item in the pattern items. 2401 * @param[in] ether_type 2402 * Type in the ethernet layer header (including dot1q). 2403 * @param[in] acc_mask 2404 * Acceptable mask, if NULL default internal default mask 2405 * will be used to check whether item fields are supported. 2406 * @param[in] range_accepted 2407 * True if range of values is accepted for specific fields, false otherwise. 2408 * @param[out] error 2409 * Pointer to error structure. 2410 * 2411 * @return 2412 * 0 on success, a negative errno value otherwise and rte_errno is set. 2413 */ 2414 int 2415 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2416 uint64_t item_flags, 2417 uint64_t last_item, 2418 uint16_t ether_type, 2419 const struct rte_flow_item_ipv4 *acc_mask, 2420 bool range_accepted, 2421 struct rte_flow_error *error) 2422 { 2423 const struct rte_flow_item_ipv4 *mask = item->mask; 2424 const struct rte_flow_item_ipv4 *spec = item->spec; 2425 const struct rte_flow_item_ipv4 nic_mask = { 2426 .hdr = { 2427 .src_addr = RTE_BE32(0xffffffff), 2428 .dst_addr = RTE_BE32(0xffffffff), 2429 .type_of_service = 0xff, 2430 .next_proto_id = 0xff, 2431 }, 2432 }; 2433 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2434 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2435 MLX5_FLOW_LAYER_OUTER_L3; 2436 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2437 MLX5_FLOW_LAYER_OUTER_L4; 2438 int ret; 2439 uint8_t next_proto = 0xFF; 2440 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2441 MLX5_FLOW_LAYER_OUTER_VLAN | 2442 MLX5_FLOW_LAYER_INNER_VLAN); 2443 2444 if ((last_item & l2_vlan) && ether_type && 2445 ether_type != RTE_ETHER_TYPE_IPV4) 2446 return rte_flow_error_set(error, EINVAL, 2447 RTE_FLOW_ERROR_TYPE_ITEM, item, 2448 "IPv4 cannot follow L2/VLAN layer " 2449 "which ether type is not IPv4"); 2450 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2451 if (mask && spec) 2452 next_proto = mask->hdr.next_proto_id & 2453 spec->hdr.next_proto_id; 2454 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2455 return rte_flow_error_set(error, EINVAL, 2456 RTE_FLOW_ERROR_TYPE_ITEM, 2457 item, 2458 "multiple tunnel " 2459 "not supported"); 2460 } 2461 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2462 return rte_flow_error_set(error, EINVAL, 2463 RTE_FLOW_ERROR_TYPE_ITEM, item, 2464 "wrong tunnel type - IPv6 specified " 2465 "but IPv4 item provided"); 2466 if (item_flags & l3m) 2467 return rte_flow_error_set(error, ENOTSUP, 2468 RTE_FLOW_ERROR_TYPE_ITEM, item, 2469 "multiple L3 layers not supported"); 2470 else if (item_flags & l4m) 2471 return rte_flow_error_set(error, EINVAL, 2472 RTE_FLOW_ERROR_TYPE_ITEM, item, 2473 "L3 cannot follow an L4 layer."); 2474 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2475 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2476 return rte_flow_error_set(error, EINVAL, 2477 RTE_FLOW_ERROR_TYPE_ITEM, item, 2478 "L3 cannot follow an NVGRE layer."); 2479 if (!mask) 2480 mask = &rte_flow_item_ipv4_mask; 2481 else if (mask->hdr.next_proto_id != 0 && 2482 mask->hdr.next_proto_id != 0xff) 2483 return rte_flow_error_set(error, EINVAL, 2484 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2485 "partial mask is not supported" 2486 " for protocol"); 2487 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2488 acc_mask ? (const uint8_t *)acc_mask 2489 : (const uint8_t *)&nic_mask, 2490 sizeof(struct rte_flow_item_ipv4), 2491 range_accepted, error); 2492 if (ret < 0) 2493 return ret; 2494 return 0; 2495 } 2496 2497 /** 2498 * Validate IPV6 item. 2499 * 2500 * @param[in] item 2501 * Item specification. 2502 * @param[in] item_flags 2503 * Bit-fields that holds the items detected until now. 2504 * @param[in] last_item 2505 * Previous validated item in the pattern items. 2506 * @param[in] ether_type 2507 * Type in the ethernet layer header (including dot1q). 2508 * @param[in] acc_mask 2509 * Acceptable mask, if NULL default internal default mask 2510 * will be used to check whether item fields are supported. 2511 * @param[out] error 2512 * Pointer to error structure. 2513 * 2514 * @return 2515 * 0 on success, a negative errno value otherwise and rte_errno is set. 2516 */ 2517 int 2518 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2519 uint64_t item_flags, 2520 uint64_t last_item, 2521 uint16_t ether_type, 2522 const struct rte_flow_item_ipv6 *acc_mask, 2523 struct rte_flow_error *error) 2524 { 2525 const struct rte_flow_item_ipv6 *mask = item->mask; 2526 const struct rte_flow_item_ipv6 *spec = item->spec; 2527 const struct rte_flow_item_ipv6 nic_mask = { 2528 .hdr = { 2529 .src_addr = 2530 "\xff\xff\xff\xff\xff\xff\xff\xff" 2531 "\xff\xff\xff\xff\xff\xff\xff\xff", 2532 .dst_addr = 2533 "\xff\xff\xff\xff\xff\xff\xff\xff" 2534 "\xff\xff\xff\xff\xff\xff\xff\xff", 2535 .vtc_flow = RTE_BE32(0xffffffff), 2536 .proto = 0xff, 2537 }, 2538 }; 2539 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2540 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2541 MLX5_FLOW_LAYER_OUTER_L3; 2542 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2543 MLX5_FLOW_LAYER_OUTER_L4; 2544 int ret; 2545 uint8_t next_proto = 0xFF; 2546 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2547 MLX5_FLOW_LAYER_OUTER_VLAN | 2548 MLX5_FLOW_LAYER_INNER_VLAN); 2549 2550 if ((last_item & l2_vlan) && ether_type && 2551 ether_type != RTE_ETHER_TYPE_IPV6) 2552 return rte_flow_error_set(error, EINVAL, 2553 RTE_FLOW_ERROR_TYPE_ITEM, item, 2554 "IPv6 cannot follow L2/VLAN layer " 2555 "which ether type is not IPv6"); 2556 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2557 next_proto = spec->hdr.proto; 2558 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2559 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2560 return rte_flow_error_set(error, EINVAL, 2561 RTE_FLOW_ERROR_TYPE_ITEM, 2562 item, 2563 "multiple tunnel " 2564 "not supported"); 2565 } 2566 if (next_proto == IPPROTO_HOPOPTS || 2567 next_proto == IPPROTO_ROUTING || 2568 next_proto == IPPROTO_FRAGMENT || 2569 next_proto == IPPROTO_ESP || 2570 next_proto == IPPROTO_AH || 2571 next_proto == IPPROTO_DSTOPTS) 2572 return rte_flow_error_set(error, EINVAL, 2573 RTE_FLOW_ERROR_TYPE_ITEM, item, 2574 "IPv6 proto (next header) should " 2575 "not be set as extension header"); 2576 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2577 return rte_flow_error_set(error, EINVAL, 2578 RTE_FLOW_ERROR_TYPE_ITEM, item, 2579 "wrong tunnel type - IPv4 specified " 2580 "but IPv6 item provided"); 2581 if (item_flags & l3m) 2582 return rte_flow_error_set(error, ENOTSUP, 2583 RTE_FLOW_ERROR_TYPE_ITEM, item, 2584 "multiple L3 layers not supported"); 2585 else if (item_flags & l4m) 2586 return rte_flow_error_set(error, EINVAL, 2587 RTE_FLOW_ERROR_TYPE_ITEM, item, 2588 "L3 cannot follow an L4 layer."); 2589 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2590 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2591 return rte_flow_error_set(error, EINVAL, 2592 RTE_FLOW_ERROR_TYPE_ITEM, item, 2593 "L3 cannot follow an NVGRE layer."); 2594 if (!mask) 2595 mask = &rte_flow_item_ipv6_mask; 2596 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2597 acc_mask ? (const uint8_t *)acc_mask 2598 : (const uint8_t *)&nic_mask, 2599 sizeof(struct rte_flow_item_ipv6), 2600 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2601 if (ret < 0) 2602 return ret; 2603 return 0; 2604 } 2605 2606 /** 2607 * Validate UDP item. 2608 * 2609 * @param[in] item 2610 * Item specification. 2611 * @param[in] item_flags 2612 * Bit-fields that holds the items detected until now. 2613 * @param[in] target_protocol 2614 * The next protocol in the previous item. 2615 * @param[in] flow_mask 2616 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2617 * @param[out] error 2618 * Pointer to error structure. 2619 * 2620 * @return 2621 * 0 on success, a negative errno value otherwise and rte_errno is set. 2622 */ 2623 int 2624 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2625 uint64_t item_flags, 2626 uint8_t target_protocol, 2627 struct rte_flow_error *error) 2628 { 2629 const struct rte_flow_item_udp *mask = item->mask; 2630 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2631 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2632 MLX5_FLOW_LAYER_OUTER_L3; 2633 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2634 MLX5_FLOW_LAYER_OUTER_L4; 2635 int ret; 2636 2637 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2638 return rte_flow_error_set(error, EINVAL, 2639 RTE_FLOW_ERROR_TYPE_ITEM, item, 2640 "protocol filtering not compatible" 2641 " with UDP layer"); 2642 if (!(item_flags & l3m)) 2643 return rte_flow_error_set(error, EINVAL, 2644 RTE_FLOW_ERROR_TYPE_ITEM, item, 2645 "L3 is mandatory to filter on L4"); 2646 if (item_flags & l4m) 2647 return rte_flow_error_set(error, EINVAL, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "multiple L4 layers not supported"); 2650 if (!mask) 2651 mask = &rte_flow_item_udp_mask; 2652 ret = mlx5_flow_item_acceptable 2653 (item, (const uint8_t *)mask, 2654 (const uint8_t *)&rte_flow_item_udp_mask, 2655 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2656 error); 2657 if (ret < 0) 2658 return ret; 2659 return 0; 2660 } 2661 2662 /** 2663 * Validate TCP item. 2664 * 2665 * @param[in] item 2666 * Item specification. 2667 * @param[in] item_flags 2668 * Bit-fields that holds the items detected until now. 2669 * @param[in] target_protocol 2670 * The next protocol in the previous item. 2671 * @param[out] error 2672 * Pointer to error structure. 2673 * 2674 * @return 2675 * 0 on success, a negative errno value otherwise and rte_errno is set. 2676 */ 2677 int 2678 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2679 uint64_t item_flags, 2680 uint8_t target_protocol, 2681 const struct rte_flow_item_tcp *flow_mask, 2682 struct rte_flow_error *error) 2683 { 2684 const struct rte_flow_item_tcp *mask = item->mask; 2685 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2686 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2687 MLX5_FLOW_LAYER_OUTER_L3; 2688 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2689 MLX5_FLOW_LAYER_OUTER_L4; 2690 int ret; 2691 2692 MLX5_ASSERT(flow_mask); 2693 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2694 return rte_flow_error_set(error, EINVAL, 2695 RTE_FLOW_ERROR_TYPE_ITEM, item, 2696 "protocol filtering not compatible" 2697 " with TCP layer"); 2698 if (!(item_flags & l3m)) 2699 return rte_flow_error_set(error, EINVAL, 2700 RTE_FLOW_ERROR_TYPE_ITEM, item, 2701 "L3 is mandatory to filter on L4"); 2702 if (item_flags & l4m) 2703 return rte_flow_error_set(error, EINVAL, 2704 RTE_FLOW_ERROR_TYPE_ITEM, item, 2705 "multiple L4 layers not supported"); 2706 if (!mask) 2707 mask = &rte_flow_item_tcp_mask; 2708 ret = mlx5_flow_item_acceptable 2709 (item, (const uint8_t *)mask, 2710 (const uint8_t *)flow_mask, 2711 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2712 error); 2713 if (ret < 0) 2714 return ret; 2715 return 0; 2716 } 2717 2718 /** 2719 * Validate VXLAN item. 2720 * 2721 * @param[in] dev 2722 * Pointer to the Ethernet device structure. 2723 * @param[in] udp_dport 2724 * UDP destination port 2725 * @param[in] item 2726 * Item specification. 2727 * @param[in] item_flags 2728 * Bit-fields that holds the items detected until now. 2729 * @param[in] attr 2730 * Flow rule attributes. 2731 * @param[out] error 2732 * Pointer to error structure. 2733 * 2734 * @return 2735 * 0 on success, a negative errno value otherwise and rte_errno is set. 2736 */ 2737 int 2738 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2739 uint16_t udp_dport, 2740 const struct rte_flow_item *item, 2741 uint64_t item_flags, 2742 const struct rte_flow_attr *attr, 2743 struct rte_flow_error *error) 2744 { 2745 const struct rte_flow_item_vxlan *spec = item->spec; 2746 const struct rte_flow_item_vxlan *mask = item->mask; 2747 int ret; 2748 struct mlx5_priv *priv = dev->data->dev_private; 2749 union vni { 2750 uint32_t vlan_id; 2751 uint8_t vni[4]; 2752 } id = { .vlan_id = 0, }; 2753 const struct rte_flow_item_vxlan nic_mask = { 2754 .vni = "\xff\xff\xff", 2755 .rsvd1 = 0xff, 2756 }; 2757 const struct rte_flow_item_vxlan *valid_mask; 2758 2759 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2760 return rte_flow_error_set(error, ENOTSUP, 2761 RTE_FLOW_ERROR_TYPE_ITEM, item, 2762 "multiple tunnel layers not" 2763 " supported"); 2764 valid_mask = &rte_flow_item_vxlan_mask; 2765 /* 2766 * Verify only UDPv4 is present as defined in 2767 * https://tools.ietf.org/html/rfc7348 2768 */ 2769 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2770 return rte_flow_error_set(error, EINVAL, 2771 RTE_FLOW_ERROR_TYPE_ITEM, item, 2772 "no outer UDP layer found"); 2773 if (!mask) 2774 mask = &rte_flow_item_vxlan_mask; 2775 2776 if (priv->sh->steering_format_version != 2777 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2778 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2779 /* FDB domain & NIC domain non-zero group */ 2780 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2781 valid_mask = &nic_mask; 2782 /* Group zero in NIC domain */ 2783 if (!attr->group && !attr->transfer && 2784 priv->sh->tunnel_header_0_1) 2785 valid_mask = &nic_mask; 2786 } 2787 ret = mlx5_flow_item_acceptable 2788 (item, (const uint8_t *)mask, 2789 (const uint8_t *)valid_mask, 2790 sizeof(struct rte_flow_item_vxlan), 2791 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2792 if (ret < 0) 2793 return ret; 2794 if (spec) { 2795 memcpy(&id.vni[1], spec->vni, 3); 2796 memcpy(&id.vni[1], mask->vni, 3); 2797 } 2798 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2799 return rte_flow_error_set(error, ENOTSUP, 2800 RTE_FLOW_ERROR_TYPE_ITEM, item, 2801 "VXLAN tunnel must be fully defined"); 2802 return 0; 2803 } 2804 2805 /** 2806 * Validate VXLAN_GPE item. 2807 * 2808 * @param[in] item 2809 * Item specification. 2810 * @param[in] item_flags 2811 * Bit-fields that holds the items detected until now. 2812 * @param[in] priv 2813 * Pointer to the private data structure. 2814 * @param[in] target_protocol 2815 * The next protocol in the previous item. 2816 * @param[out] error 2817 * Pointer to error structure. 2818 * 2819 * @return 2820 * 0 on success, a negative errno value otherwise and rte_errno is set. 2821 */ 2822 int 2823 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2824 uint64_t item_flags, 2825 struct rte_eth_dev *dev, 2826 struct rte_flow_error *error) 2827 { 2828 struct mlx5_priv *priv = dev->data->dev_private; 2829 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2830 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2831 int ret; 2832 union vni { 2833 uint32_t vlan_id; 2834 uint8_t vni[4]; 2835 } id = { .vlan_id = 0, }; 2836 2837 if (!priv->sh->config.l3_vxlan_en) 2838 return rte_flow_error_set(error, ENOTSUP, 2839 RTE_FLOW_ERROR_TYPE_ITEM, item, 2840 "L3 VXLAN is not enabled by device" 2841 " parameter and/or not configured in" 2842 " firmware"); 2843 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2844 return rte_flow_error_set(error, ENOTSUP, 2845 RTE_FLOW_ERROR_TYPE_ITEM, item, 2846 "multiple tunnel layers not" 2847 " supported"); 2848 /* 2849 * Verify only UDPv4 is present as defined in 2850 * https://tools.ietf.org/html/rfc7348 2851 */ 2852 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2853 return rte_flow_error_set(error, EINVAL, 2854 RTE_FLOW_ERROR_TYPE_ITEM, item, 2855 "no outer UDP layer found"); 2856 if (!mask) 2857 mask = &rte_flow_item_vxlan_gpe_mask; 2858 ret = mlx5_flow_item_acceptable 2859 (item, (const uint8_t *)mask, 2860 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2861 sizeof(struct rte_flow_item_vxlan_gpe), 2862 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2863 if (ret < 0) 2864 return ret; 2865 if (spec) { 2866 if (spec->protocol) 2867 return rte_flow_error_set(error, ENOTSUP, 2868 RTE_FLOW_ERROR_TYPE_ITEM, 2869 item, 2870 "VxLAN-GPE protocol" 2871 " not supported"); 2872 memcpy(&id.vni[1], spec->vni, 3); 2873 memcpy(&id.vni[1], mask->vni, 3); 2874 } 2875 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2876 return rte_flow_error_set(error, ENOTSUP, 2877 RTE_FLOW_ERROR_TYPE_ITEM, item, 2878 "VXLAN-GPE tunnel must be fully" 2879 " defined"); 2880 return 0; 2881 } 2882 /** 2883 * Validate GRE Key item. 2884 * 2885 * @param[in] item 2886 * Item specification. 2887 * @param[in] item_flags 2888 * Bit flags to mark detected items. 2889 * @param[in] gre_item 2890 * Pointer to gre_item 2891 * @param[out] error 2892 * Pointer to error structure. 2893 * 2894 * @return 2895 * 0 on success, a negative errno value otherwise and rte_errno is set. 2896 */ 2897 int 2898 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2899 uint64_t item_flags, 2900 const struct rte_flow_item *gre_item, 2901 struct rte_flow_error *error) 2902 { 2903 const rte_be32_t *mask = item->mask; 2904 int ret = 0; 2905 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2906 const struct rte_flow_item_gre *gre_spec; 2907 const struct rte_flow_item_gre *gre_mask; 2908 2909 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2910 return rte_flow_error_set(error, ENOTSUP, 2911 RTE_FLOW_ERROR_TYPE_ITEM, item, 2912 "Multiple GRE key not support"); 2913 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2914 return rte_flow_error_set(error, ENOTSUP, 2915 RTE_FLOW_ERROR_TYPE_ITEM, item, 2916 "No preceding GRE header"); 2917 if (item_flags & MLX5_FLOW_LAYER_INNER) 2918 return rte_flow_error_set(error, ENOTSUP, 2919 RTE_FLOW_ERROR_TYPE_ITEM, item, 2920 "GRE key following a wrong item"); 2921 gre_mask = gre_item->mask; 2922 if (!gre_mask) 2923 gre_mask = &rte_flow_item_gre_mask; 2924 gre_spec = gre_item->spec; 2925 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2926 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2927 return rte_flow_error_set(error, EINVAL, 2928 RTE_FLOW_ERROR_TYPE_ITEM, item, 2929 "Key bit must be on"); 2930 2931 if (!mask) 2932 mask = &gre_key_default_mask; 2933 ret = mlx5_flow_item_acceptable 2934 (item, (const uint8_t *)mask, 2935 (const uint8_t *)&gre_key_default_mask, 2936 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2937 return ret; 2938 } 2939 2940 /** 2941 * Validate GRE optional item. 2942 * 2943 * @param[in] dev 2944 * Pointer to the Ethernet device structure. 2945 * @param[in] item 2946 * Item specification. 2947 * @param[in] item_flags 2948 * Bit flags to mark detected items. 2949 * @param[in] attr 2950 * Flow rule attributes. 2951 * @param[in] gre_item 2952 * Pointer to gre_item 2953 * @param[out] error 2954 * Pointer to error structure. 2955 * 2956 * @return 2957 * 0 on success, a negative errno value otherwise and rte_errno is set. 2958 */ 2959 int 2960 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 2961 const struct rte_flow_item *item, 2962 uint64_t item_flags, 2963 const struct rte_flow_attr *attr, 2964 const struct rte_flow_item *gre_item, 2965 struct rte_flow_error *error) 2966 { 2967 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 2968 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 2969 const struct rte_flow_item_gre_opt *spec = item->spec; 2970 const struct rte_flow_item_gre_opt *mask = item->mask; 2971 struct mlx5_priv *priv = dev->data->dev_private; 2972 int ret = 0; 2973 struct rte_flow_item_gre_opt nic_mask = { 2974 .checksum_rsvd = { 2975 .checksum = RTE_BE16(UINT16_MAX), 2976 .reserved1 = 0x0, 2977 }, 2978 .key = { 2979 .key = RTE_BE32(UINT32_MAX), 2980 }, 2981 .sequence = { 2982 .sequence = RTE_BE32(UINT32_MAX), 2983 }, 2984 }; 2985 2986 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2987 return rte_flow_error_set(error, ENOTSUP, 2988 RTE_FLOW_ERROR_TYPE_ITEM, item, 2989 "No preceding GRE header"); 2990 if (item_flags & MLX5_FLOW_LAYER_INNER) 2991 return rte_flow_error_set(error, ENOTSUP, 2992 RTE_FLOW_ERROR_TYPE_ITEM, item, 2993 "GRE option following a wrong item"); 2994 if (!spec || !mask) 2995 return rte_flow_error_set(error, EINVAL, 2996 RTE_FLOW_ERROR_TYPE_ITEM, item, 2997 "At least one field gre_option(checksum/key/sequence) must be specified"); 2998 if (!gre_mask) 2999 gre_mask = &rte_flow_item_gre_mask; 3000 if (mask->checksum_rsvd.checksum) 3001 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3002 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3003 return rte_flow_error_set(error, EINVAL, 3004 RTE_FLOW_ERROR_TYPE_ITEM, 3005 item, 3006 "Checksum bit must be on"); 3007 if (mask->key.key) 3008 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3009 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3010 return rte_flow_error_set(error, EINVAL, 3011 RTE_FLOW_ERROR_TYPE_ITEM, 3012 item, "Key bit must be on"); 3013 if (mask->sequence.sequence) 3014 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3015 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3016 return rte_flow_error_set(error, EINVAL, 3017 RTE_FLOW_ERROR_TYPE_ITEM, 3018 item, 3019 "Sequence bit must be on"); 3020 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3021 if (priv->sh->steering_format_version == 3022 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3023 ((attr->group || attr->transfer) && 3024 !priv->sh->misc5_cap) || 3025 (!(priv->sh->tunnel_header_0_1 && 3026 priv->sh->tunnel_header_2_3) && 3027 !attr->group && !attr->transfer)) 3028 return rte_flow_error_set(error, EINVAL, 3029 RTE_FLOW_ERROR_TYPE_ITEM, 3030 item, 3031 "Checksum/Sequence not supported"); 3032 } 3033 ret = mlx5_flow_item_acceptable 3034 (item, (const uint8_t *)mask, 3035 (const uint8_t *)&nic_mask, 3036 sizeof(struct rte_flow_item_gre_opt), 3037 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3038 return ret; 3039 } 3040 3041 /** 3042 * Validate GRE item. 3043 * 3044 * @param[in] item 3045 * Item specification. 3046 * @param[in] item_flags 3047 * Bit flags to mark detected items. 3048 * @param[in] target_protocol 3049 * The next protocol in the previous item. 3050 * @param[out] error 3051 * Pointer to error structure. 3052 * 3053 * @return 3054 * 0 on success, a negative errno value otherwise and rte_errno is set. 3055 */ 3056 int 3057 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3058 uint64_t item_flags, 3059 uint8_t target_protocol, 3060 struct rte_flow_error *error) 3061 { 3062 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3063 const struct rte_flow_item_gre *mask = item->mask; 3064 int ret; 3065 const struct rte_flow_item_gre nic_mask = { 3066 .c_rsvd0_ver = RTE_BE16(0xB000), 3067 .protocol = RTE_BE16(UINT16_MAX), 3068 }; 3069 3070 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3071 return rte_flow_error_set(error, EINVAL, 3072 RTE_FLOW_ERROR_TYPE_ITEM, item, 3073 "protocol filtering not compatible" 3074 " with this GRE layer"); 3075 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3076 return rte_flow_error_set(error, ENOTSUP, 3077 RTE_FLOW_ERROR_TYPE_ITEM, item, 3078 "multiple tunnel layers not" 3079 " supported"); 3080 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3081 return rte_flow_error_set(error, ENOTSUP, 3082 RTE_FLOW_ERROR_TYPE_ITEM, item, 3083 "L3 Layer is missing"); 3084 if (!mask) 3085 mask = &rte_flow_item_gre_mask; 3086 ret = mlx5_flow_item_acceptable 3087 (item, (const uint8_t *)mask, 3088 (const uint8_t *)&nic_mask, 3089 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3090 error); 3091 if (ret < 0) 3092 return ret; 3093 #ifndef HAVE_MLX5DV_DR 3094 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3095 if (spec && (spec->protocol & mask->protocol)) 3096 return rte_flow_error_set(error, ENOTSUP, 3097 RTE_FLOW_ERROR_TYPE_ITEM, item, 3098 "without MPLS support the" 3099 " specification cannot be used for" 3100 " filtering"); 3101 #endif 3102 #endif 3103 return 0; 3104 } 3105 3106 /** 3107 * Validate Geneve item. 3108 * 3109 * @param[in] item 3110 * Item specification. 3111 * @param[in] itemFlags 3112 * Bit-fields that holds the items detected until now. 3113 * @param[in] enPriv 3114 * Pointer to the private data structure. 3115 * @param[out] error 3116 * Pointer to error structure. 3117 * 3118 * @return 3119 * 0 on success, a negative errno value otherwise and rte_errno is set. 3120 */ 3121 3122 int 3123 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3124 uint64_t item_flags, 3125 struct rte_eth_dev *dev, 3126 struct rte_flow_error *error) 3127 { 3128 struct mlx5_priv *priv = dev->data->dev_private; 3129 const struct rte_flow_item_geneve *spec = item->spec; 3130 const struct rte_flow_item_geneve *mask = item->mask; 3131 int ret; 3132 uint16_t gbhdr; 3133 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3134 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3135 const struct rte_flow_item_geneve nic_mask = { 3136 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3137 .vni = "\xff\xff\xff", 3138 .protocol = RTE_BE16(UINT16_MAX), 3139 }; 3140 3141 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3142 return rte_flow_error_set(error, ENOTSUP, 3143 RTE_FLOW_ERROR_TYPE_ITEM, item, 3144 "L3 Geneve is not enabled by device" 3145 " parameter and/or not configured in" 3146 " firmware"); 3147 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3148 return rte_flow_error_set(error, ENOTSUP, 3149 RTE_FLOW_ERROR_TYPE_ITEM, item, 3150 "multiple tunnel layers not" 3151 " supported"); 3152 /* 3153 * Verify only UDPv4 is present as defined in 3154 * https://tools.ietf.org/html/rfc7348 3155 */ 3156 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3157 return rte_flow_error_set(error, EINVAL, 3158 RTE_FLOW_ERROR_TYPE_ITEM, item, 3159 "no outer UDP layer found"); 3160 if (!mask) 3161 mask = &rte_flow_item_geneve_mask; 3162 ret = mlx5_flow_item_acceptable 3163 (item, (const uint8_t *)mask, 3164 (const uint8_t *)&nic_mask, 3165 sizeof(struct rte_flow_item_geneve), 3166 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3167 if (ret) 3168 return ret; 3169 if (spec) { 3170 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3171 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3172 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3173 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3174 return rte_flow_error_set(error, ENOTSUP, 3175 RTE_FLOW_ERROR_TYPE_ITEM, 3176 item, 3177 "Geneve protocol unsupported" 3178 " fields are being used"); 3179 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3180 return rte_flow_error_set 3181 (error, ENOTSUP, 3182 RTE_FLOW_ERROR_TYPE_ITEM, 3183 item, 3184 "Unsupported Geneve options length"); 3185 } 3186 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3187 return rte_flow_error_set 3188 (error, ENOTSUP, 3189 RTE_FLOW_ERROR_TYPE_ITEM, item, 3190 "Geneve tunnel must be fully defined"); 3191 return 0; 3192 } 3193 3194 /** 3195 * Validate Geneve TLV option item. 3196 * 3197 * @param[in] item 3198 * Item specification. 3199 * @param[in] last_item 3200 * Previous validated item in the pattern items. 3201 * @param[in] geneve_item 3202 * Previous GENEVE item specification. 3203 * @param[in] dev 3204 * Pointer to the rte_eth_dev structure. 3205 * @param[out] error 3206 * Pointer to error structure. 3207 * 3208 * @return 3209 * 0 on success, a negative errno value otherwise and rte_errno is set. 3210 */ 3211 int 3212 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3213 uint64_t last_item, 3214 const struct rte_flow_item *geneve_item, 3215 struct rte_eth_dev *dev, 3216 struct rte_flow_error *error) 3217 { 3218 struct mlx5_priv *priv = dev->data->dev_private; 3219 struct mlx5_dev_ctx_shared *sh = priv->sh; 3220 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3221 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3222 uint8_t data_max_supported = 3223 hca_attr->max_geneve_tlv_option_data_len * 4; 3224 const struct rte_flow_item_geneve *geneve_spec; 3225 const struct rte_flow_item_geneve *geneve_mask; 3226 const struct rte_flow_item_geneve_opt *spec = item->spec; 3227 const struct rte_flow_item_geneve_opt *mask = item->mask; 3228 unsigned int i; 3229 unsigned int data_len; 3230 uint8_t tlv_option_len; 3231 uint16_t optlen_m, optlen_v; 3232 const struct rte_flow_item_geneve_opt full_mask = { 3233 .option_class = RTE_BE16(0xffff), 3234 .option_type = 0xff, 3235 .option_len = 0x1f, 3236 }; 3237 3238 if (!mask) 3239 mask = &rte_flow_item_geneve_opt_mask; 3240 if (!spec) 3241 return rte_flow_error_set 3242 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3243 "Geneve TLV opt class/type/length must be specified"); 3244 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3245 return rte_flow_error_set 3246 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3247 "Geneve TLV opt length exceeds the limit (31)"); 3248 /* Check if class type and length masks are full. */ 3249 if (full_mask.option_class != mask->option_class || 3250 full_mask.option_type != mask->option_type || 3251 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3252 return rte_flow_error_set 3253 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3254 "Geneve TLV opt class/type/length masks must be full"); 3255 /* Check if length is supported */ 3256 if ((uint32_t)spec->option_len > 3257 hca_attr->max_geneve_tlv_option_data_len) 3258 return rte_flow_error_set 3259 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3260 "Geneve TLV opt length not supported"); 3261 if (hca_attr->max_geneve_tlv_options > 1) 3262 DRV_LOG(DEBUG, 3263 "max_geneve_tlv_options supports more than 1 option"); 3264 /* Check GENEVE item preceding. */ 3265 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3266 return rte_flow_error_set 3267 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3268 "Geneve opt item must be preceded with Geneve item"); 3269 geneve_spec = geneve_item->spec; 3270 geneve_mask = geneve_item->mask ? geneve_item->mask : 3271 &rte_flow_item_geneve_mask; 3272 /* Check if GENEVE TLV option size doesn't exceed option length */ 3273 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3274 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3275 tlv_option_len = spec->option_len & mask->option_len; 3276 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3277 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3278 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3279 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3280 if ((optlen_v & optlen_m) <= tlv_option_len) 3281 return rte_flow_error_set 3282 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3283 "GENEVE TLV option length exceeds optlen"); 3284 } 3285 /* Check if length is 0 or data is 0. */ 3286 if (spec->data == NULL || spec->option_len == 0) 3287 return rte_flow_error_set 3288 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3289 "Geneve TLV opt with zero data/length not supported"); 3290 /* Check not all data & mask are 0. */ 3291 data_len = spec->option_len * 4; 3292 if (mask->data == NULL) { 3293 for (i = 0; i < data_len; i++) 3294 if (spec->data[i]) 3295 break; 3296 if (i == data_len) 3297 return rte_flow_error_set(error, ENOTSUP, 3298 RTE_FLOW_ERROR_TYPE_ITEM, item, 3299 "Can't match on Geneve option data 0"); 3300 } else { 3301 for (i = 0; i < data_len; i++) 3302 if (spec->data[i] & mask->data[i]) 3303 break; 3304 if (i == data_len) 3305 return rte_flow_error_set(error, ENOTSUP, 3306 RTE_FLOW_ERROR_TYPE_ITEM, item, 3307 "Can't match on Geneve option data and mask 0"); 3308 /* Check data mask supported. */ 3309 for (i = data_max_supported; i < data_len ; i++) 3310 if (mask->data[i]) 3311 return rte_flow_error_set(error, ENOTSUP, 3312 RTE_FLOW_ERROR_TYPE_ITEM, item, 3313 "Data mask is of unsupported size"); 3314 } 3315 /* Check GENEVE option is supported in NIC. */ 3316 if (!hca_attr->geneve_tlv_opt) 3317 return rte_flow_error_set 3318 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3319 "Geneve TLV opt not supported"); 3320 /* Check if we already have geneve option with different type/class. */ 3321 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3322 geneve_opt_resource = sh->geneve_tlv_option_resource; 3323 if (geneve_opt_resource != NULL) 3324 if (geneve_opt_resource->option_class != spec->option_class || 3325 geneve_opt_resource->option_type != spec->option_type || 3326 geneve_opt_resource->length != spec->option_len) { 3327 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3328 return rte_flow_error_set(error, ENOTSUP, 3329 RTE_FLOW_ERROR_TYPE_ITEM, item, 3330 "Only one Geneve TLV option supported"); 3331 } 3332 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3333 return 0; 3334 } 3335 3336 /** 3337 * Validate MPLS item. 3338 * 3339 * @param[in] dev 3340 * Pointer to the rte_eth_dev structure. 3341 * @param[in] item 3342 * Item specification. 3343 * @param[in] item_flags 3344 * Bit-fields that holds the items detected until now. 3345 * @param[in] prev_layer 3346 * The protocol layer indicated in previous item. 3347 * @param[out] error 3348 * Pointer to error structure. 3349 * 3350 * @return 3351 * 0 on success, a negative errno value otherwise and rte_errno is set. 3352 */ 3353 int 3354 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3355 const struct rte_flow_item *item __rte_unused, 3356 uint64_t item_flags __rte_unused, 3357 uint64_t prev_layer __rte_unused, 3358 struct rte_flow_error *error) 3359 { 3360 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3361 const struct rte_flow_item_mpls *mask = item->mask; 3362 struct mlx5_priv *priv = dev->data->dev_private; 3363 int ret; 3364 3365 if (!priv->sh->dev_cap.mpls_en) 3366 return rte_flow_error_set(error, ENOTSUP, 3367 RTE_FLOW_ERROR_TYPE_ITEM, item, 3368 "MPLS not supported or" 3369 " disabled in firmware" 3370 " configuration."); 3371 /* MPLS over UDP, GRE is allowed */ 3372 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3373 MLX5_FLOW_LAYER_GRE | 3374 MLX5_FLOW_LAYER_GRE_KEY))) 3375 return rte_flow_error_set(error, EINVAL, 3376 RTE_FLOW_ERROR_TYPE_ITEM, item, 3377 "protocol filtering not compatible" 3378 " with MPLS layer"); 3379 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3380 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3381 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3382 return rte_flow_error_set(error, ENOTSUP, 3383 RTE_FLOW_ERROR_TYPE_ITEM, item, 3384 "multiple tunnel layers not" 3385 " supported"); 3386 if (!mask) 3387 mask = &rte_flow_item_mpls_mask; 3388 ret = mlx5_flow_item_acceptable 3389 (item, (const uint8_t *)mask, 3390 (const uint8_t *)&rte_flow_item_mpls_mask, 3391 sizeof(struct rte_flow_item_mpls), 3392 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3393 if (ret < 0) 3394 return ret; 3395 return 0; 3396 #else 3397 return rte_flow_error_set(error, ENOTSUP, 3398 RTE_FLOW_ERROR_TYPE_ITEM, item, 3399 "MPLS is not supported by Verbs, please" 3400 " update."); 3401 #endif 3402 } 3403 3404 /** 3405 * Validate NVGRE item. 3406 * 3407 * @param[in] item 3408 * Item specification. 3409 * @param[in] item_flags 3410 * Bit flags to mark detected items. 3411 * @param[in] target_protocol 3412 * The next protocol in the previous item. 3413 * @param[out] error 3414 * Pointer to error structure. 3415 * 3416 * @return 3417 * 0 on success, a negative errno value otherwise and rte_errno is set. 3418 */ 3419 int 3420 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3421 uint64_t item_flags, 3422 uint8_t target_protocol, 3423 struct rte_flow_error *error) 3424 { 3425 const struct rte_flow_item_nvgre *mask = item->mask; 3426 int ret; 3427 3428 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3429 return rte_flow_error_set(error, EINVAL, 3430 RTE_FLOW_ERROR_TYPE_ITEM, item, 3431 "protocol filtering not compatible" 3432 " with this GRE layer"); 3433 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3434 return rte_flow_error_set(error, ENOTSUP, 3435 RTE_FLOW_ERROR_TYPE_ITEM, item, 3436 "multiple tunnel layers not" 3437 " supported"); 3438 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3439 return rte_flow_error_set(error, ENOTSUP, 3440 RTE_FLOW_ERROR_TYPE_ITEM, item, 3441 "L3 Layer is missing"); 3442 if (!mask) 3443 mask = &rte_flow_item_nvgre_mask; 3444 ret = mlx5_flow_item_acceptable 3445 (item, (const uint8_t *)mask, 3446 (const uint8_t *)&rte_flow_item_nvgre_mask, 3447 sizeof(struct rte_flow_item_nvgre), 3448 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3449 if (ret < 0) 3450 return ret; 3451 return 0; 3452 } 3453 3454 /** 3455 * Validate eCPRI item. 3456 * 3457 * @param[in] item 3458 * Item specification. 3459 * @param[in] item_flags 3460 * Bit-fields that holds the items detected until now. 3461 * @param[in] last_item 3462 * Previous validated item in the pattern items. 3463 * @param[in] ether_type 3464 * Type in the ethernet layer header (including dot1q). 3465 * @param[in] acc_mask 3466 * Acceptable mask, if NULL default internal default mask 3467 * will be used to check whether item fields are supported. 3468 * @param[out] error 3469 * Pointer to error structure. 3470 * 3471 * @return 3472 * 0 on success, a negative errno value otherwise and rte_errno is set. 3473 */ 3474 int 3475 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3476 uint64_t item_flags, 3477 uint64_t last_item, 3478 uint16_t ether_type, 3479 const struct rte_flow_item_ecpri *acc_mask, 3480 struct rte_flow_error *error) 3481 { 3482 const struct rte_flow_item_ecpri *mask = item->mask; 3483 const struct rte_flow_item_ecpri nic_mask = { 3484 .hdr = { 3485 .common = { 3486 .u32 = 3487 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3488 .type = 0xFF, 3489 }).u32), 3490 }, 3491 .dummy[0] = 0xFFFFFFFF, 3492 }, 3493 }; 3494 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3495 MLX5_FLOW_LAYER_OUTER_VLAN); 3496 struct rte_flow_item_ecpri mask_lo; 3497 3498 if (!(last_item & outer_l2_vlan) && 3499 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3500 return rte_flow_error_set(error, EINVAL, 3501 RTE_FLOW_ERROR_TYPE_ITEM, item, 3502 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3503 if ((last_item & outer_l2_vlan) && ether_type && 3504 ether_type != RTE_ETHER_TYPE_ECPRI) 3505 return rte_flow_error_set(error, EINVAL, 3506 RTE_FLOW_ERROR_TYPE_ITEM, item, 3507 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3508 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3509 return rte_flow_error_set(error, EINVAL, 3510 RTE_FLOW_ERROR_TYPE_ITEM, item, 3511 "eCPRI with tunnel is not supported right now"); 3512 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3513 return rte_flow_error_set(error, ENOTSUP, 3514 RTE_FLOW_ERROR_TYPE_ITEM, item, 3515 "multiple L3 layers not supported"); 3516 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3517 return rte_flow_error_set(error, EINVAL, 3518 RTE_FLOW_ERROR_TYPE_ITEM, item, 3519 "eCPRI cannot coexist with a TCP layer"); 3520 /* In specification, eCPRI could be over UDP layer. */ 3521 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3522 return rte_flow_error_set(error, EINVAL, 3523 RTE_FLOW_ERROR_TYPE_ITEM, item, 3524 "eCPRI over UDP layer is not yet supported right now"); 3525 /* Mask for type field in common header could be zero. */ 3526 if (!mask) 3527 mask = &rte_flow_item_ecpri_mask; 3528 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3529 /* Input mask is in big-endian format. */ 3530 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3531 return rte_flow_error_set(error, EINVAL, 3532 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3533 "partial mask is not supported for protocol"); 3534 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3535 return rte_flow_error_set(error, EINVAL, 3536 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3537 "message header mask must be after a type mask"); 3538 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3539 acc_mask ? (const uint8_t *)acc_mask 3540 : (const uint8_t *)&nic_mask, 3541 sizeof(struct rte_flow_item_ecpri), 3542 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3543 } 3544 3545 static int 3546 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3547 const struct rte_flow_attr *attr __rte_unused, 3548 const struct rte_flow_item items[] __rte_unused, 3549 const struct rte_flow_action actions[] __rte_unused, 3550 bool external __rte_unused, 3551 int hairpin __rte_unused, 3552 struct rte_flow_error *error) 3553 { 3554 return rte_flow_error_set(error, ENOTSUP, 3555 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3556 } 3557 3558 static struct mlx5_flow * 3559 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3560 const struct rte_flow_attr *attr __rte_unused, 3561 const struct rte_flow_item items[] __rte_unused, 3562 const struct rte_flow_action actions[] __rte_unused, 3563 struct rte_flow_error *error) 3564 { 3565 rte_flow_error_set(error, ENOTSUP, 3566 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3567 return NULL; 3568 } 3569 3570 static int 3571 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3572 struct mlx5_flow *dev_flow __rte_unused, 3573 const struct rte_flow_attr *attr __rte_unused, 3574 const struct rte_flow_item items[] __rte_unused, 3575 const struct rte_flow_action actions[] __rte_unused, 3576 struct rte_flow_error *error) 3577 { 3578 return rte_flow_error_set(error, ENOTSUP, 3579 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3580 } 3581 3582 static int 3583 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3584 struct rte_flow *flow __rte_unused, 3585 struct rte_flow_error *error) 3586 { 3587 return rte_flow_error_set(error, ENOTSUP, 3588 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3589 } 3590 3591 static void 3592 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3593 struct rte_flow *flow __rte_unused) 3594 { 3595 } 3596 3597 static void 3598 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3599 struct rte_flow *flow __rte_unused) 3600 { 3601 } 3602 3603 static int 3604 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3605 struct rte_flow *flow __rte_unused, 3606 const struct rte_flow_action *actions __rte_unused, 3607 void *data __rte_unused, 3608 struct rte_flow_error *error) 3609 { 3610 return rte_flow_error_set(error, ENOTSUP, 3611 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3612 } 3613 3614 static int 3615 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3616 uint32_t domains __rte_unused, 3617 uint32_t flags __rte_unused) 3618 { 3619 return 0; 3620 } 3621 3622 /* Void driver to protect from null pointer reference. */ 3623 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3624 .validate = flow_null_validate, 3625 .prepare = flow_null_prepare, 3626 .translate = flow_null_translate, 3627 .apply = flow_null_apply, 3628 .remove = flow_null_remove, 3629 .destroy = flow_null_destroy, 3630 .query = flow_null_query, 3631 .sync_domain = flow_null_sync_domain, 3632 }; 3633 3634 /** 3635 * Select flow driver type according to flow attributes and device 3636 * configuration. 3637 * 3638 * @param[in] dev 3639 * Pointer to the dev structure. 3640 * @param[in] attr 3641 * Pointer to the flow attributes. 3642 * 3643 * @return 3644 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3645 */ 3646 static enum mlx5_flow_drv_type 3647 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3648 { 3649 struct mlx5_priv *priv = dev->data->dev_private; 3650 /* The OS can determine first a specific flow type (DV, VERBS) */ 3651 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3652 3653 if (type != MLX5_FLOW_TYPE_MAX) 3654 return type; 3655 /* 3656 * Currently when dv_flow_en == 2, only HW steering engine is 3657 * supported. New engines can also be chosen here if ready. 3658 */ 3659 if (priv->sh->config.dv_flow_en == 2) 3660 return MLX5_FLOW_TYPE_HW; 3661 /* If no OS specific type - continue with DV/VERBS selection */ 3662 if (attr->transfer && priv->sh->config.dv_esw_en) 3663 type = MLX5_FLOW_TYPE_DV; 3664 if (!attr->transfer) 3665 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3666 MLX5_FLOW_TYPE_VERBS; 3667 return type; 3668 } 3669 3670 #define flow_get_drv_ops(type) flow_drv_ops[type] 3671 3672 /** 3673 * Flow driver validation API. This abstracts calling driver specific functions. 3674 * The type of flow driver is determined according to flow attributes. 3675 * 3676 * @param[in] dev 3677 * Pointer to the dev structure. 3678 * @param[in] attr 3679 * Pointer to the flow attributes. 3680 * @param[in] items 3681 * Pointer to the list of items. 3682 * @param[in] actions 3683 * Pointer to the list of actions. 3684 * @param[in] external 3685 * This flow rule is created by request external to PMD. 3686 * @param[in] hairpin 3687 * Number of hairpin TX actions, 0 means classic flow. 3688 * @param[out] error 3689 * Pointer to the error structure. 3690 * 3691 * @return 3692 * 0 on success, a negative errno value otherwise and rte_errno is set. 3693 */ 3694 static inline int 3695 flow_drv_validate(struct rte_eth_dev *dev, 3696 const struct rte_flow_attr *attr, 3697 const struct rte_flow_item items[], 3698 const struct rte_flow_action actions[], 3699 bool external, int hairpin, struct rte_flow_error *error) 3700 { 3701 const struct mlx5_flow_driver_ops *fops; 3702 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3703 3704 fops = flow_get_drv_ops(type); 3705 return fops->validate(dev, attr, items, actions, external, 3706 hairpin, error); 3707 } 3708 3709 /** 3710 * Flow driver preparation API. This abstracts calling driver specific 3711 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3712 * calculates the size of memory required for device flow, allocates the memory, 3713 * initializes the device flow and returns the pointer. 3714 * 3715 * @note 3716 * This function initializes device flow structure such as dv or verbs in 3717 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3718 * rest. For example, adding returning device flow to flow->dev_flow list and 3719 * setting backward reference to the flow should be done out of this function. 3720 * layers field is not filled either. 3721 * 3722 * @param[in] dev 3723 * Pointer to the dev structure. 3724 * @param[in] attr 3725 * Pointer to the flow attributes. 3726 * @param[in] items 3727 * Pointer to the list of items. 3728 * @param[in] actions 3729 * Pointer to the list of actions. 3730 * @param[in] flow_idx 3731 * This memory pool index to the flow. 3732 * @param[out] error 3733 * Pointer to the error structure. 3734 * 3735 * @return 3736 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3737 */ 3738 static inline struct mlx5_flow * 3739 flow_drv_prepare(struct rte_eth_dev *dev, 3740 const struct rte_flow *flow, 3741 const struct rte_flow_attr *attr, 3742 const struct rte_flow_item items[], 3743 const struct rte_flow_action actions[], 3744 uint32_t flow_idx, 3745 struct rte_flow_error *error) 3746 { 3747 const struct mlx5_flow_driver_ops *fops; 3748 enum mlx5_flow_drv_type type = flow->drv_type; 3749 struct mlx5_flow *mlx5_flow = NULL; 3750 3751 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3752 fops = flow_get_drv_ops(type); 3753 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3754 if (mlx5_flow) 3755 mlx5_flow->flow_idx = flow_idx; 3756 return mlx5_flow; 3757 } 3758 3759 /** 3760 * Flow driver translation API. This abstracts calling driver specific 3761 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3762 * translates a generic flow into a driver flow. flow_drv_prepare() must 3763 * precede. 3764 * 3765 * @note 3766 * dev_flow->layers could be filled as a result of parsing during translation 3767 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3768 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3769 * flow->actions could be overwritten even though all the expanded dev_flows 3770 * have the same actions. 3771 * 3772 * @param[in] dev 3773 * Pointer to the rte dev structure. 3774 * @param[in, out] dev_flow 3775 * Pointer to the mlx5 flow. 3776 * @param[in] attr 3777 * Pointer to the flow attributes. 3778 * @param[in] items 3779 * Pointer to the list of items. 3780 * @param[in] actions 3781 * Pointer to the list of actions. 3782 * @param[out] error 3783 * Pointer to the error structure. 3784 * 3785 * @return 3786 * 0 on success, a negative errno value otherwise and rte_errno is set. 3787 */ 3788 static inline int 3789 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3790 const struct rte_flow_attr *attr, 3791 const struct rte_flow_item items[], 3792 const struct rte_flow_action actions[], 3793 struct rte_flow_error *error) 3794 { 3795 const struct mlx5_flow_driver_ops *fops; 3796 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3797 3798 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3799 fops = flow_get_drv_ops(type); 3800 return fops->translate(dev, dev_flow, attr, items, actions, error); 3801 } 3802 3803 /** 3804 * Flow driver apply API. This abstracts calling driver specific functions. 3805 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3806 * translated driver flows on to device. flow_drv_translate() must precede. 3807 * 3808 * @param[in] dev 3809 * Pointer to Ethernet device structure. 3810 * @param[in, out] flow 3811 * Pointer to flow structure. 3812 * @param[out] error 3813 * Pointer to error structure. 3814 * 3815 * @return 3816 * 0 on success, a negative errno value otherwise and rte_errno is set. 3817 */ 3818 static inline int 3819 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3820 struct rte_flow_error *error) 3821 { 3822 const struct mlx5_flow_driver_ops *fops; 3823 enum mlx5_flow_drv_type type = flow->drv_type; 3824 3825 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3826 fops = flow_get_drv_ops(type); 3827 return fops->apply(dev, flow, error); 3828 } 3829 3830 /** 3831 * Flow driver destroy API. This abstracts calling driver specific functions. 3832 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3833 * on device and releases resources of the flow. 3834 * 3835 * @param[in] dev 3836 * Pointer to Ethernet device. 3837 * @param[in, out] flow 3838 * Pointer to flow structure. 3839 */ 3840 static inline void 3841 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3842 { 3843 const struct mlx5_flow_driver_ops *fops; 3844 enum mlx5_flow_drv_type type = flow->drv_type; 3845 3846 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3847 fops = flow_get_drv_ops(type); 3848 fops->destroy(dev, flow); 3849 } 3850 3851 /** 3852 * Flow driver find RSS policy tbl API. This abstracts calling driver 3853 * specific functions. Parent flow (rte_flow) should have driver 3854 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3855 * 3856 * @param[in] dev 3857 * Pointer to Ethernet device. 3858 * @param[in, out] flow 3859 * Pointer to flow structure. 3860 * @param[in] policy 3861 * Pointer to meter policy table. 3862 * @param[in] rss_desc 3863 * Pointer to rss_desc 3864 */ 3865 static struct mlx5_flow_meter_sub_policy * 3866 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3867 struct rte_flow *flow, 3868 struct mlx5_flow_meter_policy *policy, 3869 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3870 { 3871 const struct mlx5_flow_driver_ops *fops; 3872 enum mlx5_flow_drv_type type = flow->drv_type; 3873 3874 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3875 fops = flow_get_drv_ops(type); 3876 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3877 } 3878 3879 /** 3880 * Flow driver color tag rule API. This abstracts calling driver 3881 * specific functions. Parent flow (rte_flow) should have driver 3882 * type (drv_type). It will create the color tag rules in hierarchy meter. 3883 * 3884 * @param[in] dev 3885 * Pointer to Ethernet device. 3886 * @param[in, out] flow 3887 * Pointer to flow structure. 3888 * @param[in] fm 3889 * Pointer to flow meter structure. 3890 * @param[in] src_port 3891 * The src port this extra rule should use. 3892 * @param[in] item 3893 * The src port id match item. 3894 * @param[out] error 3895 * Pointer to error structure. 3896 */ 3897 static int 3898 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3899 struct rte_flow *flow, 3900 struct mlx5_flow_meter_info *fm, 3901 int32_t src_port, 3902 const struct rte_flow_item *item, 3903 struct rte_flow_error *error) 3904 { 3905 const struct mlx5_flow_driver_ops *fops; 3906 enum mlx5_flow_drv_type type = flow->drv_type; 3907 3908 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3909 fops = flow_get_drv_ops(type); 3910 return fops->meter_hierarchy_rule_create(dev, fm, 3911 src_port, item, error); 3912 } 3913 3914 /** 3915 * Get RSS action from the action list. 3916 * 3917 * @param[in] dev 3918 * Pointer to Ethernet device. 3919 * @param[in] actions 3920 * Pointer to the list of actions. 3921 * @param[in] flow 3922 * Parent flow structure pointer. 3923 * 3924 * @return 3925 * Pointer to the RSS action if exist, else return NULL. 3926 */ 3927 static const struct rte_flow_action_rss* 3928 flow_get_rss_action(struct rte_eth_dev *dev, 3929 const struct rte_flow_action actions[]) 3930 { 3931 struct mlx5_priv *priv = dev->data->dev_private; 3932 const struct rte_flow_action_rss *rss = NULL; 3933 struct mlx5_meter_policy_action_container *acg; 3934 struct mlx5_meter_policy_action_container *acy; 3935 3936 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3937 switch (actions->type) { 3938 case RTE_FLOW_ACTION_TYPE_RSS: 3939 rss = actions->conf; 3940 break; 3941 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3942 { 3943 const struct rte_flow_action_sample *sample = 3944 actions->conf; 3945 const struct rte_flow_action *act = sample->actions; 3946 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3947 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3948 rss = act->conf; 3949 break; 3950 } 3951 case RTE_FLOW_ACTION_TYPE_METER: 3952 { 3953 uint32_t mtr_idx; 3954 struct mlx5_flow_meter_info *fm; 3955 struct mlx5_flow_meter_policy *policy; 3956 const struct rte_flow_action_meter *mtr = actions->conf; 3957 3958 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3959 if (fm && !fm->def_policy) { 3960 policy = mlx5_flow_meter_policy_find(dev, 3961 fm->policy_id, NULL); 3962 MLX5_ASSERT(policy); 3963 if (policy->is_hierarchy) { 3964 policy = 3965 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3966 policy); 3967 if (!policy) 3968 return NULL; 3969 } 3970 if (policy->is_rss) { 3971 acg = 3972 &policy->act_cnt[RTE_COLOR_GREEN]; 3973 acy = 3974 &policy->act_cnt[RTE_COLOR_YELLOW]; 3975 if (acg->fate_action == 3976 MLX5_FLOW_FATE_SHARED_RSS) 3977 rss = acg->rss->conf; 3978 else if (acy->fate_action == 3979 MLX5_FLOW_FATE_SHARED_RSS) 3980 rss = acy->rss->conf; 3981 } 3982 } 3983 break; 3984 } 3985 default: 3986 break; 3987 } 3988 } 3989 return rss; 3990 } 3991 3992 /** 3993 * Get ASO age action by index. 3994 * 3995 * @param[in] dev 3996 * Pointer to the Ethernet device structure. 3997 * @param[in] age_idx 3998 * Index to the ASO age action. 3999 * 4000 * @return 4001 * The specified ASO age action. 4002 */ 4003 struct mlx5_aso_age_action* 4004 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4005 { 4006 uint16_t pool_idx = age_idx & UINT16_MAX; 4007 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4008 struct mlx5_priv *priv = dev->data->dev_private; 4009 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4010 struct mlx5_aso_age_pool *pool; 4011 4012 rte_rwlock_read_lock(&mng->resize_rwl); 4013 pool = mng->pools[pool_idx]; 4014 rte_rwlock_read_unlock(&mng->resize_rwl); 4015 return &pool->actions[offset - 1]; 4016 } 4017 4018 /* maps indirect action to translated direct in some actions array */ 4019 struct mlx5_translated_action_handle { 4020 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4021 int index; /**< Index in related array of rte_flow_action. */ 4022 }; 4023 4024 /** 4025 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4026 * direct action if translation possible. 4027 * This functionality used to run same execution path for both direct and 4028 * indirect actions on flow create. All necessary preparations for indirect 4029 * action handling should be performed on *handle* actions list returned 4030 * from this call. 4031 * 4032 * @param[in] dev 4033 * Pointer to Ethernet device. 4034 * @param[in] actions 4035 * List of actions to translate. 4036 * @param[out] handle 4037 * List to store translated indirect action object handles. 4038 * @param[in, out] indir_n 4039 * Size of *handle* array. On return should be updated with number of 4040 * indirect actions retrieved from the *actions* list. 4041 * @param[out] translated_actions 4042 * List of actions where all indirect actions were translated to direct 4043 * if possible. NULL if no translation took place. 4044 * @param[out] error 4045 * Pointer to the error structure. 4046 * 4047 * @return 4048 * 0 on success, a negative errno value otherwise and rte_errno is set. 4049 */ 4050 static int 4051 flow_action_handles_translate(struct rte_eth_dev *dev, 4052 const struct rte_flow_action actions[], 4053 struct mlx5_translated_action_handle *handle, 4054 int *indir_n, 4055 struct rte_flow_action **translated_actions, 4056 struct rte_flow_error *error) 4057 { 4058 struct mlx5_priv *priv = dev->data->dev_private; 4059 struct rte_flow_action *translated = NULL; 4060 size_t actions_size; 4061 int n; 4062 int copied_n = 0; 4063 struct mlx5_translated_action_handle *handle_end = NULL; 4064 4065 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4066 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4067 continue; 4068 if (copied_n == *indir_n) { 4069 return rte_flow_error_set 4070 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4071 NULL, "too many shared actions"); 4072 } 4073 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4074 sizeof(actions[n].conf)); 4075 handle[copied_n].index = n; 4076 copied_n++; 4077 } 4078 n++; 4079 *indir_n = copied_n; 4080 if (!copied_n) 4081 return 0; 4082 actions_size = sizeof(struct rte_flow_action) * n; 4083 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4084 if (!translated) { 4085 rte_errno = ENOMEM; 4086 return -ENOMEM; 4087 } 4088 memcpy(translated, actions, actions_size); 4089 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4090 struct mlx5_shared_action_rss *shared_rss; 4091 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4092 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4093 uint32_t idx = act_idx & 4094 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4095 4096 switch (type) { 4097 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4098 shared_rss = mlx5_ipool_get 4099 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4100 translated[handle->index].type = 4101 RTE_FLOW_ACTION_TYPE_RSS; 4102 translated[handle->index].conf = 4103 &shared_rss->origin; 4104 break; 4105 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4106 translated[handle->index].type = 4107 (enum rte_flow_action_type) 4108 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4109 translated[handle->index].conf = (void *)(uintptr_t)idx; 4110 break; 4111 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4112 if (priv->sh->flow_hit_aso_en) { 4113 translated[handle->index].type = 4114 (enum rte_flow_action_type) 4115 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4116 translated[handle->index].conf = 4117 (void *)(uintptr_t)idx; 4118 break; 4119 } 4120 /* Fall-through */ 4121 case MLX5_INDIRECT_ACTION_TYPE_CT: 4122 if (priv->sh->ct_aso_en) { 4123 translated[handle->index].type = 4124 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4125 translated[handle->index].conf = 4126 (void *)(uintptr_t)idx; 4127 break; 4128 } 4129 /* Fall-through */ 4130 default: 4131 mlx5_free(translated); 4132 return rte_flow_error_set 4133 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4134 NULL, "invalid indirect action type"); 4135 } 4136 } 4137 *translated_actions = translated; 4138 return 0; 4139 } 4140 4141 /** 4142 * Get Shared RSS action from the action list. 4143 * 4144 * @param[in] dev 4145 * Pointer to Ethernet device. 4146 * @param[in] shared 4147 * Pointer to the list of actions. 4148 * @param[in] shared_n 4149 * Actions list length. 4150 * 4151 * @return 4152 * The MLX5 RSS action ID if exists, otherwise return 0. 4153 */ 4154 static uint32_t 4155 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4156 struct mlx5_translated_action_handle *handle, 4157 int shared_n) 4158 { 4159 struct mlx5_translated_action_handle *handle_end; 4160 struct mlx5_priv *priv = dev->data->dev_private; 4161 struct mlx5_shared_action_rss *shared_rss; 4162 4163 4164 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4165 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4166 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4167 uint32_t idx = act_idx & 4168 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4169 switch (type) { 4170 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4171 shared_rss = mlx5_ipool_get 4172 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4173 idx); 4174 __atomic_add_fetch(&shared_rss->refcnt, 1, 4175 __ATOMIC_RELAXED); 4176 return idx; 4177 default: 4178 break; 4179 } 4180 } 4181 return 0; 4182 } 4183 4184 static unsigned int 4185 find_graph_root(uint32_t rss_level) 4186 { 4187 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4188 MLX5_EXPANSION_ROOT_OUTER; 4189 } 4190 4191 /** 4192 * Get layer flags from the prefix flow. 4193 * 4194 * Some flows may be split to several subflows, the prefix subflow gets the 4195 * match items and the suffix sub flow gets the actions. 4196 * Some actions need the user defined match item flags to get the detail for 4197 * the action. 4198 * This function helps the suffix flow to get the item layer flags from prefix 4199 * subflow. 4200 * 4201 * @param[in] dev_flow 4202 * Pointer the created prefix subflow. 4203 * 4204 * @return 4205 * The layers get from prefix subflow. 4206 */ 4207 static inline uint64_t 4208 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4209 { 4210 uint64_t layers = 0; 4211 4212 /* 4213 * Layers bits could be localization, but usually the compiler will 4214 * help to do the optimization work for source code. 4215 * If no decap actions, use the layers directly. 4216 */ 4217 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4218 return dev_flow->handle->layers; 4219 /* Convert L3 layers with decap action. */ 4220 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4221 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4222 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4223 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4224 /* Convert L4 layers with decap action. */ 4225 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4226 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4227 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4228 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4229 return layers; 4230 } 4231 4232 /** 4233 * Get metadata split action information. 4234 * 4235 * @param[in] actions 4236 * Pointer to the list of actions. 4237 * @param[out] qrss 4238 * Pointer to the return pointer. 4239 * @param[out] qrss_type 4240 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4241 * if no QUEUE/RSS is found. 4242 * @param[out] encap_idx 4243 * Pointer to the index of the encap action if exists, otherwise the last 4244 * action index. 4245 * 4246 * @return 4247 * Total number of actions. 4248 */ 4249 static int 4250 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4251 const struct rte_flow_action **qrss, 4252 int *encap_idx) 4253 { 4254 const struct rte_flow_action_raw_encap *raw_encap; 4255 int actions_n = 0; 4256 int raw_decap_idx = -1; 4257 4258 *encap_idx = -1; 4259 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4260 switch (actions->type) { 4261 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4262 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4263 *encap_idx = actions_n; 4264 break; 4265 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4266 raw_decap_idx = actions_n; 4267 break; 4268 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4269 raw_encap = actions->conf; 4270 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4271 *encap_idx = raw_decap_idx != -1 ? 4272 raw_decap_idx : actions_n; 4273 break; 4274 case RTE_FLOW_ACTION_TYPE_QUEUE: 4275 case RTE_FLOW_ACTION_TYPE_RSS: 4276 *qrss = actions; 4277 break; 4278 default: 4279 break; 4280 } 4281 actions_n++; 4282 } 4283 if (*encap_idx == -1) 4284 *encap_idx = actions_n; 4285 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4286 return actions_n + 1; 4287 } 4288 4289 /** 4290 * Check if the action will change packet. 4291 * 4292 * @param dev 4293 * Pointer to Ethernet device. 4294 * @param[in] type 4295 * action type. 4296 * 4297 * @return 4298 * true if action will change packet, false otherwise. 4299 */ 4300 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4301 enum rte_flow_action_type type) 4302 { 4303 struct mlx5_priv *priv = dev->data->dev_private; 4304 4305 switch (type) { 4306 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4307 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4308 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4309 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4310 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4311 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4312 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4313 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4314 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4315 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4316 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4317 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4318 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4319 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4320 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4321 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4322 case RTE_FLOW_ACTION_TYPE_SET_META: 4323 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4324 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4325 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4326 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4327 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4328 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4329 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4330 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4331 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4332 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4333 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4334 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4335 return true; 4336 case RTE_FLOW_ACTION_TYPE_FLAG: 4337 case RTE_FLOW_ACTION_TYPE_MARK: 4338 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4339 return true; 4340 else 4341 return false; 4342 default: 4343 return false; 4344 } 4345 } 4346 4347 /** 4348 * Check meter action from the action list. 4349 * 4350 * @param dev 4351 * Pointer to Ethernet device. 4352 * @param[in] actions 4353 * Pointer to the list of actions. 4354 * @param[out] has_mtr 4355 * Pointer to the meter exist flag. 4356 * @param[out] has_modify 4357 * Pointer to the flag showing there's packet change action. 4358 * @param[out] meter_id 4359 * Pointer to the meter id. 4360 * 4361 * @return 4362 * Total number of actions. 4363 */ 4364 static int 4365 flow_check_meter_action(struct rte_eth_dev *dev, 4366 const struct rte_flow_action actions[], 4367 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4368 { 4369 const struct rte_flow_action_meter *mtr = NULL; 4370 int actions_n = 0; 4371 4372 MLX5_ASSERT(has_mtr); 4373 *has_mtr = false; 4374 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4375 switch (actions->type) { 4376 case RTE_FLOW_ACTION_TYPE_METER: 4377 mtr = actions->conf; 4378 *meter_id = mtr->mtr_id; 4379 *has_mtr = true; 4380 break; 4381 default: 4382 break; 4383 } 4384 if (!*has_mtr) 4385 *has_modify |= flow_check_modify_action_type(dev, 4386 actions->type); 4387 actions_n++; 4388 } 4389 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4390 return actions_n + 1; 4391 } 4392 4393 /** 4394 * Check if the flow should be split due to hairpin. 4395 * The reason for the split is that in current HW we can't 4396 * support encap and push-vlan on Rx, so if a flow contains 4397 * these actions we move it to Tx. 4398 * 4399 * @param dev 4400 * Pointer to Ethernet device. 4401 * @param[in] attr 4402 * Flow rule attributes. 4403 * @param[in] actions 4404 * Associated actions (list terminated by the END action). 4405 * 4406 * @return 4407 * > 0 the number of actions and the flow should be split, 4408 * 0 when no split required. 4409 */ 4410 static int 4411 flow_check_hairpin_split(struct rte_eth_dev *dev, 4412 const struct rte_flow_attr *attr, 4413 const struct rte_flow_action actions[]) 4414 { 4415 int queue_action = 0; 4416 int action_n = 0; 4417 int split = 0; 4418 const struct rte_flow_action_queue *queue; 4419 const struct rte_flow_action_rss *rss; 4420 const struct rte_flow_action_raw_encap *raw_encap; 4421 const struct rte_eth_hairpin_conf *conf; 4422 4423 if (!attr->ingress) 4424 return 0; 4425 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4426 switch (actions->type) { 4427 case RTE_FLOW_ACTION_TYPE_QUEUE: 4428 queue = actions->conf; 4429 if (queue == NULL) 4430 return 0; 4431 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4432 if (conf == NULL || conf->tx_explicit != 0) 4433 return 0; 4434 queue_action = 1; 4435 action_n++; 4436 break; 4437 case RTE_FLOW_ACTION_TYPE_RSS: 4438 rss = actions->conf; 4439 if (rss == NULL || rss->queue_num == 0) 4440 return 0; 4441 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4442 if (conf == NULL || conf->tx_explicit != 0) 4443 return 0; 4444 queue_action = 1; 4445 action_n++; 4446 break; 4447 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4448 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4449 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4450 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4451 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4452 split++; 4453 action_n++; 4454 break; 4455 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4456 raw_encap = actions->conf; 4457 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4458 split++; 4459 action_n++; 4460 break; 4461 default: 4462 action_n++; 4463 break; 4464 } 4465 } 4466 if (split && queue_action) 4467 return action_n; 4468 return 0; 4469 } 4470 4471 /* Declare flow create/destroy prototype in advance. */ 4472 static uint32_t 4473 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4474 const struct rte_flow_attr *attr, 4475 const struct rte_flow_item items[], 4476 const struct rte_flow_action actions[], 4477 bool external, struct rte_flow_error *error); 4478 4479 static void 4480 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4481 uint32_t flow_idx); 4482 4483 int 4484 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4485 struct mlx5_list_entry *entry, void *cb_ctx) 4486 { 4487 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4488 struct mlx5_flow_mreg_copy_resource *mcp_res = 4489 container_of(entry, typeof(*mcp_res), hlist_ent); 4490 4491 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4492 } 4493 4494 struct mlx5_list_entry * 4495 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4496 { 4497 struct rte_eth_dev *dev = tool_ctx; 4498 struct mlx5_priv *priv = dev->data->dev_private; 4499 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4500 struct mlx5_flow_mreg_copy_resource *mcp_res; 4501 struct rte_flow_error *error = ctx->error; 4502 uint32_t idx = 0; 4503 int ret; 4504 uint32_t mark_id = *(uint32_t *)(ctx->data); 4505 struct rte_flow_attr attr = { 4506 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4507 .ingress = 1, 4508 }; 4509 struct mlx5_rte_flow_item_tag tag_spec = { 4510 .data = mark_id, 4511 }; 4512 struct rte_flow_item items[] = { 4513 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4514 }; 4515 struct rte_flow_action_mark ftag = { 4516 .id = mark_id, 4517 }; 4518 struct mlx5_flow_action_copy_mreg cp_mreg = { 4519 .dst = REG_B, 4520 .src = REG_NON, 4521 }; 4522 struct rte_flow_action_jump jump = { 4523 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4524 }; 4525 struct rte_flow_action actions[] = { 4526 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4527 }; 4528 4529 /* Fill the register fields in the flow. */ 4530 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4531 if (ret < 0) 4532 return NULL; 4533 tag_spec.id = ret; 4534 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4535 if (ret < 0) 4536 return NULL; 4537 cp_mreg.src = ret; 4538 /* Provide the full width of FLAG specific value. */ 4539 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4540 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4541 /* Build a new flow. */ 4542 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4543 items[0] = (struct rte_flow_item){ 4544 .type = (enum rte_flow_item_type) 4545 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4546 .spec = &tag_spec, 4547 }; 4548 items[1] = (struct rte_flow_item){ 4549 .type = RTE_FLOW_ITEM_TYPE_END, 4550 }; 4551 actions[0] = (struct rte_flow_action){ 4552 .type = (enum rte_flow_action_type) 4553 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4554 .conf = &ftag, 4555 }; 4556 actions[1] = (struct rte_flow_action){ 4557 .type = (enum rte_flow_action_type) 4558 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4559 .conf = &cp_mreg, 4560 }; 4561 actions[2] = (struct rte_flow_action){ 4562 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4563 .conf = &jump, 4564 }; 4565 actions[3] = (struct rte_flow_action){ 4566 .type = RTE_FLOW_ACTION_TYPE_END, 4567 }; 4568 } else { 4569 /* Default rule, wildcard match. */ 4570 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4571 items[0] = (struct rte_flow_item){ 4572 .type = RTE_FLOW_ITEM_TYPE_END, 4573 }; 4574 actions[0] = (struct rte_flow_action){ 4575 .type = (enum rte_flow_action_type) 4576 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4577 .conf = &cp_mreg, 4578 }; 4579 actions[1] = (struct rte_flow_action){ 4580 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4581 .conf = &jump, 4582 }; 4583 actions[2] = (struct rte_flow_action){ 4584 .type = RTE_FLOW_ACTION_TYPE_END, 4585 }; 4586 } 4587 /* Build a new entry. */ 4588 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4589 if (!mcp_res) { 4590 rte_errno = ENOMEM; 4591 return NULL; 4592 } 4593 mcp_res->idx = idx; 4594 mcp_res->mark_id = mark_id; 4595 /* 4596 * The copy Flows are not included in any list. There 4597 * ones are referenced from other Flows and can not 4598 * be applied, removed, deleted in arbitrary order 4599 * by list traversing. 4600 */ 4601 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4602 &attr, items, actions, false, error); 4603 if (!mcp_res->rix_flow) { 4604 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4605 return NULL; 4606 } 4607 return &mcp_res->hlist_ent; 4608 } 4609 4610 struct mlx5_list_entry * 4611 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4612 void *cb_ctx __rte_unused) 4613 { 4614 struct rte_eth_dev *dev = tool_ctx; 4615 struct mlx5_priv *priv = dev->data->dev_private; 4616 struct mlx5_flow_mreg_copy_resource *mcp_res; 4617 uint32_t idx = 0; 4618 4619 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4620 if (!mcp_res) { 4621 rte_errno = ENOMEM; 4622 return NULL; 4623 } 4624 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4625 mcp_res->idx = idx; 4626 return &mcp_res->hlist_ent; 4627 } 4628 4629 void 4630 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4631 { 4632 struct mlx5_flow_mreg_copy_resource *mcp_res = 4633 container_of(entry, typeof(*mcp_res), hlist_ent); 4634 struct rte_eth_dev *dev = tool_ctx; 4635 struct mlx5_priv *priv = dev->data->dev_private; 4636 4637 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4638 } 4639 4640 /** 4641 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4642 * 4643 * As mark_id is unique, if there's already a registered flow for the mark_id, 4644 * return by increasing the reference counter of the resource. Otherwise, create 4645 * the resource (mcp_res) and flow. 4646 * 4647 * Flow looks like, 4648 * - If ingress port is ANY and reg_c[1] is mark_id, 4649 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4650 * 4651 * For default flow (zero mark_id), flow is like, 4652 * - If ingress port is ANY, 4653 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4654 * 4655 * @param dev 4656 * Pointer to Ethernet device. 4657 * @param mark_id 4658 * ID of MARK action, zero means default flow for META. 4659 * @param[out] error 4660 * Perform verbose error reporting if not NULL. 4661 * 4662 * @return 4663 * Associated resource on success, NULL otherwise and rte_errno is set. 4664 */ 4665 static struct mlx5_flow_mreg_copy_resource * 4666 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4667 struct rte_flow_error *error) 4668 { 4669 struct mlx5_priv *priv = dev->data->dev_private; 4670 struct mlx5_list_entry *entry; 4671 struct mlx5_flow_cb_ctx ctx = { 4672 .dev = dev, 4673 .error = error, 4674 .data = &mark_id, 4675 }; 4676 4677 /* Check if already registered. */ 4678 MLX5_ASSERT(priv->mreg_cp_tbl); 4679 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4680 if (!entry) 4681 return NULL; 4682 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4683 hlist_ent); 4684 } 4685 4686 void 4687 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4688 { 4689 struct mlx5_flow_mreg_copy_resource *mcp_res = 4690 container_of(entry, typeof(*mcp_res), hlist_ent); 4691 struct rte_eth_dev *dev = tool_ctx; 4692 struct mlx5_priv *priv = dev->data->dev_private; 4693 4694 MLX5_ASSERT(mcp_res->rix_flow); 4695 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4696 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4697 } 4698 4699 /** 4700 * Release flow in RX_CP_TBL. 4701 * 4702 * @param dev 4703 * Pointer to Ethernet device. 4704 * @flow 4705 * Parent flow for wich copying is provided. 4706 */ 4707 static void 4708 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4709 struct rte_flow *flow) 4710 { 4711 struct mlx5_flow_mreg_copy_resource *mcp_res; 4712 struct mlx5_priv *priv = dev->data->dev_private; 4713 4714 if (!flow->rix_mreg_copy) 4715 return; 4716 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4717 flow->rix_mreg_copy); 4718 if (!mcp_res || !priv->mreg_cp_tbl) 4719 return; 4720 MLX5_ASSERT(mcp_res->rix_flow); 4721 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4722 flow->rix_mreg_copy = 0; 4723 } 4724 4725 /** 4726 * Remove the default copy action from RX_CP_TBL. 4727 * 4728 * This functions is called in the mlx5_dev_start(). No thread safe 4729 * is guaranteed. 4730 * 4731 * @param dev 4732 * Pointer to Ethernet device. 4733 */ 4734 static void 4735 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4736 { 4737 struct mlx5_list_entry *entry; 4738 struct mlx5_priv *priv = dev->data->dev_private; 4739 struct mlx5_flow_cb_ctx ctx; 4740 uint32_t mark_id; 4741 4742 /* Check if default flow is registered. */ 4743 if (!priv->mreg_cp_tbl) 4744 return; 4745 mark_id = MLX5_DEFAULT_COPY_ID; 4746 ctx.data = &mark_id; 4747 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4748 if (!entry) 4749 return; 4750 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4751 } 4752 4753 /** 4754 * Add the default copy action in in RX_CP_TBL. 4755 * 4756 * This functions is called in the mlx5_dev_start(). No thread safe 4757 * is guaranteed. 4758 * 4759 * @param dev 4760 * Pointer to Ethernet device. 4761 * @param[out] error 4762 * Perform verbose error reporting if not NULL. 4763 * 4764 * @return 4765 * 0 for success, negative value otherwise and rte_errno is set. 4766 */ 4767 static int 4768 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4769 struct rte_flow_error *error) 4770 { 4771 struct mlx5_priv *priv = dev->data->dev_private; 4772 struct mlx5_flow_mreg_copy_resource *mcp_res; 4773 struct mlx5_flow_cb_ctx ctx; 4774 uint32_t mark_id; 4775 4776 /* Check whether extensive metadata feature is engaged. */ 4777 if (!priv->sh->config.dv_flow_en || 4778 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4779 !mlx5_flow_ext_mreg_supported(dev) || 4780 !priv->sh->dv_regc0_mask) 4781 return 0; 4782 /* 4783 * Add default mreg copy flow may be called multiple time, but 4784 * only be called once in stop. Avoid register it twice. 4785 */ 4786 mark_id = MLX5_DEFAULT_COPY_ID; 4787 ctx.data = &mark_id; 4788 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4789 return 0; 4790 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4791 if (!mcp_res) 4792 return -rte_errno; 4793 return 0; 4794 } 4795 4796 /** 4797 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4798 * 4799 * All the flow having Q/RSS action should be split by 4800 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4801 * performs the following, 4802 * - CQE->flow_tag := reg_c[1] (MARK) 4803 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4804 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4805 * but there should be a flow per each MARK ID set by MARK action. 4806 * 4807 * For the aforementioned reason, if there's a MARK action in flow's action 4808 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4809 * the MARK ID to CQE's flow_tag like, 4810 * - If reg_c[1] is mark_id, 4811 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4812 * 4813 * For SET_META action which stores value in reg_c[0], as the destination is 4814 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4815 * MARK ID means the default flow. The default flow looks like, 4816 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4817 * 4818 * @param dev 4819 * Pointer to Ethernet device. 4820 * @param flow 4821 * Pointer to flow structure. 4822 * @param[in] actions 4823 * Pointer to the list of actions. 4824 * @param[out] error 4825 * Perform verbose error reporting if not NULL. 4826 * 4827 * @return 4828 * 0 on success, negative value otherwise and rte_errno is set. 4829 */ 4830 static int 4831 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4832 struct rte_flow *flow, 4833 const struct rte_flow_action *actions, 4834 struct rte_flow_error *error) 4835 { 4836 struct mlx5_priv *priv = dev->data->dev_private; 4837 struct mlx5_sh_config *config = &priv->sh->config; 4838 struct mlx5_flow_mreg_copy_resource *mcp_res; 4839 const struct rte_flow_action_mark *mark; 4840 4841 /* Check whether extensive metadata feature is engaged. */ 4842 if (!config->dv_flow_en || 4843 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4844 !mlx5_flow_ext_mreg_supported(dev) || 4845 !priv->sh->dv_regc0_mask) 4846 return 0; 4847 /* Find MARK action. */ 4848 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4849 switch (actions->type) { 4850 case RTE_FLOW_ACTION_TYPE_FLAG: 4851 mcp_res = flow_mreg_add_copy_action 4852 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4853 if (!mcp_res) 4854 return -rte_errno; 4855 flow->rix_mreg_copy = mcp_res->idx; 4856 return 0; 4857 case RTE_FLOW_ACTION_TYPE_MARK: 4858 mark = (const struct rte_flow_action_mark *) 4859 actions->conf; 4860 mcp_res = 4861 flow_mreg_add_copy_action(dev, mark->id, error); 4862 if (!mcp_res) 4863 return -rte_errno; 4864 flow->rix_mreg_copy = mcp_res->idx; 4865 return 0; 4866 default: 4867 break; 4868 } 4869 } 4870 return 0; 4871 } 4872 4873 #define MLX5_MAX_SPLIT_ACTIONS 24 4874 #define MLX5_MAX_SPLIT_ITEMS 24 4875 4876 /** 4877 * Split the hairpin flow. 4878 * Since HW can't support encap and push-vlan on Rx, we move these 4879 * actions to Tx. 4880 * If the count action is after the encap then we also 4881 * move the count action. in this case the count will also measure 4882 * the outer bytes. 4883 * 4884 * @param dev 4885 * Pointer to Ethernet device. 4886 * @param[in] actions 4887 * Associated actions (list terminated by the END action). 4888 * @param[out] actions_rx 4889 * Rx flow actions. 4890 * @param[out] actions_tx 4891 * Tx flow actions.. 4892 * @param[out] pattern_tx 4893 * The pattern items for the Tx flow. 4894 * @param[out] flow_id 4895 * The flow ID connected to this flow. 4896 * 4897 * @return 4898 * 0 on success. 4899 */ 4900 static int 4901 flow_hairpin_split(struct rte_eth_dev *dev, 4902 const struct rte_flow_action actions[], 4903 struct rte_flow_action actions_rx[], 4904 struct rte_flow_action actions_tx[], 4905 struct rte_flow_item pattern_tx[], 4906 uint32_t flow_id) 4907 { 4908 const struct rte_flow_action_raw_encap *raw_encap; 4909 const struct rte_flow_action_raw_decap *raw_decap; 4910 struct mlx5_rte_flow_action_set_tag *set_tag; 4911 struct rte_flow_action *tag_action; 4912 struct mlx5_rte_flow_item_tag *tag_item; 4913 struct rte_flow_item *item; 4914 char *addr; 4915 int encap = 0; 4916 4917 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4918 switch (actions->type) { 4919 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4920 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4921 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4922 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4923 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4924 rte_memcpy(actions_tx, actions, 4925 sizeof(struct rte_flow_action)); 4926 actions_tx++; 4927 break; 4928 case RTE_FLOW_ACTION_TYPE_COUNT: 4929 if (encap) { 4930 rte_memcpy(actions_tx, actions, 4931 sizeof(struct rte_flow_action)); 4932 actions_tx++; 4933 } else { 4934 rte_memcpy(actions_rx, actions, 4935 sizeof(struct rte_flow_action)); 4936 actions_rx++; 4937 } 4938 break; 4939 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4940 raw_encap = actions->conf; 4941 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4942 memcpy(actions_tx, actions, 4943 sizeof(struct rte_flow_action)); 4944 actions_tx++; 4945 encap = 1; 4946 } else { 4947 rte_memcpy(actions_rx, actions, 4948 sizeof(struct rte_flow_action)); 4949 actions_rx++; 4950 } 4951 break; 4952 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4953 raw_decap = actions->conf; 4954 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4955 memcpy(actions_tx, actions, 4956 sizeof(struct rte_flow_action)); 4957 actions_tx++; 4958 } else { 4959 rte_memcpy(actions_rx, actions, 4960 sizeof(struct rte_flow_action)); 4961 actions_rx++; 4962 } 4963 break; 4964 default: 4965 rte_memcpy(actions_rx, actions, 4966 sizeof(struct rte_flow_action)); 4967 actions_rx++; 4968 break; 4969 } 4970 } 4971 /* Add set meta action and end action for the Rx flow. */ 4972 tag_action = actions_rx; 4973 tag_action->type = (enum rte_flow_action_type) 4974 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4975 actions_rx++; 4976 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4977 actions_rx++; 4978 set_tag = (void *)actions_rx; 4979 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4980 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4981 .data = flow_id, 4982 }; 4983 MLX5_ASSERT(set_tag->id > REG_NON); 4984 tag_action->conf = set_tag; 4985 /* Create Tx item list. */ 4986 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4987 addr = (void *)&pattern_tx[2]; 4988 item = pattern_tx; 4989 item->type = (enum rte_flow_item_type) 4990 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4991 tag_item = (void *)addr; 4992 tag_item->data = flow_id; 4993 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4994 MLX5_ASSERT(set_tag->id > REG_NON); 4995 item->spec = tag_item; 4996 addr += sizeof(struct mlx5_rte_flow_item_tag); 4997 tag_item = (void *)addr; 4998 tag_item->data = UINT32_MAX; 4999 tag_item->id = UINT16_MAX; 5000 item->mask = tag_item; 5001 item->last = NULL; 5002 item++; 5003 item->type = RTE_FLOW_ITEM_TYPE_END; 5004 return 0; 5005 } 5006 5007 /** 5008 * The last stage of splitting chain, just creates the subflow 5009 * without any modification. 5010 * 5011 * @param[in] dev 5012 * Pointer to Ethernet device. 5013 * @param[in] flow 5014 * Parent flow structure pointer. 5015 * @param[in, out] sub_flow 5016 * Pointer to return the created subflow, may be NULL. 5017 * @param[in] attr 5018 * Flow rule attributes. 5019 * @param[in] items 5020 * Pattern specification (list terminated by the END pattern item). 5021 * @param[in] actions 5022 * Associated actions (list terminated by the END action). 5023 * @param[in] flow_split_info 5024 * Pointer to flow split info structure. 5025 * @param[out] error 5026 * Perform verbose error reporting if not NULL. 5027 * @return 5028 * 0 on success, negative value otherwise 5029 */ 5030 static int 5031 flow_create_split_inner(struct rte_eth_dev *dev, 5032 struct rte_flow *flow, 5033 struct mlx5_flow **sub_flow, 5034 const struct rte_flow_attr *attr, 5035 const struct rte_flow_item items[], 5036 const struct rte_flow_action actions[], 5037 struct mlx5_flow_split_info *flow_split_info, 5038 struct rte_flow_error *error) 5039 { 5040 struct mlx5_flow *dev_flow; 5041 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5042 5043 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5044 flow_split_info->flow_idx, error); 5045 if (!dev_flow) 5046 return -rte_errno; 5047 dev_flow->flow = flow; 5048 dev_flow->external = flow_split_info->external; 5049 dev_flow->skip_scale = flow_split_info->skip_scale; 5050 /* Subflow object was created, we must include one in the list. */ 5051 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5052 dev_flow->handle, next); 5053 /* 5054 * If dev_flow is as one of the suffix flow, some actions in suffix 5055 * flow may need some user defined item layer flags, and pass the 5056 * Metadata rxq mark flag to suffix flow as well. 5057 */ 5058 if (flow_split_info->prefix_layers) 5059 dev_flow->handle->layers = flow_split_info->prefix_layers; 5060 if (flow_split_info->prefix_mark) { 5061 MLX5_ASSERT(wks); 5062 wks->mark = 1; 5063 } 5064 if (sub_flow) 5065 *sub_flow = dev_flow; 5066 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5067 dev_flow->dv.table_id = flow_split_info->table_id; 5068 #endif 5069 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5070 } 5071 5072 /** 5073 * Get the sub policy of a meter. 5074 * 5075 * @param[in] dev 5076 * Pointer to Ethernet device. 5077 * @param[in] flow 5078 * Parent flow structure pointer. 5079 * @param wks 5080 * Pointer to thread flow work space. 5081 * @param[in] attr 5082 * Flow rule attributes. 5083 * @param[in] items 5084 * Pattern specification (list terminated by the END pattern item). 5085 * @param[out] error 5086 * Perform verbose error reporting if not NULL. 5087 * 5088 * @return 5089 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5090 */ 5091 static struct mlx5_flow_meter_sub_policy * 5092 get_meter_sub_policy(struct rte_eth_dev *dev, 5093 struct rte_flow *flow, 5094 struct mlx5_flow_workspace *wks, 5095 const struct rte_flow_attr *attr, 5096 const struct rte_flow_item items[], 5097 struct rte_flow_error *error) 5098 { 5099 struct mlx5_flow_meter_policy *policy; 5100 struct mlx5_flow_meter_policy *final_policy; 5101 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5102 5103 policy = wks->policy; 5104 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5105 if (final_policy->is_rss || final_policy->is_queue) { 5106 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5107 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5108 uint32_t i; 5109 5110 /* 5111 * This is a tmp dev_flow, 5112 * no need to register any matcher for it in translate. 5113 */ 5114 wks->skip_matcher_reg = 1; 5115 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5116 struct mlx5_flow dev_flow = {0}; 5117 struct mlx5_flow_handle dev_handle = { {0} }; 5118 uint8_t fate = final_policy->act_cnt[i].fate_action; 5119 5120 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5121 const struct rte_flow_action_rss *rss_act = 5122 final_policy->act_cnt[i].rss->conf; 5123 struct rte_flow_action rss_actions[2] = { 5124 [0] = { 5125 .type = RTE_FLOW_ACTION_TYPE_RSS, 5126 .conf = rss_act, 5127 }, 5128 [1] = { 5129 .type = RTE_FLOW_ACTION_TYPE_END, 5130 .conf = NULL, 5131 } 5132 }; 5133 5134 dev_flow.handle = &dev_handle; 5135 dev_flow.ingress = attr->ingress; 5136 dev_flow.flow = flow; 5137 dev_flow.external = 0; 5138 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5139 dev_flow.dv.transfer = attr->transfer; 5140 #endif 5141 /** 5142 * Translate RSS action to get rss hash fields. 5143 */ 5144 if (flow_drv_translate(dev, &dev_flow, attr, 5145 items, rss_actions, error)) 5146 goto exit; 5147 rss_desc_v[i] = wks->rss_desc; 5148 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5149 rss_desc_v[i].hash_fields = 5150 dev_flow.hash_fields; 5151 rss_desc_v[i].queue_num = 5152 rss_desc_v[i].hash_fields ? 5153 rss_desc_v[i].queue_num : 1; 5154 rss_desc_v[i].tunnel = 5155 !!(dev_flow.handle->layers & 5156 MLX5_FLOW_LAYER_TUNNEL); 5157 /* Use the RSS queues in the containers. */ 5158 rss_desc_v[i].queue = 5159 (uint16_t *)(uintptr_t)rss_act->queue; 5160 rss_desc[i] = &rss_desc_v[i]; 5161 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5162 /* This is queue action. */ 5163 rss_desc_v[i] = wks->rss_desc; 5164 rss_desc_v[i].key_len = 0; 5165 rss_desc_v[i].hash_fields = 0; 5166 rss_desc_v[i].queue = 5167 &final_policy->act_cnt[i].queue; 5168 rss_desc_v[i].queue_num = 1; 5169 rss_desc[i] = &rss_desc_v[i]; 5170 } else { 5171 rss_desc[i] = NULL; 5172 } 5173 } 5174 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5175 flow, policy, rss_desc); 5176 } else { 5177 enum mlx5_meter_domain mtr_domain = 5178 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5179 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5180 MLX5_MTR_DOMAIN_INGRESS); 5181 sub_policy = policy->sub_policys[mtr_domain][0]; 5182 } 5183 if (!sub_policy) 5184 rte_flow_error_set(error, EINVAL, 5185 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5186 "Failed to get meter sub-policy."); 5187 exit: 5188 return sub_policy; 5189 } 5190 5191 /** 5192 * Split the meter flow. 5193 * 5194 * As meter flow will split to three sub flow, other than meter 5195 * action, the other actions make sense to only meter accepts 5196 * the packet. If it need to be dropped, no other additional 5197 * actions should be take. 5198 * 5199 * One kind of special action which decapsulates the L3 tunnel 5200 * header will be in the prefix sub flow, as not to take the 5201 * L3 tunnel header into account. 5202 * 5203 * @param[in] dev 5204 * Pointer to Ethernet device. 5205 * @param[in] flow 5206 * Parent flow structure pointer. 5207 * @param wks 5208 * Pointer to thread flow work space. 5209 * @param[in] attr 5210 * Flow rule attributes. 5211 * @param[in] items 5212 * Pattern specification (list terminated by the END pattern item). 5213 * @param[out] sfx_items 5214 * Suffix flow match items (list terminated by the END pattern item). 5215 * @param[in] actions 5216 * Associated actions (list terminated by the END action). 5217 * @param[out] actions_sfx 5218 * Suffix flow actions. 5219 * @param[out] actions_pre 5220 * Prefix flow actions. 5221 * @param[out] mtr_flow_id 5222 * Pointer to meter flow id. 5223 * @param[out] error 5224 * Perform verbose error reporting if not NULL. 5225 * 5226 * @return 5227 * 0 on success, a negative errno value otherwise and rte_errno is set. 5228 */ 5229 static int 5230 flow_meter_split_prep(struct rte_eth_dev *dev, 5231 struct rte_flow *flow, 5232 struct mlx5_flow_workspace *wks, 5233 const struct rte_flow_attr *attr, 5234 const struct rte_flow_item items[], 5235 struct rte_flow_item sfx_items[], 5236 const struct rte_flow_action actions[], 5237 struct rte_flow_action actions_sfx[], 5238 struct rte_flow_action actions_pre[], 5239 uint32_t *mtr_flow_id, 5240 struct rte_flow_error *error) 5241 { 5242 struct mlx5_priv *priv = dev->data->dev_private; 5243 struct mlx5_flow_meter_info *fm = wks->fm; 5244 struct rte_flow_action *tag_action = NULL; 5245 struct rte_flow_item *tag_item; 5246 struct mlx5_rte_flow_action_set_tag *set_tag; 5247 const struct rte_flow_action_raw_encap *raw_encap; 5248 const struct rte_flow_action_raw_decap *raw_decap; 5249 struct mlx5_rte_flow_item_tag *tag_item_spec; 5250 struct mlx5_rte_flow_item_tag *tag_item_mask; 5251 uint32_t tag_id = 0; 5252 struct rte_flow_item *vlan_item_dst = NULL; 5253 const struct rte_flow_item *vlan_item_src = NULL; 5254 const struct rte_flow_item *orig_items = items; 5255 struct rte_flow_action *hw_mtr_action; 5256 struct rte_flow_action *action_pre_head = NULL; 5257 int32_t flow_src_port = priv->representor_id; 5258 bool mtr_first; 5259 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5260 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5261 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5262 uint32_t flow_id = 0; 5263 uint32_t flow_id_reversed = 0; 5264 uint8_t flow_id_bits = 0; 5265 int shift; 5266 5267 /* Prepare the suffix subflow items. */ 5268 tag_item = sfx_items++; 5269 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5270 struct mlx5_priv *port_priv; 5271 const struct rte_flow_item_port_id *pid_v; 5272 int item_type = items->type; 5273 5274 switch (item_type) { 5275 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5276 pid_v = items->spec; 5277 MLX5_ASSERT(pid_v); 5278 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5279 if (!port_priv) 5280 return rte_flow_error_set(error, 5281 rte_errno, 5282 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5283 pid_v, 5284 "Failed to get port info."); 5285 flow_src_port = port_priv->representor_id; 5286 if (!fm->def_policy && wks->policy->is_hierarchy && 5287 flow_src_port != priv->representor_id) { 5288 if (flow_drv_mtr_hierarchy_rule_create(dev, 5289 flow, fm, 5290 flow_src_port, 5291 items, 5292 error)) 5293 return -rte_errno; 5294 } 5295 memcpy(sfx_items, items, sizeof(*sfx_items)); 5296 sfx_items++; 5297 break; 5298 case RTE_FLOW_ITEM_TYPE_VLAN: 5299 /* Determine if copy vlan item below. */ 5300 vlan_item_src = items; 5301 vlan_item_dst = sfx_items++; 5302 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5303 break; 5304 default: 5305 break; 5306 } 5307 } 5308 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5309 sfx_items++; 5310 mtr_first = priv->sh->meter_aso_en && 5311 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5312 /* For ASO meter, meter must be before tag in TX direction. */ 5313 if (mtr_first) { 5314 action_pre_head = actions_pre++; 5315 /* Leave space for tag action. */ 5316 tag_action = actions_pre++; 5317 } 5318 /* Prepare the actions for prefix and suffix flow. */ 5319 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5320 struct rte_flow_action *action_cur = NULL; 5321 5322 switch (actions->type) { 5323 case RTE_FLOW_ACTION_TYPE_METER: 5324 if (mtr_first) { 5325 action_cur = action_pre_head; 5326 } else { 5327 /* Leave space for tag action. */ 5328 tag_action = actions_pre++; 5329 action_cur = actions_pre++; 5330 } 5331 break; 5332 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5333 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5334 action_cur = actions_pre++; 5335 break; 5336 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5337 raw_encap = actions->conf; 5338 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5339 action_cur = actions_pre++; 5340 break; 5341 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5342 raw_decap = actions->conf; 5343 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5344 action_cur = actions_pre++; 5345 break; 5346 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5347 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5348 if (vlan_item_dst && vlan_item_src) { 5349 memcpy(vlan_item_dst, vlan_item_src, 5350 sizeof(*vlan_item_dst)); 5351 /* 5352 * Convert to internal match item, it is used 5353 * for vlan push and set vid. 5354 */ 5355 vlan_item_dst->type = (enum rte_flow_item_type) 5356 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5357 } 5358 break; 5359 default: 5360 break; 5361 } 5362 if (!action_cur) 5363 action_cur = (fm->def_policy) ? 5364 actions_sfx++ : actions_pre++; 5365 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5366 } 5367 /* Add end action to the actions. */ 5368 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5369 if (priv->sh->meter_aso_en) { 5370 /** 5371 * For ASO meter, need to add an extra jump action explicitly, 5372 * to jump from meter to policer table. 5373 */ 5374 struct mlx5_flow_meter_sub_policy *sub_policy; 5375 struct mlx5_flow_tbl_data_entry *tbl_data; 5376 5377 if (!fm->def_policy) { 5378 sub_policy = get_meter_sub_policy(dev, flow, wks, 5379 attr, orig_items, 5380 error); 5381 if (!sub_policy) 5382 return -rte_errno; 5383 } else { 5384 enum mlx5_meter_domain mtr_domain = 5385 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5386 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5387 MLX5_MTR_DOMAIN_INGRESS); 5388 5389 sub_policy = 5390 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5391 } 5392 tbl_data = container_of(sub_policy->tbl_rsc, 5393 struct mlx5_flow_tbl_data_entry, tbl); 5394 hw_mtr_action = actions_pre++; 5395 hw_mtr_action->type = (enum rte_flow_action_type) 5396 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5397 hw_mtr_action->conf = tbl_data->jump.action; 5398 } 5399 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5400 actions_pre++; 5401 if (!tag_action) 5402 return rte_flow_error_set(error, ENOMEM, 5403 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5404 NULL, "No tag action space."); 5405 if (!mtr_flow_id) { 5406 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5407 goto exit; 5408 } 5409 /* Only default-policy Meter creates mtr flow id. */ 5410 if (fm->def_policy) { 5411 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5412 if (!tag_id) 5413 return rte_flow_error_set(error, ENOMEM, 5414 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5415 "Failed to allocate meter flow id."); 5416 flow_id = tag_id - 1; 5417 flow_id_bits = (!flow_id) ? 1 : 5418 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5419 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5420 mtr_reg_bits) { 5421 mlx5_ipool_free(fm->flow_ipool, tag_id); 5422 return rte_flow_error_set(error, EINVAL, 5423 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5424 "Meter flow id exceeds max limit."); 5425 } 5426 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5427 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5428 } 5429 /* Build tag actions and items for meter_id/meter flow_id. */ 5430 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5431 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5432 tag_item_mask = tag_item_spec + 1; 5433 /* Both flow_id and meter_id share the same register. */ 5434 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5435 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5436 0, error), 5437 .offset = mtr_id_offset, 5438 .length = mtr_reg_bits, 5439 .data = flow->meter, 5440 }; 5441 /* 5442 * The color Reg bits used by flow_id are growing from 5443 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5444 */ 5445 for (shift = 0; shift < flow_id_bits; shift++) 5446 flow_id_reversed = (flow_id_reversed << 1) | 5447 ((flow_id >> shift) & 0x1); 5448 set_tag->data |= 5449 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5450 tag_item_spec->id = set_tag->id; 5451 tag_item_spec->data = set_tag->data << mtr_id_offset; 5452 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5453 tag_action->type = (enum rte_flow_action_type) 5454 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5455 tag_action->conf = set_tag; 5456 tag_item->type = (enum rte_flow_item_type) 5457 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5458 tag_item->spec = tag_item_spec; 5459 tag_item->last = NULL; 5460 tag_item->mask = tag_item_mask; 5461 exit: 5462 if (mtr_flow_id) 5463 *mtr_flow_id = tag_id; 5464 return 0; 5465 } 5466 5467 /** 5468 * Split action list having QUEUE/RSS for metadata register copy. 5469 * 5470 * Once Q/RSS action is detected in user's action list, the flow action 5471 * should be split in order to copy metadata registers, which will happen in 5472 * RX_CP_TBL like, 5473 * - CQE->flow_tag := reg_c[1] (MARK) 5474 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5475 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5476 * This is because the last action of each flow must be a terminal action 5477 * (QUEUE, RSS or DROP). 5478 * 5479 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5480 * stored and kept in the mlx5_flow structure per each sub_flow. 5481 * 5482 * The Q/RSS action is replaced with, 5483 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5484 * And the following JUMP action is added at the end, 5485 * - JUMP, to RX_CP_TBL. 5486 * 5487 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5488 * flow_create_split_metadata() routine. The flow will look like, 5489 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5490 * 5491 * @param dev 5492 * Pointer to Ethernet device. 5493 * @param[out] split_actions 5494 * Pointer to store split actions to jump to CP_TBL. 5495 * @param[in] actions 5496 * Pointer to the list of original flow actions. 5497 * @param[in] qrss 5498 * Pointer to the Q/RSS action. 5499 * @param[in] actions_n 5500 * Number of original actions. 5501 * @param[in] mtr_sfx 5502 * Check if it is in meter suffix table. 5503 * @param[out] error 5504 * Perform verbose error reporting if not NULL. 5505 * 5506 * @return 5507 * non-zero unique flow_id on success, otherwise 0 and 5508 * error/rte_error are set. 5509 */ 5510 static uint32_t 5511 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5512 struct rte_flow_action *split_actions, 5513 const struct rte_flow_action *actions, 5514 const struct rte_flow_action *qrss, 5515 int actions_n, int mtr_sfx, 5516 struct rte_flow_error *error) 5517 { 5518 struct mlx5_priv *priv = dev->data->dev_private; 5519 struct mlx5_rte_flow_action_set_tag *set_tag; 5520 struct rte_flow_action_jump *jump; 5521 const int qrss_idx = qrss - actions; 5522 uint32_t flow_id = 0; 5523 int ret = 0; 5524 5525 /* 5526 * Given actions will be split 5527 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5528 * - Add jump to mreg CP_TBL. 5529 * As a result, there will be one more action. 5530 */ 5531 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5532 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5533 ++actions_n; 5534 set_tag = (void *)(split_actions + actions_n); 5535 /* 5536 * If we are not the meter suffix flow, add the tag action. 5537 * Since meter suffix flow already has the tag added. 5538 */ 5539 if (!mtr_sfx) { 5540 /* 5541 * Allocate the new subflow ID. This one is unique within 5542 * device and not shared with representors. Otherwise, 5543 * we would have to resolve multi-thread access synch 5544 * issue. Each flow on the shared device is appended 5545 * with source vport identifier, so the resulting 5546 * flows will be unique in the shared (by master and 5547 * representors) domain even if they have coinciding 5548 * IDs. 5549 */ 5550 mlx5_ipool_malloc(priv->sh->ipool 5551 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5552 if (!flow_id) 5553 return rte_flow_error_set(error, ENOMEM, 5554 RTE_FLOW_ERROR_TYPE_ACTION, 5555 NULL, "can't allocate id " 5556 "for split Q/RSS subflow"); 5557 /* Internal SET_TAG action to set flow ID. */ 5558 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5559 .data = flow_id, 5560 }; 5561 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5562 if (ret < 0) 5563 return ret; 5564 set_tag->id = ret; 5565 /* Construct new actions array. */ 5566 /* Replace QUEUE/RSS action. */ 5567 split_actions[qrss_idx] = (struct rte_flow_action){ 5568 .type = (enum rte_flow_action_type) 5569 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5570 .conf = set_tag, 5571 }; 5572 } else { 5573 /* 5574 * If we are the suffix flow of meter, tag already exist. 5575 * Set the QUEUE/RSS action to void. 5576 */ 5577 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5578 } 5579 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5580 jump = (void *)(set_tag + 1); 5581 *jump = (struct rte_flow_action_jump){ 5582 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5583 }; 5584 split_actions[actions_n - 2] = (struct rte_flow_action){ 5585 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5586 .conf = jump, 5587 }; 5588 split_actions[actions_n - 1] = (struct rte_flow_action){ 5589 .type = RTE_FLOW_ACTION_TYPE_END, 5590 }; 5591 return flow_id; 5592 } 5593 5594 /** 5595 * Extend the given action list for Tx metadata copy. 5596 * 5597 * Copy the given action list to the ext_actions and add flow metadata register 5598 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5599 * 5600 * @param[out] ext_actions 5601 * Pointer to the extended action list. 5602 * @param[in] actions 5603 * Pointer to the list of actions. 5604 * @param[in] actions_n 5605 * Number of actions in the list. 5606 * @param[out] error 5607 * Perform verbose error reporting if not NULL. 5608 * @param[in] encap_idx 5609 * The encap action index. 5610 * 5611 * @return 5612 * 0 on success, negative value otherwise 5613 */ 5614 static int 5615 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5616 struct rte_flow_action *ext_actions, 5617 const struct rte_flow_action *actions, 5618 int actions_n, struct rte_flow_error *error, 5619 int encap_idx) 5620 { 5621 struct mlx5_flow_action_copy_mreg *cp_mreg = 5622 (struct mlx5_flow_action_copy_mreg *) 5623 (ext_actions + actions_n + 1); 5624 int ret; 5625 5626 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5627 if (ret < 0) 5628 return ret; 5629 cp_mreg->dst = ret; 5630 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5631 if (ret < 0) 5632 return ret; 5633 cp_mreg->src = ret; 5634 if (encap_idx != 0) 5635 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5636 if (encap_idx == actions_n - 1) { 5637 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5638 .type = (enum rte_flow_action_type) 5639 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5640 .conf = cp_mreg, 5641 }; 5642 ext_actions[actions_n] = (struct rte_flow_action){ 5643 .type = RTE_FLOW_ACTION_TYPE_END, 5644 }; 5645 } else { 5646 ext_actions[encap_idx] = (struct rte_flow_action){ 5647 .type = (enum rte_flow_action_type) 5648 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5649 .conf = cp_mreg, 5650 }; 5651 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5652 sizeof(*ext_actions) * (actions_n - encap_idx)); 5653 } 5654 return 0; 5655 } 5656 5657 /** 5658 * Check the match action from the action list. 5659 * 5660 * @param[in] actions 5661 * Pointer to the list of actions. 5662 * @param[in] attr 5663 * Flow rule attributes. 5664 * @param[in] action 5665 * The action to be check if exist. 5666 * @param[out] match_action_pos 5667 * Pointer to the position of the matched action if exists, otherwise is -1. 5668 * @param[out] qrss_action_pos 5669 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5670 * @param[out] modify_after_mirror 5671 * Pointer to the flag of modify action after FDB mirroring. 5672 * 5673 * @return 5674 * > 0 the total number of actions. 5675 * 0 if not found match action in action list. 5676 */ 5677 static int 5678 flow_check_match_action(const struct rte_flow_action actions[], 5679 const struct rte_flow_attr *attr, 5680 enum rte_flow_action_type action, 5681 int *match_action_pos, int *qrss_action_pos, 5682 int *modify_after_mirror) 5683 { 5684 const struct rte_flow_action_sample *sample; 5685 const struct rte_flow_action_raw_decap *decap; 5686 int actions_n = 0; 5687 uint32_t ratio = 0; 5688 int sub_type = 0; 5689 int flag = 0; 5690 int fdb_mirror = 0; 5691 5692 *match_action_pos = -1; 5693 *qrss_action_pos = -1; 5694 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5695 if (actions->type == action) { 5696 flag = 1; 5697 *match_action_pos = actions_n; 5698 } 5699 switch (actions->type) { 5700 case RTE_FLOW_ACTION_TYPE_QUEUE: 5701 case RTE_FLOW_ACTION_TYPE_RSS: 5702 *qrss_action_pos = actions_n; 5703 break; 5704 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5705 sample = actions->conf; 5706 ratio = sample->ratio; 5707 sub_type = ((const struct rte_flow_action *) 5708 (sample->actions))->type; 5709 if (ratio == 1 && attr->transfer) 5710 fdb_mirror = 1; 5711 break; 5712 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5713 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5714 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5715 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5716 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5717 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5718 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5719 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5720 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5721 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5722 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5723 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5724 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5725 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5726 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5727 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5728 case RTE_FLOW_ACTION_TYPE_FLAG: 5729 case RTE_FLOW_ACTION_TYPE_MARK: 5730 case RTE_FLOW_ACTION_TYPE_SET_META: 5731 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5732 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5733 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5734 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5735 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5736 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5737 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5738 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5739 case RTE_FLOW_ACTION_TYPE_METER: 5740 if (fdb_mirror) 5741 *modify_after_mirror = 1; 5742 break; 5743 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5744 decap = actions->conf; 5745 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5746 ; 5747 actions_n++; 5748 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5749 const struct rte_flow_action_raw_encap *encap = 5750 actions->conf; 5751 if (decap->size <= 5752 MLX5_ENCAPSULATION_DECISION_SIZE && 5753 encap->size > 5754 MLX5_ENCAPSULATION_DECISION_SIZE) 5755 /* L3 encap. */ 5756 break; 5757 } 5758 if (fdb_mirror) 5759 *modify_after_mirror = 1; 5760 break; 5761 default: 5762 break; 5763 } 5764 actions_n++; 5765 } 5766 if (flag && fdb_mirror && !*modify_after_mirror) { 5767 /* FDB mirroring uses the destination array to implement 5768 * instead of FLOW_SAMPLER object. 5769 */ 5770 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5771 flag = 0; 5772 } 5773 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5774 return flag ? actions_n + 1 : 0; 5775 } 5776 5777 #define SAMPLE_SUFFIX_ITEM 2 5778 5779 /** 5780 * Split the sample flow. 5781 * 5782 * As sample flow will split to two sub flow, sample flow with 5783 * sample action, the other actions will move to new suffix flow. 5784 * 5785 * Also add unique tag id with tag action in the sample flow, 5786 * the same tag id will be as match in the suffix flow. 5787 * 5788 * @param dev 5789 * Pointer to Ethernet device. 5790 * @param[in] add_tag 5791 * Add extra tag action flag. 5792 * @param[out] sfx_items 5793 * Suffix flow match items (list terminated by the END pattern item). 5794 * @param[in] actions 5795 * Associated actions (list terminated by the END action). 5796 * @param[out] actions_sfx 5797 * Suffix flow actions. 5798 * @param[out] actions_pre 5799 * Prefix flow actions. 5800 * @param[in] actions_n 5801 * The total number of actions. 5802 * @param[in] sample_action_pos 5803 * The sample action position. 5804 * @param[in] qrss_action_pos 5805 * The Queue/RSS action position. 5806 * @param[in] jump_table 5807 * Add extra jump action flag. 5808 * @param[out] error 5809 * Perform verbose error reporting if not NULL. 5810 * 5811 * @return 5812 * 0 on success, or unique flow_id, a negative errno value 5813 * otherwise and rte_errno is set. 5814 */ 5815 static int 5816 flow_sample_split_prep(struct rte_eth_dev *dev, 5817 int add_tag, 5818 struct rte_flow_item sfx_items[], 5819 const struct rte_flow_action actions[], 5820 struct rte_flow_action actions_sfx[], 5821 struct rte_flow_action actions_pre[], 5822 int actions_n, 5823 int sample_action_pos, 5824 int qrss_action_pos, 5825 int jump_table, 5826 struct rte_flow_error *error) 5827 { 5828 struct mlx5_priv *priv = dev->data->dev_private; 5829 struct mlx5_rte_flow_action_set_tag *set_tag; 5830 struct mlx5_rte_flow_item_tag *tag_spec; 5831 struct mlx5_rte_flow_item_tag *tag_mask; 5832 struct rte_flow_action_jump *jump_action; 5833 uint32_t tag_id = 0; 5834 int index; 5835 int append_index = 0; 5836 int ret; 5837 5838 if (sample_action_pos < 0) 5839 return rte_flow_error_set(error, EINVAL, 5840 RTE_FLOW_ERROR_TYPE_ACTION, 5841 NULL, "invalid position of sample " 5842 "action in list"); 5843 /* Prepare the actions for prefix and suffix flow. */ 5844 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5845 index = qrss_action_pos; 5846 /* Put the preceding the Queue/RSS action into prefix flow. */ 5847 if (index != 0) 5848 memcpy(actions_pre, actions, 5849 sizeof(struct rte_flow_action) * index); 5850 /* Put others preceding the sample action into prefix flow. */ 5851 if (sample_action_pos > index + 1) 5852 memcpy(actions_pre + index, actions + index + 1, 5853 sizeof(struct rte_flow_action) * 5854 (sample_action_pos - index - 1)); 5855 index = sample_action_pos - 1; 5856 /* Put Queue/RSS action into Suffix flow. */ 5857 memcpy(actions_sfx, actions + qrss_action_pos, 5858 sizeof(struct rte_flow_action)); 5859 actions_sfx++; 5860 } else { 5861 index = sample_action_pos; 5862 if (index != 0) 5863 memcpy(actions_pre, actions, 5864 sizeof(struct rte_flow_action) * index); 5865 } 5866 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5867 * For CX6DX and above, metadata registers Cx preserve their value, 5868 * add an extra tag action for NIC-RX and E-Switch Domain. 5869 */ 5870 if (add_tag) { 5871 /* Prepare the prefix tag action. */ 5872 append_index++; 5873 set_tag = (void *)(actions_pre + actions_n + append_index); 5874 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5875 if (ret < 0) 5876 return ret; 5877 mlx5_ipool_malloc(priv->sh->ipool 5878 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5879 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5880 .id = ret, 5881 .data = tag_id, 5882 }; 5883 /* Prepare the suffix subflow items. */ 5884 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5885 tag_spec->data = tag_id; 5886 tag_spec->id = set_tag->id; 5887 tag_mask = tag_spec + 1; 5888 tag_mask->data = UINT32_MAX; 5889 sfx_items[0] = (struct rte_flow_item){ 5890 .type = (enum rte_flow_item_type) 5891 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5892 .spec = tag_spec, 5893 .last = NULL, 5894 .mask = tag_mask, 5895 }; 5896 sfx_items[1] = (struct rte_flow_item){ 5897 .type = (enum rte_flow_item_type) 5898 RTE_FLOW_ITEM_TYPE_END, 5899 }; 5900 /* Prepare the tag action in prefix subflow. */ 5901 actions_pre[index++] = 5902 (struct rte_flow_action){ 5903 .type = (enum rte_flow_action_type) 5904 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5905 .conf = set_tag, 5906 }; 5907 } 5908 memcpy(actions_pre + index, actions + sample_action_pos, 5909 sizeof(struct rte_flow_action)); 5910 index += 1; 5911 /* For the modify action after the sample action in E-Switch mirroring, 5912 * Add the extra jump action in prefix subflow and jump into the next 5913 * table, then do the modify action in the new table. 5914 */ 5915 if (jump_table) { 5916 /* Prepare the prefix jump action. */ 5917 append_index++; 5918 jump_action = (void *)(actions_pre + actions_n + append_index); 5919 jump_action->group = jump_table; 5920 actions_pre[index++] = 5921 (struct rte_flow_action){ 5922 .type = (enum rte_flow_action_type) 5923 RTE_FLOW_ACTION_TYPE_JUMP, 5924 .conf = jump_action, 5925 }; 5926 } 5927 actions_pre[index] = (struct rte_flow_action){ 5928 .type = (enum rte_flow_action_type) 5929 RTE_FLOW_ACTION_TYPE_END, 5930 }; 5931 /* Put the actions after sample into Suffix flow. */ 5932 memcpy(actions_sfx, actions + sample_action_pos + 1, 5933 sizeof(struct rte_flow_action) * 5934 (actions_n - sample_action_pos - 1)); 5935 return tag_id; 5936 } 5937 5938 /** 5939 * The splitting for metadata feature. 5940 * 5941 * - Q/RSS action on NIC Rx should be split in order to pass by 5942 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5943 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5944 * 5945 * - All the actions on NIC Tx should have a mreg copy action to 5946 * copy reg_a from WQE to reg_c[0]. 5947 * 5948 * @param dev 5949 * Pointer to Ethernet device. 5950 * @param[in] flow 5951 * Parent flow structure pointer. 5952 * @param[in] attr 5953 * Flow rule attributes. 5954 * @param[in] items 5955 * Pattern specification (list terminated by the END pattern item). 5956 * @param[in] actions 5957 * Associated actions (list terminated by the END action). 5958 * @param[in] flow_split_info 5959 * Pointer to flow split info structure. 5960 * @param[out] error 5961 * Perform verbose error reporting if not NULL. 5962 * @return 5963 * 0 on success, negative value otherwise 5964 */ 5965 static int 5966 flow_create_split_metadata(struct rte_eth_dev *dev, 5967 struct rte_flow *flow, 5968 const struct rte_flow_attr *attr, 5969 const struct rte_flow_item items[], 5970 const struct rte_flow_action actions[], 5971 struct mlx5_flow_split_info *flow_split_info, 5972 struct rte_flow_error *error) 5973 { 5974 struct mlx5_priv *priv = dev->data->dev_private; 5975 struct mlx5_sh_config *config = &priv->sh->config; 5976 const struct rte_flow_action *qrss = NULL; 5977 struct rte_flow_action *ext_actions = NULL; 5978 struct mlx5_flow *dev_flow = NULL; 5979 uint32_t qrss_id = 0; 5980 int mtr_sfx = 0; 5981 size_t act_size; 5982 int actions_n; 5983 int encap_idx; 5984 int ret; 5985 5986 /* Check whether extensive metadata feature is engaged. */ 5987 if (!config->dv_flow_en || 5988 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5989 !mlx5_flow_ext_mreg_supported(dev)) 5990 return flow_create_split_inner(dev, flow, NULL, attr, items, 5991 actions, flow_split_info, error); 5992 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5993 &encap_idx); 5994 if (qrss) { 5995 /* Exclude hairpin flows from splitting. */ 5996 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5997 const struct rte_flow_action_queue *queue; 5998 5999 queue = qrss->conf; 6000 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6001 qrss = NULL; 6002 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6003 const struct rte_flow_action_rss *rss; 6004 6005 rss = qrss->conf; 6006 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6007 qrss = NULL; 6008 } 6009 } 6010 if (qrss) { 6011 /* Check if it is in meter suffix table. */ 6012 mtr_sfx = attr->group == (attr->transfer ? 6013 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6014 MLX5_FLOW_TABLE_LEVEL_METER); 6015 /* 6016 * Q/RSS action on NIC Rx should be split in order to pass by 6017 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6018 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6019 */ 6020 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6021 sizeof(struct rte_flow_action_set_tag) + 6022 sizeof(struct rte_flow_action_jump); 6023 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6024 SOCKET_ID_ANY); 6025 if (!ext_actions) 6026 return rte_flow_error_set(error, ENOMEM, 6027 RTE_FLOW_ERROR_TYPE_ACTION, 6028 NULL, "no memory to split " 6029 "metadata flow"); 6030 /* 6031 * Create the new actions list with removed Q/RSS action 6032 * and appended set tag and jump to register copy table 6033 * (RX_CP_TBL). We should preallocate unique tag ID here 6034 * in advance, because it is needed for set tag action. 6035 */ 6036 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6037 qrss, actions_n, 6038 mtr_sfx, error); 6039 if (!mtr_sfx && !qrss_id) { 6040 ret = -rte_errno; 6041 goto exit; 6042 } 6043 } else if (attr->egress && !attr->transfer) { 6044 /* 6045 * All the actions on NIC Tx should have a metadata register 6046 * copy action to copy reg_a from WQE to reg_c[meta] 6047 */ 6048 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6049 sizeof(struct mlx5_flow_action_copy_mreg); 6050 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6051 SOCKET_ID_ANY); 6052 if (!ext_actions) 6053 return rte_flow_error_set(error, ENOMEM, 6054 RTE_FLOW_ERROR_TYPE_ACTION, 6055 NULL, "no memory to split " 6056 "metadata flow"); 6057 /* Create the action list appended with copy register. */ 6058 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6059 actions_n, error, encap_idx); 6060 if (ret < 0) 6061 goto exit; 6062 } 6063 /* Add the unmodified original or prefix subflow. */ 6064 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6065 items, ext_actions ? ext_actions : 6066 actions, flow_split_info, error); 6067 if (ret < 0) 6068 goto exit; 6069 MLX5_ASSERT(dev_flow); 6070 if (qrss) { 6071 const struct rte_flow_attr q_attr = { 6072 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6073 .ingress = 1, 6074 }; 6075 /* Internal PMD action to set register. */ 6076 struct mlx5_rte_flow_item_tag q_tag_spec = { 6077 .data = qrss_id, 6078 .id = REG_NON, 6079 }; 6080 struct rte_flow_item q_items[] = { 6081 { 6082 .type = (enum rte_flow_item_type) 6083 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6084 .spec = &q_tag_spec, 6085 .last = NULL, 6086 .mask = NULL, 6087 }, 6088 { 6089 .type = RTE_FLOW_ITEM_TYPE_END, 6090 }, 6091 }; 6092 struct rte_flow_action q_actions[] = { 6093 { 6094 .type = qrss->type, 6095 .conf = qrss->conf, 6096 }, 6097 { 6098 .type = RTE_FLOW_ACTION_TYPE_END, 6099 }, 6100 }; 6101 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6102 6103 /* 6104 * Configure the tag item only if there is no meter subflow. 6105 * Since tag is already marked in the meter suffix subflow 6106 * we can just use the meter suffix items as is. 6107 */ 6108 if (qrss_id) { 6109 /* Not meter subflow. */ 6110 MLX5_ASSERT(!mtr_sfx); 6111 /* 6112 * Put unique id in prefix flow due to it is destroyed 6113 * after suffix flow and id will be freed after there 6114 * is no actual flows with this id and identifier 6115 * reallocation becomes possible (for example, for 6116 * other flows in other threads). 6117 */ 6118 dev_flow->handle->split_flow_id = qrss_id; 6119 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6120 error); 6121 if (ret < 0) 6122 goto exit; 6123 q_tag_spec.id = ret; 6124 } 6125 dev_flow = NULL; 6126 /* Add suffix subflow to execute Q/RSS. */ 6127 flow_split_info->prefix_layers = layers; 6128 flow_split_info->prefix_mark = 0; 6129 flow_split_info->table_id = 0; 6130 ret = flow_create_split_inner(dev, flow, &dev_flow, 6131 &q_attr, mtr_sfx ? items : 6132 q_items, q_actions, 6133 flow_split_info, error); 6134 if (ret < 0) 6135 goto exit; 6136 /* qrss ID should be freed if failed. */ 6137 qrss_id = 0; 6138 MLX5_ASSERT(dev_flow); 6139 } 6140 6141 exit: 6142 /* 6143 * We do not destroy the partially created sub_flows in case of error. 6144 * These ones are included into parent flow list and will be destroyed 6145 * by flow_drv_destroy. 6146 */ 6147 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6148 qrss_id); 6149 mlx5_free(ext_actions); 6150 return ret; 6151 } 6152 6153 /** 6154 * Create meter internal drop flow with the original pattern. 6155 * 6156 * @param dev 6157 * Pointer to Ethernet device. 6158 * @param[in] flow 6159 * Parent flow structure pointer. 6160 * @param[in] attr 6161 * Flow rule attributes. 6162 * @param[in] items 6163 * Pattern specification (list terminated by the END pattern item). 6164 * @param[in] flow_split_info 6165 * Pointer to flow split info structure. 6166 * @param[in] fm 6167 * Pointer to flow meter structure. 6168 * @param[out] error 6169 * Perform verbose error reporting if not NULL. 6170 * @return 6171 * 0 on success, negative value otherwise 6172 */ 6173 static uint32_t 6174 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6175 struct rte_flow *flow, 6176 const struct rte_flow_attr *attr, 6177 const struct rte_flow_item items[], 6178 struct mlx5_flow_split_info *flow_split_info, 6179 struct mlx5_flow_meter_info *fm, 6180 struct rte_flow_error *error) 6181 { 6182 struct mlx5_flow *dev_flow = NULL; 6183 struct rte_flow_attr drop_attr = *attr; 6184 struct rte_flow_action drop_actions[3]; 6185 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6186 6187 MLX5_ASSERT(fm->drop_cnt); 6188 drop_actions[0].type = 6189 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6190 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6191 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6192 drop_actions[1].conf = NULL; 6193 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6194 drop_actions[2].conf = NULL; 6195 drop_split_info.external = false; 6196 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6197 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6198 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6199 return flow_create_split_inner(dev, flow, &dev_flow, 6200 &drop_attr, items, drop_actions, 6201 &drop_split_info, error); 6202 } 6203 6204 /** 6205 * The splitting for meter feature. 6206 * 6207 * - The meter flow will be split to two flows as prefix and 6208 * suffix flow. The packets make sense only it pass the prefix 6209 * meter action. 6210 * 6211 * - Reg_C_5 is used for the packet to match betweend prefix and 6212 * suffix flow. 6213 * 6214 * @param dev 6215 * Pointer to Ethernet device. 6216 * @param[in] flow 6217 * Parent flow structure pointer. 6218 * @param[in] attr 6219 * Flow rule attributes. 6220 * @param[in] items 6221 * Pattern specification (list terminated by the END pattern item). 6222 * @param[in] actions 6223 * Associated actions (list terminated by the END action). 6224 * @param[in] flow_split_info 6225 * Pointer to flow split info structure. 6226 * @param[out] error 6227 * Perform verbose error reporting if not NULL. 6228 * @return 6229 * 0 on success, negative value otherwise 6230 */ 6231 static int 6232 flow_create_split_meter(struct rte_eth_dev *dev, 6233 struct rte_flow *flow, 6234 const struct rte_flow_attr *attr, 6235 const struct rte_flow_item items[], 6236 const struct rte_flow_action actions[], 6237 struct mlx5_flow_split_info *flow_split_info, 6238 struct rte_flow_error *error) 6239 { 6240 struct mlx5_priv *priv = dev->data->dev_private; 6241 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6242 struct rte_flow_action *sfx_actions = NULL; 6243 struct rte_flow_action *pre_actions = NULL; 6244 struct rte_flow_item *sfx_items = NULL; 6245 struct mlx5_flow *dev_flow = NULL; 6246 struct rte_flow_attr sfx_attr = *attr; 6247 struct mlx5_flow_meter_info *fm = NULL; 6248 uint8_t skip_scale_restore; 6249 bool has_mtr = false; 6250 bool has_modify = false; 6251 bool set_mtr_reg = true; 6252 bool is_mtr_hierarchy = false; 6253 uint32_t meter_id = 0; 6254 uint32_t mtr_idx = 0; 6255 uint32_t mtr_flow_id = 0; 6256 size_t act_size; 6257 size_t item_size; 6258 int actions_n = 0; 6259 int ret = 0; 6260 6261 if (priv->mtr_en) 6262 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6263 &has_modify, &meter_id); 6264 if (has_mtr) { 6265 if (flow->meter) { 6266 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6267 if (!fm) 6268 return rte_flow_error_set(error, EINVAL, 6269 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6270 NULL, "Meter not found."); 6271 } else { 6272 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6273 if (!fm) 6274 return rte_flow_error_set(error, EINVAL, 6275 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6276 NULL, "Meter not found."); 6277 ret = mlx5_flow_meter_attach(priv, fm, 6278 &sfx_attr, error); 6279 if (ret) 6280 return -rte_errno; 6281 flow->meter = mtr_idx; 6282 } 6283 MLX5_ASSERT(wks); 6284 wks->fm = fm; 6285 if (!fm->def_policy) { 6286 wks->policy = mlx5_flow_meter_policy_find(dev, 6287 fm->policy_id, 6288 NULL); 6289 MLX5_ASSERT(wks->policy); 6290 if (wks->policy->is_hierarchy) { 6291 wks->final_policy = 6292 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6293 wks->policy); 6294 if (!wks->final_policy) 6295 return rte_flow_error_set(error, 6296 EINVAL, 6297 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6298 "Failed to find terminal policy of hierarchy."); 6299 is_mtr_hierarchy = true; 6300 } 6301 } 6302 /* 6303 * If it isn't default-policy Meter, and 6304 * 1. There's no action in flow to change 6305 * packet (modify/encap/decap etc.), OR 6306 * 2. No drop count needed for this meter. 6307 * 3. It's not meter hierarchy. 6308 * Then no need to use regC to save meter id anymore. 6309 */ 6310 if (!fm->def_policy && !is_mtr_hierarchy && 6311 (!has_modify || !fm->drop_cnt)) 6312 set_mtr_reg = false; 6313 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6314 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6315 sizeof(struct mlx5_rte_flow_action_set_tag); 6316 /* Suffix items: tag, vlan, port id, end. */ 6317 #define METER_SUFFIX_ITEM 4 6318 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6319 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6320 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6321 0, SOCKET_ID_ANY); 6322 if (!sfx_actions) 6323 return rte_flow_error_set(error, ENOMEM, 6324 RTE_FLOW_ERROR_TYPE_ACTION, 6325 NULL, "no memory to split " 6326 "meter flow"); 6327 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6328 act_size); 6329 /* There's no suffix flow for meter of non-default policy. */ 6330 if (!fm->def_policy) 6331 pre_actions = sfx_actions + 1; 6332 else 6333 pre_actions = sfx_actions + actions_n; 6334 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6335 items, sfx_items, actions, 6336 sfx_actions, pre_actions, 6337 (set_mtr_reg ? &mtr_flow_id : NULL), 6338 error); 6339 if (ret) { 6340 ret = -rte_errno; 6341 goto exit; 6342 } 6343 /* Add the prefix subflow. */ 6344 skip_scale_restore = flow_split_info->skip_scale; 6345 flow_split_info->skip_scale |= 6346 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6347 ret = flow_create_split_inner(dev, flow, &dev_flow, 6348 attr, items, pre_actions, 6349 flow_split_info, error); 6350 flow_split_info->skip_scale = skip_scale_restore; 6351 if (ret) { 6352 if (mtr_flow_id) 6353 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6354 ret = -rte_errno; 6355 goto exit; 6356 } 6357 if (mtr_flow_id) { 6358 dev_flow->handle->split_flow_id = mtr_flow_id; 6359 dev_flow->handle->is_meter_flow_id = 1; 6360 } 6361 if (!fm->def_policy) { 6362 if (!set_mtr_reg && fm->drop_cnt) 6363 ret = 6364 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6365 &sfx_attr, items, 6366 flow_split_info, 6367 fm, error); 6368 goto exit; 6369 } 6370 /* Setting the sfx group atrr. */ 6371 sfx_attr.group = sfx_attr.transfer ? 6372 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6373 MLX5_FLOW_TABLE_LEVEL_METER; 6374 flow_split_info->prefix_layers = 6375 flow_get_prefix_layer_flags(dev_flow); 6376 flow_split_info->prefix_mark |= wks->mark; 6377 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6378 } 6379 /* Add the prefix subflow. */ 6380 ret = flow_create_split_metadata(dev, flow, 6381 &sfx_attr, sfx_items ? 6382 sfx_items : items, 6383 sfx_actions ? sfx_actions : actions, 6384 flow_split_info, error); 6385 exit: 6386 if (sfx_actions) 6387 mlx5_free(sfx_actions); 6388 return ret; 6389 } 6390 6391 /** 6392 * The splitting for sample feature. 6393 * 6394 * Once Sample action is detected in the action list, the flow actions should 6395 * be split into prefix sub flow and suffix sub flow. 6396 * 6397 * The original items remain in the prefix sub flow, all actions preceding the 6398 * sample action and the sample action itself will be copied to the prefix 6399 * sub flow, the actions following the sample action will be copied to the 6400 * suffix sub flow, Queue action always be located in the suffix sub flow. 6401 * 6402 * In order to make the packet from prefix sub flow matches with suffix sub 6403 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6404 * flow uses tag item with the unique flow id. 6405 * 6406 * @param dev 6407 * Pointer to Ethernet device. 6408 * @param[in] flow 6409 * Parent flow structure pointer. 6410 * @param[in] attr 6411 * Flow rule attributes. 6412 * @param[in] items 6413 * Pattern specification (list terminated by the END pattern item). 6414 * @param[in] actions 6415 * Associated actions (list terminated by the END action). 6416 * @param[in] flow_split_info 6417 * Pointer to flow split info structure. 6418 * @param[out] error 6419 * Perform verbose error reporting if not NULL. 6420 * @return 6421 * 0 on success, negative value otherwise 6422 */ 6423 static int 6424 flow_create_split_sample(struct rte_eth_dev *dev, 6425 struct rte_flow *flow, 6426 const struct rte_flow_attr *attr, 6427 const struct rte_flow_item items[], 6428 const struct rte_flow_action actions[], 6429 struct mlx5_flow_split_info *flow_split_info, 6430 struct rte_flow_error *error) 6431 { 6432 struct mlx5_priv *priv = dev->data->dev_private; 6433 struct rte_flow_action *sfx_actions = NULL; 6434 struct rte_flow_action *pre_actions = NULL; 6435 struct rte_flow_item *sfx_items = NULL; 6436 struct mlx5_flow *dev_flow = NULL; 6437 struct rte_flow_attr sfx_attr = *attr; 6438 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6439 struct mlx5_flow_dv_sample_resource *sample_res; 6440 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6441 struct mlx5_flow_tbl_resource *sfx_tbl; 6442 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6443 #endif 6444 size_t act_size; 6445 size_t item_size; 6446 uint32_t fdb_tx = 0; 6447 int32_t tag_id = 0; 6448 int actions_n = 0; 6449 int sample_action_pos; 6450 int qrss_action_pos; 6451 int add_tag = 0; 6452 int modify_after_mirror = 0; 6453 uint16_t jump_table = 0; 6454 const uint32_t next_ft_step = 1; 6455 int ret = 0; 6456 6457 if (priv->sampler_en) 6458 actions_n = flow_check_match_action(actions, attr, 6459 RTE_FLOW_ACTION_TYPE_SAMPLE, 6460 &sample_action_pos, &qrss_action_pos, 6461 &modify_after_mirror); 6462 if (actions_n) { 6463 /* The prefix actions must includes sample, tag, end. */ 6464 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6465 + sizeof(struct mlx5_rte_flow_action_set_tag); 6466 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6467 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6468 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6469 item_size), 0, SOCKET_ID_ANY); 6470 if (!sfx_actions) 6471 return rte_flow_error_set(error, ENOMEM, 6472 RTE_FLOW_ERROR_TYPE_ACTION, 6473 NULL, "no memory to split " 6474 "sample flow"); 6475 /* The representor_id is UINT16_MAX for uplink. */ 6476 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6477 /* 6478 * When reg_c_preserve is set, metadata registers Cx preserve 6479 * their value even through packet duplication. 6480 */ 6481 add_tag = (!fdb_tx || 6482 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6483 if (add_tag) 6484 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6485 + act_size); 6486 if (modify_after_mirror) 6487 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6488 next_ft_step; 6489 pre_actions = sfx_actions + actions_n; 6490 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6491 actions, sfx_actions, 6492 pre_actions, actions_n, 6493 sample_action_pos, 6494 qrss_action_pos, jump_table, 6495 error); 6496 if (tag_id < 0 || (add_tag && !tag_id)) { 6497 ret = -rte_errno; 6498 goto exit; 6499 } 6500 if (modify_after_mirror) 6501 flow_split_info->skip_scale = 6502 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6503 /* Add the prefix subflow. */ 6504 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6505 items, pre_actions, 6506 flow_split_info, error); 6507 if (ret) { 6508 ret = -rte_errno; 6509 goto exit; 6510 } 6511 dev_flow->handle->split_flow_id = tag_id; 6512 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6513 if (!modify_after_mirror) { 6514 /* Set the sfx group attr. */ 6515 sample_res = (struct mlx5_flow_dv_sample_resource *) 6516 dev_flow->dv.sample_res; 6517 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6518 sample_res->normal_path_tbl; 6519 sfx_tbl_data = container_of(sfx_tbl, 6520 struct mlx5_flow_tbl_data_entry, 6521 tbl); 6522 sfx_attr.group = sfx_attr.transfer ? 6523 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6524 } else { 6525 MLX5_ASSERT(attr->transfer); 6526 sfx_attr.group = jump_table; 6527 } 6528 flow_split_info->prefix_layers = 6529 flow_get_prefix_layer_flags(dev_flow); 6530 MLX5_ASSERT(wks); 6531 flow_split_info->prefix_mark |= wks->mark; 6532 /* Suffix group level already be scaled with factor, set 6533 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6534 * again in translation. 6535 */ 6536 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6537 #endif 6538 } 6539 /* Add the suffix subflow. */ 6540 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6541 sfx_items ? sfx_items : items, 6542 sfx_actions ? sfx_actions : actions, 6543 flow_split_info, error); 6544 exit: 6545 if (sfx_actions) 6546 mlx5_free(sfx_actions); 6547 return ret; 6548 } 6549 6550 /** 6551 * Split the flow to subflow set. The splitters might be linked 6552 * in the chain, like this: 6553 * flow_create_split_outer() calls: 6554 * flow_create_split_meter() calls: 6555 * flow_create_split_metadata(meter_subflow_0) calls: 6556 * flow_create_split_inner(metadata_subflow_0) 6557 * flow_create_split_inner(metadata_subflow_1) 6558 * flow_create_split_inner(metadata_subflow_2) 6559 * flow_create_split_metadata(meter_subflow_1) calls: 6560 * flow_create_split_inner(metadata_subflow_0) 6561 * flow_create_split_inner(metadata_subflow_1) 6562 * flow_create_split_inner(metadata_subflow_2) 6563 * 6564 * This provide flexible way to add new levels of flow splitting. 6565 * The all of successfully created subflows are included to the 6566 * parent flow dev_flow list. 6567 * 6568 * @param dev 6569 * Pointer to Ethernet device. 6570 * @param[in] flow 6571 * Parent flow structure pointer. 6572 * @param[in] attr 6573 * Flow rule attributes. 6574 * @param[in] items 6575 * Pattern specification (list terminated by the END pattern item). 6576 * @param[in] actions 6577 * Associated actions (list terminated by the END action). 6578 * @param[in] flow_split_info 6579 * Pointer to flow split info structure. 6580 * @param[out] error 6581 * Perform verbose error reporting if not NULL. 6582 * @return 6583 * 0 on success, negative value otherwise 6584 */ 6585 static int 6586 flow_create_split_outer(struct rte_eth_dev *dev, 6587 struct rte_flow *flow, 6588 const struct rte_flow_attr *attr, 6589 const struct rte_flow_item items[], 6590 const struct rte_flow_action actions[], 6591 struct mlx5_flow_split_info *flow_split_info, 6592 struct rte_flow_error *error) 6593 { 6594 int ret; 6595 6596 ret = flow_create_split_sample(dev, flow, attr, items, 6597 actions, flow_split_info, error); 6598 MLX5_ASSERT(ret <= 0); 6599 return ret; 6600 } 6601 6602 static inline struct mlx5_flow_tunnel * 6603 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6604 { 6605 struct mlx5_flow_tunnel *tunnel; 6606 6607 #pragma GCC diagnostic push 6608 #pragma GCC diagnostic ignored "-Wcast-qual" 6609 tunnel = (typeof(tunnel))flow->tunnel; 6610 #pragma GCC diagnostic pop 6611 6612 return tunnel; 6613 } 6614 6615 /** 6616 * Adjust flow RSS workspace if needed. 6617 * 6618 * @param wks 6619 * Pointer to thread flow work space. 6620 * @param rss_desc 6621 * Pointer to RSS descriptor. 6622 * @param[in] nrssq_num 6623 * New RSS queue number. 6624 * 6625 * @return 6626 * 0 on success, -1 otherwise and rte_errno is set. 6627 */ 6628 static int 6629 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6630 struct mlx5_flow_rss_desc *rss_desc, 6631 uint32_t nrssq_num) 6632 { 6633 if (likely(nrssq_num <= wks->rssq_num)) 6634 return 0; 6635 rss_desc->queue = realloc(rss_desc->queue, 6636 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6637 if (!rss_desc->queue) { 6638 rte_errno = ENOMEM; 6639 return -1; 6640 } 6641 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6642 return 0; 6643 } 6644 6645 /** 6646 * Create a flow and add it to @p list. 6647 * 6648 * @param dev 6649 * Pointer to Ethernet device. 6650 * @param list 6651 * Pointer to a TAILQ flow list. If this parameter NULL, 6652 * no list insertion occurred, flow is just created, 6653 * this is caller's responsibility to track the 6654 * created flow. 6655 * @param[in] attr 6656 * Flow rule attributes. 6657 * @param[in] items 6658 * Pattern specification (list terminated by the END pattern item). 6659 * @param[in] actions 6660 * Associated actions (list terminated by the END action). 6661 * @param[in] external 6662 * This flow rule is created by request external to PMD. 6663 * @param[out] error 6664 * Perform verbose error reporting if not NULL. 6665 * 6666 * @return 6667 * A flow index on success, 0 otherwise and rte_errno is set. 6668 */ 6669 static uint32_t 6670 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6671 const struct rte_flow_attr *attr, 6672 const struct rte_flow_item items[], 6673 const struct rte_flow_action original_actions[], 6674 bool external, struct rte_flow_error *error) 6675 { 6676 struct mlx5_priv *priv = dev->data->dev_private; 6677 struct rte_flow *flow = NULL; 6678 struct mlx5_flow *dev_flow; 6679 const struct rte_flow_action_rss *rss = NULL; 6680 struct mlx5_translated_action_handle 6681 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6682 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6683 union { 6684 struct mlx5_flow_expand_rss buf; 6685 uint8_t buffer[4096]; 6686 } expand_buffer; 6687 union { 6688 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6689 uint8_t buffer[2048]; 6690 } actions_rx; 6691 union { 6692 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6693 uint8_t buffer[2048]; 6694 } actions_hairpin_tx; 6695 union { 6696 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6697 uint8_t buffer[2048]; 6698 } items_tx; 6699 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6700 struct mlx5_flow_rss_desc *rss_desc; 6701 const struct rte_flow_action *p_actions_rx; 6702 uint32_t i; 6703 uint32_t idx = 0; 6704 int hairpin_flow; 6705 struct rte_flow_attr attr_tx = { .priority = 0 }; 6706 const struct rte_flow_action *actions; 6707 struct rte_flow_action *translated_actions = NULL; 6708 struct mlx5_flow_tunnel *tunnel; 6709 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6710 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6711 struct mlx5_flow_split_info flow_split_info = { 6712 .external = !!external, 6713 .skip_scale = 0, 6714 .flow_idx = 0, 6715 .prefix_mark = 0, 6716 .prefix_layers = 0, 6717 .table_id = 0 6718 }; 6719 int ret; 6720 6721 MLX5_ASSERT(wks); 6722 rss_desc = &wks->rss_desc; 6723 ret = flow_action_handles_translate(dev, original_actions, 6724 indir_actions, 6725 &indir_actions_n, 6726 &translated_actions, error); 6727 if (ret < 0) { 6728 MLX5_ASSERT(translated_actions == NULL); 6729 return 0; 6730 } 6731 actions = translated_actions ? translated_actions : original_actions; 6732 p_actions_rx = actions; 6733 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6734 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6735 external, hairpin_flow, error); 6736 if (ret < 0) 6737 goto error_before_hairpin_split; 6738 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6739 if (!flow) { 6740 rte_errno = ENOMEM; 6741 goto error_before_hairpin_split; 6742 } 6743 if (hairpin_flow > 0) { 6744 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6745 rte_errno = EINVAL; 6746 goto error_before_hairpin_split; 6747 } 6748 flow_hairpin_split(dev, actions, actions_rx.actions, 6749 actions_hairpin_tx.actions, items_tx.items, 6750 idx); 6751 p_actions_rx = actions_rx.actions; 6752 } 6753 flow_split_info.flow_idx = idx; 6754 flow->drv_type = flow_get_drv_type(dev, attr); 6755 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6756 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6757 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6758 /* RSS Action only works on NIC RX domain */ 6759 if (attr->ingress && !attr->transfer) 6760 rss = flow_get_rss_action(dev, p_actions_rx); 6761 if (rss) { 6762 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6763 return 0; 6764 /* 6765 * The following information is required by 6766 * mlx5_flow_hashfields_adjust() in advance. 6767 */ 6768 rss_desc->level = rss->level; 6769 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6770 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6771 } 6772 flow->dev_handles = 0; 6773 if (rss && rss->types) { 6774 unsigned int graph_root; 6775 6776 graph_root = find_graph_root(rss->level); 6777 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6778 items, rss->types, 6779 mlx5_support_expansion, graph_root); 6780 MLX5_ASSERT(ret > 0 && 6781 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6782 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6783 for (i = 0; i < buf->entries; ++i) 6784 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6785 } 6786 } else { 6787 buf->entries = 1; 6788 buf->entry[0].pattern = (void *)(uintptr_t)items; 6789 } 6790 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6791 indir_actions_n); 6792 for (i = 0; i < buf->entries; ++i) { 6793 /* Initialize flow split data. */ 6794 flow_split_info.prefix_layers = 0; 6795 flow_split_info.prefix_mark = 0; 6796 flow_split_info.skip_scale = 0; 6797 /* 6798 * The splitter may create multiple dev_flows, 6799 * depending on configuration. In the simplest 6800 * case it just creates unmodified original flow. 6801 */ 6802 ret = flow_create_split_outer(dev, flow, attr, 6803 buf->entry[i].pattern, 6804 p_actions_rx, &flow_split_info, 6805 error); 6806 if (ret < 0) 6807 goto error; 6808 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6809 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6810 p_actions_rx, 6811 idx, 6812 wks->flows[0].tunnel, 6813 &default_miss_ctx, 6814 error); 6815 if (ret < 0) { 6816 mlx5_free(default_miss_ctx.queue); 6817 goto error; 6818 } 6819 } 6820 } 6821 /* Create the tx flow. */ 6822 if (hairpin_flow) { 6823 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6824 attr_tx.ingress = 0; 6825 attr_tx.egress = 1; 6826 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6827 actions_hairpin_tx.actions, 6828 idx, error); 6829 if (!dev_flow) 6830 goto error; 6831 dev_flow->flow = flow; 6832 dev_flow->external = 0; 6833 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6834 dev_flow->handle, next); 6835 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6836 items_tx.items, 6837 actions_hairpin_tx.actions, error); 6838 if (ret < 0) 6839 goto error; 6840 } 6841 /* 6842 * Update the metadata register copy table. If extensive 6843 * metadata feature is enabled and registers are supported 6844 * we might create the extra rte_flow for each unique 6845 * MARK/FLAG action ID. 6846 * 6847 * The table is updated for ingress Flows only, because 6848 * the egress Flows belong to the different device and 6849 * copy table should be updated in peer NIC Rx domain. 6850 */ 6851 if (attr->ingress && 6852 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6853 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6854 if (ret) 6855 goto error; 6856 } 6857 /* 6858 * If the flow is external (from application) OR device is started, 6859 * OR mreg discover, then apply immediately. 6860 */ 6861 if (external || dev->data->dev_started || 6862 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6863 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6864 ret = flow_drv_apply(dev, flow, error); 6865 if (ret < 0) 6866 goto error; 6867 } 6868 flow->type = type; 6869 flow_rxq_flags_set(dev, flow); 6870 rte_free(translated_actions); 6871 tunnel = flow_tunnel_from_rule(wks->flows); 6872 if (tunnel) { 6873 flow->tunnel = 1; 6874 flow->tunnel_id = tunnel->tunnel_id; 6875 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6876 mlx5_free(default_miss_ctx.queue); 6877 } 6878 mlx5_flow_pop_thread_workspace(); 6879 return idx; 6880 error: 6881 MLX5_ASSERT(flow); 6882 ret = rte_errno; /* Save rte_errno before cleanup. */ 6883 flow_mreg_del_copy_action(dev, flow); 6884 flow_drv_destroy(dev, flow); 6885 if (rss_desc->shared_rss) 6886 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6887 mlx5_ipool_get 6888 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6889 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6890 mlx5_ipool_free(priv->flows[type], idx); 6891 rte_errno = ret; /* Restore rte_errno. */ 6892 ret = rte_errno; 6893 rte_errno = ret; 6894 mlx5_flow_pop_thread_workspace(); 6895 error_before_hairpin_split: 6896 rte_free(translated_actions); 6897 return 0; 6898 } 6899 6900 /** 6901 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6902 * incoming packets to table 1. 6903 * 6904 * Other flow rules, requested for group n, will be created in 6905 * e-switch table n+1. 6906 * Jump action to e-switch group n will be created to group n+1. 6907 * 6908 * Used when working in switchdev mode, to utilise advantages of table 1 6909 * and above. 6910 * 6911 * @param dev 6912 * Pointer to Ethernet device. 6913 * 6914 * @return 6915 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6916 */ 6917 struct rte_flow * 6918 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6919 { 6920 const struct rte_flow_attr attr = { 6921 .group = 0, 6922 .priority = 0, 6923 .ingress = 1, 6924 .egress = 0, 6925 .transfer = 1, 6926 }; 6927 const struct rte_flow_item pattern = { 6928 .type = RTE_FLOW_ITEM_TYPE_END, 6929 }; 6930 struct rte_flow_action_jump jump = { 6931 .group = 1, 6932 }; 6933 const struct rte_flow_action actions[] = { 6934 { 6935 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6936 .conf = &jump, 6937 }, 6938 { 6939 .type = RTE_FLOW_ACTION_TYPE_END, 6940 }, 6941 }; 6942 struct rte_flow_error error; 6943 6944 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6945 &attr, &pattern, 6946 actions, false, &error); 6947 } 6948 6949 /** 6950 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6951 * and sq number, directs all packets to peer vport. 6952 * 6953 * @param dev 6954 * Pointer to Ethernet device. 6955 * @param txq 6956 * Txq index. 6957 * 6958 * @return 6959 * Flow ID on success, 0 otherwise and rte_errno is set. 6960 */ 6961 uint32_t 6962 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6963 { 6964 struct rte_flow_attr attr = { 6965 .group = 0, 6966 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6967 .ingress = 1, 6968 .egress = 0, 6969 .transfer = 1, 6970 }; 6971 struct rte_flow_item_port_id port_spec = { 6972 .id = MLX5_PORT_ESW_MGR, 6973 }; 6974 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6975 .queue = txq, 6976 }; 6977 struct rte_flow_item pattern[] = { 6978 { 6979 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6980 .spec = &port_spec, 6981 }, 6982 { 6983 .type = (enum rte_flow_item_type) 6984 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6985 .spec = &txq_spec, 6986 }, 6987 { 6988 .type = RTE_FLOW_ITEM_TYPE_END, 6989 }, 6990 }; 6991 struct rte_flow_action_jump jump = { 6992 .group = 1, 6993 }; 6994 struct rte_flow_action_port_id port = { 6995 .id = dev->data->port_id, 6996 }; 6997 struct rte_flow_action actions[] = { 6998 { 6999 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7000 .conf = &jump, 7001 }, 7002 { 7003 .type = RTE_FLOW_ACTION_TYPE_END, 7004 }, 7005 }; 7006 struct rte_flow_error error; 7007 7008 /* 7009 * Creates group 0, highest priority jump flow. 7010 * Matches txq to bypass kernel packets. 7011 */ 7012 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7013 false, &error) == 0) 7014 return 0; 7015 /* Create group 1, lowest priority redirect flow for txq. */ 7016 attr.group = 1; 7017 actions[0].conf = &port; 7018 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7019 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7020 actions, false, &error); 7021 } 7022 7023 /** 7024 * Validate a flow supported by the NIC. 7025 * 7026 * @see rte_flow_validate() 7027 * @see rte_flow_ops 7028 */ 7029 int 7030 mlx5_flow_validate(struct rte_eth_dev *dev, 7031 const struct rte_flow_attr *attr, 7032 const struct rte_flow_item items[], 7033 const struct rte_flow_action original_actions[], 7034 struct rte_flow_error *error) 7035 { 7036 int hairpin_flow; 7037 struct mlx5_translated_action_handle 7038 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7039 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7040 const struct rte_flow_action *actions; 7041 struct rte_flow_action *translated_actions = NULL; 7042 int ret = flow_action_handles_translate(dev, original_actions, 7043 indir_actions, 7044 &indir_actions_n, 7045 &translated_actions, error); 7046 7047 if (ret) 7048 return ret; 7049 actions = translated_actions ? translated_actions : original_actions; 7050 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7051 ret = flow_drv_validate(dev, attr, items, actions, 7052 true, hairpin_flow, error); 7053 rte_free(translated_actions); 7054 return ret; 7055 } 7056 7057 /** 7058 * Create a flow. 7059 * 7060 * @see rte_flow_create() 7061 * @see rte_flow_ops 7062 */ 7063 struct rte_flow * 7064 mlx5_flow_create(struct rte_eth_dev *dev, 7065 const struct rte_flow_attr *attr, 7066 const struct rte_flow_item items[], 7067 const struct rte_flow_action actions[], 7068 struct rte_flow_error *error) 7069 { 7070 struct mlx5_priv *priv = dev->data->dev_private; 7071 7072 if (priv->sh->config.dv_flow_en == 2) { 7073 rte_flow_error_set(error, ENOTSUP, 7074 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7075 NULL, 7076 "Flow non-Q creation not supported"); 7077 return NULL; 7078 } 7079 /* 7080 * If the device is not started yet, it is not allowed to created a 7081 * flow from application. PMD default flows and traffic control flows 7082 * are not affected. 7083 */ 7084 if (unlikely(!dev->data->dev_started)) { 7085 DRV_LOG(DEBUG, "port %u is not started when " 7086 "inserting a flow", dev->data->port_id); 7087 rte_flow_error_set(error, ENODEV, 7088 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7089 NULL, 7090 "port not started"); 7091 return NULL; 7092 } 7093 7094 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7095 attr, items, actions, 7096 true, error); 7097 } 7098 7099 /** 7100 * Destroy a flow in a list. 7101 * 7102 * @param dev 7103 * Pointer to Ethernet device. 7104 * @param[in] flow_idx 7105 * Index of flow to destroy. 7106 */ 7107 static void 7108 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7109 uint32_t flow_idx) 7110 { 7111 struct mlx5_priv *priv = dev->data->dev_private; 7112 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7113 7114 if (!flow) 7115 return; 7116 MLX5_ASSERT(flow->type == type); 7117 /* 7118 * Update RX queue flags only if port is started, otherwise it is 7119 * already clean. 7120 */ 7121 if (dev->data->dev_started) 7122 flow_rxq_flags_trim(dev, flow); 7123 flow_drv_destroy(dev, flow); 7124 if (flow->tunnel) { 7125 struct mlx5_flow_tunnel *tunnel; 7126 7127 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7128 RTE_VERIFY(tunnel); 7129 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7130 mlx5_flow_tunnel_free(dev, tunnel); 7131 } 7132 flow_mreg_del_copy_action(dev, flow); 7133 mlx5_ipool_free(priv->flows[type], flow_idx); 7134 } 7135 7136 /** 7137 * Destroy all flows. 7138 * 7139 * @param dev 7140 * Pointer to Ethernet device. 7141 * @param type 7142 * Flow type to be flushed. 7143 * @param active 7144 * If flushing is called actively. 7145 */ 7146 void 7147 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7148 bool active) 7149 { 7150 struct mlx5_priv *priv = dev->data->dev_private; 7151 uint32_t num_flushed = 0, fidx = 1; 7152 struct rte_flow *flow; 7153 7154 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7155 if (priv->sh->config.dv_flow_en == 2 && 7156 type == MLX5_FLOW_TYPE_GEN) { 7157 flow_hw_q_flow_flush(dev, NULL); 7158 return; 7159 } 7160 #endif 7161 7162 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7163 flow_list_destroy(dev, type, fidx); 7164 num_flushed++; 7165 } 7166 if (active) { 7167 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7168 dev->data->port_id, num_flushed); 7169 } 7170 } 7171 7172 /** 7173 * Stop all default actions for flows. 7174 * 7175 * @param dev 7176 * Pointer to Ethernet device. 7177 */ 7178 void 7179 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7180 { 7181 flow_mreg_del_default_copy_action(dev); 7182 flow_rxq_flags_clear(dev); 7183 } 7184 7185 /** 7186 * Start all default actions for flows. 7187 * 7188 * @param dev 7189 * Pointer to Ethernet device. 7190 * @return 7191 * 0 on success, a negative errno value otherwise and rte_errno is set. 7192 */ 7193 int 7194 mlx5_flow_start_default(struct rte_eth_dev *dev) 7195 { 7196 struct rte_flow_error error; 7197 7198 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7199 return flow_mreg_add_default_copy_action(dev, &error); 7200 } 7201 7202 /** 7203 * Release key of thread specific flow workspace data. 7204 */ 7205 void 7206 flow_release_workspace(void *data) 7207 { 7208 struct mlx5_flow_workspace *wks = data; 7209 struct mlx5_flow_workspace *next; 7210 7211 while (wks) { 7212 next = wks->next; 7213 free(wks->rss_desc.queue); 7214 free(wks); 7215 wks = next; 7216 } 7217 } 7218 7219 /** 7220 * Get thread specific current flow workspace. 7221 * 7222 * @return pointer to thread specific flow workspace data, NULL on error. 7223 */ 7224 struct mlx5_flow_workspace* 7225 mlx5_flow_get_thread_workspace(void) 7226 { 7227 struct mlx5_flow_workspace *data; 7228 7229 data = mlx5_flow_os_get_specific_workspace(); 7230 MLX5_ASSERT(data && data->inuse); 7231 if (!data || !data->inuse) 7232 DRV_LOG(ERR, "flow workspace not initialized."); 7233 return data; 7234 } 7235 7236 /** 7237 * Allocate and init new flow workspace. 7238 * 7239 * @return pointer to flow workspace data, NULL on error. 7240 */ 7241 static struct mlx5_flow_workspace* 7242 flow_alloc_thread_workspace(void) 7243 { 7244 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7245 7246 if (!data) { 7247 DRV_LOG(ERR, "Failed to allocate flow workspace " 7248 "memory."); 7249 return NULL; 7250 } 7251 data->rss_desc.queue = calloc(1, 7252 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7253 if (!data->rss_desc.queue) 7254 goto err; 7255 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7256 return data; 7257 err: 7258 free(data->rss_desc.queue); 7259 free(data); 7260 return NULL; 7261 } 7262 7263 /** 7264 * Get new thread specific flow workspace. 7265 * 7266 * If current workspace inuse, create new one and set as current. 7267 * 7268 * @return pointer to thread specific flow workspace data, NULL on error. 7269 */ 7270 static struct mlx5_flow_workspace* 7271 mlx5_flow_push_thread_workspace(void) 7272 { 7273 struct mlx5_flow_workspace *curr; 7274 struct mlx5_flow_workspace *data; 7275 7276 curr = mlx5_flow_os_get_specific_workspace(); 7277 if (!curr) { 7278 data = flow_alloc_thread_workspace(); 7279 if (!data) 7280 return NULL; 7281 } else if (!curr->inuse) { 7282 data = curr; 7283 } else if (curr->next) { 7284 data = curr->next; 7285 } else { 7286 data = flow_alloc_thread_workspace(); 7287 if (!data) 7288 return NULL; 7289 curr->next = data; 7290 data->prev = curr; 7291 } 7292 data->inuse = 1; 7293 data->flow_idx = 0; 7294 /* Set as current workspace */ 7295 if (mlx5_flow_os_set_specific_workspace(data)) 7296 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7297 return data; 7298 } 7299 7300 /** 7301 * Close current thread specific flow workspace. 7302 * 7303 * If previous workspace available, set it as current. 7304 * 7305 * @return pointer to thread specific flow workspace data, NULL on error. 7306 */ 7307 static void 7308 mlx5_flow_pop_thread_workspace(void) 7309 { 7310 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7311 7312 if (!data) 7313 return; 7314 if (!data->inuse) { 7315 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7316 return; 7317 } 7318 data->inuse = 0; 7319 if (!data->prev) 7320 return; 7321 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7322 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7323 } 7324 7325 /** 7326 * Verify the flow list is empty 7327 * 7328 * @param dev 7329 * Pointer to Ethernet device. 7330 * 7331 * @return the number of flows not released. 7332 */ 7333 int 7334 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7335 { 7336 struct mlx5_priv *priv = dev->data->dev_private; 7337 struct rte_flow *flow; 7338 uint32_t idx = 0; 7339 int ret = 0, i; 7340 7341 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7342 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7343 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7344 dev->data->port_id, (void *)flow); 7345 ret++; 7346 } 7347 } 7348 return ret; 7349 } 7350 7351 /** 7352 * Enable default hairpin egress flow. 7353 * 7354 * @param dev 7355 * Pointer to Ethernet device. 7356 * @param queue 7357 * The queue index. 7358 * 7359 * @return 7360 * 0 on success, a negative errno value otherwise and rte_errno is set. 7361 */ 7362 int 7363 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7364 uint32_t queue) 7365 { 7366 const struct rte_flow_attr attr = { 7367 .egress = 1, 7368 .priority = 0, 7369 }; 7370 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7371 .queue = queue, 7372 }; 7373 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7374 .queue = UINT32_MAX, 7375 }; 7376 struct rte_flow_item items[] = { 7377 { 7378 .type = (enum rte_flow_item_type) 7379 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7380 .spec = &queue_spec, 7381 .last = NULL, 7382 .mask = &queue_mask, 7383 }, 7384 { 7385 .type = RTE_FLOW_ITEM_TYPE_END, 7386 }, 7387 }; 7388 struct rte_flow_action_jump jump = { 7389 .group = MLX5_HAIRPIN_TX_TABLE, 7390 }; 7391 struct rte_flow_action actions[2]; 7392 uint32_t flow_idx; 7393 struct rte_flow_error error; 7394 7395 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7396 actions[0].conf = &jump; 7397 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7398 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7399 &attr, items, actions, false, &error); 7400 if (!flow_idx) { 7401 DRV_LOG(DEBUG, 7402 "Failed to create ctrl flow: rte_errno(%d)," 7403 " type(%d), message(%s)", 7404 rte_errno, error.type, 7405 error.message ? error.message : " (no stated reason)"); 7406 return -rte_errno; 7407 } 7408 return 0; 7409 } 7410 7411 /** 7412 * Enable a control flow configured from the control plane. 7413 * 7414 * @param dev 7415 * Pointer to Ethernet device. 7416 * @param eth_spec 7417 * An Ethernet flow spec to apply. 7418 * @param eth_mask 7419 * An Ethernet flow mask to apply. 7420 * @param vlan_spec 7421 * A VLAN flow spec to apply. 7422 * @param vlan_mask 7423 * A VLAN flow mask to apply. 7424 * 7425 * @return 7426 * 0 on success, a negative errno value otherwise and rte_errno is set. 7427 */ 7428 int 7429 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7430 struct rte_flow_item_eth *eth_spec, 7431 struct rte_flow_item_eth *eth_mask, 7432 struct rte_flow_item_vlan *vlan_spec, 7433 struct rte_flow_item_vlan *vlan_mask) 7434 { 7435 struct mlx5_priv *priv = dev->data->dev_private; 7436 const struct rte_flow_attr attr = { 7437 .ingress = 1, 7438 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7439 }; 7440 struct rte_flow_item items[] = { 7441 { 7442 .type = RTE_FLOW_ITEM_TYPE_ETH, 7443 .spec = eth_spec, 7444 .last = NULL, 7445 .mask = eth_mask, 7446 }, 7447 { 7448 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7449 RTE_FLOW_ITEM_TYPE_END, 7450 .spec = vlan_spec, 7451 .last = NULL, 7452 .mask = vlan_mask, 7453 }, 7454 { 7455 .type = RTE_FLOW_ITEM_TYPE_END, 7456 }, 7457 }; 7458 uint16_t queue[priv->reta_idx_n]; 7459 struct rte_flow_action_rss action_rss = { 7460 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7461 .level = 0, 7462 .types = priv->rss_conf.rss_hf, 7463 .key_len = priv->rss_conf.rss_key_len, 7464 .queue_num = priv->reta_idx_n, 7465 .key = priv->rss_conf.rss_key, 7466 .queue = queue, 7467 }; 7468 struct rte_flow_action actions[] = { 7469 { 7470 .type = RTE_FLOW_ACTION_TYPE_RSS, 7471 .conf = &action_rss, 7472 }, 7473 { 7474 .type = RTE_FLOW_ACTION_TYPE_END, 7475 }, 7476 }; 7477 uint32_t flow_idx; 7478 struct rte_flow_error error; 7479 unsigned int i; 7480 7481 if (!priv->reta_idx_n || !priv->rxqs_n) { 7482 return 0; 7483 } 7484 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7485 action_rss.types = 0; 7486 for (i = 0; i != priv->reta_idx_n; ++i) 7487 queue[i] = (*priv->reta_idx)[i]; 7488 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7489 &attr, items, actions, false, &error); 7490 if (!flow_idx) 7491 return -rte_errno; 7492 return 0; 7493 } 7494 7495 /** 7496 * Enable a flow control configured from the control plane. 7497 * 7498 * @param dev 7499 * Pointer to Ethernet device. 7500 * @param eth_spec 7501 * An Ethernet flow spec to apply. 7502 * @param eth_mask 7503 * An Ethernet flow mask to apply. 7504 * 7505 * @return 7506 * 0 on success, a negative errno value otherwise and rte_errno is set. 7507 */ 7508 int 7509 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7510 struct rte_flow_item_eth *eth_spec, 7511 struct rte_flow_item_eth *eth_mask) 7512 { 7513 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7514 } 7515 7516 /** 7517 * Create default miss flow rule matching lacp traffic 7518 * 7519 * @param dev 7520 * Pointer to Ethernet device. 7521 * @param eth_spec 7522 * An Ethernet flow spec to apply. 7523 * 7524 * @return 7525 * 0 on success, a negative errno value otherwise and rte_errno is set. 7526 */ 7527 int 7528 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7529 { 7530 /* 7531 * The LACP matching is done by only using ether type since using 7532 * a multicast dst mac causes kernel to give low priority to this flow. 7533 */ 7534 static const struct rte_flow_item_eth lacp_spec = { 7535 .type = RTE_BE16(0x8809), 7536 }; 7537 static const struct rte_flow_item_eth lacp_mask = { 7538 .type = 0xffff, 7539 }; 7540 const struct rte_flow_attr attr = { 7541 .ingress = 1, 7542 }; 7543 struct rte_flow_item items[] = { 7544 { 7545 .type = RTE_FLOW_ITEM_TYPE_ETH, 7546 .spec = &lacp_spec, 7547 .mask = &lacp_mask, 7548 }, 7549 { 7550 .type = RTE_FLOW_ITEM_TYPE_END, 7551 }, 7552 }; 7553 struct rte_flow_action actions[] = { 7554 { 7555 .type = (enum rte_flow_action_type) 7556 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7557 }, 7558 { 7559 .type = RTE_FLOW_ACTION_TYPE_END, 7560 }, 7561 }; 7562 struct rte_flow_error error; 7563 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7564 &attr, items, actions, 7565 false, &error); 7566 7567 if (!flow_idx) 7568 return -rte_errno; 7569 return 0; 7570 } 7571 7572 /** 7573 * Destroy a flow. 7574 * 7575 * @see rte_flow_destroy() 7576 * @see rte_flow_ops 7577 */ 7578 int 7579 mlx5_flow_destroy(struct rte_eth_dev *dev, 7580 struct rte_flow *flow, 7581 struct rte_flow_error *error __rte_unused) 7582 { 7583 struct mlx5_priv *priv = dev->data->dev_private; 7584 7585 if (priv->sh->config.dv_flow_en == 2) 7586 return rte_flow_error_set(error, ENOTSUP, 7587 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7588 NULL, 7589 "Flow non-Q destruction not supported"); 7590 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7591 (uintptr_t)(void *)flow); 7592 return 0; 7593 } 7594 7595 /** 7596 * Destroy all flows. 7597 * 7598 * @see rte_flow_flush() 7599 * @see rte_flow_ops 7600 */ 7601 int 7602 mlx5_flow_flush(struct rte_eth_dev *dev, 7603 struct rte_flow_error *error __rte_unused) 7604 { 7605 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7606 return 0; 7607 } 7608 7609 /** 7610 * Isolated mode. 7611 * 7612 * @see rte_flow_isolate() 7613 * @see rte_flow_ops 7614 */ 7615 int 7616 mlx5_flow_isolate(struct rte_eth_dev *dev, 7617 int enable, 7618 struct rte_flow_error *error) 7619 { 7620 struct mlx5_priv *priv = dev->data->dev_private; 7621 7622 if (dev->data->dev_started) { 7623 rte_flow_error_set(error, EBUSY, 7624 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7625 NULL, 7626 "port must be stopped first"); 7627 return -rte_errno; 7628 } 7629 priv->isolated = !!enable; 7630 if (enable) 7631 dev->dev_ops = &mlx5_dev_ops_isolate; 7632 else 7633 dev->dev_ops = &mlx5_dev_ops; 7634 7635 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7636 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7637 7638 return 0; 7639 } 7640 7641 /** 7642 * Query a flow. 7643 * 7644 * @see rte_flow_query() 7645 * @see rte_flow_ops 7646 */ 7647 static int 7648 flow_drv_query(struct rte_eth_dev *dev, 7649 uint32_t flow_idx, 7650 const struct rte_flow_action *actions, 7651 void *data, 7652 struct rte_flow_error *error) 7653 { 7654 struct mlx5_priv *priv = dev->data->dev_private; 7655 const struct mlx5_flow_driver_ops *fops; 7656 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7657 flow_idx); 7658 enum mlx5_flow_drv_type ftype; 7659 7660 if (!flow) { 7661 return rte_flow_error_set(error, ENOENT, 7662 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7663 NULL, 7664 "invalid flow handle"); 7665 } 7666 ftype = flow->drv_type; 7667 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7668 fops = flow_get_drv_ops(ftype); 7669 7670 return fops->query(dev, flow, actions, data, error); 7671 } 7672 7673 /** 7674 * Query a flow. 7675 * 7676 * @see rte_flow_query() 7677 * @see rte_flow_ops 7678 */ 7679 int 7680 mlx5_flow_query(struct rte_eth_dev *dev, 7681 struct rte_flow *flow, 7682 const struct rte_flow_action *actions, 7683 void *data, 7684 struct rte_flow_error *error) 7685 { 7686 int ret; 7687 struct mlx5_priv *priv = dev->data->dev_private; 7688 7689 if (priv->sh->config.dv_flow_en == 2) 7690 return rte_flow_error_set(error, ENOTSUP, 7691 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7692 NULL, 7693 "Flow non-Q query not supported"); 7694 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7695 error); 7696 if (ret < 0) 7697 return ret; 7698 return 0; 7699 } 7700 7701 /** 7702 * Get rte_flow callbacks. 7703 * 7704 * @param dev 7705 * Pointer to Ethernet device structure. 7706 * @param ops 7707 * Pointer to operation-specific structure. 7708 * 7709 * @return 0 7710 */ 7711 int 7712 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7713 const struct rte_flow_ops **ops) 7714 { 7715 *ops = &mlx5_flow_ops; 7716 return 0; 7717 } 7718 7719 /** 7720 * Validate meter policy actions. 7721 * Dispatcher for action type specific validation. 7722 * 7723 * @param[in] dev 7724 * Pointer to the Ethernet device structure. 7725 * @param[in] action 7726 * The meter policy action object to validate. 7727 * @param[in] attr 7728 * Attributes of flow to determine steering domain. 7729 * @param[out] is_rss 7730 * Is RSS or not. 7731 * @param[out] domain_bitmap 7732 * Domain bitmap. 7733 * @param[out] is_def_policy 7734 * Is default policy or not. 7735 * @param[out] error 7736 * Perform verbose error reporting if not NULL. Initialized in case of 7737 * error only. 7738 * 7739 * @return 7740 * 0 on success, otherwise negative errno value. 7741 */ 7742 int 7743 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7744 const struct rte_flow_action *actions[RTE_COLORS], 7745 struct rte_flow_attr *attr, 7746 bool *is_rss, 7747 uint8_t *domain_bitmap, 7748 uint8_t *policy_mode, 7749 struct rte_mtr_error *error) 7750 { 7751 const struct mlx5_flow_driver_ops *fops; 7752 7753 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7754 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7755 domain_bitmap, policy_mode, error); 7756 } 7757 7758 /** 7759 * Destroy the meter table set. 7760 * 7761 * @param[in] dev 7762 * Pointer to Ethernet device. 7763 * @param[in] mtr_policy 7764 * Meter policy struct. 7765 */ 7766 void 7767 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7768 struct mlx5_flow_meter_policy *mtr_policy) 7769 { 7770 const struct mlx5_flow_driver_ops *fops; 7771 7772 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7773 fops->destroy_mtr_acts(dev, mtr_policy); 7774 } 7775 7776 /** 7777 * Create policy action, lock free, 7778 * (mutex should be acquired by caller). 7779 * Dispatcher for action type specific call. 7780 * 7781 * @param[in] dev 7782 * Pointer to the Ethernet device structure. 7783 * @param[in] mtr_policy 7784 * Meter policy struct. 7785 * @param[in] action 7786 * Action specification used to create meter actions. 7787 * @param[out] error 7788 * Perform verbose error reporting if not NULL. Initialized in case of 7789 * error only. 7790 * 7791 * @return 7792 * 0 on success, otherwise negative errno value. 7793 */ 7794 int 7795 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7796 struct mlx5_flow_meter_policy *mtr_policy, 7797 const struct rte_flow_action *actions[RTE_COLORS], 7798 struct rte_mtr_error *error) 7799 { 7800 const struct mlx5_flow_driver_ops *fops; 7801 7802 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7803 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7804 } 7805 7806 /** 7807 * Create policy rules, lock free, 7808 * (mutex should be acquired by caller). 7809 * Dispatcher for action type specific call. 7810 * 7811 * @param[in] dev 7812 * Pointer to the Ethernet device structure. 7813 * @param[in] mtr_policy 7814 * Meter policy struct. 7815 * 7816 * @return 7817 * 0 on success, -1 otherwise. 7818 */ 7819 int 7820 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7821 struct mlx5_flow_meter_policy *mtr_policy) 7822 { 7823 const struct mlx5_flow_driver_ops *fops; 7824 7825 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7826 return fops->create_policy_rules(dev, mtr_policy); 7827 } 7828 7829 /** 7830 * Destroy policy rules, lock free, 7831 * (mutex should be acquired by caller). 7832 * Dispatcher for action type specific call. 7833 * 7834 * @param[in] dev 7835 * Pointer to the Ethernet device structure. 7836 * @param[in] mtr_policy 7837 * Meter policy struct. 7838 */ 7839 void 7840 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7841 struct mlx5_flow_meter_policy *mtr_policy) 7842 { 7843 const struct mlx5_flow_driver_ops *fops; 7844 7845 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7846 fops->destroy_policy_rules(dev, mtr_policy); 7847 } 7848 7849 /** 7850 * Destroy the default policy table set. 7851 * 7852 * @param[in] dev 7853 * Pointer to Ethernet device. 7854 */ 7855 void 7856 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7857 { 7858 const struct mlx5_flow_driver_ops *fops; 7859 7860 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7861 fops->destroy_def_policy(dev); 7862 } 7863 7864 /** 7865 * Destroy the default policy table set. 7866 * 7867 * @param[in] dev 7868 * Pointer to Ethernet device. 7869 * 7870 * @return 7871 * 0 on success, -1 otherwise. 7872 */ 7873 int 7874 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7875 { 7876 const struct mlx5_flow_driver_ops *fops; 7877 7878 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7879 return fops->create_def_policy(dev); 7880 } 7881 7882 /** 7883 * Create the needed meter and suffix tables. 7884 * 7885 * @param[in] dev 7886 * Pointer to Ethernet device. 7887 * 7888 * @return 7889 * 0 on success, -1 otherwise. 7890 */ 7891 int 7892 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7893 struct mlx5_flow_meter_info *fm, 7894 uint32_t mtr_idx, 7895 uint8_t domain_bitmap) 7896 { 7897 const struct mlx5_flow_driver_ops *fops; 7898 7899 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7900 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7901 } 7902 7903 /** 7904 * Destroy the meter table set. 7905 * 7906 * @param[in] dev 7907 * Pointer to Ethernet device. 7908 * @param[in] tbl 7909 * Pointer to the meter table set. 7910 */ 7911 void 7912 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7913 struct mlx5_flow_meter_info *fm) 7914 { 7915 const struct mlx5_flow_driver_ops *fops; 7916 7917 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7918 fops->destroy_mtr_tbls(dev, fm); 7919 } 7920 7921 /** 7922 * Destroy the global meter drop table. 7923 * 7924 * @param[in] dev 7925 * Pointer to Ethernet device. 7926 */ 7927 void 7928 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7929 { 7930 const struct mlx5_flow_driver_ops *fops; 7931 7932 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7933 fops->destroy_mtr_drop_tbls(dev); 7934 } 7935 7936 /** 7937 * Destroy the sub policy table with RX queue. 7938 * 7939 * @param[in] dev 7940 * Pointer to Ethernet device. 7941 * @param[in] mtr_policy 7942 * Pointer to meter policy table. 7943 */ 7944 void 7945 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7946 struct mlx5_flow_meter_policy *mtr_policy) 7947 { 7948 const struct mlx5_flow_driver_ops *fops; 7949 7950 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7951 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7952 } 7953 7954 /** 7955 * Allocate the needed aso flow meter id. 7956 * 7957 * @param[in] dev 7958 * Pointer to Ethernet device. 7959 * 7960 * @return 7961 * Index to aso flow meter on success, NULL otherwise. 7962 */ 7963 uint32_t 7964 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7965 { 7966 const struct mlx5_flow_driver_ops *fops; 7967 7968 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7969 return fops->create_meter(dev); 7970 } 7971 7972 /** 7973 * Free the aso flow meter id. 7974 * 7975 * @param[in] dev 7976 * Pointer to Ethernet device. 7977 * @param[in] mtr_idx 7978 * Index to aso flow meter to be free. 7979 * 7980 * @return 7981 * 0 on success. 7982 */ 7983 void 7984 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7985 { 7986 const struct mlx5_flow_driver_ops *fops; 7987 7988 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7989 fops->free_meter(dev, mtr_idx); 7990 } 7991 7992 /** 7993 * Allocate a counter. 7994 * 7995 * @param[in] dev 7996 * Pointer to Ethernet device structure. 7997 * 7998 * @return 7999 * Index to allocated counter on success, 0 otherwise. 8000 */ 8001 uint32_t 8002 mlx5_counter_alloc(struct rte_eth_dev *dev) 8003 { 8004 const struct mlx5_flow_driver_ops *fops; 8005 struct rte_flow_attr attr = { .transfer = 0 }; 8006 8007 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8008 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8009 return fops->counter_alloc(dev); 8010 } 8011 DRV_LOG(ERR, 8012 "port %u counter allocate is not supported.", 8013 dev->data->port_id); 8014 return 0; 8015 } 8016 8017 /** 8018 * Free a counter. 8019 * 8020 * @param[in] dev 8021 * Pointer to Ethernet device structure. 8022 * @param[in] cnt 8023 * Index to counter to be free. 8024 */ 8025 void 8026 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8027 { 8028 const struct mlx5_flow_driver_ops *fops; 8029 struct rte_flow_attr attr = { .transfer = 0 }; 8030 8031 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8032 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8033 fops->counter_free(dev, cnt); 8034 return; 8035 } 8036 DRV_LOG(ERR, 8037 "port %u counter free is not supported.", 8038 dev->data->port_id); 8039 } 8040 8041 /** 8042 * Query counter statistics. 8043 * 8044 * @param[in] dev 8045 * Pointer to Ethernet device structure. 8046 * @param[in] cnt 8047 * Index to counter to query. 8048 * @param[in] clear 8049 * Set to clear counter statistics. 8050 * @param[out] pkts 8051 * The counter hits packets number to save. 8052 * @param[out] bytes 8053 * The counter hits bytes number to save. 8054 * 8055 * @return 8056 * 0 on success, a negative errno value otherwise. 8057 */ 8058 int 8059 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8060 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8061 { 8062 const struct mlx5_flow_driver_ops *fops; 8063 struct rte_flow_attr attr = { .transfer = 0 }; 8064 8065 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8066 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8067 return fops->counter_query(dev, cnt, clear, pkts, 8068 bytes, action); 8069 } 8070 DRV_LOG(ERR, 8071 "port %u counter query is not supported.", 8072 dev->data->port_id); 8073 return -ENOTSUP; 8074 } 8075 8076 /** 8077 * Get information about HWS pre-configurable resources. 8078 * 8079 * @param[in] dev 8080 * Pointer to the rte_eth_dev structure. 8081 * @param[out] port_info 8082 * Pointer to port information. 8083 * @param[out] queue_info 8084 * Pointer to queue information. 8085 * @param[out] error 8086 * Pointer to error structure. 8087 * 8088 * @return 8089 * 0 on success, a negative errno value otherwise and rte_errno is set. 8090 */ 8091 static int 8092 mlx5_flow_info_get(struct rte_eth_dev *dev, 8093 struct rte_flow_port_info *port_info, 8094 struct rte_flow_queue_info *queue_info, 8095 struct rte_flow_error *error) 8096 { 8097 const struct mlx5_flow_driver_ops *fops; 8098 8099 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8100 return rte_flow_error_set(error, ENOTSUP, 8101 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8102 NULL, 8103 "info get with incorrect steering mode"); 8104 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8105 return fops->info_get(dev, port_info, queue_info, error); 8106 } 8107 8108 /** 8109 * Configure port HWS resources. 8110 * 8111 * @param[in] dev 8112 * Pointer to the rte_eth_dev structure. 8113 * @param[in] port_attr 8114 * Port configuration attributes. 8115 * @param[in] nb_queue 8116 * Number of queue. 8117 * @param[in] queue_attr 8118 * Array that holds attributes for each flow queue. 8119 * @param[out] error 8120 * Pointer to error structure. 8121 * 8122 * @return 8123 * 0 on success, a negative errno value otherwise and rte_errno is set. 8124 */ 8125 static int 8126 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8127 const struct rte_flow_port_attr *port_attr, 8128 uint16_t nb_queue, 8129 const struct rte_flow_queue_attr *queue_attr[], 8130 struct rte_flow_error *error) 8131 { 8132 const struct mlx5_flow_driver_ops *fops; 8133 8134 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8135 return rte_flow_error_set(error, ENOTSUP, 8136 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8137 NULL, 8138 "port configure with incorrect steering mode"); 8139 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8140 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8141 } 8142 8143 /** 8144 * Create flow item template. 8145 * 8146 * @param[in] dev 8147 * Pointer to the rte_eth_dev structure. 8148 * @param[in] attr 8149 * Pointer to the item template attributes. 8150 * @param[in] items 8151 * The template item pattern. 8152 * @param[out] error 8153 * Pointer to error structure. 8154 * 8155 * @return 8156 * 0 on success, a negative errno value otherwise and rte_errno is set. 8157 */ 8158 static struct rte_flow_pattern_template * 8159 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8160 const struct rte_flow_pattern_template_attr *attr, 8161 const struct rte_flow_item items[], 8162 struct rte_flow_error *error) 8163 { 8164 const struct mlx5_flow_driver_ops *fops; 8165 8166 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8167 rte_flow_error_set(error, ENOTSUP, 8168 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8169 NULL, 8170 "pattern create with incorrect steering mode"); 8171 return NULL; 8172 } 8173 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8174 return fops->pattern_template_create(dev, attr, items, error); 8175 } 8176 8177 /** 8178 * Destroy flow item template. 8179 * 8180 * @param[in] dev 8181 * Pointer to the rte_eth_dev structure. 8182 * @param[in] template 8183 * Pointer to the item template to be destroyed. 8184 * @param[out] error 8185 * Pointer to error structure. 8186 * 8187 * @return 8188 * 0 on success, a negative errno value otherwise and rte_errno is set. 8189 */ 8190 static int 8191 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8192 struct rte_flow_pattern_template *template, 8193 struct rte_flow_error *error) 8194 { 8195 const struct mlx5_flow_driver_ops *fops; 8196 8197 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8198 return rte_flow_error_set(error, ENOTSUP, 8199 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8200 NULL, 8201 "pattern destroy with incorrect steering mode"); 8202 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8203 return fops->pattern_template_destroy(dev, template, error); 8204 } 8205 8206 /** 8207 * Create flow item template. 8208 * 8209 * @param[in] dev 8210 * Pointer to the rte_eth_dev structure. 8211 * @param[in] attr 8212 * Pointer to the action template attributes. 8213 * @param[in] actions 8214 * Associated actions (list terminated by the END action). 8215 * @param[in] masks 8216 * List of actions that marks which of the action's member is constant. 8217 * @param[out] error 8218 * Pointer to error structure. 8219 * 8220 * @return 8221 * 0 on success, a negative errno value otherwise and rte_errno is set. 8222 */ 8223 static struct rte_flow_actions_template * 8224 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8225 const struct rte_flow_actions_template_attr *attr, 8226 const struct rte_flow_action actions[], 8227 const struct rte_flow_action masks[], 8228 struct rte_flow_error *error) 8229 { 8230 const struct mlx5_flow_driver_ops *fops; 8231 8232 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8233 rte_flow_error_set(error, ENOTSUP, 8234 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8235 NULL, 8236 "action create with incorrect steering mode"); 8237 return NULL; 8238 } 8239 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8240 return fops->actions_template_create(dev, attr, actions, masks, error); 8241 } 8242 8243 /** 8244 * Destroy flow action template. 8245 * 8246 * @param[in] dev 8247 * Pointer to the rte_eth_dev structure. 8248 * @param[in] template 8249 * Pointer to the action template to be destroyed. 8250 * @param[out] error 8251 * Pointer to error structure. 8252 * 8253 * @return 8254 * 0 on success, a negative errno value otherwise and rte_errno is set. 8255 */ 8256 static int 8257 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8258 struct rte_flow_actions_template *template, 8259 struct rte_flow_error *error) 8260 { 8261 const struct mlx5_flow_driver_ops *fops; 8262 8263 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8264 return rte_flow_error_set(error, ENOTSUP, 8265 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8266 NULL, 8267 "action destroy with incorrect steering mode"); 8268 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8269 return fops->actions_template_destroy(dev, template, error); 8270 } 8271 8272 /** 8273 * Create flow table. 8274 * 8275 * @param[in] dev 8276 * Pointer to the rte_eth_dev structure. 8277 * @param[in] attr 8278 * Pointer to the table attributes. 8279 * @param[in] item_templates 8280 * Item template array to be binded to the table. 8281 * @param[in] nb_item_templates 8282 * Number of item template. 8283 * @param[in] action_templates 8284 * Action template array to be binded to the table. 8285 * @param[in] nb_action_templates 8286 * Number of action template. 8287 * @param[out] error 8288 * Pointer to error structure. 8289 * 8290 * @return 8291 * Table on success, NULL otherwise and rte_errno is set. 8292 */ 8293 static struct rte_flow_template_table * 8294 mlx5_flow_table_create(struct rte_eth_dev *dev, 8295 const struct rte_flow_template_table_attr *attr, 8296 struct rte_flow_pattern_template *item_templates[], 8297 uint8_t nb_item_templates, 8298 struct rte_flow_actions_template *action_templates[], 8299 uint8_t nb_action_templates, 8300 struct rte_flow_error *error) 8301 { 8302 const struct mlx5_flow_driver_ops *fops; 8303 8304 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8305 rte_flow_error_set(error, ENOTSUP, 8306 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8307 NULL, 8308 "table create with incorrect steering mode"); 8309 return NULL; 8310 } 8311 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8312 return fops->template_table_create(dev, 8313 attr, 8314 item_templates, 8315 nb_item_templates, 8316 action_templates, 8317 nb_action_templates, 8318 error); 8319 } 8320 8321 /** 8322 * PMD destroy flow table. 8323 * 8324 * @param[in] dev 8325 * Pointer to the rte_eth_dev structure. 8326 * @param[in] table 8327 * Pointer to the table to be destroyed. 8328 * @param[out] error 8329 * Pointer to error structure. 8330 * 8331 * @return 8332 * 0 on success, a negative errno value otherwise and rte_errno is set. 8333 */ 8334 static int 8335 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8336 struct rte_flow_template_table *table, 8337 struct rte_flow_error *error) 8338 { 8339 const struct mlx5_flow_driver_ops *fops; 8340 8341 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8342 return rte_flow_error_set(error, ENOTSUP, 8343 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8344 NULL, 8345 "table destroy with incorrect steering mode"); 8346 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8347 return fops->template_table_destroy(dev, table, error); 8348 } 8349 8350 /** 8351 * Enqueue flow creation. 8352 * 8353 * @param[in] dev 8354 * Pointer to the rte_eth_dev structure. 8355 * @param[in] queue_id 8356 * The queue to create the flow. 8357 * @param[in] attr 8358 * Pointer to the flow operation attributes. 8359 * @param[in] items 8360 * Items with flow spec value. 8361 * @param[in] pattern_template_index 8362 * The item pattern flow follows from the table. 8363 * @param[in] actions 8364 * Action with flow spec value. 8365 * @param[in] action_template_index 8366 * The action pattern flow follows from the table. 8367 * @param[in] user_data 8368 * Pointer to the user_data. 8369 * @param[out] error 8370 * Pointer to error structure. 8371 * 8372 * @return 8373 * Flow pointer on success, NULL otherwise and rte_errno is set. 8374 */ 8375 static struct rte_flow * 8376 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8377 uint32_t queue_id, 8378 const struct rte_flow_op_attr *attr, 8379 struct rte_flow_template_table *table, 8380 const struct rte_flow_item items[], 8381 uint8_t pattern_template_index, 8382 const struct rte_flow_action actions[], 8383 uint8_t action_template_index, 8384 void *user_data, 8385 struct rte_flow_error *error) 8386 { 8387 const struct mlx5_flow_driver_ops *fops; 8388 8389 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8390 rte_flow_error_set(error, ENOTSUP, 8391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8392 NULL, 8393 "flow_q create with incorrect steering mode"); 8394 return NULL; 8395 } 8396 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8397 return fops->async_flow_create(dev, queue_id, attr, table, 8398 items, pattern_template_index, 8399 actions, action_template_index, 8400 user_data, error); 8401 } 8402 8403 /** 8404 * Enqueue flow destruction. 8405 * 8406 * @param[in] dev 8407 * Pointer to the rte_eth_dev structure. 8408 * @param[in] queue 8409 * The queue to destroy the flow. 8410 * @param[in] attr 8411 * Pointer to the flow operation attributes. 8412 * @param[in] flow 8413 * Pointer to the flow to be destroyed. 8414 * @param[in] user_data 8415 * Pointer to the user_data. 8416 * @param[out] error 8417 * Pointer to error structure. 8418 * 8419 * @return 8420 * 0 on success, negative value otherwise and rte_errno is set. 8421 */ 8422 static int 8423 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8424 uint32_t queue, 8425 const struct rte_flow_op_attr *attr, 8426 struct rte_flow *flow, 8427 void *user_data, 8428 struct rte_flow_error *error) 8429 { 8430 const struct mlx5_flow_driver_ops *fops; 8431 8432 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8433 return rte_flow_error_set(error, ENOTSUP, 8434 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8435 NULL, 8436 "flow_q destroy with incorrect steering mode"); 8437 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8438 return fops->async_flow_destroy(dev, queue, attr, flow, 8439 user_data, error); 8440 } 8441 8442 /** 8443 * Pull the enqueued flows. 8444 * 8445 * @param[in] dev 8446 * Pointer to the rte_eth_dev structure. 8447 * @param[in] queue 8448 * The queue to pull the result. 8449 * @param[in/out] res 8450 * Array to save the results. 8451 * @param[in] n_res 8452 * Available result with the array. 8453 * @param[out] error 8454 * Pointer to error structure. 8455 * 8456 * @return 8457 * Result number on success, negative value otherwise and rte_errno is set. 8458 */ 8459 static int 8460 mlx5_flow_pull(struct rte_eth_dev *dev, 8461 uint32_t queue, 8462 struct rte_flow_op_result res[], 8463 uint16_t n_res, 8464 struct rte_flow_error *error) 8465 { 8466 const struct mlx5_flow_driver_ops *fops; 8467 8468 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8469 return rte_flow_error_set(error, ENOTSUP, 8470 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8471 NULL, 8472 "flow_q pull with incorrect steering mode"); 8473 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8474 return fops->pull(dev, queue, res, n_res, error); 8475 } 8476 8477 /** 8478 * Push the enqueued flows. 8479 * 8480 * @param[in] dev 8481 * Pointer to the rte_eth_dev structure. 8482 * @param[in] queue 8483 * The queue to push the flows. 8484 * @param[out] error 8485 * Pointer to error structure. 8486 * 8487 * @return 8488 * 0 on success, negative value otherwise and rte_errno is set. 8489 */ 8490 static int 8491 mlx5_flow_push(struct rte_eth_dev *dev, 8492 uint32_t queue, 8493 struct rte_flow_error *error) 8494 { 8495 const struct mlx5_flow_driver_ops *fops; 8496 8497 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8498 return rte_flow_error_set(error, ENOTSUP, 8499 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8500 NULL, 8501 "flow_q push with incorrect steering mode"); 8502 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8503 return fops->push(dev, queue, error); 8504 } 8505 8506 /** 8507 * Create shared action. 8508 * 8509 * @param[in] dev 8510 * Pointer to the rte_eth_dev structure. 8511 * @param[in] queue 8512 * Which queue to be used.. 8513 * @param[in] attr 8514 * Operation attribute. 8515 * @param[in] conf 8516 * Indirect action configuration. 8517 * @param[in] action 8518 * rte_flow action detail. 8519 * @param[in] user_data 8520 * Pointer to the user_data. 8521 * @param[out] error 8522 * Pointer to error structure. 8523 * 8524 * @return 8525 * Action handle on success, NULL otherwise and rte_errno is set. 8526 */ 8527 static struct rte_flow_action_handle * 8528 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8529 const struct rte_flow_op_attr *attr, 8530 const struct rte_flow_indir_action_conf *conf, 8531 const struct rte_flow_action *action, 8532 void *user_data, 8533 struct rte_flow_error *error) 8534 { 8535 const struct mlx5_flow_driver_ops *fops = 8536 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8537 8538 return fops->async_action_create(dev, queue, attr, conf, action, 8539 user_data, error); 8540 } 8541 8542 /** 8543 * Update shared action. 8544 * 8545 * @param[in] dev 8546 * Pointer to the rte_eth_dev structure. 8547 * @param[in] queue 8548 * Which queue to be used.. 8549 * @param[in] attr 8550 * Operation attribute. 8551 * @param[in] handle 8552 * Action handle to be updated. 8553 * @param[in] update 8554 * Update value. 8555 * @param[in] user_data 8556 * Pointer to the user_data. 8557 * @param[out] error 8558 * Pointer to error structure. 8559 * 8560 * @return 8561 * 0 on success, negative value otherwise and rte_errno is set. 8562 */ 8563 static int 8564 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8565 const struct rte_flow_op_attr *attr, 8566 struct rte_flow_action_handle *handle, 8567 const void *update, 8568 void *user_data, 8569 struct rte_flow_error *error) 8570 { 8571 const struct mlx5_flow_driver_ops *fops = 8572 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8573 8574 return fops->async_action_update(dev, queue, attr, handle, 8575 update, user_data, error); 8576 } 8577 8578 /** 8579 * Destroy shared action. 8580 * 8581 * @param[in] dev 8582 * Pointer to the rte_eth_dev structure. 8583 * @param[in] queue 8584 * Which queue to be used.. 8585 * @param[in] attr 8586 * Operation attribute. 8587 * @param[in] handle 8588 * Action handle to be destroyed. 8589 * @param[in] user_data 8590 * Pointer to the user_data. 8591 * @param[out] error 8592 * Pointer to error structure. 8593 * 8594 * @return 8595 * 0 on success, negative value otherwise and rte_errno is set. 8596 */ 8597 static int 8598 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 8599 const struct rte_flow_op_attr *attr, 8600 struct rte_flow_action_handle *handle, 8601 void *user_data, 8602 struct rte_flow_error *error) 8603 { 8604 const struct mlx5_flow_driver_ops *fops = 8605 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8606 8607 return fops->async_action_destroy(dev, queue, attr, handle, 8608 user_data, error); 8609 } 8610 8611 /** 8612 * Allocate a new memory for the counter values wrapped by all the needed 8613 * management. 8614 * 8615 * @param[in] sh 8616 * Pointer to mlx5_dev_ctx_shared object. 8617 * 8618 * @return 8619 * 0 on success, a negative errno value otherwise. 8620 */ 8621 static int 8622 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8623 { 8624 struct mlx5_counter_stats_mem_mng *mem_mng; 8625 volatile struct flow_counter_stats *raw_data; 8626 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8627 int size = (sizeof(struct flow_counter_stats) * 8628 MLX5_COUNTERS_PER_POOL + 8629 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8630 sizeof(struct mlx5_counter_stats_mem_mng); 8631 size_t pgsize = rte_mem_page_size(); 8632 uint8_t *mem; 8633 int ret; 8634 int i; 8635 8636 if (pgsize == (size_t)-1) { 8637 DRV_LOG(ERR, "Failed to get mem page size"); 8638 rte_errno = ENOMEM; 8639 return -ENOMEM; 8640 } 8641 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8642 if (!mem) { 8643 rte_errno = ENOMEM; 8644 return -ENOMEM; 8645 } 8646 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8647 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8648 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8649 sh->cdev->pdn, mem, size, 8650 &mem_mng->wm); 8651 if (ret) { 8652 rte_errno = errno; 8653 mlx5_free(mem); 8654 return -rte_errno; 8655 } 8656 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8657 raw_data = (volatile struct flow_counter_stats *)mem; 8658 for (i = 0; i < raws_n; ++i) { 8659 mem_mng->raws[i].mem_mng = mem_mng; 8660 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8661 } 8662 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8663 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8664 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8665 next); 8666 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8667 sh->cmng.mem_mng = mem_mng; 8668 return 0; 8669 } 8670 8671 /** 8672 * Set the statistic memory to the new counter pool. 8673 * 8674 * @param[in] sh 8675 * Pointer to mlx5_dev_ctx_shared object. 8676 * @param[in] pool 8677 * Pointer to the pool to set the statistic memory. 8678 * 8679 * @return 8680 * 0 on success, a negative errno value otherwise. 8681 */ 8682 static int 8683 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8684 struct mlx5_flow_counter_pool *pool) 8685 { 8686 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8687 /* Resize statistic memory once used out. */ 8688 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8689 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8690 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8691 return -1; 8692 } 8693 rte_spinlock_lock(&pool->sl); 8694 pool->raw = cmng->mem_mng->raws + pool->index % 8695 MLX5_CNT_CONTAINER_RESIZE; 8696 rte_spinlock_unlock(&pool->sl); 8697 pool->raw_hw = NULL; 8698 return 0; 8699 } 8700 8701 #define MLX5_POOL_QUERY_FREQ_US 1000000 8702 8703 /** 8704 * Set the periodic procedure for triggering asynchronous batch queries for all 8705 * the counter pools. 8706 * 8707 * @param[in] sh 8708 * Pointer to mlx5_dev_ctx_shared object. 8709 */ 8710 void 8711 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8712 { 8713 uint32_t pools_n, us; 8714 8715 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8716 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8717 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8718 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8719 sh->cmng.query_thread_on = 0; 8720 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8721 } else { 8722 sh->cmng.query_thread_on = 1; 8723 } 8724 } 8725 8726 /** 8727 * The periodic procedure for triggering asynchronous batch queries for all the 8728 * counter pools. This function is probably called by the host thread. 8729 * 8730 * @param[in] arg 8731 * The parameter for the alarm process. 8732 */ 8733 void 8734 mlx5_flow_query_alarm(void *arg) 8735 { 8736 struct mlx5_dev_ctx_shared *sh = arg; 8737 int ret; 8738 uint16_t pool_index = sh->cmng.pool_index; 8739 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8740 struct mlx5_flow_counter_pool *pool; 8741 uint16_t n_valid; 8742 8743 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8744 goto set_alarm; 8745 rte_spinlock_lock(&cmng->pool_update_sl); 8746 pool = cmng->pools[pool_index]; 8747 n_valid = cmng->n_valid; 8748 rte_spinlock_unlock(&cmng->pool_update_sl); 8749 /* Set the statistic memory to the new created pool. */ 8750 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8751 goto set_alarm; 8752 if (pool->raw_hw) 8753 /* There is a pool query in progress. */ 8754 goto set_alarm; 8755 pool->raw_hw = 8756 LIST_FIRST(&sh->cmng.free_stat_raws); 8757 if (!pool->raw_hw) 8758 /* No free counter statistics raw memory. */ 8759 goto set_alarm; 8760 /* 8761 * Identify the counters released between query trigger and query 8762 * handle more efficiently. The counter released in this gap period 8763 * should wait for a new round of query as the new arrived packets 8764 * will not be taken into account. 8765 */ 8766 pool->query_gen++; 8767 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8768 MLX5_COUNTERS_PER_POOL, 8769 NULL, NULL, 8770 pool->raw_hw->mem_mng->wm.lkey, 8771 (void *)(uintptr_t) 8772 pool->raw_hw->data, 8773 sh->devx_comp, 8774 (uint64_t)(uintptr_t)pool); 8775 if (ret) { 8776 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8777 " %d", pool->min_dcs->id); 8778 pool->raw_hw = NULL; 8779 goto set_alarm; 8780 } 8781 LIST_REMOVE(pool->raw_hw, next); 8782 sh->cmng.pending_queries++; 8783 pool_index++; 8784 if (pool_index >= n_valid) 8785 pool_index = 0; 8786 set_alarm: 8787 sh->cmng.pool_index = pool_index; 8788 mlx5_set_query_alarm(sh); 8789 } 8790 8791 /** 8792 * Check and callback event for new aged flow in the counter pool 8793 * 8794 * @param[in] sh 8795 * Pointer to mlx5_dev_ctx_shared object. 8796 * @param[in] pool 8797 * Pointer to Current counter pool. 8798 */ 8799 static void 8800 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8801 struct mlx5_flow_counter_pool *pool) 8802 { 8803 struct mlx5_priv *priv; 8804 struct mlx5_flow_counter *cnt; 8805 struct mlx5_age_info *age_info; 8806 struct mlx5_age_param *age_param; 8807 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8808 struct mlx5_counter_stats_raw *prev = pool->raw; 8809 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8810 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8811 uint16_t expected = AGE_CANDIDATE; 8812 uint32_t i; 8813 8814 pool->time_of_last_age_check = curr_time; 8815 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8816 cnt = MLX5_POOL_GET_CNT(pool, i); 8817 age_param = MLX5_CNT_TO_AGE(cnt); 8818 if (__atomic_load_n(&age_param->state, 8819 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8820 continue; 8821 if (cur->data[i].hits != prev->data[i].hits) { 8822 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8823 __ATOMIC_RELAXED); 8824 continue; 8825 } 8826 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8827 time_delta, 8828 __ATOMIC_RELAXED) <= age_param->timeout) 8829 continue; 8830 /** 8831 * Hold the lock first, or if between the 8832 * state AGE_TMOUT and tailq operation the 8833 * release happened, the release procedure 8834 * may delete a non-existent tailq node. 8835 */ 8836 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8837 age_info = GET_PORT_AGE_INFO(priv); 8838 rte_spinlock_lock(&age_info->aged_sl); 8839 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8840 AGE_TMOUT, false, 8841 __ATOMIC_RELAXED, 8842 __ATOMIC_RELAXED)) { 8843 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8844 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8845 } 8846 rte_spinlock_unlock(&age_info->aged_sl); 8847 } 8848 mlx5_age_event_prepare(sh); 8849 } 8850 8851 /** 8852 * Handler for the HW respond about ready values from an asynchronous batch 8853 * query. This function is probably called by the host thread. 8854 * 8855 * @param[in] sh 8856 * The pointer to the shared device context. 8857 * @param[in] async_id 8858 * The Devx async ID. 8859 * @param[in] status 8860 * The status of the completion. 8861 */ 8862 void 8863 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8864 uint64_t async_id, int status) 8865 { 8866 struct mlx5_flow_counter_pool *pool = 8867 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8868 struct mlx5_counter_stats_raw *raw_to_free; 8869 uint8_t query_gen = pool->query_gen ^ 1; 8870 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8871 enum mlx5_counter_type cnt_type = 8872 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8873 MLX5_COUNTER_TYPE_ORIGIN; 8874 8875 if (unlikely(status)) { 8876 raw_to_free = pool->raw_hw; 8877 } else { 8878 raw_to_free = pool->raw; 8879 if (pool->is_aged) 8880 mlx5_flow_aging_check(sh, pool); 8881 rte_spinlock_lock(&pool->sl); 8882 pool->raw = pool->raw_hw; 8883 rte_spinlock_unlock(&pool->sl); 8884 /* Be sure the new raw counters data is updated in memory. */ 8885 rte_io_wmb(); 8886 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8887 rte_spinlock_lock(&cmng->csl[cnt_type]); 8888 TAILQ_CONCAT(&cmng->counters[cnt_type], 8889 &pool->counters[query_gen], next); 8890 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8891 } 8892 } 8893 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8894 pool->raw_hw = NULL; 8895 sh->cmng.pending_queries--; 8896 } 8897 8898 static int 8899 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8900 const struct flow_grp_info *grp_info, 8901 struct rte_flow_error *error) 8902 { 8903 if (grp_info->transfer && grp_info->external && 8904 grp_info->fdb_def_rule) { 8905 if (group == UINT32_MAX) 8906 return rte_flow_error_set 8907 (error, EINVAL, 8908 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8909 NULL, 8910 "group index not supported"); 8911 *table = group + 1; 8912 } else { 8913 *table = group; 8914 } 8915 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8916 return 0; 8917 } 8918 8919 /** 8920 * Translate the rte_flow group index to HW table value. 8921 * 8922 * If tunnel offload is disabled, all group ids converted to flow table 8923 * id using the standard method. 8924 * If tunnel offload is enabled, group id can be converted using the 8925 * standard or tunnel conversion method. Group conversion method 8926 * selection depends on flags in `grp_info` parameter: 8927 * - Internal (grp_info.external == 0) groups conversion uses the 8928 * standard method. 8929 * - Group ids in JUMP action converted with the tunnel conversion. 8930 * - Group id in rule attribute conversion depends on a rule type and 8931 * group id value: 8932 * ** non zero group attributes converted with the tunnel method 8933 * ** zero group attribute in non-tunnel rule is converted using the 8934 * standard method - there's only one root table 8935 * ** zero group attribute in steer tunnel rule is converted with the 8936 * standard method - single root table 8937 * ** zero group attribute in match tunnel rule is a special OvS 8938 * case: that value is used for portability reasons. That group 8939 * id is converted with the tunnel conversion method. 8940 * 8941 * @param[in] dev 8942 * Port device 8943 * @param[in] tunnel 8944 * PMD tunnel offload object 8945 * @param[in] group 8946 * rte_flow group index value. 8947 * @param[out] table 8948 * HW table value. 8949 * @param[in] grp_info 8950 * flags used for conversion 8951 * @param[out] error 8952 * Pointer to error structure. 8953 * 8954 * @return 8955 * 0 on success, a negative errno value otherwise and rte_errno is set. 8956 */ 8957 int 8958 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8959 const struct mlx5_flow_tunnel *tunnel, 8960 uint32_t group, uint32_t *table, 8961 const struct flow_grp_info *grp_info, 8962 struct rte_flow_error *error) 8963 { 8964 int ret; 8965 bool standard_translation; 8966 8967 if (!grp_info->skip_scale && grp_info->external && 8968 group < MLX5_MAX_TABLES_EXTERNAL) 8969 group *= MLX5_FLOW_TABLE_FACTOR; 8970 if (is_tunnel_offload_active(dev)) { 8971 standard_translation = !grp_info->external || 8972 grp_info->std_tbl_fix; 8973 } else { 8974 standard_translation = true; 8975 } 8976 DRV_LOG(DEBUG, 8977 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8978 dev->data->port_id, group, grp_info->transfer, 8979 grp_info->external, grp_info->fdb_def_rule, 8980 standard_translation ? "STANDARD" : "TUNNEL"); 8981 if (standard_translation) 8982 ret = flow_group_to_table(dev->data->port_id, group, table, 8983 grp_info, error); 8984 else 8985 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8986 table, error); 8987 8988 return ret; 8989 } 8990 8991 /** 8992 * Discover availability of metadata reg_c's. 8993 * 8994 * Iteratively use test flows to check availability. 8995 * 8996 * @param[in] dev 8997 * Pointer to the Ethernet device structure. 8998 * 8999 * @return 9000 * 0 on success, a negative errno value otherwise and rte_errno is set. 9001 */ 9002 int 9003 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9004 { 9005 struct mlx5_priv *priv = dev->data->dev_private; 9006 enum modify_reg idx; 9007 int n = 0; 9008 9009 /* reg_c[0] and reg_c[1] are reserved. */ 9010 priv->sh->flow_mreg_c[n++] = REG_C_0; 9011 priv->sh->flow_mreg_c[n++] = REG_C_1; 9012 /* Discover availability of other reg_c's. */ 9013 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9014 struct rte_flow_attr attr = { 9015 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9016 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9017 .ingress = 1, 9018 }; 9019 struct rte_flow_item items[] = { 9020 [0] = { 9021 .type = RTE_FLOW_ITEM_TYPE_END, 9022 }, 9023 }; 9024 struct rte_flow_action actions[] = { 9025 [0] = { 9026 .type = (enum rte_flow_action_type) 9027 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9028 .conf = &(struct mlx5_flow_action_copy_mreg){ 9029 .src = REG_C_1, 9030 .dst = idx, 9031 }, 9032 }, 9033 [1] = { 9034 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9035 .conf = &(struct rte_flow_action_jump){ 9036 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9037 }, 9038 }, 9039 [2] = { 9040 .type = RTE_FLOW_ACTION_TYPE_END, 9041 }, 9042 }; 9043 uint32_t flow_idx; 9044 struct rte_flow *flow; 9045 struct rte_flow_error error; 9046 9047 if (!priv->sh->config.dv_flow_en) 9048 break; 9049 /* Create internal flow, validation skips copy action. */ 9050 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9051 items, actions, false, &error); 9052 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9053 flow_idx); 9054 if (!flow) 9055 continue; 9056 priv->sh->flow_mreg_c[n++] = idx; 9057 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9058 } 9059 for (; n < MLX5_MREG_C_NUM; ++n) 9060 priv->sh->flow_mreg_c[n] = REG_NON; 9061 priv->sh->metadata_regc_check_flag = 1; 9062 return 0; 9063 } 9064 9065 int 9066 save_dump_file(const uint8_t *data, uint32_t size, 9067 uint32_t type, uint64_t id, void *arg, FILE *file) 9068 { 9069 char line[BUF_SIZE]; 9070 uint32_t out = 0; 9071 uint32_t k; 9072 uint32_t actions_num; 9073 struct rte_flow_query_count *count; 9074 9075 memset(line, 0, BUF_SIZE); 9076 switch (type) { 9077 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9078 actions_num = *(uint32_t *)(arg); 9079 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9080 type, id, actions_num); 9081 break; 9082 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9083 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9084 type, id); 9085 break; 9086 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9087 count = (struct rte_flow_query_count *)arg; 9088 fprintf(file, 9089 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9090 type, id, count->hits, count->bytes); 9091 return 0; 9092 default: 9093 return -1; 9094 } 9095 9096 for (k = 0; k < size; k++) { 9097 /* Make sure we do not overrun the line buffer length. */ 9098 if (out >= BUF_SIZE - 4) { 9099 line[out] = '\0'; 9100 break; 9101 } 9102 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9103 (data[k]) & 0xff); 9104 } 9105 fprintf(file, "%s\n", line); 9106 return 0; 9107 } 9108 9109 int 9110 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9111 struct rte_flow_query_count *count, struct rte_flow_error *error) 9112 { 9113 struct rte_flow_action action[2]; 9114 enum mlx5_flow_drv_type ftype; 9115 const struct mlx5_flow_driver_ops *fops; 9116 9117 if (!flow) { 9118 return rte_flow_error_set(error, ENOENT, 9119 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9120 NULL, 9121 "invalid flow handle"); 9122 } 9123 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9124 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9125 if (flow->counter) { 9126 memset(count, 0, sizeof(struct rte_flow_query_count)); 9127 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9128 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9129 ftype < MLX5_FLOW_TYPE_MAX); 9130 fops = flow_get_drv_ops(ftype); 9131 return fops->query(dev, flow, action, count, error); 9132 } 9133 return -1; 9134 } 9135 9136 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9137 /** 9138 * Dump flow ipool data to file 9139 * 9140 * @param[in] dev 9141 * The pointer to Ethernet device. 9142 * @param[in] file 9143 * A pointer to a file for output. 9144 * @param[out] error 9145 * Perform verbose error reporting if not NULL. PMDs initialize this 9146 * structure in case of error only. 9147 * @return 9148 * 0 on success, a negative value otherwise. 9149 */ 9150 int 9151 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9152 struct rte_flow *flow, FILE *file, 9153 struct rte_flow_error *error) 9154 { 9155 struct mlx5_priv *priv = dev->data->dev_private; 9156 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9157 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9158 uint32_t handle_idx; 9159 struct mlx5_flow_handle *dh; 9160 struct rte_flow_query_count count; 9161 uint32_t actions_num; 9162 const uint8_t *data; 9163 size_t size; 9164 uint64_t id; 9165 uint32_t type; 9166 void *action = NULL; 9167 9168 if (!flow) { 9169 return rte_flow_error_set(error, ENOENT, 9170 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9171 NULL, 9172 "invalid flow handle"); 9173 } 9174 handle_idx = flow->dev_handles; 9175 /* query counter */ 9176 if (flow->counter && 9177 (!mlx5_counter_query(dev, flow->counter, false, 9178 &count.hits, &count.bytes, &action)) && action) { 9179 id = (uint64_t)(uintptr_t)action; 9180 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9181 save_dump_file(NULL, 0, type, 9182 id, (void *)&count, file); 9183 } 9184 9185 while (handle_idx) { 9186 dh = mlx5_ipool_get(priv->sh->ipool 9187 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9188 if (!dh) 9189 continue; 9190 handle_idx = dh->next.next; 9191 9192 /* Get modify_hdr and encap_decap buf from ipools. */ 9193 encap_decap = NULL; 9194 modify_hdr = dh->dvh.modify_hdr; 9195 9196 if (dh->dvh.rix_encap_decap) { 9197 encap_decap = mlx5_ipool_get(priv->sh->ipool 9198 [MLX5_IPOOL_DECAP_ENCAP], 9199 dh->dvh.rix_encap_decap); 9200 } 9201 if (modify_hdr) { 9202 data = (const uint8_t *)modify_hdr->actions; 9203 size = (size_t)(modify_hdr->actions_num) * 8; 9204 id = (uint64_t)(uintptr_t)modify_hdr->action; 9205 actions_num = modify_hdr->actions_num; 9206 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9207 save_dump_file(data, size, type, id, 9208 (void *)(&actions_num), file); 9209 } 9210 if (encap_decap) { 9211 data = encap_decap->buf; 9212 size = encap_decap->size; 9213 id = (uint64_t)(uintptr_t)encap_decap->action; 9214 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9215 save_dump_file(data, size, type, 9216 id, NULL, file); 9217 } 9218 } 9219 return 0; 9220 } 9221 9222 /** 9223 * Dump all flow's encap_decap/modify_hdr/counter data to file 9224 * 9225 * @param[in] dev 9226 * The pointer to Ethernet device. 9227 * @param[in] file 9228 * A pointer to a file for output. 9229 * @param[out] error 9230 * Perform verbose error reporting if not NULL. PMDs initialize this 9231 * structure in case of error only. 9232 * @return 9233 * 0 on success, a negative value otherwise. 9234 */ 9235 static int 9236 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9237 FILE *file, struct rte_flow_error *error __rte_unused) 9238 { 9239 struct mlx5_priv *priv = dev->data->dev_private; 9240 struct mlx5_dev_ctx_shared *sh = priv->sh; 9241 struct mlx5_hlist *h; 9242 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9243 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9244 struct rte_flow_query_count count; 9245 uint32_t actions_num; 9246 const uint8_t *data; 9247 size_t size; 9248 uint64_t id; 9249 uint32_t type; 9250 uint32_t i; 9251 uint32_t j; 9252 struct mlx5_list_inconst *l_inconst; 9253 struct mlx5_list_entry *e; 9254 int lcore_index; 9255 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 9256 uint32_t max; 9257 void *action; 9258 9259 /* encap_decap hlist is lcore_share, get global core cache. */ 9260 i = MLX5_LIST_GLOBAL; 9261 h = sh->encaps_decaps; 9262 if (h) { 9263 for (j = 0; j <= h->mask; j++) { 9264 l_inconst = &h->buckets[j].l; 9265 if (!l_inconst || !l_inconst->cache[i]) 9266 continue; 9267 9268 e = LIST_FIRST(&l_inconst->cache[i]->h); 9269 while (e) { 9270 encap_decap = 9271 (struct mlx5_flow_dv_encap_decap_resource *)e; 9272 data = encap_decap->buf; 9273 size = encap_decap->size; 9274 id = (uint64_t)(uintptr_t)encap_decap->action; 9275 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9276 save_dump_file(data, size, type, 9277 id, NULL, file); 9278 e = LIST_NEXT(e, next); 9279 } 9280 } 9281 } 9282 9283 /* get modify_hdr */ 9284 h = sh->modify_cmds; 9285 if (h) { 9286 lcore_index = rte_lcore_index(rte_lcore_id()); 9287 if (unlikely(lcore_index == -1)) { 9288 lcore_index = MLX5_LIST_NLCORE; 9289 rte_spinlock_lock(&h->l_const.lcore_lock); 9290 } 9291 i = lcore_index; 9292 9293 for (j = 0; j <= h->mask; j++) { 9294 l_inconst = &h->buckets[j].l; 9295 if (!l_inconst || !l_inconst->cache[i]) 9296 continue; 9297 9298 e = LIST_FIRST(&l_inconst->cache[i]->h); 9299 while (e) { 9300 modify_hdr = 9301 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9302 data = (const uint8_t *)modify_hdr->actions; 9303 size = (size_t)(modify_hdr->actions_num) * 8; 9304 actions_num = modify_hdr->actions_num; 9305 id = (uint64_t)(uintptr_t)modify_hdr->action; 9306 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9307 save_dump_file(data, size, type, id, 9308 (void *)(&actions_num), file); 9309 e = LIST_NEXT(e, next); 9310 } 9311 } 9312 9313 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9314 rte_spinlock_unlock(&h->l_const.lcore_lock); 9315 } 9316 9317 /* get counter */ 9318 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9319 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9320 for (j = 1; j <= max; j++) { 9321 action = NULL; 9322 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9323 &count.bytes, &action)) && action) { 9324 id = (uint64_t)(uintptr_t)action; 9325 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9326 save_dump_file(NULL, 0, type, 9327 id, (void *)&count, file); 9328 } 9329 } 9330 return 0; 9331 } 9332 #endif 9333 9334 /** 9335 * Dump flow raw hw data to file 9336 * 9337 * @param[in] dev 9338 * The pointer to Ethernet device. 9339 * @param[in] file 9340 * A pointer to a file for output. 9341 * @param[out] error 9342 * Perform verbose error reporting if not NULL. PMDs initialize this 9343 * structure in case of error only. 9344 * @return 9345 * 0 on success, a negative value otherwise. 9346 */ 9347 int 9348 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9349 FILE *file, 9350 struct rte_flow_error *error __rte_unused) 9351 { 9352 struct mlx5_priv *priv = dev->data->dev_private; 9353 struct mlx5_dev_ctx_shared *sh = priv->sh; 9354 uint32_t handle_idx; 9355 int ret; 9356 struct mlx5_flow_handle *dh; 9357 struct rte_flow *flow; 9358 9359 if (!sh->config.dv_flow_en) { 9360 if (fputs("device dv flow disabled\n", file) <= 0) 9361 return -errno; 9362 return -ENOTSUP; 9363 } 9364 9365 /* dump all */ 9366 if (!flow_idx) { 9367 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9368 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9369 return -EINVAL; 9370 #endif 9371 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9372 sh->rx_domain, 9373 sh->tx_domain, file); 9374 } 9375 /* dump one */ 9376 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9377 (uintptr_t)(void *)flow_idx); 9378 if (!flow) 9379 return -EINVAL; 9380 9381 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9382 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9383 #endif 9384 handle_idx = flow->dev_handles; 9385 while (handle_idx) { 9386 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9387 handle_idx); 9388 if (!dh) 9389 return -ENOENT; 9390 if (dh->drv_flow) { 9391 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9392 file); 9393 if (ret) 9394 return -ENOENT; 9395 } 9396 handle_idx = dh->next.next; 9397 } 9398 return 0; 9399 } 9400 9401 /** 9402 * Get aged-out flows. 9403 * 9404 * @param[in] dev 9405 * Pointer to the Ethernet device structure. 9406 * @param[in] context 9407 * The address of an array of pointers to the aged-out flows contexts. 9408 * @param[in] nb_countexts 9409 * The length of context array pointers. 9410 * @param[out] error 9411 * Perform verbose error reporting if not NULL. Initialized in case of 9412 * error only. 9413 * 9414 * @return 9415 * how many contexts get in success, otherwise negative errno value. 9416 * if nb_contexts is 0, return the amount of all aged contexts. 9417 * if nb_contexts is not 0 , return the amount of aged flows reported 9418 * in the context array. 9419 */ 9420 int 9421 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9422 uint32_t nb_contexts, struct rte_flow_error *error) 9423 { 9424 const struct mlx5_flow_driver_ops *fops; 9425 struct rte_flow_attr attr = { .transfer = 0 }; 9426 9427 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 9428 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 9429 return fops->get_aged_flows(dev, contexts, nb_contexts, 9430 error); 9431 } 9432 DRV_LOG(ERR, 9433 "port %u get aged flows is not supported.", 9434 dev->data->port_id); 9435 return -ENOTSUP; 9436 } 9437 9438 /* Wrapper for driver action_validate op callback */ 9439 static int 9440 flow_drv_action_validate(struct rte_eth_dev *dev, 9441 const struct rte_flow_indir_action_conf *conf, 9442 const struct rte_flow_action *action, 9443 const struct mlx5_flow_driver_ops *fops, 9444 struct rte_flow_error *error) 9445 { 9446 static const char err_msg[] = "indirect action validation unsupported"; 9447 9448 if (!fops->action_validate) { 9449 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9450 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9451 NULL, err_msg); 9452 return -rte_errno; 9453 } 9454 return fops->action_validate(dev, conf, action, error); 9455 } 9456 9457 /** 9458 * Destroys the shared action by handle. 9459 * 9460 * @param dev 9461 * Pointer to Ethernet device structure. 9462 * @param[in] handle 9463 * Handle for the indirect action object to be destroyed. 9464 * @param[out] error 9465 * Perform verbose error reporting if not NULL. PMDs initialize this 9466 * structure in case of error only. 9467 * 9468 * @return 9469 * 0 on success, a negative errno value otherwise and rte_errno is set. 9470 * 9471 * @note: wrapper for driver action_create op callback. 9472 */ 9473 static int 9474 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9475 struct rte_flow_action_handle *handle, 9476 struct rte_flow_error *error) 9477 { 9478 static const char err_msg[] = "indirect action destruction unsupported"; 9479 struct rte_flow_attr attr = { .transfer = 0 }; 9480 const struct mlx5_flow_driver_ops *fops = 9481 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9482 9483 if (!fops->action_destroy) { 9484 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9485 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9486 NULL, err_msg); 9487 return -rte_errno; 9488 } 9489 return fops->action_destroy(dev, handle, error); 9490 } 9491 9492 /* Wrapper for driver action_destroy op callback */ 9493 static int 9494 flow_drv_action_update(struct rte_eth_dev *dev, 9495 struct rte_flow_action_handle *handle, 9496 const void *update, 9497 const struct mlx5_flow_driver_ops *fops, 9498 struct rte_flow_error *error) 9499 { 9500 static const char err_msg[] = "indirect action update unsupported"; 9501 9502 if (!fops->action_update) { 9503 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9504 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9505 NULL, err_msg); 9506 return -rte_errno; 9507 } 9508 return fops->action_update(dev, handle, update, error); 9509 } 9510 9511 /* Wrapper for driver action_destroy op callback */ 9512 static int 9513 flow_drv_action_query(struct rte_eth_dev *dev, 9514 const struct rte_flow_action_handle *handle, 9515 void *data, 9516 const struct mlx5_flow_driver_ops *fops, 9517 struct rte_flow_error *error) 9518 { 9519 static const char err_msg[] = "indirect action query unsupported"; 9520 9521 if (!fops->action_query) { 9522 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9523 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9524 NULL, err_msg); 9525 return -rte_errno; 9526 } 9527 return fops->action_query(dev, handle, data, error); 9528 } 9529 9530 /** 9531 * Create indirect action for reuse in multiple flow rules. 9532 * 9533 * @param dev 9534 * Pointer to Ethernet device structure. 9535 * @param conf 9536 * Pointer to indirect action object configuration. 9537 * @param[in] action 9538 * Action configuration for indirect action object creation. 9539 * @param[out] error 9540 * Perform verbose error reporting if not NULL. PMDs initialize this 9541 * structure in case of error only. 9542 * @return 9543 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9544 */ 9545 static struct rte_flow_action_handle * 9546 mlx5_action_handle_create(struct rte_eth_dev *dev, 9547 const struct rte_flow_indir_action_conf *conf, 9548 const struct rte_flow_action *action, 9549 struct rte_flow_error *error) 9550 { 9551 static const char err_msg[] = "indirect action creation unsupported"; 9552 struct rte_flow_attr attr = { .transfer = 0 }; 9553 const struct mlx5_flow_driver_ops *fops = 9554 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9555 9556 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9557 return NULL; 9558 if (!fops->action_create) { 9559 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9560 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9561 NULL, err_msg); 9562 return NULL; 9563 } 9564 return fops->action_create(dev, conf, action, error); 9565 } 9566 9567 /** 9568 * Updates inplace the indirect action configuration pointed by *handle* 9569 * with the configuration provided as *update* argument. 9570 * The update of the indirect action configuration effects all flow rules 9571 * reusing the action via handle. 9572 * 9573 * @param dev 9574 * Pointer to Ethernet device structure. 9575 * @param[in] handle 9576 * Handle for the indirect action to be updated. 9577 * @param[in] update 9578 * Action specification used to modify the action pointed by handle. 9579 * *update* could be of same type with the action pointed by the *handle* 9580 * handle argument, or some other structures like a wrapper, depending on 9581 * the indirect action type. 9582 * @param[out] error 9583 * Perform verbose error reporting if not NULL. PMDs initialize this 9584 * structure in case of error only. 9585 * 9586 * @return 9587 * 0 on success, a negative errno value otherwise and rte_errno is set. 9588 */ 9589 static int 9590 mlx5_action_handle_update(struct rte_eth_dev *dev, 9591 struct rte_flow_action_handle *handle, 9592 const void *update, 9593 struct rte_flow_error *error) 9594 { 9595 struct rte_flow_attr attr = { .transfer = 0 }; 9596 const struct mlx5_flow_driver_ops *fops = 9597 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9598 int ret; 9599 9600 ret = flow_drv_action_validate(dev, NULL, 9601 (const struct rte_flow_action *)update, fops, error); 9602 if (ret) 9603 return ret; 9604 return flow_drv_action_update(dev, handle, update, fops, 9605 error); 9606 } 9607 9608 /** 9609 * Query the indirect action by handle. 9610 * 9611 * This function allows retrieving action-specific data such as counters. 9612 * Data is gathered by special action which may be present/referenced in 9613 * more than one flow rule definition. 9614 * 9615 * see @RTE_FLOW_ACTION_TYPE_COUNT 9616 * 9617 * @param dev 9618 * Pointer to Ethernet device structure. 9619 * @param[in] handle 9620 * Handle for the indirect action to query. 9621 * @param[in, out] data 9622 * Pointer to storage for the associated query data type. 9623 * @param[out] error 9624 * Perform verbose error reporting if not NULL. PMDs initialize this 9625 * structure in case of error only. 9626 * 9627 * @return 9628 * 0 on success, a negative errno value otherwise and rte_errno is set. 9629 */ 9630 static int 9631 mlx5_action_handle_query(struct rte_eth_dev *dev, 9632 const struct rte_flow_action_handle *handle, 9633 void *data, 9634 struct rte_flow_error *error) 9635 { 9636 struct rte_flow_attr attr = { .transfer = 0 }; 9637 const struct mlx5_flow_driver_ops *fops = 9638 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9639 9640 return flow_drv_action_query(dev, handle, data, fops, error); 9641 } 9642 9643 /** 9644 * Destroy all indirect actions (shared RSS). 9645 * 9646 * @param dev 9647 * Pointer to Ethernet device. 9648 * 9649 * @return 9650 * 0 on success, a negative errno value otherwise and rte_errno is set. 9651 */ 9652 int 9653 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9654 { 9655 struct rte_flow_error error; 9656 struct mlx5_priv *priv = dev->data->dev_private; 9657 struct mlx5_shared_action_rss *shared_rss; 9658 int ret = 0; 9659 uint32_t idx; 9660 9661 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9662 priv->rss_shared_actions, idx, shared_rss, next) { 9663 ret |= mlx5_action_handle_destroy(dev, 9664 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9665 } 9666 return ret; 9667 } 9668 9669 /** 9670 * Validate existing indirect actions against current device configuration 9671 * and attach them to device resources. 9672 * 9673 * @param dev 9674 * Pointer to Ethernet device. 9675 * 9676 * @return 9677 * 0 on success, a negative errno value otherwise and rte_errno is set. 9678 */ 9679 int 9680 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9681 { 9682 struct mlx5_priv *priv = dev->data->dev_private; 9683 int ret = 0; 9684 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9685 9686 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9687 const char *message; 9688 uint32_t queue_idx; 9689 9690 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9691 ind_tbl->queues_n, 9692 &message, &queue_idx); 9693 if (ret != 0) { 9694 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9695 dev->data->port_id, ind_tbl->queues[queue_idx], 9696 message); 9697 break; 9698 } 9699 } 9700 if (ret != 0) 9701 return ret; 9702 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9703 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9704 if (ret != 0) { 9705 DRV_LOG(ERR, "Port %u could not attach " 9706 "indirection table obj %p", 9707 dev->data->port_id, (void *)ind_tbl); 9708 goto error; 9709 } 9710 } 9711 9712 return 0; 9713 error: 9714 ind_tbl_last = ind_tbl; 9715 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9716 if (ind_tbl == ind_tbl_last) 9717 break; 9718 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9719 DRV_LOG(CRIT, "Port %u could not detach " 9720 "indirection table obj %p on rollback", 9721 dev->data->port_id, (void *)ind_tbl); 9722 } 9723 return ret; 9724 } 9725 9726 /** 9727 * Detach indirect actions of the device from its resources. 9728 * 9729 * @param dev 9730 * Pointer to Ethernet device. 9731 * 9732 * @return 9733 * 0 on success, a negative errno value otherwise and rte_errno is set. 9734 */ 9735 int 9736 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9737 { 9738 struct mlx5_priv *priv = dev->data->dev_private; 9739 int ret = 0; 9740 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9741 9742 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9743 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9744 if (ret != 0) { 9745 DRV_LOG(ERR, "Port %u could not detach " 9746 "indirection table obj %p", 9747 dev->data->port_id, (void *)ind_tbl); 9748 goto error; 9749 } 9750 } 9751 return 0; 9752 error: 9753 ind_tbl_last = ind_tbl; 9754 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9755 if (ind_tbl == ind_tbl_last) 9756 break; 9757 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9758 DRV_LOG(CRIT, "Port %u could not attach " 9759 "indirection table obj %p on rollback", 9760 dev->data->port_id, (void *)ind_tbl); 9761 } 9762 return ret; 9763 } 9764 9765 #ifndef HAVE_MLX5DV_DR 9766 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9767 #else 9768 #define MLX5_DOMAIN_SYNC_FLOW \ 9769 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9770 #endif 9771 9772 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9773 { 9774 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9775 const struct mlx5_flow_driver_ops *fops; 9776 int ret; 9777 struct rte_flow_attr attr = { .transfer = 0 }; 9778 9779 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9780 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9781 if (ret > 0) 9782 ret = -ret; 9783 return ret; 9784 } 9785 9786 const struct mlx5_flow_tunnel * 9787 mlx5_get_tof(const struct rte_flow_item *item, 9788 const struct rte_flow_action *action, 9789 enum mlx5_tof_rule_type *rule_type) 9790 { 9791 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9792 if (item->type == (typeof(item->type)) 9793 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9794 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9795 return flow_items_to_tunnel(item); 9796 } 9797 } 9798 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9799 if (action->type == (typeof(action->type)) 9800 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9801 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9802 return flow_actions_to_tunnel(action); 9803 } 9804 } 9805 return NULL; 9806 } 9807 9808 /** 9809 * tunnel offload functionality is defined for DV environment only 9810 */ 9811 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9812 __extension__ 9813 union tunnel_offload_mark { 9814 uint32_t val; 9815 struct { 9816 uint32_t app_reserve:8; 9817 uint32_t table_id:15; 9818 uint32_t transfer:1; 9819 uint32_t _unused_:8; 9820 }; 9821 }; 9822 9823 static bool 9824 mlx5_access_tunnel_offload_db 9825 (struct rte_eth_dev *dev, 9826 bool (*match)(struct rte_eth_dev *, 9827 struct mlx5_flow_tunnel *, const void *), 9828 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9829 void (*miss)(struct rte_eth_dev *, void *), 9830 void *ctx, bool lock_op); 9831 9832 static int 9833 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9834 struct rte_flow *flow, 9835 const struct rte_flow_attr *attr, 9836 const struct rte_flow_action *app_actions, 9837 uint32_t flow_idx, 9838 const struct mlx5_flow_tunnel *tunnel, 9839 struct tunnel_default_miss_ctx *ctx, 9840 struct rte_flow_error *error) 9841 { 9842 struct mlx5_priv *priv = dev->data->dev_private; 9843 struct mlx5_flow *dev_flow; 9844 struct rte_flow_attr miss_attr = *attr; 9845 const struct rte_flow_item miss_items[2] = { 9846 { 9847 .type = RTE_FLOW_ITEM_TYPE_ETH, 9848 .spec = NULL, 9849 .last = NULL, 9850 .mask = NULL 9851 }, 9852 { 9853 .type = RTE_FLOW_ITEM_TYPE_END, 9854 .spec = NULL, 9855 .last = NULL, 9856 .mask = NULL 9857 } 9858 }; 9859 union tunnel_offload_mark mark_id; 9860 struct rte_flow_action_mark miss_mark; 9861 struct rte_flow_action miss_actions[3] = { 9862 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9863 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9864 }; 9865 const struct rte_flow_action_jump *jump_data; 9866 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9867 struct flow_grp_info grp_info = { 9868 .external = 1, 9869 .transfer = attr->transfer, 9870 .fdb_def_rule = !!priv->fdb_def_rule, 9871 .std_tbl_fix = 0, 9872 }; 9873 int ret; 9874 9875 if (!attr->transfer) { 9876 uint32_t q_size; 9877 9878 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9879 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9880 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9881 0, SOCKET_ID_ANY); 9882 if (!ctx->queue) 9883 return rte_flow_error_set 9884 (error, ENOMEM, 9885 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9886 NULL, "invalid default miss RSS"); 9887 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9888 ctx->action_rss.level = 0, 9889 ctx->action_rss.types = priv->rss_conf.rss_hf, 9890 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9891 ctx->action_rss.queue_num = priv->reta_idx_n, 9892 ctx->action_rss.key = priv->rss_conf.rss_key, 9893 ctx->action_rss.queue = ctx->queue; 9894 if (!priv->reta_idx_n || !priv->rxqs_n) 9895 return rte_flow_error_set 9896 (error, EINVAL, 9897 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9898 NULL, "invalid port configuration"); 9899 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9900 ctx->action_rss.types = 0; 9901 for (i = 0; i != priv->reta_idx_n; ++i) 9902 ctx->queue[i] = (*priv->reta_idx)[i]; 9903 } else { 9904 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9905 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9906 } 9907 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9908 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9909 jump_data = app_actions->conf; 9910 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9911 miss_attr.group = jump_data->group; 9912 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9913 &flow_table, &grp_info, error); 9914 if (ret) 9915 return rte_flow_error_set(error, EINVAL, 9916 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9917 NULL, "invalid tunnel id"); 9918 mark_id.app_reserve = 0; 9919 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9920 mark_id.transfer = !!attr->transfer; 9921 mark_id._unused_ = 0; 9922 miss_mark.id = mark_id.val; 9923 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9924 miss_items, miss_actions, flow_idx, error); 9925 if (!dev_flow) 9926 return -rte_errno; 9927 dev_flow->flow = flow; 9928 dev_flow->external = true; 9929 dev_flow->tunnel = tunnel; 9930 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9931 /* Subflow object was created, we must include one in the list. */ 9932 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9933 dev_flow->handle, next); 9934 DRV_LOG(DEBUG, 9935 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9936 dev->data->port_id, tunnel->app_tunnel.type, 9937 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9938 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9939 miss_actions, error); 9940 if (!ret) 9941 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9942 error); 9943 9944 return ret; 9945 } 9946 9947 static const struct mlx5_flow_tbl_data_entry * 9948 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9949 { 9950 struct mlx5_priv *priv = dev->data->dev_private; 9951 struct mlx5_dev_ctx_shared *sh = priv->sh; 9952 struct mlx5_list_entry *he; 9953 union tunnel_offload_mark mbits = { .val = mark }; 9954 union mlx5_flow_tbl_key table_key = { 9955 { 9956 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9957 .id = 0, 9958 .reserved = 0, 9959 .dummy = 0, 9960 .is_fdb = !!mbits.transfer, 9961 .is_egress = 0, 9962 } 9963 }; 9964 struct mlx5_flow_cb_ctx ctx = { 9965 .data = &table_key.v64, 9966 }; 9967 9968 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9969 return he ? 9970 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9971 } 9972 9973 static void 9974 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9975 struct mlx5_list_entry *entry) 9976 { 9977 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9978 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9979 9980 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9981 tunnel_flow_tbl_to_id(tte->flow_table)); 9982 mlx5_free(tte); 9983 } 9984 9985 static int 9986 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9987 struct mlx5_list_entry *entry, void *cb_ctx) 9988 { 9989 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9990 union tunnel_tbl_key tbl = { 9991 .val = *(uint64_t *)(ctx->data), 9992 }; 9993 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9994 9995 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9996 } 9997 9998 static struct mlx5_list_entry * 9999 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10000 { 10001 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10002 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10003 struct tunnel_tbl_entry *tte; 10004 union tunnel_tbl_key tbl = { 10005 .val = *(uint64_t *)(ctx->data), 10006 }; 10007 10008 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10009 sizeof(*tte), 0, 10010 SOCKET_ID_ANY); 10011 if (!tte) 10012 goto err; 10013 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10014 &tte->flow_table); 10015 if (tte->flow_table >= MLX5_MAX_TABLES) { 10016 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10017 tte->flow_table); 10018 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10019 tte->flow_table); 10020 goto err; 10021 } else if (!tte->flow_table) { 10022 goto err; 10023 } 10024 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10025 tte->tunnel_id = tbl.tunnel_id; 10026 tte->group = tbl.group; 10027 return &tte->hash; 10028 err: 10029 if (tte) 10030 mlx5_free(tte); 10031 return NULL; 10032 } 10033 10034 static struct mlx5_list_entry * 10035 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10036 struct mlx5_list_entry *oentry, 10037 void *cb_ctx __rte_unused) 10038 { 10039 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10040 0, SOCKET_ID_ANY); 10041 10042 if (!tte) 10043 return NULL; 10044 memcpy(tte, oentry, sizeof(*tte)); 10045 return &tte->hash; 10046 } 10047 10048 static void 10049 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10050 struct mlx5_list_entry *entry) 10051 { 10052 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10053 10054 mlx5_free(tte); 10055 } 10056 10057 static uint32_t 10058 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10059 const struct mlx5_flow_tunnel *tunnel, 10060 uint32_t group, uint32_t *table, 10061 struct rte_flow_error *error) 10062 { 10063 struct mlx5_list_entry *he; 10064 struct tunnel_tbl_entry *tte; 10065 union tunnel_tbl_key key = { 10066 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10067 .group = group 10068 }; 10069 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10070 struct mlx5_hlist *group_hash; 10071 struct mlx5_flow_cb_ctx ctx = { 10072 .data = &key.val, 10073 }; 10074 10075 group_hash = tunnel ? tunnel->groups : thub->groups; 10076 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10077 if (!he) 10078 return rte_flow_error_set(error, EINVAL, 10079 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10080 NULL, 10081 "tunnel group index not supported"); 10082 tte = container_of(he, typeof(*tte), hash); 10083 *table = tte->flow_table; 10084 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10085 dev->data->port_id, key.tunnel_id, group, *table); 10086 return 0; 10087 } 10088 10089 static void 10090 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10091 struct mlx5_flow_tunnel *tunnel) 10092 { 10093 struct mlx5_priv *priv = dev->data->dev_private; 10094 struct mlx5_indexed_pool *ipool; 10095 10096 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10097 dev->data->port_id, tunnel->tunnel_id); 10098 LIST_REMOVE(tunnel, chain); 10099 mlx5_hlist_destroy(tunnel->groups); 10100 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10101 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10102 } 10103 10104 static bool 10105 mlx5_access_tunnel_offload_db 10106 (struct rte_eth_dev *dev, 10107 bool (*match)(struct rte_eth_dev *, 10108 struct mlx5_flow_tunnel *, const void *), 10109 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10110 void (*miss)(struct rte_eth_dev *, void *), 10111 void *ctx, bool lock_op) 10112 { 10113 bool verdict = false; 10114 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10115 struct mlx5_flow_tunnel *tunnel; 10116 10117 rte_spinlock_lock(&thub->sl); 10118 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10119 verdict = match(dev, tunnel, (const void *)ctx); 10120 if (verdict) 10121 break; 10122 } 10123 if (!lock_op) 10124 rte_spinlock_unlock(&thub->sl); 10125 if (verdict && hit) 10126 hit(dev, tunnel, ctx); 10127 if (!verdict && miss) 10128 miss(dev, ctx); 10129 if (lock_op) 10130 rte_spinlock_unlock(&thub->sl); 10131 10132 return verdict; 10133 } 10134 10135 struct tunnel_db_find_tunnel_id_ctx { 10136 uint32_t tunnel_id; 10137 struct mlx5_flow_tunnel *tunnel; 10138 }; 10139 10140 static bool 10141 find_tunnel_id_match(struct rte_eth_dev *dev, 10142 struct mlx5_flow_tunnel *tunnel, const void *x) 10143 { 10144 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10145 10146 RTE_SET_USED(dev); 10147 return tunnel->tunnel_id == ctx->tunnel_id; 10148 } 10149 10150 static void 10151 find_tunnel_id_hit(struct rte_eth_dev *dev, 10152 struct mlx5_flow_tunnel *tunnel, void *x) 10153 { 10154 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10155 RTE_SET_USED(dev); 10156 ctx->tunnel = tunnel; 10157 } 10158 10159 static struct mlx5_flow_tunnel * 10160 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10161 { 10162 struct tunnel_db_find_tunnel_id_ctx ctx = { 10163 .tunnel_id = id, 10164 }; 10165 10166 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10167 find_tunnel_id_hit, NULL, &ctx, true); 10168 10169 return ctx.tunnel; 10170 } 10171 10172 static struct mlx5_flow_tunnel * 10173 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10174 const struct rte_flow_tunnel *app_tunnel) 10175 { 10176 struct mlx5_priv *priv = dev->data->dev_private; 10177 struct mlx5_indexed_pool *ipool; 10178 struct mlx5_flow_tunnel *tunnel; 10179 uint32_t id; 10180 10181 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10182 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10183 if (!tunnel) 10184 return NULL; 10185 if (id >= MLX5_MAX_TUNNELS) { 10186 mlx5_ipool_free(ipool, id); 10187 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10188 return NULL; 10189 } 10190 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10191 priv->sh, 10192 mlx5_flow_tunnel_grp2tbl_create_cb, 10193 mlx5_flow_tunnel_grp2tbl_match_cb, 10194 mlx5_flow_tunnel_grp2tbl_remove_cb, 10195 mlx5_flow_tunnel_grp2tbl_clone_cb, 10196 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10197 if (!tunnel->groups) { 10198 mlx5_ipool_free(ipool, id); 10199 return NULL; 10200 } 10201 /* initiate new PMD tunnel */ 10202 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10203 tunnel->tunnel_id = id; 10204 tunnel->action.type = (typeof(tunnel->action.type)) 10205 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10206 tunnel->action.conf = tunnel; 10207 tunnel->item.type = (typeof(tunnel->item.type)) 10208 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10209 tunnel->item.spec = tunnel; 10210 tunnel->item.last = NULL; 10211 tunnel->item.mask = NULL; 10212 10213 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10214 dev->data->port_id, tunnel->tunnel_id); 10215 10216 return tunnel; 10217 } 10218 10219 struct tunnel_db_get_tunnel_ctx { 10220 const struct rte_flow_tunnel *app_tunnel; 10221 struct mlx5_flow_tunnel *tunnel; 10222 }; 10223 10224 static bool get_tunnel_match(struct rte_eth_dev *dev, 10225 struct mlx5_flow_tunnel *tunnel, const void *x) 10226 { 10227 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10228 10229 RTE_SET_USED(dev); 10230 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10231 sizeof(*ctx->app_tunnel)); 10232 } 10233 10234 static void get_tunnel_hit(struct rte_eth_dev *dev, 10235 struct mlx5_flow_tunnel *tunnel, void *x) 10236 { 10237 /* called under tunnel spinlock protection */ 10238 struct tunnel_db_get_tunnel_ctx *ctx = x; 10239 10240 RTE_SET_USED(dev); 10241 tunnel->refctn++; 10242 ctx->tunnel = tunnel; 10243 } 10244 10245 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10246 { 10247 /* called under tunnel spinlock protection */ 10248 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10249 struct tunnel_db_get_tunnel_ctx *ctx = x; 10250 10251 rte_spinlock_unlock(&thub->sl); 10252 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10253 rte_spinlock_lock(&thub->sl); 10254 if (ctx->tunnel) { 10255 ctx->tunnel->refctn = 1; 10256 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10257 } 10258 } 10259 10260 10261 static int 10262 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10263 const struct rte_flow_tunnel *app_tunnel, 10264 struct mlx5_flow_tunnel **tunnel) 10265 { 10266 struct tunnel_db_get_tunnel_ctx ctx = { 10267 .app_tunnel = app_tunnel, 10268 }; 10269 10270 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10271 get_tunnel_miss, &ctx, true); 10272 *tunnel = ctx.tunnel; 10273 return ctx.tunnel ? 0 : -ENOMEM; 10274 } 10275 10276 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10277 { 10278 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10279 10280 if (!thub) 10281 return; 10282 if (!LIST_EMPTY(&thub->tunnels)) 10283 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10284 mlx5_hlist_destroy(thub->groups); 10285 mlx5_free(thub); 10286 } 10287 10288 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10289 { 10290 int err; 10291 struct mlx5_flow_tunnel_hub *thub; 10292 10293 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10294 0, SOCKET_ID_ANY); 10295 if (!thub) 10296 return -ENOMEM; 10297 LIST_INIT(&thub->tunnels); 10298 rte_spinlock_init(&thub->sl); 10299 thub->groups = mlx5_hlist_create("flow groups", 64, 10300 false, true, sh, 10301 mlx5_flow_tunnel_grp2tbl_create_cb, 10302 mlx5_flow_tunnel_grp2tbl_match_cb, 10303 mlx5_flow_tunnel_grp2tbl_remove_cb, 10304 mlx5_flow_tunnel_grp2tbl_clone_cb, 10305 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10306 if (!thub->groups) { 10307 err = -rte_errno; 10308 goto err; 10309 } 10310 sh->tunnel_hub = thub; 10311 10312 return 0; 10313 10314 err: 10315 if (thub->groups) 10316 mlx5_hlist_destroy(thub->groups); 10317 if (thub) 10318 mlx5_free(thub); 10319 return err; 10320 } 10321 10322 static inline int 10323 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10324 struct rte_flow_tunnel *tunnel, 10325 struct rte_flow_error *error) 10326 { 10327 struct mlx5_priv *priv = dev->data->dev_private; 10328 10329 if (!priv->sh->config.dv_flow_en) 10330 return rte_flow_error_set(error, ENOTSUP, 10331 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10332 "flow DV interface is off"); 10333 if (!is_tunnel_offload_active(dev)) 10334 return rte_flow_error_set(error, ENOTSUP, 10335 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10336 "tunnel offload was not activated"); 10337 if (!tunnel) 10338 return rte_flow_error_set(error, EINVAL, 10339 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10340 "no application tunnel"); 10341 switch (tunnel->type) { 10342 default: 10343 return rte_flow_error_set(error, EINVAL, 10344 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10345 "unsupported tunnel type"); 10346 case RTE_FLOW_ITEM_TYPE_VXLAN: 10347 case RTE_FLOW_ITEM_TYPE_GRE: 10348 case RTE_FLOW_ITEM_TYPE_NVGRE: 10349 case RTE_FLOW_ITEM_TYPE_GENEVE: 10350 break; 10351 } 10352 return 0; 10353 } 10354 10355 static int 10356 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10357 struct rte_flow_tunnel *app_tunnel, 10358 struct rte_flow_action **actions, 10359 uint32_t *num_of_actions, 10360 struct rte_flow_error *error) 10361 { 10362 struct mlx5_flow_tunnel *tunnel; 10363 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10364 10365 if (ret) 10366 return ret; 10367 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10368 if (ret < 0) { 10369 return rte_flow_error_set(error, ret, 10370 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10371 "failed to initialize pmd tunnel"); 10372 } 10373 *actions = &tunnel->action; 10374 *num_of_actions = 1; 10375 return 0; 10376 } 10377 10378 static int 10379 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10380 struct rte_flow_tunnel *app_tunnel, 10381 struct rte_flow_item **items, 10382 uint32_t *num_of_items, 10383 struct rte_flow_error *error) 10384 { 10385 struct mlx5_flow_tunnel *tunnel; 10386 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10387 10388 if (ret) 10389 return ret; 10390 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10391 if (ret < 0) { 10392 return rte_flow_error_set(error, ret, 10393 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10394 "failed to initialize pmd tunnel"); 10395 } 10396 *items = &tunnel->item; 10397 *num_of_items = 1; 10398 return 0; 10399 } 10400 10401 struct tunnel_db_element_release_ctx { 10402 struct rte_flow_item *items; 10403 struct rte_flow_action *actions; 10404 uint32_t num_elements; 10405 struct rte_flow_error *error; 10406 int ret; 10407 }; 10408 10409 static bool 10410 tunnel_element_release_match(struct rte_eth_dev *dev, 10411 struct mlx5_flow_tunnel *tunnel, const void *x) 10412 { 10413 const struct tunnel_db_element_release_ctx *ctx = x; 10414 10415 RTE_SET_USED(dev); 10416 if (ctx->num_elements != 1) 10417 return false; 10418 else if (ctx->items) 10419 return ctx->items == &tunnel->item; 10420 else if (ctx->actions) 10421 return ctx->actions == &tunnel->action; 10422 10423 return false; 10424 } 10425 10426 static void 10427 tunnel_element_release_hit(struct rte_eth_dev *dev, 10428 struct mlx5_flow_tunnel *tunnel, void *x) 10429 { 10430 struct tunnel_db_element_release_ctx *ctx = x; 10431 ctx->ret = 0; 10432 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10433 mlx5_flow_tunnel_free(dev, tunnel); 10434 } 10435 10436 static void 10437 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10438 { 10439 struct tunnel_db_element_release_ctx *ctx = x; 10440 RTE_SET_USED(dev); 10441 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10442 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10443 "invalid argument"); 10444 } 10445 10446 static int 10447 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10448 struct rte_flow_item *pmd_items, 10449 uint32_t num_items, struct rte_flow_error *err) 10450 { 10451 struct tunnel_db_element_release_ctx ctx = { 10452 .items = pmd_items, 10453 .actions = NULL, 10454 .num_elements = num_items, 10455 .error = err, 10456 }; 10457 10458 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10459 tunnel_element_release_hit, 10460 tunnel_element_release_miss, &ctx, false); 10461 10462 return ctx.ret; 10463 } 10464 10465 static int 10466 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10467 struct rte_flow_action *pmd_actions, 10468 uint32_t num_actions, struct rte_flow_error *err) 10469 { 10470 struct tunnel_db_element_release_ctx ctx = { 10471 .items = NULL, 10472 .actions = pmd_actions, 10473 .num_elements = num_actions, 10474 .error = err, 10475 }; 10476 10477 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10478 tunnel_element_release_hit, 10479 tunnel_element_release_miss, &ctx, false); 10480 10481 return ctx.ret; 10482 } 10483 10484 static int 10485 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10486 struct rte_mbuf *m, 10487 struct rte_flow_restore_info *info, 10488 struct rte_flow_error *err) 10489 { 10490 uint64_t ol_flags = m->ol_flags; 10491 const struct mlx5_flow_tbl_data_entry *tble; 10492 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10493 10494 if (!is_tunnel_offload_active(dev)) { 10495 info->flags = 0; 10496 return 0; 10497 } 10498 10499 if ((ol_flags & mask) != mask) 10500 goto err; 10501 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10502 if (!tble) { 10503 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10504 dev->data->port_id, m->hash.fdir.hi); 10505 goto err; 10506 } 10507 MLX5_ASSERT(tble->tunnel); 10508 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10509 info->group_id = tble->group_id; 10510 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10511 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10512 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10513 10514 return 0; 10515 10516 err: 10517 return rte_flow_error_set(err, EINVAL, 10518 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10519 "failed to get restore info"); 10520 } 10521 10522 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10523 static int 10524 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10525 __rte_unused struct rte_flow_tunnel *app_tunnel, 10526 __rte_unused struct rte_flow_action **actions, 10527 __rte_unused uint32_t *num_of_actions, 10528 __rte_unused struct rte_flow_error *error) 10529 { 10530 return -ENOTSUP; 10531 } 10532 10533 static int 10534 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10535 __rte_unused struct rte_flow_tunnel *app_tunnel, 10536 __rte_unused struct rte_flow_item **items, 10537 __rte_unused uint32_t *num_of_items, 10538 __rte_unused struct rte_flow_error *error) 10539 { 10540 return -ENOTSUP; 10541 } 10542 10543 static int 10544 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10545 __rte_unused struct rte_flow_item *pmd_items, 10546 __rte_unused uint32_t num_items, 10547 __rte_unused struct rte_flow_error *err) 10548 { 10549 return -ENOTSUP; 10550 } 10551 10552 static int 10553 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10554 __rte_unused struct rte_flow_action *pmd_action, 10555 __rte_unused uint32_t num_actions, 10556 __rte_unused struct rte_flow_error *err) 10557 { 10558 return -ENOTSUP; 10559 } 10560 10561 static int 10562 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10563 __rte_unused struct rte_mbuf *m, 10564 __rte_unused struct rte_flow_restore_info *i, 10565 __rte_unused struct rte_flow_error *err) 10566 { 10567 return -ENOTSUP; 10568 } 10569 10570 static int 10571 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10572 __rte_unused struct rte_flow *flow, 10573 __rte_unused const struct rte_flow_attr *attr, 10574 __rte_unused const struct rte_flow_action *actions, 10575 __rte_unused uint32_t flow_idx, 10576 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10577 __rte_unused struct tunnel_default_miss_ctx *ctx, 10578 __rte_unused struct rte_flow_error *error) 10579 { 10580 return -ENOTSUP; 10581 } 10582 10583 static struct mlx5_flow_tunnel * 10584 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10585 __rte_unused uint32_t id) 10586 { 10587 return NULL; 10588 } 10589 10590 static void 10591 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10592 __rte_unused struct mlx5_flow_tunnel *tunnel) 10593 { 10594 } 10595 10596 static uint32_t 10597 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10598 __rte_unused const struct mlx5_flow_tunnel *t, 10599 __rte_unused uint32_t group, 10600 __rte_unused uint32_t *table, 10601 struct rte_flow_error *error) 10602 { 10603 return rte_flow_error_set(error, ENOTSUP, 10604 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10605 "tunnel offload requires DV support"); 10606 } 10607 10608 void 10609 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10610 __rte_unused uint16_t port_id) 10611 { 10612 } 10613 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10614 10615 /* Flex flow item API */ 10616 static struct rte_flow_item_flex_handle * 10617 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10618 const struct rte_flow_item_flex_conf *conf, 10619 struct rte_flow_error *error) 10620 { 10621 static const char err_msg[] = "flex item creation unsupported"; 10622 struct rte_flow_attr attr = { .transfer = 0 }; 10623 const struct mlx5_flow_driver_ops *fops = 10624 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10625 10626 if (!fops->item_create) { 10627 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10628 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10629 NULL, err_msg); 10630 return NULL; 10631 } 10632 return fops->item_create(dev, conf, error); 10633 } 10634 10635 static int 10636 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10637 const struct rte_flow_item_flex_handle *handle, 10638 struct rte_flow_error *error) 10639 { 10640 static const char err_msg[] = "flex item release unsupported"; 10641 struct rte_flow_attr attr = { .transfer = 0 }; 10642 const struct mlx5_flow_driver_ops *fops = 10643 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10644 10645 if (!fops->item_release) { 10646 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10647 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10648 NULL, err_msg); 10649 return -rte_errno; 10650 } 10651 return fops->item_release(dev, handle, error); 10652 } 10653 10654 static void 10655 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10656 { 10657 int ret; 10658 struct rte_flow_error error; 10659 10660 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10661 char *item_name; 10662 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10663 sizeof(item_name), 10664 (void *)(uintptr_t)item->type, &error); 10665 if (ret > 0) 10666 printf("%s ", item_name); 10667 else 10668 printf("%d\n", (int)item->type); 10669 } 10670 printf("END\n"); 10671 } 10672 10673 static int 10674 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10675 { 10676 const struct rte_flow_item_udp *spec = udp_item->spec; 10677 const struct rte_flow_item_udp *mask = udp_item->mask; 10678 uint16_t udp_dport = 0; 10679 10680 if (spec != NULL) { 10681 if (!mask) 10682 mask = &rte_flow_item_udp_mask; 10683 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10684 mask->hdr.dst_port); 10685 } 10686 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10687 } 10688 10689 static const struct mlx5_flow_expand_node * 10690 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10691 unsigned int item_idx, 10692 const struct mlx5_flow_expand_node graph[], 10693 const struct mlx5_flow_expand_node *node) 10694 { 10695 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10696 10697 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10698 node != NULL && 10699 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10700 /* 10701 * The expansion node is VXLAN and it is also the last 10702 * expandable item in the pattern, so need to continue 10703 * expansion of the inner tunnel. 10704 */ 10705 MLX5_ASSERT(item_idx > 0); 10706 prev_item = pattern + item_idx - 1; 10707 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10708 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10709 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10710 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10711 } 10712 return node; 10713 } 10714 10715 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10716 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10717 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10718 }; 10719 10720 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10721 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10722 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10723 { 9, 10, 11 }, { 12, 13, 14 }, 10724 }; 10725 10726 /** 10727 * Discover the number of available flow priorities. 10728 * 10729 * @param dev 10730 * Ethernet device. 10731 * 10732 * @return 10733 * On success, number of available flow priorities. 10734 * On failure, a negative errno-style code and rte_errno is set. 10735 */ 10736 int 10737 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10738 { 10739 static const uint16_t vprio[] = {8, 16}; 10740 const struct mlx5_priv *priv = dev->data->dev_private; 10741 const struct mlx5_flow_driver_ops *fops; 10742 enum mlx5_flow_drv_type type; 10743 int ret; 10744 10745 type = mlx5_flow_os_get_type(); 10746 if (type == MLX5_FLOW_TYPE_MAX) { 10747 type = MLX5_FLOW_TYPE_VERBS; 10748 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10749 type = MLX5_FLOW_TYPE_DV; 10750 } 10751 fops = flow_get_drv_ops(type); 10752 if (fops->discover_priorities == NULL) { 10753 DRV_LOG(ERR, "Priority discovery not supported"); 10754 rte_errno = ENOTSUP; 10755 return -rte_errno; 10756 } 10757 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10758 if (ret < 0) 10759 return ret; 10760 switch (ret) { 10761 case 8: 10762 ret = RTE_DIM(priority_map_3); 10763 break; 10764 case 16: 10765 ret = RTE_DIM(priority_map_5); 10766 break; 10767 default: 10768 rte_errno = ENOTSUP; 10769 DRV_LOG(ERR, 10770 "port %u maximum priority: %d expected 8/16", 10771 dev->data->port_id, ret); 10772 return -rte_errno; 10773 } 10774 DRV_LOG(INFO, "port %u supported flow priorities:" 10775 " 0-%d for ingress or egress root table," 10776 " 0-%d for non-root table or transfer root table.", 10777 dev->data->port_id, ret - 2, 10778 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10779 return ret; 10780 } 10781 10782 /** 10783 * Adjust flow priority based on the highest layer and the request priority. 10784 * 10785 * @param[in] dev 10786 * Pointer to the Ethernet device structure. 10787 * @param[in] priority 10788 * The rule base priority. 10789 * @param[in] subpriority 10790 * The priority based on the items. 10791 * 10792 * @return 10793 * The new priority. 10794 */ 10795 uint32_t 10796 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10797 uint32_t subpriority) 10798 { 10799 uint32_t res = 0; 10800 struct mlx5_priv *priv = dev->data->dev_private; 10801 10802 switch (priv->sh->flow_max_priority) { 10803 case RTE_DIM(priority_map_3): 10804 res = priority_map_3[priority][subpriority]; 10805 break; 10806 case RTE_DIM(priority_map_5): 10807 res = priority_map_5[priority][subpriority]; 10808 break; 10809 } 10810 return res; 10811 } 10812