1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 static int 809 mlx5_flow_info_get(struct rte_eth_dev *dev, 810 struct rte_flow_port_info *port_info, 811 struct rte_flow_queue_info *queue_info, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_port_configure(struct rte_eth_dev *dev, 815 const struct rte_flow_port_attr *port_attr, 816 uint16_t nb_queue, 817 const struct rte_flow_queue_attr *queue_attr[], 818 struct rte_flow_error *err); 819 820 static struct rte_flow_pattern_template * 821 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 822 const struct rte_flow_pattern_template_attr *attr, 823 const struct rte_flow_item items[], 824 struct rte_flow_error *error); 825 826 static int 827 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 828 struct rte_flow_pattern_template *template, 829 struct rte_flow_error *error); 830 831 static const struct rte_flow_ops mlx5_flow_ops = { 832 .validate = mlx5_flow_validate, 833 .create = mlx5_flow_create, 834 .destroy = mlx5_flow_destroy, 835 .flush = mlx5_flow_flush, 836 .isolate = mlx5_flow_isolate, 837 .query = mlx5_flow_query, 838 .dev_dump = mlx5_flow_dev_dump, 839 .get_aged_flows = mlx5_flow_get_aged_flows, 840 .action_handle_create = mlx5_action_handle_create, 841 .action_handle_destroy = mlx5_action_handle_destroy, 842 .action_handle_update = mlx5_action_handle_update, 843 .action_handle_query = mlx5_action_handle_query, 844 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 845 .tunnel_match = mlx5_flow_tunnel_match, 846 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 847 .tunnel_item_release = mlx5_flow_tunnel_item_release, 848 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 849 .flex_item_create = mlx5_flow_flex_item_create, 850 .flex_item_release = mlx5_flow_flex_item_release, 851 .info_get = mlx5_flow_info_get, 852 .configure = mlx5_flow_port_configure, 853 .pattern_template_create = mlx5_flow_pattern_template_create, 854 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 855 }; 856 857 /* Tunnel information. */ 858 struct mlx5_flow_tunnel_info { 859 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 860 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 861 }; 862 863 static struct mlx5_flow_tunnel_info tunnels_info[] = { 864 { 865 .tunnel = MLX5_FLOW_LAYER_VXLAN, 866 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 867 }, 868 { 869 .tunnel = MLX5_FLOW_LAYER_GENEVE, 870 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 871 }, 872 { 873 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 874 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 875 }, 876 { 877 .tunnel = MLX5_FLOW_LAYER_GRE, 878 .ptype = RTE_PTYPE_TUNNEL_GRE, 879 }, 880 { 881 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 882 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 883 }, 884 { 885 .tunnel = MLX5_FLOW_LAYER_MPLS, 886 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 887 }, 888 { 889 .tunnel = MLX5_FLOW_LAYER_NVGRE, 890 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 891 }, 892 { 893 .tunnel = MLX5_FLOW_LAYER_IPIP, 894 .ptype = RTE_PTYPE_TUNNEL_IP, 895 }, 896 { 897 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 898 .ptype = RTE_PTYPE_TUNNEL_IP, 899 }, 900 { 901 .tunnel = MLX5_FLOW_LAYER_GTP, 902 .ptype = RTE_PTYPE_TUNNEL_GTPU, 903 }, 904 }; 905 906 907 908 /** 909 * Translate tag ID to register. 910 * 911 * @param[in] dev 912 * Pointer to the Ethernet device structure. 913 * @param[in] feature 914 * The feature that request the register. 915 * @param[in] id 916 * The request register ID. 917 * @param[out] error 918 * Error description in case of any. 919 * 920 * @return 921 * The request register on success, a negative errno 922 * value otherwise and rte_errno is set. 923 */ 924 int 925 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 926 enum mlx5_feature_name feature, 927 uint32_t id, 928 struct rte_flow_error *error) 929 { 930 struct mlx5_priv *priv = dev->data->dev_private; 931 struct mlx5_sh_config *config = &priv->sh->config; 932 enum modify_reg start_reg; 933 bool skip_mtr_reg = false; 934 935 switch (feature) { 936 case MLX5_HAIRPIN_RX: 937 return REG_B; 938 case MLX5_HAIRPIN_TX: 939 return REG_A; 940 case MLX5_METADATA_RX: 941 switch (config->dv_xmeta_en) { 942 case MLX5_XMETA_MODE_LEGACY: 943 return REG_B; 944 case MLX5_XMETA_MODE_META16: 945 return REG_C_0; 946 case MLX5_XMETA_MODE_META32: 947 return REG_C_1; 948 } 949 break; 950 case MLX5_METADATA_TX: 951 return REG_A; 952 case MLX5_METADATA_FDB: 953 switch (config->dv_xmeta_en) { 954 case MLX5_XMETA_MODE_LEGACY: 955 return REG_NON; 956 case MLX5_XMETA_MODE_META16: 957 return REG_C_0; 958 case MLX5_XMETA_MODE_META32: 959 return REG_C_1; 960 } 961 break; 962 case MLX5_FLOW_MARK: 963 switch (config->dv_xmeta_en) { 964 case MLX5_XMETA_MODE_LEGACY: 965 return REG_NON; 966 case MLX5_XMETA_MODE_META16: 967 return REG_C_1; 968 case MLX5_XMETA_MODE_META32: 969 return REG_C_0; 970 } 971 break; 972 case MLX5_MTR_ID: 973 /* 974 * If meter color and meter id share one register, flow match 975 * should use the meter color register for match. 976 */ 977 if (priv->mtr_reg_share) 978 return priv->mtr_color_reg; 979 else 980 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 981 REG_C_3; 982 case MLX5_MTR_COLOR: 983 case MLX5_ASO_FLOW_HIT: 984 case MLX5_ASO_CONNTRACK: 985 case MLX5_SAMPLE_ID: 986 /* All features use the same REG_C. */ 987 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 988 return priv->mtr_color_reg; 989 case MLX5_COPY_MARK: 990 /* 991 * Metadata COPY_MARK register using is in meter suffix sub 992 * flow while with meter. It's safe to share the same register. 993 */ 994 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 995 case MLX5_APP_TAG: 996 /* 997 * If meter is enable, it will engage the register for color 998 * match and flow match. If meter color match is not using the 999 * REG_C_2, need to skip the REG_C_x be used by meter color 1000 * match. 1001 * If meter is disable, free to use all available registers. 1002 */ 1003 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1004 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1005 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1006 if (id > (uint32_t)(REG_C_7 - start_reg)) 1007 return rte_flow_error_set(error, EINVAL, 1008 RTE_FLOW_ERROR_TYPE_ITEM, 1009 NULL, "invalid tag id"); 1010 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1011 return rte_flow_error_set(error, ENOTSUP, 1012 RTE_FLOW_ERROR_TYPE_ITEM, 1013 NULL, "unsupported tag id"); 1014 /* 1015 * This case means meter is using the REG_C_x great than 2. 1016 * Take care not to conflict with meter color REG_C_x. 1017 * If the available index REG_C_y >= REG_C_x, skip the 1018 * color register. 1019 */ 1020 if (skip_mtr_reg && priv->sh->flow_mreg_c 1021 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1022 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1023 return rte_flow_error_set(error, EINVAL, 1024 RTE_FLOW_ERROR_TYPE_ITEM, 1025 NULL, "invalid tag id"); 1026 if (priv->sh->flow_mreg_c 1027 [id + 1 + start_reg - REG_C_0] != REG_NON) 1028 return priv->sh->flow_mreg_c 1029 [id + 1 + start_reg - REG_C_0]; 1030 return rte_flow_error_set(error, ENOTSUP, 1031 RTE_FLOW_ERROR_TYPE_ITEM, 1032 NULL, "unsupported tag id"); 1033 } 1034 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1035 } 1036 MLX5_ASSERT(false); 1037 return rte_flow_error_set(error, EINVAL, 1038 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1039 NULL, "invalid feature name"); 1040 } 1041 1042 /** 1043 * Check extensive flow metadata register support. 1044 * 1045 * @param dev 1046 * Pointer to rte_eth_dev structure. 1047 * 1048 * @return 1049 * True if device supports extensive flow metadata register, otherwise false. 1050 */ 1051 bool 1052 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1053 { 1054 struct mlx5_priv *priv = dev->data->dev_private; 1055 1056 /* 1057 * Having available reg_c can be regarded inclusively as supporting 1058 * extensive flow metadata register, which could mean, 1059 * - metadata register copy action by modify header. 1060 * - 16 modify header actions is supported. 1061 * - reg_c's are preserved across different domain (FDB and NIC) on 1062 * packet loopback by flow lookup miss. 1063 */ 1064 return priv->sh->flow_mreg_c[2] != REG_NON; 1065 } 1066 1067 /** 1068 * Get the lowest priority. 1069 * 1070 * @param[in] dev 1071 * Pointer to the Ethernet device structure. 1072 * @param[in] attributes 1073 * Pointer to device flow rule attributes. 1074 * 1075 * @return 1076 * The value of lowest priority of flow. 1077 */ 1078 uint32_t 1079 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1080 const struct rte_flow_attr *attr) 1081 { 1082 struct mlx5_priv *priv = dev->data->dev_private; 1083 1084 if (!attr->group && !attr->transfer) 1085 return priv->sh->flow_max_priority - 2; 1086 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1087 } 1088 1089 /** 1090 * Calculate matcher priority of the flow. 1091 * 1092 * @param[in] dev 1093 * Pointer to the Ethernet device structure. 1094 * @param[in] attr 1095 * Pointer to device flow rule attributes. 1096 * @param[in] subpriority 1097 * The priority based on the items. 1098 * @param[in] external 1099 * Flow is user flow. 1100 * @return 1101 * The matcher priority of the flow. 1102 */ 1103 uint16_t 1104 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1105 const struct rte_flow_attr *attr, 1106 uint32_t subpriority, bool external) 1107 { 1108 uint16_t priority = (uint16_t)attr->priority; 1109 struct mlx5_priv *priv = dev->data->dev_private; 1110 1111 if (!attr->group && !attr->transfer) { 1112 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1113 priority = priv->sh->flow_max_priority - 1; 1114 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1115 } else if (!external && attr->transfer && attr->group == 0 && 1116 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1117 return (priv->sh->flow_max_priority - 1) * 3; 1118 } 1119 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1120 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1121 return priority * 3 + subpriority; 1122 } 1123 1124 /** 1125 * Verify the @p item specifications (spec, last, mask) are compatible with the 1126 * NIC capabilities. 1127 * 1128 * @param[in] item 1129 * Item specification. 1130 * @param[in] mask 1131 * @p item->mask or flow default bit-masks. 1132 * @param[in] nic_mask 1133 * Bit-masks covering supported fields by the NIC to compare with user mask. 1134 * @param[in] size 1135 * Bit-masks size in bytes. 1136 * @param[in] range_accepted 1137 * True if range of values is accepted for specific fields, false otherwise. 1138 * @param[out] error 1139 * Pointer to error structure. 1140 * 1141 * @return 1142 * 0 on success, a negative errno value otherwise and rte_errno is set. 1143 */ 1144 int 1145 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1146 const uint8_t *mask, 1147 const uint8_t *nic_mask, 1148 unsigned int size, 1149 bool range_accepted, 1150 struct rte_flow_error *error) 1151 { 1152 unsigned int i; 1153 1154 MLX5_ASSERT(nic_mask); 1155 for (i = 0; i < size; ++i) 1156 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1157 return rte_flow_error_set(error, ENOTSUP, 1158 RTE_FLOW_ERROR_TYPE_ITEM, 1159 item, 1160 "mask enables non supported" 1161 " bits"); 1162 if (!item->spec && (item->mask || item->last)) 1163 return rte_flow_error_set(error, EINVAL, 1164 RTE_FLOW_ERROR_TYPE_ITEM, item, 1165 "mask/last without a spec is not" 1166 " supported"); 1167 if (item->spec && item->last && !range_accepted) { 1168 uint8_t spec[size]; 1169 uint8_t last[size]; 1170 unsigned int i; 1171 int ret; 1172 1173 for (i = 0; i < size; ++i) { 1174 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1175 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1176 } 1177 ret = memcmp(spec, last, size); 1178 if (ret != 0) 1179 return rte_flow_error_set(error, EINVAL, 1180 RTE_FLOW_ERROR_TYPE_ITEM, 1181 item, 1182 "range is not valid"); 1183 } 1184 return 0; 1185 } 1186 1187 /** 1188 * Adjust the hash fields according to the @p flow information. 1189 * 1190 * @param[in] dev_flow. 1191 * Pointer to the mlx5_flow. 1192 * @param[in] tunnel 1193 * 1 when the hash field is for a tunnel item. 1194 * @param[in] layer_types 1195 * RTE_ETH_RSS_* types. 1196 * @param[in] hash_fields 1197 * Item hash fields. 1198 * 1199 * @return 1200 * The hash fields that should be used. 1201 */ 1202 uint64_t 1203 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1204 int tunnel __rte_unused, uint64_t layer_types, 1205 uint64_t hash_fields) 1206 { 1207 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1208 int rss_request_inner = rss_desc->level >= 2; 1209 1210 /* Check RSS hash level for tunnel. */ 1211 if (tunnel && rss_request_inner) 1212 hash_fields |= IBV_RX_HASH_INNER; 1213 else if (tunnel || rss_request_inner) 1214 return 0; 1215 #endif 1216 /* Check if requested layer matches RSS hash fields. */ 1217 if (!(rss_desc->types & layer_types)) 1218 return 0; 1219 return hash_fields; 1220 } 1221 1222 /** 1223 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1224 * if several tunnel rules are used on this queue, the tunnel ptype will be 1225 * cleared. 1226 * 1227 * @param rxq_ctrl 1228 * Rx queue to update. 1229 */ 1230 static void 1231 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1232 { 1233 unsigned int i; 1234 uint32_t tunnel_ptype = 0; 1235 1236 /* Look up for the ptype to use. */ 1237 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1238 if (!rxq_ctrl->flow_tunnels_n[i]) 1239 continue; 1240 if (!tunnel_ptype) { 1241 tunnel_ptype = tunnels_info[i].ptype; 1242 } else { 1243 tunnel_ptype = 0; 1244 break; 1245 } 1246 } 1247 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1248 } 1249 1250 /** 1251 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1252 * flow. 1253 * 1254 * @param[in] dev 1255 * Pointer to the Ethernet device structure. 1256 * @param[in] dev_handle 1257 * Pointer to device flow handle structure. 1258 */ 1259 void 1260 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1261 struct mlx5_flow_handle *dev_handle) 1262 { 1263 struct mlx5_priv *priv = dev->data->dev_private; 1264 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1265 struct mlx5_ind_table_obj *ind_tbl = NULL; 1266 unsigned int i; 1267 1268 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1269 struct mlx5_hrxq *hrxq; 1270 1271 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1272 dev_handle->rix_hrxq); 1273 if (hrxq) 1274 ind_tbl = hrxq->ind_table; 1275 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1276 struct mlx5_shared_action_rss *shared_rss; 1277 1278 shared_rss = mlx5_ipool_get 1279 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1280 dev_handle->rix_srss); 1281 if (shared_rss) 1282 ind_tbl = shared_rss->ind_tbl; 1283 } 1284 if (!ind_tbl) 1285 return; 1286 for (i = 0; i != ind_tbl->queues_n; ++i) { 1287 int idx = ind_tbl->queues[i]; 1288 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1289 1290 MLX5_ASSERT(rxq_ctrl != NULL); 1291 if (rxq_ctrl == NULL) 1292 continue; 1293 /* 1294 * To support metadata register copy on Tx loopback, 1295 * this must be always enabled (metadata may arive 1296 * from other port - not from local flows only. 1297 */ 1298 if (tunnel) { 1299 unsigned int j; 1300 1301 /* Increase the counter matching the flow. */ 1302 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1303 if ((tunnels_info[j].tunnel & 1304 dev_handle->layers) == 1305 tunnels_info[j].tunnel) { 1306 rxq_ctrl->flow_tunnels_n[j]++; 1307 break; 1308 } 1309 } 1310 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1311 } 1312 } 1313 } 1314 1315 static void 1316 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1317 { 1318 struct mlx5_priv *priv = dev->data->dev_private; 1319 struct mlx5_rxq_ctrl *rxq_ctrl; 1320 1321 if (priv->mark_enabled) 1322 return; 1323 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1324 rxq_ctrl->rxq.mark = 1; 1325 } 1326 priv->mark_enabled = 1; 1327 } 1328 1329 /** 1330 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1331 * 1332 * @param[in] dev 1333 * Pointer to the Ethernet device structure. 1334 * @param[in] flow 1335 * Pointer to flow structure. 1336 */ 1337 static void 1338 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1339 { 1340 struct mlx5_priv *priv = dev->data->dev_private; 1341 uint32_t handle_idx; 1342 struct mlx5_flow_handle *dev_handle; 1343 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1344 1345 MLX5_ASSERT(wks); 1346 if (wks->mark) 1347 flow_rxq_mark_flag_set(dev); 1348 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1349 handle_idx, dev_handle, next) 1350 flow_drv_rxq_flags_set(dev, dev_handle); 1351 } 1352 1353 /** 1354 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1355 * device flow if no other flow uses it with the same kind of request. 1356 * 1357 * @param dev 1358 * Pointer to Ethernet device. 1359 * @param[in] dev_handle 1360 * Pointer to the device flow handle structure. 1361 */ 1362 static void 1363 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1364 struct mlx5_flow_handle *dev_handle) 1365 { 1366 struct mlx5_priv *priv = dev->data->dev_private; 1367 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1368 struct mlx5_ind_table_obj *ind_tbl = NULL; 1369 unsigned int i; 1370 1371 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1372 struct mlx5_hrxq *hrxq; 1373 1374 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1375 dev_handle->rix_hrxq); 1376 if (hrxq) 1377 ind_tbl = hrxq->ind_table; 1378 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1379 struct mlx5_shared_action_rss *shared_rss; 1380 1381 shared_rss = mlx5_ipool_get 1382 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1383 dev_handle->rix_srss); 1384 if (shared_rss) 1385 ind_tbl = shared_rss->ind_tbl; 1386 } 1387 if (!ind_tbl) 1388 return; 1389 MLX5_ASSERT(dev->data->dev_started); 1390 for (i = 0; i != ind_tbl->queues_n; ++i) { 1391 int idx = ind_tbl->queues[i]; 1392 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1393 1394 MLX5_ASSERT(rxq_ctrl != NULL); 1395 if (rxq_ctrl == NULL) 1396 continue; 1397 if (tunnel) { 1398 unsigned int j; 1399 1400 /* Decrease the counter matching the flow. */ 1401 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1402 if ((tunnels_info[j].tunnel & 1403 dev_handle->layers) == 1404 tunnels_info[j].tunnel) { 1405 rxq_ctrl->flow_tunnels_n[j]--; 1406 break; 1407 } 1408 } 1409 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1410 } 1411 } 1412 } 1413 1414 /** 1415 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1416 * @p flow if no other flow uses it with the same kind of request. 1417 * 1418 * @param dev 1419 * Pointer to Ethernet device. 1420 * @param[in] flow 1421 * Pointer to the flow. 1422 */ 1423 static void 1424 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1425 { 1426 struct mlx5_priv *priv = dev->data->dev_private; 1427 uint32_t handle_idx; 1428 struct mlx5_flow_handle *dev_handle; 1429 1430 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1431 handle_idx, dev_handle, next) 1432 flow_drv_rxq_flags_trim(dev, dev_handle); 1433 } 1434 1435 /** 1436 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1437 * 1438 * @param dev 1439 * Pointer to Ethernet device. 1440 */ 1441 static void 1442 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1443 { 1444 struct mlx5_priv *priv = dev->data->dev_private; 1445 unsigned int i; 1446 1447 for (i = 0; i != priv->rxqs_n; ++i) { 1448 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1449 unsigned int j; 1450 1451 if (rxq == NULL || rxq->ctrl == NULL) 1452 continue; 1453 rxq->ctrl->rxq.mark = 0; 1454 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1455 rxq->ctrl->flow_tunnels_n[j] = 0; 1456 rxq->ctrl->rxq.tunnel = 0; 1457 } 1458 priv->mark_enabled = 0; 1459 } 1460 1461 /** 1462 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1463 * 1464 * @param[in] dev 1465 * Pointer to the Ethernet device structure. 1466 */ 1467 void 1468 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1469 { 1470 struct mlx5_priv *priv = dev->data->dev_private; 1471 unsigned int i; 1472 1473 for (i = 0; i != priv->rxqs_n; ++i) { 1474 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1475 struct mlx5_rxq_data *data; 1476 1477 if (rxq == NULL || rxq->ctrl == NULL) 1478 continue; 1479 data = &rxq->ctrl->rxq; 1480 if (!rte_flow_dynf_metadata_avail()) { 1481 data->dynf_meta = 0; 1482 data->flow_meta_mask = 0; 1483 data->flow_meta_offset = -1; 1484 data->flow_meta_port_mask = 0; 1485 } else { 1486 data->dynf_meta = 1; 1487 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1488 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1489 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1490 } 1491 } 1492 } 1493 1494 /* 1495 * return a pointer to the desired action in the list of actions. 1496 * 1497 * @param[in] actions 1498 * The list of actions to search the action in. 1499 * @param[in] action 1500 * The action to find. 1501 * 1502 * @return 1503 * Pointer to the action in the list, if found. NULL otherwise. 1504 */ 1505 const struct rte_flow_action * 1506 mlx5_flow_find_action(const struct rte_flow_action *actions, 1507 enum rte_flow_action_type action) 1508 { 1509 if (actions == NULL) 1510 return NULL; 1511 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1512 if (actions->type == action) 1513 return actions; 1514 return NULL; 1515 } 1516 1517 /* 1518 * Validate the flag action. 1519 * 1520 * @param[in] action_flags 1521 * Bit-fields that holds the actions detected until now. 1522 * @param[in] attr 1523 * Attributes of flow that includes this action. 1524 * @param[out] error 1525 * Pointer to error structure. 1526 * 1527 * @return 1528 * 0 on success, a negative errno value otherwise and rte_errno is set. 1529 */ 1530 int 1531 mlx5_flow_validate_action_flag(uint64_t action_flags, 1532 const struct rte_flow_attr *attr, 1533 struct rte_flow_error *error) 1534 { 1535 if (action_flags & MLX5_FLOW_ACTION_MARK) 1536 return rte_flow_error_set(error, EINVAL, 1537 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1538 "can't mark and flag in same flow"); 1539 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1540 return rte_flow_error_set(error, EINVAL, 1541 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1542 "can't have 2 flag" 1543 " actions in same flow"); 1544 if (attr->egress) 1545 return rte_flow_error_set(error, ENOTSUP, 1546 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1547 "flag action not supported for " 1548 "egress"); 1549 return 0; 1550 } 1551 1552 /* 1553 * Validate the mark action. 1554 * 1555 * @param[in] action 1556 * Pointer to the queue action. 1557 * @param[in] action_flags 1558 * Bit-fields that holds the actions detected until now. 1559 * @param[in] attr 1560 * Attributes of flow that includes this action. 1561 * @param[out] error 1562 * Pointer to error structure. 1563 * 1564 * @return 1565 * 0 on success, a negative errno value otherwise and rte_errno is set. 1566 */ 1567 int 1568 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1569 uint64_t action_flags, 1570 const struct rte_flow_attr *attr, 1571 struct rte_flow_error *error) 1572 { 1573 const struct rte_flow_action_mark *mark = action->conf; 1574 1575 if (!mark) 1576 return rte_flow_error_set(error, EINVAL, 1577 RTE_FLOW_ERROR_TYPE_ACTION, 1578 action, 1579 "configuration cannot be null"); 1580 if (mark->id >= MLX5_FLOW_MARK_MAX) 1581 return rte_flow_error_set(error, EINVAL, 1582 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1583 &mark->id, 1584 "mark id must in 0 <= id < " 1585 RTE_STR(MLX5_FLOW_MARK_MAX)); 1586 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1587 return rte_flow_error_set(error, EINVAL, 1588 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1589 "can't flag and mark in same flow"); 1590 if (action_flags & MLX5_FLOW_ACTION_MARK) 1591 return rte_flow_error_set(error, EINVAL, 1592 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1593 "can't have 2 mark actions in same" 1594 " flow"); 1595 if (attr->egress) 1596 return rte_flow_error_set(error, ENOTSUP, 1597 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1598 "mark action not supported for " 1599 "egress"); 1600 return 0; 1601 } 1602 1603 /* 1604 * Validate the drop action. 1605 * 1606 * @param[in] action_flags 1607 * Bit-fields that holds the actions detected until now. 1608 * @param[in] attr 1609 * Attributes of flow that includes this action. 1610 * @param[out] error 1611 * Pointer to error structure. 1612 * 1613 * @return 1614 * 0 on success, a negative errno value otherwise and rte_errno is set. 1615 */ 1616 int 1617 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1618 const struct rte_flow_attr *attr, 1619 struct rte_flow_error *error) 1620 { 1621 if (attr->egress) 1622 return rte_flow_error_set(error, ENOTSUP, 1623 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1624 "drop action not supported for " 1625 "egress"); 1626 return 0; 1627 } 1628 1629 /* 1630 * Validate the queue action. 1631 * 1632 * @param[in] action 1633 * Pointer to the queue action. 1634 * @param[in] action_flags 1635 * Bit-fields that holds the actions detected until now. 1636 * @param[in] dev 1637 * Pointer to the Ethernet device structure. 1638 * @param[in] attr 1639 * Attributes of flow that includes this action. 1640 * @param[out] error 1641 * Pointer to error structure. 1642 * 1643 * @return 1644 * 0 on success, a negative errno value otherwise and rte_errno is set. 1645 */ 1646 int 1647 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1648 uint64_t action_flags, 1649 struct rte_eth_dev *dev, 1650 const struct rte_flow_attr *attr, 1651 struct rte_flow_error *error) 1652 { 1653 struct mlx5_priv *priv = dev->data->dev_private; 1654 const struct rte_flow_action_queue *queue = action->conf; 1655 1656 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1657 return rte_flow_error_set(error, EINVAL, 1658 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1659 "can't have 2 fate actions in" 1660 " same flow"); 1661 if (!priv->rxqs_n) 1662 return rte_flow_error_set(error, EINVAL, 1663 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1664 NULL, "No Rx queues configured"); 1665 if (queue->index >= priv->rxqs_n) 1666 return rte_flow_error_set(error, EINVAL, 1667 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1668 &queue->index, 1669 "queue index out of range"); 1670 if (mlx5_rxq_get(dev, queue->index) == NULL) 1671 return rte_flow_error_set(error, EINVAL, 1672 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1673 &queue->index, 1674 "queue is not configured"); 1675 if (attr->egress) 1676 return rte_flow_error_set(error, ENOTSUP, 1677 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1678 "queue action not supported for " 1679 "egress"); 1680 return 0; 1681 } 1682 1683 /** 1684 * Validate queue numbers for device RSS. 1685 * 1686 * @param[in] dev 1687 * Configured device. 1688 * @param[in] queues 1689 * Array of queue numbers. 1690 * @param[in] queues_n 1691 * Size of the @p queues array. 1692 * @param[out] error 1693 * On error, filled with a textual error description. 1694 * @param[out] queue 1695 * On error, filled with an offending queue index in @p queues array. 1696 * 1697 * @return 1698 * 0 on success, a negative errno code on error. 1699 */ 1700 static int 1701 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1702 const uint16_t *queues, uint32_t queues_n, 1703 const char **error, uint32_t *queue_idx) 1704 { 1705 const struct mlx5_priv *priv = dev->data->dev_private; 1706 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1707 uint32_t i; 1708 1709 for (i = 0; i != queues_n; ++i) { 1710 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1711 queues[i]); 1712 1713 if (queues[i] >= priv->rxqs_n) { 1714 *error = "queue index out of range"; 1715 *queue_idx = i; 1716 return -EINVAL; 1717 } 1718 if (rxq_ctrl == NULL) { 1719 *error = "queue is not configured"; 1720 *queue_idx = i; 1721 return -EINVAL; 1722 } 1723 if (i == 0) 1724 rxq_type = rxq_ctrl->type; 1725 if (rxq_type != rxq_ctrl->type) { 1726 *error = "combining hairpin and regular RSS queues is not supported"; 1727 *queue_idx = i; 1728 return -ENOTSUP; 1729 } 1730 } 1731 return 0; 1732 } 1733 1734 /* 1735 * Validate the rss action. 1736 * 1737 * @param[in] dev 1738 * Pointer to the Ethernet device structure. 1739 * @param[in] action 1740 * Pointer to the queue action. 1741 * @param[out] error 1742 * Pointer to error structure. 1743 * 1744 * @return 1745 * 0 on success, a negative errno value otherwise and rte_errno is set. 1746 */ 1747 int 1748 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1749 const struct rte_flow_action *action, 1750 struct rte_flow_error *error) 1751 { 1752 struct mlx5_priv *priv = dev->data->dev_private; 1753 const struct rte_flow_action_rss *rss = action->conf; 1754 int ret; 1755 const char *message; 1756 uint32_t queue_idx; 1757 1758 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1759 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1760 return rte_flow_error_set(error, ENOTSUP, 1761 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1762 &rss->func, 1763 "RSS hash function not supported"); 1764 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1765 if (rss->level > 2) 1766 #else 1767 if (rss->level > 1) 1768 #endif 1769 return rte_flow_error_set(error, ENOTSUP, 1770 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1771 &rss->level, 1772 "tunnel RSS is not supported"); 1773 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1774 if (rss->key_len == 0 && rss->key != NULL) 1775 return rte_flow_error_set(error, ENOTSUP, 1776 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1777 &rss->key_len, 1778 "RSS hash key length 0"); 1779 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1780 return rte_flow_error_set(error, ENOTSUP, 1781 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1782 &rss->key_len, 1783 "RSS hash key too small"); 1784 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1785 return rte_flow_error_set(error, ENOTSUP, 1786 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1787 &rss->key_len, 1788 "RSS hash key too large"); 1789 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1790 return rte_flow_error_set(error, ENOTSUP, 1791 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1792 &rss->queue_num, 1793 "number of queues too large"); 1794 if (rss->types & MLX5_RSS_HF_MASK) 1795 return rte_flow_error_set(error, ENOTSUP, 1796 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1797 &rss->types, 1798 "some RSS protocols are not" 1799 " supported"); 1800 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1801 !(rss->types & RTE_ETH_RSS_IP)) 1802 return rte_flow_error_set(error, EINVAL, 1803 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1804 "L3 partial RSS requested but L3 RSS" 1805 " type not specified"); 1806 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1807 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1808 return rte_flow_error_set(error, EINVAL, 1809 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1810 "L4 partial RSS requested but L4 RSS" 1811 " type not specified"); 1812 if (!priv->rxqs_n) 1813 return rte_flow_error_set(error, EINVAL, 1814 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1815 NULL, "No Rx queues configured"); 1816 if (!rss->queue_num) 1817 return rte_flow_error_set(error, EINVAL, 1818 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1819 NULL, "No queues configured"); 1820 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1821 &message, &queue_idx); 1822 if (ret != 0) { 1823 return rte_flow_error_set(error, -ret, 1824 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1825 &rss->queue[queue_idx], message); 1826 } 1827 return 0; 1828 } 1829 1830 /* 1831 * Validate the rss action. 1832 * 1833 * @param[in] action 1834 * Pointer to the queue action. 1835 * @param[in] action_flags 1836 * Bit-fields that holds the actions detected until now. 1837 * @param[in] dev 1838 * Pointer to the Ethernet device structure. 1839 * @param[in] attr 1840 * Attributes of flow that includes this action. 1841 * @param[in] item_flags 1842 * Items that were detected. 1843 * @param[out] error 1844 * Pointer to error structure. 1845 * 1846 * @return 1847 * 0 on success, a negative errno value otherwise and rte_errno is set. 1848 */ 1849 int 1850 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1851 uint64_t action_flags, 1852 struct rte_eth_dev *dev, 1853 const struct rte_flow_attr *attr, 1854 uint64_t item_flags, 1855 struct rte_flow_error *error) 1856 { 1857 const struct rte_flow_action_rss *rss = action->conf; 1858 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1859 int ret; 1860 1861 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1862 return rte_flow_error_set(error, EINVAL, 1863 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1864 "can't have 2 fate actions" 1865 " in same flow"); 1866 ret = mlx5_validate_action_rss(dev, action, error); 1867 if (ret) 1868 return ret; 1869 if (attr->egress) 1870 return rte_flow_error_set(error, ENOTSUP, 1871 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1872 "rss action not supported for " 1873 "egress"); 1874 if (rss->level > 1 && !tunnel) 1875 return rte_flow_error_set(error, EINVAL, 1876 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1877 "inner RSS is not supported for " 1878 "non-tunnel flows"); 1879 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1880 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1881 return rte_flow_error_set(error, EINVAL, 1882 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1883 "RSS on eCPRI is not supported now"); 1884 } 1885 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1886 !(item_flags & 1887 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1888 rss->level > 1) 1889 return rte_flow_error_set(error, EINVAL, 1890 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1891 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1892 return 0; 1893 } 1894 1895 /* 1896 * Validate the default miss action. 1897 * 1898 * @param[in] action_flags 1899 * Bit-fields that holds the actions detected until now. 1900 * @param[out] error 1901 * Pointer to error structure. 1902 * 1903 * @return 1904 * 0 on success, a negative errno value otherwise and rte_errno is set. 1905 */ 1906 int 1907 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1908 const struct rte_flow_attr *attr, 1909 struct rte_flow_error *error) 1910 { 1911 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1912 return rte_flow_error_set(error, EINVAL, 1913 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1914 "can't have 2 fate actions in" 1915 " same flow"); 1916 if (attr->egress) 1917 return rte_flow_error_set(error, ENOTSUP, 1918 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1919 "default miss action not supported " 1920 "for egress"); 1921 if (attr->group) 1922 return rte_flow_error_set(error, ENOTSUP, 1923 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1924 "only group 0 is supported"); 1925 if (attr->transfer) 1926 return rte_flow_error_set(error, ENOTSUP, 1927 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1928 NULL, "transfer is not supported"); 1929 return 0; 1930 } 1931 1932 /* 1933 * Validate the count action. 1934 * 1935 * @param[in] dev 1936 * Pointer to the Ethernet device structure. 1937 * @param[in] attr 1938 * Attributes of flow that includes this action. 1939 * @param[out] error 1940 * Pointer to error structure. 1941 * 1942 * @return 1943 * 0 on success, a negative errno value otherwise and rte_errno is set. 1944 */ 1945 int 1946 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1947 const struct rte_flow_attr *attr, 1948 struct rte_flow_error *error) 1949 { 1950 if (attr->egress) 1951 return rte_flow_error_set(error, ENOTSUP, 1952 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1953 "count action not supported for " 1954 "egress"); 1955 return 0; 1956 } 1957 1958 /* 1959 * Validate the ASO CT action. 1960 * 1961 * @param[in] dev 1962 * Pointer to the Ethernet device structure. 1963 * @param[in] conntrack 1964 * Pointer to the CT action profile. 1965 * @param[out] error 1966 * Pointer to error structure. 1967 * 1968 * @return 1969 * 0 on success, a negative errno value otherwise and rte_errno is set. 1970 */ 1971 int 1972 mlx5_validate_action_ct(struct rte_eth_dev *dev, 1973 const struct rte_flow_action_conntrack *conntrack, 1974 struct rte_flow_error *error) 1975 { 1976 RTE_SET_USED(dev); 1977 1978 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 1979 return rte_flow_error_set(error, EINVAL, 1980 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1981 "Invalid CT state"); 1982 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 1983 return rte_flow_error_set(error, EINVAL, 1984 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1985 "Invalid last TCP packet flag"); 1986 return 0; 1987 } 1988 1989 /** 1990 * Verify the @p attributes will be correctly understood by the NIC and store 1991 * them in the @p flow if everything is correct. 1992 * 1993 * @param[in] dev 1994 * Pointer to the Ethernet device structure. 1995 * @param[in] attributes 1996 * Pointer to flow attributes 1997 * @param[out] error 1998 * Pointer to error structure. 1999 * 2000 * @return 2001 * 0 on success, a negative errno value otherwise and rte_errno is set. 2002 */ 2003 int 2004 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2005 const struct rte_flow_attr *attributes, 2006 struct rte_flow_error *error) 2007 { 2008 struct mlx5_priv *priv = dev->data->dev_private; 2009 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2010 2011 if (attributes->group) 2012 return rte_flow_error_set(error, ENOTSUP, 2013 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2014 NULL, "groups is not supported"); 2015 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2016 attributes->priority >= priority_max) 2017 return rte_flow_error_set(error, ENOTSUP, 2018 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2019 NULL, "priority out of range"); 2020 if (attributes->egress) 2021 return rte_flow_error_set(error, ENOTSUP, 2022 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2023 "egress is not supported"); 2024 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2025 return rte_flow_error_set(error, ENOTSUP, 2026 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2027 NULL, "transfer is not supported"); 2028 if (!attributes->ingress) 2029 return rte_flow_error_set(error, EINVAL, 2030 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2031 NULL, 2032 "ingress attribute is mandatory"); 2033 return 0; 2034 } 2035 2036 /** 2037 * Validate ICMP6 item. 2038 * 2039 * @param[in] item 2040 * Item specification. 2041 * @param[in] item_flags 2042 * Bit-fields that holds the items detected until now. 2043 * @param[in] ext_vlan_sup 2044 * Whether extended VLAN features are supported or not. 2045 * @param[out] error 2046 * Pointer to error structure. 2047 * 2048 * @return 2049 * 0 on success, a negative errno value otherwise and rte_errno is set. 2050 */ 2051 int 2052 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2053 uint64_t item_flags, 2054 uint8_t target_protocol, 2055 struct rte_flow_error *error) 2056 { 2057 const struct rte_flow_item_icmp6 *mask = item->mask; 2058 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2059 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2060 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2061 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2062 MLX5_FLOW_LAYER_OUTER_L4; 2063 int ret; 2064 2065 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2066 return rte_flow_error_set(error, EINVAL, 2067 RTE_FLOW_ERROR_TYPE_ITEM, item, 2068 "protocol filtering not compatible" 2069 " with ICMP6 layer"); 2070 if (!(item_flags & l3m)) 2071 return rte_flow_error_set(error, EINVAL, 2072 RTE_FLOW_ERROR_TYPE_ITEM, item, 2073 "IPv6 is mandatory to filter on" 2074 " ICMP6"); 2075 if (item_flags & l4m) 2076 return rte_flow_error_set(error, EINVAL, 2077 RTE_FLOW_ERROR_TYPE_ITEM, item, 2078 "multiple L4 layers not supported"); 2079 if (!mask) 2080 mask = &rte_flow_item_icmp6_mask; 2081 ret = mlx5_flow_item_acceptable 2082 (item, (const uint8_t *)mask, 2083 (const uint8_t *)&rte_flow_item_icmp6_mask, 2084 sizeof(struct rte_flow_item_icmp6), 2085 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2086 if (ret < 0) 2087 return ret; 2088 return 0; 2089 } 2090 2091 /** 2092 * Validate ICMP item. 2093 * 2094 * @param[in] item 2095 * Item specification. 2096 * @param[in] item_flags 2097 * Bit-fields that holds the items detected until now. 2098 * @param[out] error 2099 * Pointer to error structure. 2100 * 2101 * @return 2102 * 0 on success, a negative errno value otherwise and rte_errno is set. 2103 */ 2104 int 2105 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2106 uint64_t item_flags, 2107 uint8_t target_protocol, 2108 struct rte_flow_error *error) 2109 { 2110 const struct rte_flow_item_icmp *mask = item->mask; 2111 const struct rte_flow_item_icmp nic_mask = { 2112 .hdr.icmp_type = 0xff, 2113 .hdr.icmp_code = 0xff, 2114 .hdr.icmp_ident = RTE_BE16(0xffff), 2115 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2116 }; 2117 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2118 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2119 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2120 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2121 MLX5_FLOW_LAYER_OUTER_L4; 2122 int ret; 2123 2124 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2125 return rte_flow_error_set(error, EINVAL, 2126 RTE_FLOW_ERROR_TYPE_ITEM, item, 2127 "protocol filtering not compatible" 2128 " with ICMP layer"); 2129 if (!(item_flags & l3m)) 2130 return rte_flow_error_set(error, EINVAL, 2131 RTE_FLOW_ERROR_TYPE_ITEM, item, 2132 "IPv4 is mandatory to filter" 2133 " on ICMP"); 2134 if (item_flags & l4m) 2135 return rte_flow_error_set(error, EINVAL, 2136 RTE_FLOW_ERROR_TYPE_ITEM, item, 2137 "multiple L4 layers not supported"); 2138 if (!mask) 2139 mask = &nic_mask; 2140 ret = mlx5_flow_item_acceptable 2141 (item, (const uint8_t *)mask, 2142 (const uint8_t *)&nic_mask, 2143 sizeof(struct rte_flow_item_icmp), 2144 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2145 if (ret < 0) 2146 return ret; 2147 return 0; 2148 } 2149 2150 /** 2151 * Validate Ethernet item. 2152 * 2153 * @param[in] item 2154 * Item specification. 2155 * @param[in] item_flags 2156 * Bit-fields that holds the items detected until now. 2157 * @param[out] error 2158 * Pointer to error structure. 2159 * 2160 * @return 2161 * 0 on success, a negative errno value otherwise and rte_errno is set. 2162 */ 2163 int 2164 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2165 uint64_t item_flags, bool ext_vlan_sup, 2166 struct rte_flow_error *error) 2167 { 2168 const struct rte_flow_item_eth *mask = item->mask; 2169 const struct rte_flow_item_eth nic_mask = { 2170 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2171 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2172 .type = RTE_BE16(0xffff), 2173 .has_vlan = ext_vlan_sup ? 1 : 0, 2174 }; 2175 int ret; 2176 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2177 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2178 MLX5_FLOW_LAYER_OUTER_L2; 2179 2180 if (item_flags & ethm) 2181 return rte_flow_error_set(error, ENOTSUP, 2182 RTE_FLOW_ERROR_TYPE_ITEM, item, 2183 "multiple L2 layers not supported"); 2184 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2185 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2186 return rte_flow_error_set(error, EINVAL, 2187 RTE_FLOW_ERROR_TYPE_ITEM, item, 2188 "L2 layer should not follow " 2189 "L3 layers"); 2190 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2191 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2192 return rte_flow_error_set(error, EINVAL, 2193 RTE_FLOW_ERROR_TYPE_ITEM, item, 2194 "L2 layer should not follow VLAN"); 2195 if (item_flags & MLX5_FLOW_LAYER_GTP) 2196 return rte_flow_error_set(error, EINVAL, 2197 RTE_FLOW_ERROR_TYPE_ITEM, item, 2198 "L2 layer should not follow GTP"); 2199 if (!mask) 2200 mask = &rte_flow_item_eth_mask; 2201 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2202 (const uint8_t *)&nic_mask, 2203 sizeof(struct rte_flow_item_eth), 2204 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2205 return ret; 2206 } 2207 2208 /** 2209 * Validate VLAN item. 2210 * 2211 * @param[in] item 2212 * Item specification. 2213 * @param[in] item_flags 2214 * Bit-fields that holds the items detected until now. 2215 * @param[in] dev 2216 * Ethernet device flow is being created on. 2217 * @param[out] error 2218 * Pointer to error structure. 2219 * 2220 * @return 2221 * 0 on success, a negative errno value otherwise and rte_errno is set. 2222 */ 2223 int 2224 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2225 uint64_t item_flags, 2226 struct rte_eth_dev *dev, 2227 struct rte_flow_error *error) 2228 { 2229 const struct rte_flow_item_vlan *spec = item->spec; 2230 const struct rte_flow_item_vlan *mask = item->mask; 2231 const struct rte_flow_item_vlan nic_mask = { 2232 .tci = RTE_BE16(UINT16_MAX), 2233 .inner_type = RTE_BE16(UINT16_MAX), 2234 }; 2235 uint16_t vlan_tag = 0; 2236 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2237 int ret; 2238 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2239 MLX5_FLOW_LAYER_INNER_L4) : 2240 (MLX5_FLOW_LAYER_OUTER_L3 | 2241 MLX5_FLOW_LAYER_OUTER_L4); 2242 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2243 MLX5_FLOW_LAYER_OUTER_VLAN; 2244 2245 if (item_flags & vlanm) 2246 return rte_flow_error_set(error, EINVAL, 2247 RTE_FLOW_ERROR_TYPE_ITEM, item, 2248 "multiple VLAN layers not supported"); 2249 else if ((item_flags & l34m) != 0) 2250 return rte_flow_error_set(error, EINVAL, 2251 RTE_FLOW_ERROR_TYPE_ITEM, item, 2252 "VLAN cannot follow L3/L4 layer"); 2253 if (!mask) 2254 mask = &rte_flow_item_vlan_mask; 2255 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2256 (const uint8_t *)&nic_mask, 2257 sizeof(struct rte_flow_item_vlan), 2258 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2259 if (ret) 2260 return ret; 2261 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2262 struct mlx5_priv *priv = dev->data->dev_private; 2263 2264 if (priv->vmwa_context) { 2265 /* 2266 * Non-NULL context means we have a virtual machine 2267 * and SR-IOV enabled, we have to create VLAN interface 2268 * to make hypervisor to setup E-Switch vport 2269 * context correctly. We avoid creating the multiple 2270 * VLAN interfaces, so we cannot support VLAN tag mask. 2271 */ 2272 return rte_flow_error_set(error, EINVAL, 2273 RTE_FLOW_ERROR_TYPE_ITEM, 2274 item, 2275 "VLAN tag mask is not" 2276 " supported in virtual" 2277 " environment"); 2278 } 2279 } 2280 if (spec) { 2281 vlan_tag = spec->tci; 2282 vlan_tag &= mask->tci; 2283 } 2284 /* 2285 * From verbs perspective an empty VLAN is equivalent 2286 * to a packet without VLAN layer. 2287 */ 2288 if (!vlan_tag) 2289 return rte_flow_error_set(error, EINVAL, 2290 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2291 item->spec, 2292 "VLAN cannot be empty"); 2293 return 0; 2294 } 2295 2296 /** 2297 * Validate IPV4 item. 2298 * 2299 * @param[in] item 2300 * Item specification. 2301 * @param[in] item_flags 2302 * Bit-fields that holds the items detected until now. 2303 * @param[in] last_item 2304 * Previous validated item in the pattern items. 2305 * @param[in] ether_type 2306 * Type in the ethernet layer header (including dot1q). 2307 * @param[in] acc_mask 2308 * Acceptable mask, if NULL default internal default mask 2309 * will be used to check whether item fields are supported. 2310 * @param[in] range_accepted 2311 * True if range of values is accepted for specific fields, false otherwise. 2312 * @param[out] error 2313 * Pointer to error structure. 2314 * 2315 * @return 2316 * 0 on success, a negative errno value otherwise and rte_errno is set. 2317 */ 2318 int 2319 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2320 uint64_t item_flags, 2321 uint64_t last_item, 2322 uint16_t ether_type, 2323 const struct rte_flow_item_ipv4 *acc_mask, 2324 bool range_accepted, 2325 struct rte_flow_error *error) 2326 { 2327 const struct rte_flow_item_ipv4 *mask = item->mask; 2328 const struct rte_flow_item_ipv4 *spec = item->spec; 2329 const struct rte_flow_item_ipv4 nic_mask = { 2330 .hdr = { 2331 .src_addr = RTE_BE32(0xffffffff), 2332 .dst_addr = RTE_BE32(0xffffffff), 2333 .type_of_service = 0xff, 2334 .next_proto_id = 0xff, 2335 }, 2336 }; 2337 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2338 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2339 MLX5_FLOW_LAYER_OUTER_L3; 2340 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2341 MLX5_FLOW_LAYER_OUTER_L4; 2342 int ret; 2343 uint8_t next_proto = 0xFF; 2344 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2345 MLX5_FLOW_LAYER_OUTER_VLAN | 2346 MLX5_FLOW_LAYER_INNER_VLAN); 2347 2348 if ((last_item & l2_vlan) && ether_type && 2349 ether_type != RTE_ETHER_TYPE_IPV4) 2350 return rte_flow_error_set(error, EINVAL, 2351 RTE_FLOW_ERROR_TYPE_ITEM, item, 2352 "IPv4 cannot follow L2/VLAN layer " 2353 "which ether type is not IPv4"); 2354 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2355 if (mask && spec) 2356 next_proto = mask->hdr.next_proto_id & 2357 spec->hdr.next_proto_id; 2358 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2359 return rte_flow_error_set(error, EINVAL, 2360 RTE_FLOW_ERROR_TYPE_ITEM, 2361 item, 2362 "multiple tunnel " 2363 "not supported"); 2364 } 2365 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2366 return rte_flow_error_set(error, EINVAL, 2367 RTE_FLOW_ERROR_TYPE_ITEM, item, 2368 "wrong tunnel type - IPv6 specified " 2369 "but IPv4 item provided"); 2370 if (item_flags & l3m) 2371 return rte_flow_error_set(error, ENOTSUP, 2372 RTE_FLOW_ERROR_TYPE_ITEM, item, 2373 "multiple L3 layers not supported"); 2374 else if (item_flags & l4m) 2375 return rte_flow_error_set(error, EINVAL, 2376 RTE_FLOW_ERROR_TYPE_ITEM, item, 2377 "L3 cannot follow an L4 layer."); 2378 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2379 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2380 return rte_flow_error_set(error, EINVAL, 2381 RTE_FLOW_ERROR_TYPE_ITEM, item, 2382 "L3 cannot follow an NVGRE layer."); 2383 if (!mask) 2384 mask = &rte_flow_item_ipv4_mask; 2385 else if (mask->hdr.next_proto_id != 0 && 2386 mask->hdr.next_proto_id != 0xff) 2387 return rte_flow_error_set(error, EINVAL, 2388 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2389 "partial mask is not supported" 2390 " for protocol"); 2391 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2392 acc_mask ? (const uint8_t *)acc_mask 2393 : (const uint8_t *)&nic_mask, 2394 sizeof(struct rte_flow_item_ipv4), 2395 range_accepted, error); 2396 if (ret < 0) 2397 return ret; 2398 return 0; 2399 } 2400 2401 /** 2402 * Validate IPV6 item. 2403 * 2404 * @param[in] item 2405 * Item specification. 2406 * @param[in] item_flags 2407 * Bit-fields that holds the items detected until now. 2408 * @param[in] last_item 2409 * Previous validated item in the pattern items. 2410 * @param[in] ether_type 2411 * Type in the ethernet layer header (including dot1q). 2412 * @param[in] acc_mask 2413 * Acceptable mask, if NULL default internal default mask 2414 * will be used to check whether item fields are supported. 2415 * @param[out] error 2416 * Pointer to error structure. 2417 * 2418 * @return 2419 * 0 on success, a negative errno value otherwise and rte_errno is set. 2420 */ 2421 int 2422 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2423 uint64_t item_flags, 2424 uint64_t last_item, 2425 uint16_t ether_type, 2426 const struct rte_flow_item_ipv6 *acc_mask, 2427 struct rte_flow_error *error) 2428 { 2429 const struct rte_flow_item_ipv6 *mask = item->mask; 2430 const struct rte_flow_item_ipv6 *spec = item->spec; 2431 const struct rte_flow_item_ipv6 nic_mask = { 2432 .hdr = { 2433 .src_addr = 2434 "\xff\xff\xff\xff\xff\xff\xff\xff" 2435 "\xff\xff\xff\xff\xff\xff\xff\xff", 2436 .dst_addr = 2437 "\xff\xff\xff\xff\xff\xff\xff\xff" 2438 "\xff\xff\xff\xff\xff\xff\xff\xff", 2439 .vtc_flow = RTE_BE32(0xffffffff), 2440 .proto = 0xff, 2441 }, 2442 }; 2443 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2444 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2445 MLX5_FLOW_LAYER_OUTER_L3; 2446 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2447 MLX5_FLOW_LAYER_OUTER_L4; 2448 int ret; 2449 uint8_t next_proto = 0xFF; 2450 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2451 MLX5_FLOW_LAYER_OUTER_VLAN | 2452 MLX5_FLOW_LAYER_INNER_VLAN); 2453 2454 if ((last_item & l2_vlan) && ether_type && 2455 ether_type != RTE_ETHER_TYPE_IPV6) 2456 return rte_flow_error_set(error, EINVAL, 2457 RTE_FLOW_ERROR_TYPE_ITEM, item, 2458 "IPv6 cannot follow L2/VLAN layer " 2459 "which ether type is not IPv6"); 2460 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2461 next_proto = spec->hdr.proto; 2462 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2463 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2464 return rte_flow_error_set(error, EINVAL, 2465 RTE_FLOW_ERROR_TYPE_ITEM, 2466 item, 2467 "multiple tunnel " 2468 "not supported"); 2469 } 2470 if (next_proto == IPPROTO_HOPOPTS || 2471 next_proto == IPPROTO_ROUTING || 2472 next_proto == IPPROTO_FRAGMENT || 2473 next_proto == IPPROTO_ESP || 2474 next_proto == IPPROTO_AH || 2475 next_proto == IPPROTO_DSTOPTS) 2476 return rte_flow_error_set(error, EINVAL, 2477 RTE_FLOW_ERROR_TYPE_ITEM, item, 2478 "IPv6 proto (next header) should " 2479 "not be set as extension header"); 2480 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2481 return rte_flow_error_set(error, EINVAL, 2482 RTE_FLOW_ERROR_TYPE_ITEM, item, 2483 "wrong tunnel type - IPv4 specified " 2484 "but IPv6 item provided"); 2485 if (item_flags & l3m) 2486 return rte_flow_error_set(error, ENOTSUP, 2487 RTE_FLOW_ERROR_TYPE_ITEM, item, 2488 "multiple L3 layers not supported"); 2489 else if (item_flags & l4m) 2490 return rte_flow_error_set(error, EINVAL, 2491 RTE_FLOW_ERROR_TYPE_ITEM, item, 2492 "L3 cannot follow an L4 layer."); 2493 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2494 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2495 return rte_flow_error_set(error, EINVAL, 2496 RTE_FLOW_ERROR_TYPE_ITEM, item, 2497 "L3 cannot follow an NVGRE layer."); 2498 if (!mask) 2499 mask = &rte_flow_item_ipv6_mask; 2500 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2501 acc_mask ? (const uint8_t *)acc_mask 2502 : (const uint8_t *)&nic_mask, 2503 sizeof(struct rte_flow_item_ipv6), 2504 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2505 if (ret < 0) 2506 return ret; 2507 return 0; 2508 } 2509 2510 /** 2511 * Validate UDP item. 2512 * 2513 * @param[in] item 2514 * Item specification. 2515 * @param[in] item_flags 2516 * Bit-fields that holds the items detected until now. 2517 * @param[in] target_protocol 2518 * The next protocol in the previous item. 2519 * @param[in] flow_mask 2520 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2521 * @param[out] error 2522 * Pointer to error structure. 2523 * 2524 * @return 2525 * 0 on success, a negative errno value otherwise and rte_errno is set. 2526 */ 2527 int 2528 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2529 uint64_t item_flags, 2530 uint8_t target_protocol, 2531 struct rte_flow_error *error) 2532 { 2533 const struct rte_flow_item_udp *mask = item->mask; 2534 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2535 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2536 MLX5_FLOW_LAYER_OUTER_L3; 2537 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2538 MLX5_FLOW_LAYER_OUTER_L4; 2539 int ret; 2540 2541 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2542 return rte_flow_error_set(error, EINVAL, 2543 RTE_FLOW_ERROR_TYPE_ITEM, item, 2544 "protocol filtering not compatible" 2545 " with UDP layer"); 2546 if (!(item_flags & l3m)) 2547 return rte_flow_error_set(error, EINVAL, 2548 RTE_FLOW_ERROR_TYPE_ITEM, item, 2549 "L3 is mandatory to filter on L4"); 2550 if (item_flags & l4m) 2551 return rte_flow_error_set(error, EINVAL, 2552 RTE_FLOW_ERROR_TYPE_ITEM, item, 2553 "multiple L4 layers not supported"); 2554 if (!mask) 2555 mask = &rte_flow_item_udp_mask; 2556 ret = mlx5_flow_item_acceptable 2557 (item, (const uint8_t *)mask, 2558 (const uint8_t *)&rte_flow_item_udp_mask, 2559 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2560 error); 2561 if (ret < 0) 2562 return ret; 2563 return 0; 2564 } 2565 2566 /** 2567 * Validate TCP item. 2568 * 2569 * @param[in] item 2570 * Item specification. 2571 * @param[in] item_flags 2572 * Bit-fields that holds the items detected until now. 2573 * @param[in] target_protocol 2574 * The next protocol in the previous item. 2575 * @param[out] error 2576 * Pointer to error structure. 2577 * 2578 * @return 2579 * 0 on success, a negative errno value otherwise and rte_errno is set. 2580 */ 2581 int 2582 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2583 uint64_t item_flags, 2584 uint8_t target_protocol, 2585 const struct rte_flow_item_tcp *flow_mask, 2586 struct rte_flow_error *error) 2587 { 2588 const struct rte_flow_item_tcp *mask = item->mask; 2589 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2590 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2591 MLX5_FLOW_LAYER_OUTER_L3; 2592 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2593 MLX5_FLOW_LAYER_OUTER_L4; 2594 int ret; 2595 2596 MLX5_ASSERT(flow_mask); 2597 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2598 return rte_flow_error_set(error, EINVAL, 2599 RTE_FLOW_ERROR_TYPE_ITEM, item, 2600 "protocol filtering not compatible" 2601 " with TCP layer"); 2602 if (!(item_flags & l3m)) 2603 return rte_flow_error_set(error, EINVAL, 2604 RTE_FLOW_ERROR_TYPE_ITEM, item, 2605 "L3 is mandatory to filter on L4"); 2606 if (item_flags & l4m) 2607 return rte_flow_error_set(error, EINVAL, 2608 RTE_FLOW_ERROR_TYPE_ITEM, item, 2609 "multiple L4 layers not supported"); 2610 if (!mask) 2611 mask = &rte_flow_item_tcp_mask; 2612 ret = mlx5_flow_item_acceptable 2613 (item, (const uint8_t *)mask, 2614 (const uint8_t *)flow_mask, 2615 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2616 error); 2617 if (ret < 0) 2618 return ret; 2619 return 0; 2620 } 2621 2622 /** 2623 * Validate VXLAN item. 2624 * 2625 * @param[in] dev 2626 * Pointer to the Ethernet device structure. 2627 * @param[in] udp_dport 2628 * UDP destination port 2629 * @param[in] item 2630 * Item specification. 2631 * @param[in] item_flags 2632 * Bit-fields that holds the items detected until now. 2633 * @param[in] attr 2634 * Flow rule attributes. 2635 * @param[out] error 2636 * Pointer to error structure. 2637 * 2638 * @return 2639 * 0 on success, a negative errno value otherwise and rte_errno is set. 2640 */ 2641 int 2642 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2643 uint16_t udp_dport, 2644 const struct rte_flow_item *item, 2645 uint64_t item_flags, 2646 const struct rte_flow_attr *attr, 2647 struct rte_flow_error *error) 2648 { 2649 const struct rte_flow_item_vxlan *spec = item->spec; 2650 const struct rte_flow_item_vxlan *mask = item->mask; 2651 int ret; 2652 struct mlx5_priv *priv = dev->data->dev_private; 2653 union vni { 2654 uint32_t vlan_id; 2655 uint8_t vni[4]; 2656 } id = { .vlan_id = 0, }; 2657 const struct rte_flow_item_vxlan nic_mask = { 2658 .vni = "\xff\xff\xff", 2659 .rsvd1 = 0xff, 2660 }; 2661 const struct rte_flow_item_vxlan *valid_mask; 2662 2663 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2664 return rte_flow_error_set(error, ENOTSUP, 2665 RTE_FLOW_ERROR_TYPE_ITEM, item, 2666 "multiple tunnel layers not" 2667 " supported"); 2668 valid_mask = &rte_flow_item_vxlan_mask; 2669 /* 2670 * Verify only UDPv4 is present as defined in 2671 * https://tools.ietf.org/html/rfc7348 2672 */ 2673 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2674 return rte_flow_error_set(error, EINVAL, 2675 RTE_FLOW_ERROR_TYPE_ITEM, item, 2676 "no outer UDP layer found"); 2677 if (!mask) 2678 mask = &rte_flow_item_vxlan_mask; 2679 2680 if (priv->sh->steering_format_version != 2681 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2682 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2683 /* FDB domain & NIC domain non-zero group */ 2684 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2685 valid_mask = &nic_mask; 2686 /* Group zero in NIC domain */ 2687 if (!attr->group && !attr->transfer && 2688 priv->sh->tunnel_header_0_1) 2689 valid_mask = &nic_mask; 2690 } 2691 ret = mlx5_flow_item_acceptable 2692 (item, (const uint8_t *)mask, 2693 (const uint8_t *)valid_mask, 2694 sizeof(struct rte_flow_item_vxlan), 2695 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2696 if (ret < 0) 2697 return ret; 2698 if (spec) { 2699 memcpy(&id.vni[1], spec->vni, 3); 2700 memcpy(&id.vni[1], mask->vni, 3); 2701 } 2702 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2703 return rte_flow_error_set(error, ENOTSUP, 2704 RTE_FLOW_ERROR_TYPE_ITEM, item, 2705 "VXLAN tunnel must be fully defined"); 2706 return 0; 2707 } 2708 2709 /** 2710 * Validate VXLAN_GPE item. 2711 * 2712 * @param[in] item 2713 * Item specification. 2714 * @param[in] item_flags 2715 * Bit-fields that holds the items detected until now. 2716 * @param[in] priv 2717 * Pointer to the private data structure. 2718 * @param[in] target_protocol 2719 * The next protocol in the previous item. 2720 * @param[out] error 2721 * Pointer to error structure. 2722 * 2723 * @return 2724 * 0 on success, a negative errno value otherwise and rte_errno is set. 2725 */ 2726 int 2727 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2728 uint64_t item_flags, 2729 struct rte_eth_dev *dev, 2730 struct rte_flow_error *error) 2731 { 2732 struct mlx5_priv *priv = dev->data->dev_private; 2733 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2734 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2735 int ret; 2736 union vni { 2737 uint32_t vlan_id; 2738 uint8_t vni[4]; 2739 } id = { .vlan_id = 0, }; 2740 2741 if (!priv->sh->config.l3_vxlan_en) 2742 return rte_flow_error_set(error, ENOTSUP, 2743 RTE_FLOW_ERROR_TYPE_ITEM, item, 2744 "L3 VXLAN is not enabled by device" 2745 " parameter and/or not configured in" 2746 " firmware"); 2747 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2748 return rte_flow_error_set(error, ENOTSUP, 2749 RTE_FLOW_ERROR_TYPE_ITEM, item, 2750 "multiple tunnel layers not" 2751 " supported"); 2752 /* 2753 * Verify only UDPv4 is present as defined in 2754 * https://tools.ietf.org/html/rfc7348 2755 */ 2756 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2757 return rte_flow_error_set(error, EINVAL, 2758 RTE_FLOW_ERROR_TYPE_ITEM, item, 2759 "no outer UDP layer found"); 2760 if (!mask) 2761 mask = &rte_flow_item_vxlan_gpe_mask; 2762 ret = mlx5_flow_item_acceptable 2763 (item, (const uint8_t *)mask, 2764 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2765 sizeof(struct rte_flow_item_vxlan_gpe), 2766 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2767 if (ret < 0) 2768 return ret; 2769 if (spec) { 2770 if (spec->protocol) 2771 return rte_flow_error_set(error, ENOTSUP, 2772 RTE_FLOW_ERROR_TYPE_ITEM, 2773 item, 2774 "VxLAN-GPE protocol" 2775 " not supported"); 2776 memcpy(&id.vni[1], spec->vni, 3); 2777 memcpy(&id.vni[1], mask->vni, 3); 2778 } 2779 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2780 return rte_flow_error_set(error, ENOTSUP, 2781 RTE_FLOW_ERROR_TYPE_ITEM, item, 2782 "VXLAN-GPE tunnel must be fully" 2783 " defined"); 2784 return 0; 2785 } 2786 /** 2787 * Validate GRE Key item. 2788 * 2789 * @param[in] item 2790 * Item specification. 2791 * @param[in] item_flags 2792 * Bit flags to mark detected items. 2793 * @param[in] gre_item 2794 * Pointer to gre_item 2795 * @param[out] error 2796 * Pointer to error structure. 2797 * 2798 * @return 2799 * 0 on success, a negative errno value otherwise and rte_errno is set. 2800 */ 2801 int 2802 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2803 uint64_t item_flags, 2804 const struct rte_flow_item *gre_item, 2805 struct rte_flow_error *error) 2806 { 2807 const rte_be32_t *mask = item->mask; 2808 int ret = 0; 2809 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2810 const struct rte_flow_item_gre *gre_spec; 2811 const struct rte_flow_item_gre *gre_mask; 2812 2813 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2814 return rte_flow_error_set(error, ENOTSUP, 2815 RTE_FLOW_ERROR_TYPE_ITEM, item, 2816 "Multiple GRE key not support"); 2817 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2818 return rte_flow_error_set(error, ENOTSUP, 2819 RTE_FLOW_ERROR_TYPE_ITEM, item, 2820 "No preceding GRE header"); 2821 if (item_flags & MLX5_FLOW_LAYER_INNER) 2822 return rte_flow_error_set(error, ENOTSUP, 2823 RTE_FLOW_ERROR_TYPE_ITEM, item, 2824 "GRE key following a wrong item"); 2825 gre_mask = gre_item->mask; 2826 if (!gre_mask) 2827 gre_mask = &rte_flow_item_gre_mask; 2828 gre_spec = gre_item->spec; 2829 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2830 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2831 return rte_flow_error_set(error, EINVAL, 2832 RTE_FLOW_ERROR_TYPE_ITEM, item, 2833 "Key bit must be on"); 2834 2835 if (!mask) 2836 mask = &gre_key_default_mask; 2837 ret = mlx5_flow_item_acceptable 2838 (item, (const uint8_t *)mask, 2839 (const uint8_t *)&gre_key_default_mask, 2840 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2841 return ret; 2842 } 2843 2844 /** 2845 * Validate GRE item. 2846 * 2847 * @param[in] item 2848 * Item specification. 2849 * @param[in] item_flags 2850 * Bit flags to mark detected items. 2851 * @param[in] target_protocol 2852 * The next protocol in the previous item. 2853 * @param[out] error 2854 * Pointer to error structure. 2855 * 2856 * @return 2857 * 0 on success, a negative errno value otherwise and rte_errno is set. 2858 */ 2859 int 2860 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2861 uint64_t item_flags, 2862 uint8_t target_protocol, 2863 struct rte_flow_error *error) 2864 { 2865 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2866 const struct rte_flow_item_gre *mask = item->mask; 2867 int ret; 2868 const struct rte_flow_item_gre nic_mask = { 2869 .c_rsvd0_ver = RTE_BE16(0xB000), 2870 .protocol = RTE_BE16(UINT16_MAX), 2871 }; 2872 2873 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2874 return rte_flow_error_set(error, EINVAL, 2875 RTE_FLOW_ERROR_TYPE_ITEM, item, 2876 "protocol filtering not compatible" 2877 " with this GRE layer"); 2878 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2879 return rte_flow_error_set(error, ENOTSUP, 2880 RTE_FLOW_ERROR_TYPE_ITEM, item, 2881 "multiple tunnel layers not" 2882 " supported"); 2883 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2884 return rte_flow_error_set(error, ENOTSUP, 2885 RTE_FLOW_ERROR_TYPE_ITEM, item, 2886 "L3 Layer is missing"); 2887 if (!mask) 2888 mask = &rte_flow_item_gre_mask; 2889 ret = mlx5_flow_item_acceptable 2890 (item, (const uint8_t *)mask, 2891 (const uint8_t *)&nic_mask, 2892 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2893 error); 2894 if (ret < 0) 2895 return ret; 2896 #ifndef HAVE_MLX5DV_DR 2897 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2898 if (spec && (spec->protocol & mask->protocol)) 2899 return rte_flow_error_set(error, ENOTSUP, 2900 RTE_FLOW_ERROR_TYPE_ITEM, item, 2901 "without MPLS support the" 2902 " specification cannot be used for" 2903 " filtering"); 2904 #endif 2905 #endif 2906 return 0; 2907 } 2908 2909 /** 2910 * Validate Geneve item. 2911 * 2912 * @param[in] item 2913 * Item specification. 2914 * @param[in] itemFlags 2915 * Bit-fields that holds the items detected until now. 2916 * @param[in] enPriv 2917 * Pointer to the private data structure. 2918 * @param[out] error 2919 * Pointer to error structure. 2920 * 2921 * @return 2922 * 0 on success, a negative errno value otherwise and rte_errno is set. 2923 */ 2924 2925 int 2926 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2927 uint64_t item_flags, 2928 struct rte_eth_dev *dev, 2929 struct rte_flow_error *error) 2930 { 2931 struct mlx5_priv *priv = dev->data->dev_private; 2932 const struct rte_flow_item_geneve *spec = item->spec; 2933 const struct rte_flow_item_geneve *mask = item->mask; 2934 int ret; 2935 uint16_t gbhdr; 2936 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 2937 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2938 const struct rte_flow_item_geneve nic_mask = { 2939 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2940 .vni = "\xff\xff\xff", 2941 .protocol = RTE_BE16(UINT16_MAX), 2942 }; 2943 2944 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 2945 return rte_flow_error_set(error, ENOTSUP, 2946 RTE_FLOW_ERROR_TYPE_ITEM, item, 2947 "L3 Geneve is not enabled by device" 2948 " parameter and/or not configured in" 2949 " firmware"); 2950 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2951 return rte_flow_error_set(error, ENOTSUP, 2952 RTE_FLOW_ERROR_TYPE_ITEM, item, 2953 "multiple tunnel layers not" 2954 " supported"); 2955 /* 2956 * Verify only UDPv4 is present as defined in 2957 * https://tools.ietf.org/html/rfc7348 2958 */ 2959 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2960 return rte_flow_error_set(error, EINVAL, 2961 RTE_FLOW_ERROR_TYPE_ITEM, item, 2962 "no outer UDP layer found"); 2963 if (!mask) 2964 mask = &rte_flow_item_geneve_mask; 2965 ret = mlx5_flow_item_acceptable 2966 (item, (const uint8_t *)mask, 2967 (const uint8_t *)&nic_mask, 2968 sizeof(struct rte_flow_item_geneve), 2969 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2970 if (ret) 2971 return ret; 2972 if (spec) { 2973 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 2974 if (MLX5_GENEVE_VER_VAL(gbhdr) || 2975 MLX5_GENEVE_CRITO_VAL(gbhdr) || 2976 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 2977 return rte_flow_error_set(error, ENOTSUP, 2978 RTE_FLOW_ERROR_TYPE_ITEM, 2979 item, 2980 "Geneve protocol unsupported" 2981 " fields are being used"); 2982 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 2983 return rte_flow_error_set 2984 (error, ENOTSUP, 2985 RTE_FLOW_ERROR_TYPE_ITEM, 2986 item, 2987 "Unsupported Geneve options length"); 2988 } 2989 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2990 return rte_flow_error_set 2991 (error, ENOTSUP, 2992 RTE_FLOW_ERROR_TYPE_ITEM, item, 2993 "Geneve tunnel must be fully defined"); 2994 return 0; 2995 } 2996 2997 /** 2998 * Validate Geneve TLV option item. 2999 * 3000 * @param[in] item 3001 * Item specification. 3002 * @param[in] last_item 3003 * Previous validated item in the pattern items. 3004 * @param[in] geneve_item 3005 * Previous GENEVE item specification. 3006 * @param[in] dev 3007 * Pointer to the rte_eth_dev structure. 3008 * @param[out] error 3009 * Pointer to error structure. 3010 * 3011 * @return 3012 * 0 on success, a negative errno value otherwise and rte_errno is set. 3013 */ 3014 int 3015 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3016 uint64_t last_item, 3017 const struct rte_flow_item *geneve_item, 3018 struct rte_eth_dev *dev, 3019 struct rte_flow_error *error) 3020 { 3021 struct mlx5_priv *priv = dev->data->dev_private; 3022 struct mlx5_dev_ctx_shared *sh = priv->sh; 3023 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3024 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3025 uint8_t data_max_supported = 3026 hca_attr->max_geneve_tlv_option_data_len * 4; 3027 const struct rte_flow_item_geneve *geneve_spec; 3028 const struct rte_flow_item_geneve *geneve_mask; 3029 const struct rte_flow_item_geneve_opt *spec = item->spec; 3030 const struct rte_flow_item_geneve_opt *mask = item->mask; 3031 unsigned int i; 3032 unsigned int data_len; 3033 uint8_t tlv_option_len; 3034 uint16_t optlen_m, optlen_v; 3035 const struct rte_flow_item_geneve_opt full_mask = { 3036 .option_class = RTE_BE16(0xffff), 3037 .option_type = 0xff, 3038 .option_len = 0x1f, 3039 }; 3040 3041 if (!mask) 3042 mask = &rte_flow_item_geneve_opt_mask; 3043 if (!spec) 3044 return rte_flow_error_set 3045 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3046 "Geneve TLV opt class/type/length must be specified"); 3047 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3048 return rte_flow_error_set 3049 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3050 "Geneve TLV opt length exceeds the limit (31)"); 3051 /* Check if class type and length masks are full. */ 3052 if (full_mask.option_class != mask->option_class || 3053 full_mask.option_type != mask->option_type || 3054 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3055 return rte_flow_error_set 3056 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3057 "Geneve TLV opt class/type/length masks must be full"); 3058 /* Check if length is supported */ 3059 if ((uint32_t)spec->option_len > 3060 hca_attr->max_geneve_tlv_option_data_len) 3061 return rte_flow_error_set 3062 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3063 "Geneve TLV opt length not supported"); 3064 if (hca_attr->max_geneve_tlv_options > 1) 3065 DRV_LOG(DEBUG, 3066 "max_geneve_tlv_options supports more than 1 option"); 3067 /* Check GENEVE item preceding. */ 3068 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3069 return rte_flow_error_set 3070 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3071 "Geneve opt item must be preceded with Geneve item"); 3072 geneve_spec = geneve_item->spec; 3073 geneve_mask = geneve_item->mask ? geneve_item->mask : 3074 &rte_flow_item_geneve_mask; 3075 /* Check if GENEVE TLV option size doesn't exceed option length */ 3076 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3077 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3078 tlv_option_len = spec->option_len & mask->option_len; 3079 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3080 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3081 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3082 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3083 if ((optlen_v & optlen_m) <= tlv_option_len) 3084 return rte_flow_error_set 3085 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3086 "GENEVE TLV option length exceeds optlen"); 3087 } 3088 /* Check if length is 0 or data is 0. */ 3089 if (spec->data == NULL || spec->option_len == 0) 3090 return rte_flow_error_set 3091 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3092 "Geneve TLV opt with zero data/length not supported"); 3093 /* Check not all data & mask are 0. */ 3094 data_len = spec->option_len * 4; 3095 if (mask->data == NULL) { 3096 for (i = 0; i < data_len; i++) 3097 if (spec->data[i]) 3098 break; 3099 if (i == data_len) 3100 return rte_flow_error_set(error, ENOTSUP, 3101 RTE_FLOW_ERROR_TYPE_ITEM, item, 3102 "Can't match on Geneve option data 0"); 3103 } else { 3104 for (i = 0; i < data_len; i++) 3105 if (spec->data[i] & mask->data[i]) 3106 break; 3107 if (i == data_len) 3108 return rte_flow_error_set(error, ENOTSUP, 3109 RTE_FLOW_ERROR_TYPE_ITEM, item, 3110 "Can't match on Geneve option data and mask 0"); 3111 /* Check data mask supported. */ 3112 for (i = data_max_supported; i < data_len ; i++) 3113 if (mask->data[i]) 3114 return rte_flow_error_set(error, ENOTSUP, 3115 RTE_FLOW_ERROR_TYPE_ITEM, item, 3116 "Data mask is of unsupported size"); 3117 } 3118 /* Check GENEVE option is supported in NIC. */ 3119 if (!hca_attr->geneve_tlv_opt) 3120 return rte_flow_error_set 3121 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3122 "Geneve TLV opt not supported"); 3123 /* Check if we already have geneve option with different type/class. */ 3124 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3125 geneve_opt_resource = sh->geneve_tlv_option_resource; 3126 if (geneve_opt_resource != NULL) 3127 if (geneve_opt_resource->option_class != spec->option_class || 3128 geneve_opt_resource->option_type != spec->option_type || 3129 geneve_opt_resource->length != spec->option_len) { 3130 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3131 return rte_flow_error_set(error, ENOTSUP, 3132 RTE_FLOW_ERROR_TYPE_ITEM, item, 3133 "Only one Geneve TLV option supported"); 3134 } 3135 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3136 return 0; 3137 } 3138 3139 /** 3140 * Validate MPLS item. 3141 * 3142 * @param[in] dev 3143 * Pointer to the rte_eth_dev structure. 3144 * @param[in] item 3145 * Item specification. 3146 * @param[in] item_flags 3147 * Bit-fields that holds the items detected until now. 3148 * @param[in] prev_layer 3149 * The protocol layer indicated in previous item. 3150 * @param[out] error 3151 * Pointer to error structure. 3152 * 3153 * @return 3154 * 0 on success, a negative errno value otherwise and rte_errno is set. 3155 */ 3156 int 3157 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3158 const struct rte_flow_item *item __rte_unused, 3159 uint64_t item_flags __rte_unused, 3160 uint64_t prev_layer __rte_unused, 3161 struct rte_flow_error *error) 3162 { 3163 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3164 const struct rte_flow_item_mpls *mask = item->mask; 3165 struct mlx5_priv *priv = dev->data->dev_private; 3166 int ret; 3167 3168 if (!priv->sh->dev_cap.mpls_en) 3169 return rte_flow_error_set(error, ENOTSUP, 3170 RTE_FLOW_ERROR_TYPE_ITEM, item, 3171 "MPLS not supported or" 3172 " disabled in firmware" 3173 " configuration."); 3174 /* MPLS over UDP, GRE is allowed */ 3175 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3176 MLX5_FLOW_LAYER_GRE | 3177 MLX5_FLOW_LAYER_GRE_KEY))) 3178 return rte_flow_error_set(error, EINVAL, 3179 RTE_FLOW_ERROR_TYPE_ITEM, item, 3180 "protocol filtering not compatible" 3181 " with MPLS layer"); 3182 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3183 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3184 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3185 return rte_flow_error_set(error, ENOTSUP, 3186 RTE_FLOW_ERROR_TYPE_ITEM, item, 3187 "multiple tunnel layers not" 3188 " supported"); 3189 if (!mask) 3190 mask = &rte_flow_item_mpls_mask; 3191 ret = mlx5_flow_item_acceptable 3192 (item, (const uint8_t *)mask, 3193 (const uint8_t *)&rte_flow_item_mpls_mask, 3194 sizeof(struct rte_flow_item_mpls), 3195 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3196 if (ret < 0) 3197 return ret; 3198 return 0; 3199 #else 3200 return rte_flow_error_set(error, ENOTSUP, 3201 RTE_FLOW_ERROR_TYPE_ITEM, item, 3202 "MPLS is not supported by Verbs, please" 3203 " update."); 3204 #endif 3205 } 3206 3207 /** 3208 * Validate NVGRE item. 3209 * 3210 * @param[in] item 3211 * Item specification. 3212 * @param[in] item_flags 3213 * Bit flags to mark detected items. 3214 * @param[in] target_protocol 3215 * The next protocol in the previous item. 3216 * @param[out] error 3217 * Pointer to error structure. 3218 * 3219 * @return 3220 * 0 on success, a negative errno value otherwise and rte_errno is set. 3221 */ 3222 int 3223 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3224 uint64_t item_flags, 3225 uint8_t target_protocol, 3226 struct rte_flow_error *error) 3227 { 3228 const struct rte_flow_item_nvgre *mask = item->mask; 3229 int ret; 3230 3231 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3232 return rte_flow_error_set(error, EINVAL, 3233 RTE_FLOW_ERROR_TYPE_ITEM, item, 3234 "protocol filtering not compatible" 3235 " with this GRE layer"); 3236 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3237 return rte_flow_error_set(error, ENOTSUP, 3238 RTE_FLOW_ERROR_TYPE_ITEM, item, 3239 "multiple tunnel layers not" 3240 " supported"); 3241 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3242 return rte_flow_error_set(error, ENOTSUP, 3243 RTE_FLOW_ERROR_TYPE_ITEM, item, 3244 "L3 Layer is missing"); 3245 if (!mask) 3246 mask = &rte_flow_item_nvgre_mask; 3247 ret = mlx5_flow_item_acceptable 3248 (item, (const uint8_t *)mask, 3249 (const uint8_t *)&rte_flow_item_nvgre_mask, 3250 sizeof(struct rte_flow_item_nvgre), 3251 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3252 if (ret < 0) 3253 return ret; 3254 return 0; 3255 } 3256 3257 /** 3258 * Validate eCPRI item. 3259 * 3260 * @param[in] item 3261 * Item specification. 3262 * @param[in] item_flags 3263 * Bit-fields that holds the items detected until now. 3264 * @param[in] last_item 3265 * Previous validated item in the pattern items. 3266 * @param[in] ether_type 3267 * Type in the ethernet layer header (including dot1q). 3268 * @param[in] acc_mask 3269 * Acceptable mask, if NULL default internal default mask 3270 * will be used to check whether item fields are supported. 3271 * @param[out] error 3272 * Pointer to error structure. 3273 * 3274 * @return 3275 * 0 on success, a negative errno value otherwise and rte_errno is set. 3276 */ 3277 int 3278 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3279 uint64_t item_flags, 3280 uint64_t last_item, 3281 uint16_t ether_type, 3282 const struct rte_flow_item_ecpri *acc_mask, 3283 struct rte_flow_error *error) 3284 { 3285 const struct rte_flow_item_ecpri *mask = item->mask; 3286 const struct rte_flow_item_ecpri nic_mask = { 3287 .hdr = { 3288 .common = { 3289 .u32 = 3290 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3291 .type = 0xFF, 3292 }).u32), 3293 }, 3294 .dummy[0] = 0xFFFFFFFF, 3295 }, 3296 }; 3297 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3298 MLX5_FLOW_LAYER_OUTER_VLAN); 3299 struct rte_flow_item_ecpri mask_lo; 3300 3301 if (!(last_item & outer_l2_vlan) && 3302 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3303 return rte_flow_error_set(error, EINVAL, 3304 RTE_FLOW_ERROR_TYPE_ITEM, item, 3305 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3306 if ((last_item & outer_l2_vlan) && ether_type && 3307 ether_type != RTE_ETHER_TYPE_ECPRI) 3308 return rte_flow_error_set(error, EINVAL, 3309 RTE_FLOW_ERROR_TYPE_ITEM, item, 3310 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3311 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3312 return rte_flow_error_set(error, EINVAL, 3313 RTE_FLOW_ERROR_TYPE_ITEM, item, 3314 "eCPRI with tunnel is not supported right now"); 3315 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3316 return rte_flow_error_set(error, ENOTSUP, 3317 RTE_FLOW_ERROR_TYPE_ITEM, item, 3318 "multiple L3 layers not supported"); 3319 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3320 return rte_flow_error_set(error, EINVAL, 3321 RTE_FLOW_ERROR_TYPE_ITEM, item, 3322 "eCPRI cannot coexist with a TCP layer"); 3323 /* In specification, eCPRI could be over UDP layer. */ 3324 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3325 return rte_flow_error_set(error, EINVAL, 3326 RTE_FLOW_ERROR_TYPE_ITEM, item, 3327 "eCPRI over UDP layer is not yet supported right now"); 3328 /* Mask for type field in common header could be zero. */ 3329 if (!mask) 3330 mask = &rte_flow_item_ecpri_mask; 3331 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3332 /* Input mask is in big-endian format. */ 3333 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3334 return rte_flow_error_set(error, EINVAL, 3335 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3336 "partial mask is not supported for protocol"); 3337 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3338 return rte_flow_error_set(error, EINVAL, 3339 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3340 "message header mask must be after a type mask"); 3341 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3342 acc_mask ? (const uint8_t *)acc_mask 3343 : (const uint8_t *)&nic_mask, 3344 sizeof(struct rte_flow_item_ecpri), 3345 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3346 } 3347 3348 static int 3349 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3350 const struct rte_flow_attr *attr __rte_unused, 3351 const struct rte_flow_item items[] __rte_unused, 3352 const struct rte_flow_action actions[] __rte_unused, 3353 bool external __rte_unused, 3354 int hairpin __rte_unused, 3355 struct rte_flow_error *error) 3356 { 3357 return rte_flow_error_set(error, ENOTSUP, 3358 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3359 } 3360 3361 static struct mlx5_flow * 3362 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3363 const struct rte_flow_attr *attr __rte_unused, 3364 const struct rte_flow_item items[] __rte_unused, 3365 const struct rte_flow_action actions[] __rte_unused, 3366 struct rte_flow_error *error) 3367 { 3368 rte_flow_error_set(error, ENOTSUP, 3369 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3370 return NULL; 3371 } 3372 3373 static int 3374 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3375 struct mlx5_flow *dev_flow __rte_unused, 3376 const struct rte_flow_attr *attr __rte_unused, 3377 const struct rte_flow_item items[] __rte_unused, 3378 const struct rte_flow_action actions[] __rte_unused, 3379 struct rte_flow_error *error) 3380 { 3381 return rte_flow_error_set(error, ENOTSUP, 3382 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3383 } 3384 3385 static int 3386 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3387 struct rte_flow *flow __rte_unused, 3388 struct rte_flow_error *error) 3389 { 3390 return rte_flow_error_set(error, ENOTSUP, 3391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3392 } 3393 3394 static void 3395 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3396 struct rte_flow *flow __rte_unused) 3397 { 3398 } 3399 3400 static void 3401 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3402 struct rte_flow *flow __rte_unused) 3403 { 3404 } 3405 3406 static int 3407 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3408 struct rte_flow *flow __rte_unused, 3409 const struct rte_flow_action *actions __rte_unused, 3410 void *data __rte_unused, 3411 struct rte_flow_error *error) 3412 { 3413 return rte_flow_error_set(error, ENOTSUP, 3414 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3415 } 3416 3417 static int 3418 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3419 uint32_t domains __rte_unused, 3420 uint32_t flags __rte_unused) 3421 { 3422 return 0; 3423 } 3424 3425 /* Void driver to protect from null pointer reference. */ 3426 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3427 .validate = flow_null_validate, 3428 .prepare = flow_null_prepare, 3429 .translate = flow_null_translate, 3430 .apply = flow_null_apply, 3431 .remove = flow_null_remove, 3432 .destroy = flow_null_destroy, 3433 .query = flow_null_query, 3434 .sync_domain = flow_null_sync_domain, 3435 }; 3436 3437 /** 3438 * Select flow driver type according to flow attributes and device 3439 * configuration. 3440 * 3441 * @param[in] dev 3442 * Pointer to the dev structure. 3443 * @param[in] attr 3444 * Pointer to the flow attributes. 3445 * 3446 * @return 3447 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3448 */ 3449 static enum mlx5_flow_drv_type 3450 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3451 { 3452 struct mlx5_priv *priv = dev->data->dev_private; 3453 /* The OS can determine first a specific flow type (DV, VERBS) */ 3454 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3455 3456 if (type != MLX5_FLOW_TYPE_MAX) 3457 return type; 3458 /* 3459 * Currently when dv_flow_en == 2, only HW steering engine is 3460 * supported. New engines can also be chosen here if ready. 3461 */ 3462 if (priv->sh->config.dv_flow_en == 2) 3463 return MLX5_FLOW_TYPE_HW; 3464 /* If no OS specific type - continue with DV/VERBS selection */ 3465 if (attr->transfer && priv->sh->config.dv_esw_en) 3466 type = MLX5_FLOW_TYPE_DV; 3467 if (!attr->transfer) 3468 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3469 MLX5_FLOW_TYPE_VERBS; 3470 return type; 3471 } 3472 3473 #define flow_get_drv_ops(type) flow_drv_ops[type] 3474 3475 /** 3476 * Flow driver validation API. This abstracts calling driver specific functions. 3477 * The type of flow driver is determined according to flow attributes. 3478 * 3479 * @param[in] dev 3480 * Pointer to the dev structure. 3481 * @param[in] attr 3482 * Pointer to the flow attributes. 3483 * @param[in] items 3484 * Pointer to the list of items. 3485 * @param[in] actions 3486 * Pointer to the list of actions. 3487 * @param[in] external 3488 * This flow rule is created by request external to PMD. 3489 * @param[in] hairpin 3490 * Number of hairpin TX actions, 0 means classic flow. 3491 * @param[out] error 3492 * Pointer to the error structure. 3493 * 3494 * @return 3495 * 0 on success, a negative errno value otherwise and rte_errno is set. 3496 */ 3497 static inline int 3498 flow_drv_validate(struct rte_eth_dev *dev, 3499 const struct rte_flow_attr *attr, 3500 const struct rte_flow_item items[], 3501 const struct rte_flow_action actions[], 3502 bool external, int hairpin, struct rte_flow_error *error) 3503 { 3504 const struct mlx5_flow_driver_ops *fops; 3505 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3506 3507 fops = flow_get_drv_ops(type); 3508 return fops->validate(dev, attr, items, actions, external, 3509 hairpin, error); 3510 } 3511 3512 /** 3513 * Flow driver preparation API. This abstracts calling driver specific 3514 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3515 * calculates the size of memory required for device flow, allocates the memory, 3516 * initializes the device flow and returns the pointer. 3517 * 3518 * @note 3519 * This function initializes device flow structure such as dv or verbs in 3520 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3521 * rest. For example, adding returning device flow to flow->dev_flow list and 3522 * setting backward reference to the flow should be done out of this function. 3523 * layers field is not filled either. 3524 * 3525 * @param[in] dev 3526 * Pointer to the dev structure. 3527 * @param[in] attr 3528 * Pointer to the flow attributes. 3529 * @param[in] items 3530 * Pointer to the list of items. 3531 * @param[in] actions 3532 * Pointer to the list of actions. 3533 * @param[in] flow_idx 3534 * This memory pool index to the flow. 3535 * @param[out] error 3536 * Pointer to the error structure. 3537 * 3538 * @return 3539 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3540 */ 3541 static inline struct mlx5_flow * 3542 flow_drv_prepare(struct rte_eth_dev *dev, 3543 const struct rte_flow *flow, 3544 const struct rte_flow_attr *attr, 3545 const struct rte_flow_item items[], 3546 const struct rte_flow_action actions[], 3547 uint32_t flow_idx, 3548 struct rte_flow_error *error) 3549 { 3550 const struct mlx5_flow_driver_ops *fops; 3551 enum mlx5_flow_drv_type type = flow->drv_type; 3552 struct mlx5_flow *mlx5_flow = NULL; 3553 3554 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3555 fops = flow_get_drv_ops(type); 3556 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3557 if (mlx5_flow) 3558 mlx5_flow->flow_idx = flow_idx; 3559 return mlx5_flow; 3560 } 3561 3562 /** 3563 * Flow driver translation API. This abstracts calling driver specific 3564 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3565 * translates a generic flow into a driver flow. flow_drv_prepare() must 3566 * precede. 3567 * 3568 * @note 3569 * dev_flow->layers could be filled as a result of parsing during translation 3570 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3571 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3572 * flow->actions could be overwritten even though all the expanded dev_flows 3573 * have the same actions. 3574 * 3575 * @param[in] dev 3576 * Pointer to the rte dev structure. 3577 * @param[in, out] dev_flow 3578 * Pointer to the mlx5 flow. 3579 * @param[in] attr 3580 * Pointer to the flow attributes. 3581 * @param[in] items 3582 * Pointer to the list of items. 3583 * @param[in] actions 3584 * Pointer to the list of actions. 3585 * @param[out] error 3586 * Pointer to the error structure. 3587 * 3588 * @return 3589 * 0 on success, a negative errno value otherwise and rte_errno is set. 3590 */ 3591 static inline int 3592 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3593 const struct rte_flow_attr *attr, 3594 const struct rte_flow_item items[], 3595 const struct rte_flow_action actions[], 3596 struct rte_flow_error *error) 3597 { 3598 const struct mlx5_flow_driver_ops *fops; 3599 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3600 3601 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3602 fops = flow_get_drv_ops(type); 3603 return fops->translate(dev, dev_flow, attr, items, actions, error); 3604 } 3605 3606 /** 3607 * Flow driver apply API. This abstracts calling driver specific functions. 3608 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3609 * translated driver flows on to device. flow_drv_translate() must precede. 3610 * 3611 * @param[in] dev 3612 * Pointer to Ethernet device structure. 3613 * @param[in, out] flow 3614 * Pointer to flow structure. 3615 * @param[out] error 3616 * Pointer to error structure. 3617 * 3618 * @return 3619 * 0 on success, a negative errno value otherwise and rte_errno is set. 3620 */ 3621 static inline int 3622 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3623 struct rte_flow_error *error) 3624 { 3625 const struct mlx5_flow_driver_ops *fops; 3626 enum mlx5_flow_drv_type type = flow->drv_type; 3627 3628 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3629 fops = flow_get_drv_ops(type); 3630 return fops->apply(dev, flow, error); 3631 } 3632 3633 /** 3634 * Flow driver destroy API. This abstracts calling driver specific functions. 3635 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3636 * on device and releases resources of the flow. 3637 * 3638 * @param[in] dev 3639 * Pointer to Ethernet device. 3640 * @param[in, out] flow 3641 * Pointer to flow structure. 3642 */ 3643 static inline void 3644 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3645 { 3646 const struct mlx5_flow_driver_ops *fops; 3647 enum mlx5_flow_drv_type type = flow->drv_type; 3648 3649 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3650 fops = flow_get_drv_ops(type); 3651 fops->destroy(dev, flow); 3652 } 3653 3654 /** 3655 * Flow driver find RSS policy tbl API. This abstracts calling driver 3656 * specific functions. Parent flow (rte_flow) should have driver 3657 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3658 * 3659 * @param[in] dev 3660 * Pointer to Ethernet device. 3661 * @param[in, out] flow 3662 * Pointer to flow structure. 3663 * @param[in] policy 3664 * Pointer to meter policy table. 3665 * @param[in] rss_desc 3666 * Pointer to rss_desc 3667 */ 3668 static struct mlx5_flow_meter_sub_policy * 3669 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3670 struct rte_flow *flow, 3671 struct mlx5_flow_meter_policy *policy, 3672 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3673 { 3674 const struct mlx5_flow_driver_ops *fops; 3675 enum mlx5_flow_drv_type type = flow->drv_type; 3676 3677 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3678 fops = flow_get_drv_ops(type); 3679 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3680 } 3681 3682 /** 3683 * Flow driver color tag rule API. This abstracts calling driver 3684 * specific functions. Parent flow (rte_flow) should have driver 3685 * type (drv_type). It will create the color tag rules in hierarchy meter. 3686 * 3687 * @param[in] dev 3688 * Pointer to Ethernet device. 3689 * @param[in, out] flow 3690 * Pointer to flow structure. 3691 * @param[in] fm 3692 * Pointer to flow meter structure. 3693 * @param[in] src_port 3694 * The src port this extra rule should use. 3695 * @param[in] item 3696 * The src port id match item. 3697 * @param[out] error 3698 * Pointer to error structure. 3699 */ 3700 static int 3701 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3702 struct rte_flow *flow, 3703 struct mlx5_flow_meter_info *fm, 3704 int32_t src_port, 3705 const struct rte_flow_item *item, 3706 struct rte_flow_error *error) 3707 { 3708 const struct mlx5_flow_driver_ops *fops; 3709 enum mlx5_flow_drv_type type = flow->drv_type; 3710 3711 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3712 fops = flow_get_drv_ops(type); 3713 return fops->meter_hierarchy_rule_create(dev, fm, 3714 src_port, item, error); 3715 } 3716 3717 /** 3718 * Get RSS action from the action list. 3719 * 3720 * @param[in] dev 3721 * Pointer to Ethernet device. 3722 * @param[in] actions 3723 * Pointer to the list of actions. 3724 * @param[in] flow 3725 * Parent flow structure pointer. 3726 * 3727 * @return 3728 * Pointer to the RSS action if exist, else return NULL. 3729 */ 3730 static const struct rte_flow_action_rss* 3731 flow_get_rss_action(struct rte_eth_dev *dev, 3732 const struct rte_flow_action actions[]) 3733 { 3734 struct mlx5_priv *priv = dev->data->dev_private; 3735 const struct rte_flow_action_rss *rss = NULL; 3736 struct mlx5_meter_policy_action_container *acg; 3737 struct mlx5_meter_policy_action_container *acy; 3738 3739 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3740 switch (actions->type) { 3741 case RTE_FLOW_ACTION_TYPE_RSS: 3742 rss = actions->conf; 3743 break; 3744 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3745 { 3746 const struct rte_flow_action_sample *sample = 3747 actions->conf; 3748 const struct rte_flow_action *act = sample->actions; 3749 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3750 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3751 rss = act->conf; 3752 break; 3753 } 3754 case RTE_FLOW_ACTION_TYPE_METER: 3755 { 3756 uint32_t mtr_idx; 3757 struct mlx5_flow_meter_info *fm; 3758 struct mlx5_flow_meter_policy *policy; 3759 const struct rte_flow_action_meter *mtr = actions->conf; 3760 3761 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3762 if (fm && !fm->def_policy) { 3763 policy = mlx5_flow_meter_policy_find(dev, 3764 fm->policy_id, NULL); 3765 MLX5_ASSERT(policy); 3766 if (policy->is_hierarchy) { 3767 policy = 3768 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3769 policy); 3770 if (!policy) 3771 return NULL; 3772 } 3773 if (policy->is_rss) { 3774 acg = 3775 &policy->act_cnt[RTE_COLOR_GREEN]; 3776 acy = 3777 &policy->act_cnt[RTE_COLOR_YELLOW]; 3778 if (acg->fate_action == 3779 MLX5_FLOW_FATE_SHARED_RSS) 3780 rss = acg->rss->conf; 3781 else if (acy->fate_action == 3782 MLX5_FLOW_FATE_SHARED_RSS) 3783 rss = acy->rss->conf; 3784 } 3785 } 3786 break; 3787 } 3788 default: 3789 break; 3790 } 3791 } 3792 return rss; 3793 } 3794 3795 /** 3796 * Get ASO age action by index. 3797 * 3798 * @param[in] dev 3799 * Pointer to the Ethernet device structure. 3800 * @param[in] age_idx 3801 * Index to the ASO age action. 3802 * 3803 * @return 3804 * The specified ASO age action. 3805 */ 3806 struct mlx5_aso_age_action* 3807 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3808 { 3809 uint16_t pool_idx = age_idx & UINT16_MAX; 3810 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3811 struct mlx5_priv *priv = dev->data->dev_private; 3812 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3813 struct mlx5_aso_age_pool *pool; 3814 3815 rte_rwlock_read_lock(&mng->resize_rwl); 3816 pool = mng->pools[pool_idx]; 3817 rte_rwlock_read_unlock(&mng->resize_rwl); 3818 return &pool->actions[offset - 1]; 3819 } 3820 3821 /* maps indirect action to translated direct in some actions array */ 3822 struct mlx5_translated_action_handle { 3823 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3824 int index; /**< Index in related array of rte_flow_action. */ 3825 }; 3826 3827 /** 3828 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3829 * direct action if translation possible. 3830 * This functionality used to run same execution path for both direct and 3831 * indirect actions on flow create. All necessary preparations for indirect 3832 * action handling should be performed on *handle* actions list returned 3833 * from this call. 3834 * 3835 * @param[in] dev 3836 * Pointer to Ethernet device. 3837 * @param[in] actions 3838 * List of actions to translate. 3839 * @param[out] handle 3840 * List to store translated indirect action object handles. 3841 * @param[in, out] indir_n 3842 * Size of *handle* array. On return should be updated with number of 3843 * indirect actions retrieved from the *actions* list. 3844 * @param[out] translated_actions 3845 * List of actions where all indirect actions were translated to direct 3846 * if possible. NULL if no translation took place. 3847 * @param[out] error 3848 * Pointer to the error structure. 3849 * 3850 * @return 3851 * 0 on success, a negative errno value otherwise and rte_errno is set. 3852 */ 3853 static int 3854 flow_action_handles_translate(struct rte_eth_dev *dev, 3855 const struct rte_flow_action actions[], 3856 struct mlx5_translated_action_handle *handle, 3857 int *indir_n, 3858 struct rte_flow_action **translated_actions, 3859 struct rte_flow_error *error) 3860 { 3861 struct mlx5_priv *priv = dev->data->dev_private; 3862 struct rte_flow_action *translated = NULL; 3863 size_t actions_size; 3864 int n; 3865 int copied_n = 0; 3866 struct mlx5_translated_action_handle *handle_end = NULL; 3867 3868 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3869 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3870 continue; 3871 if (copied_n == *indir_n) { 3872 return rte_flow_error_set 3873 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3874 NULL, "too many shared actions"); 3875 } 3876 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3877 sizeof(actions[n].conf)); 3878 handle[copied_n].index = n; 3879 copied_n++; 3880 } 3881 n++; 3882 *indir_n = copied_n; 3883 if (!copied_n) 3884 return 0; 3885 actions_size = sizeof(struct rte_flow_action) * n; 3886 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3887 if (!translated) { 3888 rte_errno = ENOMEM; 3889 return -ENOMEM; 3890 } 3891 memcpy(translated, actions, actions_size); 3892 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3893 struct mlx5_shared_action_rss *shared_rss; 3894 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3895 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3896 uint32_t idx = act_idx & 3897 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3898 3899 switch (type) { 3900 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3901 shared_rss = mlx5_ipool_get 3902 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3903 translated[handle->index].type = 3904 RTE_FLOW_ACTION_TYPE_RSS; 3905 translated[handle->index].conf = 3906 &shared_rss->origin; 3907 break; 3908 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3909 translated[handle->index].type = 3910 (enum rte_flow_action_type) 3911 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3912 translated[handle->index].conf = (void *)(uintptr_t)idx; 3913 break; 3914 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3915 if (priv->sh->flow_hit_aso_en) { 3916 translated[handle->index].type = 3917 (enum rte_flow_action_type) 3918 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3919 translated[handle->index].conf = 3920 (void *)(uintptr_t)idx; 3921 break; 3922 } 3923 /* Fall-through */ 3924 case MLX5_INDIRECT_ACTION_TYPE_CT: 3925 if (priv->sh->ct_aso_en) { 3926 translated[handle->index].type = 3927 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3928 translated[handle->index].conf = 3929 (void *)(uintptr_t)idx; 3930 break; 3931 } 3932 /* Fall-through */ 3933 default: 3934 mlx5_free(translated); 3935 return rte_flow_error_set 3936 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3937 NULL, "invalid indirect action type"); 3938 } 3939 } 3940 *translated_actions = translated; 3941 return 0; 3942 } 3943 3944 /** 3945 * Get Shared RSS action from the action list. 3946 * 3947 * @param[in] dev 3948 * Pointer to Ethernet device. 3949 * @param[in] shared 3950 * Pointer to the list of actions. 3951 * @param[in] shared_n 3952 * Actions list length. 3953 * 3954 * @return 3955 * The MLX5 RSS action ID if exists, otherwise return 0. 3956 */ 3957 static uint32_t 3958 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3959 struct mlx5_translated_action_handle *handle, 3960 int shared_n) 3961 { 3962 struct mlx5_translated_action_handle *handle_end; 3963 struct mlx5_priv *priv = dev->data->dev_private; 3964 struct mlx5_shared_action_rss *shared_rss; 3965 3966 3967 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3968 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3969 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3970 uint32_t idx = act_idx & 3971 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3972 switch (type) { 3973 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3974 shared_rss = mlx5_ipool_get 3975 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 3976 idx); 3977 __atomic_add_fetch(&shared_rss->refcnt, 1, 3978 __ATOMIC_RELAXED); 3979 return idx; 3980 default: 3981 break; 3982 } 3983 } 3984 return 0; 3985 } 3986 3987 static unsigned int 3988 find_graph_root(uint32_t rss_level) 3989 { 3990 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 3991 MLX5_EXPANSION_ROOT_OUTER; 3992 } 3993 3994 /** 3995 * Get layer flags from the prefix flow. 3996 * 3997 * Some flows may be split to several subflows, the prefix subflow gets the 3998 * match items and the suffix sub flow gets the actions. 3999 * Some actions need the user defined match item flags to get the detail for 4000 * the action. 4001 * This function helps the suffix flow to get the item layer flags from prefix 4002 * subflow. 4003 * 4004 * @param[in] dev_flow 4005 * Pointer the created prefix subflow. 4006 * 4007 * @return 4008 * The layers get from prefix subflow. 4009 */ 4010 static inline uint64_t 4011 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4012 { 4013 uint64_t layers = 0; 4014 4015 /* 4016 * Layers bits could be localization, but usually the compiler will 4017 * help to do the optimization work for source code. 4018 * If no decap actions, use the layers directly. 4019 */ 4020 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4021 return dev_flow->handle->layers; 4022 /* Convert L3 layers with decap action. */ 4023 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4024 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4025 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4026 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4027 /* Convert L4 layers with decap action. */ 4028 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4029 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4030 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4031 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4032 return layers; 4033 } 4034 4035 /** 4036 * Get metadata split action information. 4037 * 4038 * @param[in] actions 4039 * Pointer to the list of actions. 4040 * @param[out] qrss 4041 * Pointer to the return pointer. 4042 * @param[out] qrss_type 4043 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4044 * if no QUEUE/RSS is found. 4045 * @param[out] encap_idx 4046 * Pointer to the index of the encap action if exists, otherwise the last 4047 * action index. 4048 * 4049 * @return 4050 * Total number of actions. 4051 */ 4052 static int 4053 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4054 const struct rte_flow_action **qrss, 4055 int *encap_idx) 4056 { 4057 const struct rte_flow_action_raw_encap *raw_encap; 4058 int actions_n = 0; 4059 int raw_decap_idx = -1; 4060 4061 *encap_idx = -1; 4062 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4063 switch (actions->type) { 4064 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4065 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4066 *encap_idx = actions_n; 4067 break; 4068 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4069 raw_decap_idx = actions_n; 4070 break; 4071 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4072 raw_encap = actions->conf; 4073 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4074 *encap_idx = raw_decap_idx != -1 ? 4075 raw_decap_idx : actions_n; 4076 break; 4077 case RTE_FLOW_ACTION_TYPE_QUEUE: 4078 case RTE_FLOW_ACTION_TYPE_RSS: 4079 *qrss = actions; 4080 break; 4081 default: 4082 break; 4083 } 4084 actions_n++; 4085 } 4086 if (*encap_idx == -1) 4087 *encap_idx = actions_n; 4088 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4089 return actions_n + 1; 4090 } 4091 4092 /** 4093 * Check if the action will change packet. 4094 * 4095 * @param dev 4096 * Pointer to Ethernet device. 4097 * @param[in] type 4098 * action type. 4099 * 4100 * @return 4101 * true if action will change packet, false otherwise. 4102 */ 4103 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4104 enum rte_flow_action_type type) 4105 { 4106 struct mlx5_priv *priv = dev->data->dev_private; 4107 4108 switch (type) { 4109 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4110 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4111 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4112 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4113 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4114 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4115 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4116 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4117 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4118 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4119 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4120 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4121 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4122 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4123 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4124 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4125 case RTE_FLOW_ACTION_TYPE_SET_META: 4126 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4127 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4128 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4129 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4130 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4131 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4132 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4133 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4134 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4135 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4136 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4137 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4138 return true; 4139 case RTE_FLOW_ACTION_TYPE_FLAG: 4140 case RTE_FLOW_ACTION_TYPE_MARK: 4141 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4142 return true; 4143 else 4144 return false; 4145 default: 4146 return false; 4147 } 4148 } 4149 4150 /** 4151 * Check meter action from the action list. 4152 * 4153 * @param dev 4154 * Pointer to Ethernet device. 4155 * @param[in] actions 4156 * Pointer to the list of actions. 4157 * @param[out] has_mtr 4158 * Pointer to the meter exist flag. 4159 * @param[out] has_modify 4160 * Pointer to the flag showing there's packet change action. 4161 * @param[out] meter_id 4162 * Pointer to the meter id. 4163 * 4164 * @return 4165 * Total number of actions. 4166 */ 4167 static int 4168 flow_check_meter_action(struct rte_eth_dev *dev, 4169 const struct rte_flow_action actions[], 4170 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4171 { 4172 const struct rte_flow_action_meter *mtr = NULL; 4173 int actions_n = 0; 4174 4175 MLX5_ASSERT(has_mtr); 4176 *has_mtr = false; 4177 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4178 switch (actions->type) { 4179 case RTE_FLOW_ACTION_TYPE_METER: 4180 mtr = actions->conf; 4181 *meter_id = mtr->mtr_id; 4182 *has_mtr = true; 4183 break; 4184 default: 4185 break; 4186 } 4187 if (!*has_mtr) 4188 *has_modify |= flow_check_modify_action_type(dev, 4189 actions->type); 4190 actions_n++; 4191 } 4192 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4193 return actions_n + 1; 4194 } 4195 4196 /** 4197 * Check if the flow should be split due to hairpin. 4198 * The reason for the split is that in current HW we can't 4199 * support encap and push-vlan on Rx, so if a flow contains 4200 * these actions we move it to Tx. 4201 * 4202 * @param dev 4203 * Pointer to Ethernet device. 4204 * @param[in] attr 4205 * Flow rule attributes. 4206 * @param[in] actions 4207 * Associated actions (list terminated by the END action). 4208 * 4209 * @return 4210 * > 0 the number of actions and the flow should be split, 4211 * 0 when no split required. 4212 */ 4213 static int 4214 flow_check_hairpin_split(struct rte_eth_dev *dev, 4215 const struct rte_flow_attr *attr, 4216 const struct rte_flow_action actions[]) 4217 { 4218 int queue_action = 0; 4219 int action_n = 0; 4220 int split = 0; 4221 const struct rte_flow_action_queue *queue; 4222 const struct rte_flow_action_rss *rss; 4223 const struct rte_flow_action_raw_encap *raw_encap; 4224 const struct rte_eth_hairpin_conf *conf; 4225 4226 if (!attr->ingress) 4227 return 0; 4228 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4229 switch (actions->type) { 4230 case RTE_FLOW_ACTION_TYPE_QUEUE: 4231 queue = actions->conf; 4232 if (queue == NULL) 4233 return 0; 4234 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4235 if (conf == NULL || conf->tx_explicit != 0) 4236 return 0; 4237 queue_action = 1; 4238 action_n++; 4239 break; 4240 case RTE_FLOW_ACTION_TYPE_RSS: 4241 rss = actions->conf; 4242 if (rss == NULL || rss->queue_num == 0) 4243 return 0; 4244 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4245 if (conf == NULL || conf->tx_explicit != 0) 4246 return 0; 4247 queue_action = 1; 4248 action_n++; 4249 break; 4250 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4251 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4252 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4253 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4254 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4255 split++; 4256 action_n++; 4257 break; 4258 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4259 raw_encap = actions->conf; 4260 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4261 split++; 4262 action_n++; 4263 break; 4264 default: 4265 action_n++; 4266 break; 4267 } 4268 } 4269 if (split && queue_action) 4270 return action_n; 4271 return 0; 4272 } 4273 4274 /* Declare flow create/destroy prototype in advance. */ 4275 static uint32_t 4276 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4277 const struct rte_flow_attr *attr, 4278 const struct rte_flow_item items[], 4279 const struct rte_flow_action actions[], 4280 bool external, struct rte_flow_error *error); 4281 4282 static void 4283 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4284 uint32_t flow_idx); 4285 4286 int 4287 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4288 struct mlx5_list_entry *entry, void *cb_ctx) 4289 { 4290 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4291 struct mlx5_flow_mreg_copy_resource *mcp_res = 4292 container_of(entry, typeof(*mcp_res), hlist_ent); 4293 4294 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4295 } 4296 4297 struct mlx5_list_entry * 4298 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4299 { 4300 struct rte_eth_dev *dev = tool_ctx; 4301 struct mlx5_priv *priv = dev->data->dev_private; 4302 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4303 struct mlx5_flow_mreg_copy_resource *mcp_res; 4304 struct rte_flow_error *error = ctx->error; 4305 uint32_t idx = 0; 4306 int ret; 4307 uint32_t mark_id = *(uint32_t *)(ctx->data); 4308 struct rte_flow_attr attr = { 4309 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4310 .ingress = 1, 4311 }; 4312 struct mlx5_rte_flow_item_tag tag_spec = { 4313 .data = mark_id, 4314 }; 4315 struct rte_flow_item items[] = { 4316 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4317 }; 4318 struct rte_flow_action_mark ftag = { 4319 .id = mark_id, 4320 }; 4321 struct mlx5_flow_action_copy_mreg cp_mreg = { 4322 .dst = REG_B, 4323 .src = REG_NON, 4324 }; 4325 struct rte_flow_action_jump jump = { 4326 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4327 }; 4328 struct rte_flow_action actions[] = { 4329 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4330 }; 4331 4332 /* Fill the register fields in the flow. */ 4333 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4334 if (ret < 0) 4335 return NULL; 4336 tag_spec.id = ret; 4337 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4338 if (ret < 0) 4339 return NULL; 4340 cp_mreg.src = ret; 4341 /* Provide the full width of FLAG specific value. */ 4342 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4343 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4344 /* Build a new flow. */ 4345 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4346 items[0] = (struct rte_flow_item){ 4347 .type = (enum rte_flow_item_type) 4348 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4349 .spec = &tag_spec, 4350 }; 4351 items[1] = (struct rte_flow_item){ 4352 .type = RTE_FLOW_ITEM_TYPE_END, 4353 }; 4354 actions[0] = (struct rte_flow_action){ 4355 .type = (enum rte_flow_action_type) 4356 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4357 .conf = &ftag, 4358 }; 4359 actions[1] = (struct rte_flow_action){ 4360 .type = (enum rte_flow_action_type) 4361 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4362 .conf = &cp_mreg, 4363 }; 4364 actions[2] = (struct rte_flow_action){ 4365 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4366 .conf = &jump, 4367 }; 4368 actions[3] = (struct rte_flow_action){ 4369 .type = RTE_FLOW_ACTION_TYPE_END, 4370 }; 4371 } else { 4372 /* Default rule, wildcard match. */ 4373 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4374 items[0] = (struct rte_flow_item){ 4375 .type = RTE_FLOW_ITEM_TYPE_END, 4376 }; 4377 actions[0] = (struct rte_flow_action){ 4378 .type = (enum rte_flow_action_type) 4379 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4380 .conf = &cp_mreg, 4381 }; 4382 actions[1] = (struct rte_flow_action){ 4383 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4384 .conf = &jump, 4385 }; 4386 actions[2] = (struct rte_flow_action){ 4387 .type = RTE_FLOW_ACTION_TYPE_END, 4388 }; 4389 } 4390 /* Build a new entry. */ 4391 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4392 if (!mcp_res) { 4393 rte_errno = ENOMEM; 4394 return NULL; 4395 } 4396 mcp_res->idx = idx; 4397 mcp_res->mark_id = mark_id; 4398 /* 4399 * The copy Flows are not included in any list. There 4400 * ones are referenced from other Flows and can not 4401 * be applied, removed, deleted in arbitrary order 4402 * by list traversing. 4403 */ 4404 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4405 &attr, items, actions, false, error); 4406 if (!mcp_res->rix_flow) { 4407 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4408 return NULL; 4409 } 4410 return &mcp_res->hlist_ent; 4411 } 4412 4413 struct mlx5_list_entry * 4414 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4415 void *cb_ctx __rte_unused) 4416 { 4417 struct rte_eth_dev *dev = tool_ctx; 4418 struct mlx5_priv *priv = dev->data->dev_private; 4419 struct mlx5_flow_mreg_copy_resource *mcp_res; 4420 uint32_t idx = 0; 4421 4422 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4423 if (!mcp_res) { 4424 rte_errno = ENOMEM; 4425 return NULL; 4426 } 4427 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4428 mcp_res->idx = idx; 4429 return &mcp_res->hlist_ent; 4430 } 4431 4432 void 4433 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4434 { 4435 struct mlx5_flow_mreg_copy_resource *mcp_res = 4436 container_of(entry, typeof(*mcp_res), hlist_ent); 4437 struct rte_eth_dev *dev = tool_ctx; 4438 struct mlx5_priv *priv = dev->data->dev_private; 4439 4440 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4441 } 4442 4443 /** 4444 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4445 * 4446 * As mark_id is unique, if there's already a registered flow for the mark_id, 4447 * return by increasing the reference counter of the resource. Otherwise, create 4448 * the resource (mcp_res) and flow. 4449 * 4450 * Flow looks like, 4451 * - If ingress port is ANY and reg_c[1] is mark_id, 4452 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4453 * 4454 * For default flow (zero mark_id), flow is like, 4455 * - If ingress port is ANY, 4456 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4457 * 4458 * @param dev 4459 * Pointer to Ethernet device. 4460 * @param mark_id 4461 * ID of MARK action, zero means default flow for META. 4462 * @param[out] error 4463 * Perform verbose error reporting if not NULL. 4464 * 4465 * @return 4466 * Associated resource on success, NULL otherwise and rte_errno is set. 4467 */ 4468 static struct mlx5_flow_mreg_copy_resource * 4469 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4470 struct rte_flow_error *error) 4471 { 4472 struct mlx5_priv *priv = dev->data->dev_private; 4473 struct mlx5_list_entry *entry; 4474 struct mlx5_flow_cb_ctx ctx = { 4475 .dev = dev, 4476 .error = error, 4477 .data = &mark_id, 4478 }; 4479 4480 /* Check if already registered. */ 4481 MLX5_ASSERT(priv->mreg_cp_tbl); 4482 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4483 if (!entry) 4484 return NULL; 4485 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4486 hlist_ent); 4487 } 4488 4489 void 4490 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4491 { 4492 struct mlx5_flow_mreg_copy_resource *mcp_res = 4493 container_of(entry, typeof(*mcp_res), hlist_ent); 4494 struct rte_eth_dev *dev = tool_ctx; 4495 struct mlx5_priv *priv = dev->data->dev_private; 4496 4497 MLX5_ASSERT(mcp_res->rix_flow); 4498 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4499 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4500 } 4501 4502 /** 4503 * Release flow in RX_CP_TBL. 4504 * 4505 * @param dev 4506 * Pointer to Ethernet device. 4507 * @flow 4508 * Parent flow for wich copying is provided. 4509 */ 4510 static void 4511 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4512 struct rte_flow *flow) 4513 { 4514 struct mlx5_flow_mreg_copy_resource *mcp_res; 4515 struct mlx5_priv *priv = dev->data->dev_private; 4516 4517 if (!flow->rix_mreg_copy) 4518 return; 4519 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4520 flow->rix_mreg_copy); 4521 if (!mcp_res || !priv->mreg_cp_tbl) 4522 return; 4523 MLX5_ASSERT(mcp_res->rix_flow); 4524 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4525 flow->rix_mreg_copy = 0; 4526 } 4527 4528 /** 4529 * Remove the default copy action from RX_CP_TBL. 4530 * 4531 * This functions is called in the mlx5_dev_start(). No thread safe 4532 * is guaranteed. 4533 * 4534 * @param dev 4535 * Pointer to Ethernet device. 4536 */ 4537 static void 4538 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4539 { 4540 struct mlx5_list_entry *entry; 4541 struct mlx5_priv *priv = dev->data->dev_private; 4542 struct mlx5_flow_cb_ctx ctx; 4543 uint32_t mark_id; 4544 4545 /* Check if default flow is registered. */ 4546 if (!priv->mreg_cp_tbl) 4547 return; 4548 mark_id = MLX5_DEFAULT_COPY_ID; 4549 ctx.data = &mark_id; 4550 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4551 if (!entry) 4552 return; 4553 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4554 } 4555 4556 /** 4557 * Add the default copy action in in RX_CP_TBL. 4558 * 4559 * This functions is called in the mlx5_dev_start(). No thread safe 4560 * is guaranteed. 4561 * 4562 * @param dev 4563 * Pointer to Ethernet device. 4564 * @param[out] error 4565 * Perform verbose error reporting if not NULL. 4566 * 4567 * @return 4568 * 0 for success, negative value otherwise and rte_errno is set. 4569 */ 4570 static int 4571 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4572 struct rte_flow_error *error) 4573 { 4574 struct mlx5_priv *priv = dev->data->dev_private; 4575 struct mlx5_flow_mreg_copy_resource *mcp_res; 4576 struct mlx5_flow_cb_ctx ctx; 4577 uint32_t mark_id; 4578 4579 /* Check whether extensive metadata feature is engaged. */ 4580 if (!priv->sh->config.dv_flow_en || 4581 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4582 !mlx5_flow_ext_mreg_supported(dev) || 4583 !priv->sh->dv_regc0_mask) 4584 return 0; 4585 /* 4586 * Add default mreg copy flow may be called multiple time, but 4587 * only be called once in stop. Avoid register it twice. 4588 */ 4589 mark_id = MLX5_DEFAULT_COPY_ID; 4590 ctx.data = &mark_id; 4591 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4592 return 0; 4593 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4594 if (!mcp_res) 4595 return -rte_errno; 4596 return 0; 4597 } 4598 4599 /** 4600 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4601 * 4602 * All the flow having Q/RSS action should be split by 4603 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4604 * performs the following, 4605 * - CQE->flow_tag := reg_c[1] (MARK) 4606 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4607 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4608 * but there should be a flow per each MARK ID set by MARK action. 4609 * 4610 * For the aforementioned reason, if there's a MARK action in flow's action 4611 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4612 * the MARK ID to CQE's flow_tag like, 4613 * - If reg_c[1] is mark_id, 4614 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4615 * 4616 * For SET_META action which stores value in reg_c[0], as the destination is 4617 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4618 * MARK ID means the default flow. The default flow looks like, 4619 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4620 * 4621 * @param dev 4622 * Pointer to Ethernet device. 4623 * @param flow 4624 * Pointer to flow structure. 4625 * @param[in] actions 4626 * Pointer to the list of actions. 4627 * @param[out] error 4628 * Perform verbose error reporting if not NULL. 4629 * 4630 * @return 4631 * 0 on success, negative value otherwise and rte_errno is set. 4632 */ 4633 static int 4634 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4635 struct rte_flow *flow, 4636 const struct rte_flow_action *actions, 4637 struct rte_flow_error *error) 4638 { 4639 struct mlx5_priv *priv = dev->data->dev_private; 4640 struct mlx5_sh_config *config = &priv->sh->config; 4641 struct mlx5_flow_mreg_copy_resource *mcp_res; 4642 const struct rte_flow_action_mark *mark; 4643 4644 /* Check whether extensive metadata feature is engaged. */ 4645 if (!config->dv_flow_en || 4646 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4647 !mlx5_flow_ext_mreg_supported(dev) || 4648 !priv->sh->dv_regc0_mask) 4649 return 0; 4650 /* Find MARK action. */ 4651 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4652 switch (actions->type) { 4653 case RTE_FLOW_ACTION_TYPE_FLAG: 4654 mcp_res = flow_mreg_add_copy_action 4655 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4656 if (!mcp_res) 4657 return -rte_errno; 4658 flow->rix_mreg_copy = mcp_res->idx; 4659 return 0; 4660 case RTE_FLOW_ACTION_TYPE_MARK: 4661 mark = (const struct rte_flow_action_mark *) 4662 actions->conf; 4663 mcp_res = 4664 flow_mreg_add_copy_action(dev, mark->id, error); 4665 if (!mcp_res) 4666 return -rte_errno; 4667 flow->rix_mreg_copy = mcp_res->idx; 4668 return 0; 4669 default: 4670 break; 4671 } 4672 } 4673 return 0; 4674 } 4675 4676 #define MLX5_MAX_SPLIT_ACTIONS 24 4677 #define MLX5_MAX_SPLIT_ITEMS 24 4678 4679 /** 4680 * Split the hairpin flow. 4681 * Since HW can't support encap and push-vlan on Rx, we move these 4682 * actions to Tx. 4683 * If the count action is after the encap then we also 4684 * move the count action. in this case the count will also measure 4685 * the outer bytes. 4686 * 4687 * @param dev 4688 * Pointer to Ethernet device. 4689 * @param[in] actions 4690 * Associated actions (list terminated by the END action). 4691 * @param[out] actions_rx 4692 * Rx flow actions. 4693 * @param[out] actions_tx 4694 * Tx flow actions.. 4695 * @param[out] pattern_tx 4696 * The pattern items for the Tx flow. 4697 * @param[out] flow_id 4698 * The flow ID connected to this flow. 4699 * 4700 * @return 4701 * 0 on success. 4702 */ 4703 static int 4704 flow_hairpin_split(struct rte_eth_dev *dev, 4705 const struct rte_flow_action actions[], 4706 struct rte_flow_action actions_rx[], 4707 struct rte_flow_action actions_tx[], 4708 struct rte_flow_item pattern_tx[], 4709 uint32_t flow_id) 4710 { 4711 const struct rte_flow_action_raw_encap *raw_encap; 4712 const struct rte_flow_action_raw_decap *raw_decap; 4713 struct mlx5_rte_flow_action_set_tag *set_tag; 4714 struct rte_flow_action *tag_action; 4715 struct mlx5_rte_flow_item_tag *tag_item; 4716 struct rte_flow_item *item; 4717 char *addr; 4718 int encap = 0; 4719 4720 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4721 switch (actions->type) { 4722 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4723 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4724 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4725 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4726 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4727 rte_memcpy(actions_tx, actions, 4728 sizeof(struct rte_flow_action)); 4729 actions_tx++; 4730 break; 4731 case RTE_FLOW_ACTION_TYPE_COUNT: 4732 if (encap) { 4733 rte_memcpy(actions_tx, actions, 4734 sizeof(struct rte_flow_action)); 4735 actions_tx++; 4736 } else { 4737 rte_memcpy(actions_rx, actions, 4738 sizeof(struct rte_flow_action)); 4739 actions_rx++; 4740 } 4741 break; 4742 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4743 raw_encap = actions->conf; 4744 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4745 memcpy(actions_tx, actions, 4746 sizeof(struct rte_flow_action)); 4747 actions_tx++; 4748 encap = 1; 4749 } else { 4750 rte_memcpy(actions_rx, actions, 4751 sizeof(struct rte_flow_action)); 4752 actions_rx++; 4753 } 4754 break; 4755 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4756 raw_decap = actions->conf; 4757 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4758 memcpy(actions_tx, actions, 4759 sizeof(struct rte_flow_action)); 4760 actions_tx++; 4761 } else { 4762 rte_memcpy(actions_rx, actions, 4763 sizeof(struct rte_flow_action)); 4764 actions_rx++; 4765 } 4766 break; 4767 default: 4768 rte_memcpy(actions_rx, actions, 4769 sizeof(struct rte_flow_action)); 4770 actions_rx++; 4771 break; 4772 } 4773 } 4774 /* Add set meta action and end action for the Rx flow. */ 4775 tag_action = actions_rx; 4776 tag_action->type = (enum rte_flow_action_type) 4777 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4778 actions_rx++; 4779 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4780 actions_rx++; 4781 set_tag = (void *)actions_rx; 4782 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4783 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4784 .data = flow_id, 4785 }; 4786 MLX5_ASSERT(set_tag->id > REG_NON); 4787 tag_action->conf = set_tag; 4788 /* Create Tx item list. */ 4789 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4790 addr = (void *)&pattern_tx[2]; 4791 item = pattern_tx; 4792 item->type = (enum rte_flow_item_type) 4793 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4794 tag_item = (void *)addr; 4795 tag_item->data = flow_id; 4796 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4797 MLX5_ASSERT(set_tag->id > REG_NON); 4798 item->spec = tag_item; 4799 addr += sizeof(struct mlx5_rte_flow_item_tag); 4800 tag_item = (void *)addr; 4801 tag_item->data = UINT32_MAX; 4802 tag_item->id = UINT16_MAX; 4803 item->mask = tag_item; 4804 item->last = NULL; 4805 item++; 4806 item->type = RTE_FLOW_ITEM_TYPE_END; 4807 return 0; 4808 } 4809 4810 /** 4811 * The last stage of splitting chain, just creates the subflow 4812 * without any modification. 4813 * 4814 * @param[in] dev 4815 * Pointer to Ethernet device. 4816 * @param[in] flow 4817 * Parent flow structure pointer. 4818 * @param[in, out] sub_flow 4819 * Pointer to return the created subflow, may be NULL. 4820 * @param[in] attr 4821 * Flow rule attributes. 4822 * @param[in] items 4823 * Pattern specification (list terminated by the END pattern item). 4824 * @param[in] actions 4825 * Associated actions (list terminated by the END action). 4826 * @param[in] flow_split_info 4827 * Pointer to flow split info structure. 4828 * @param[out] error 4829 * Perform verbose error reporting if not NULL. 4830 * @return 4831 * 0 on success, negative value otherwise 4832 */ 4833 static int 4834 flow_create_split_inner(struct rte_eth_dev *dev, 4835 struct rte_flow *flow, 4836 struct mlx5_flow **sub_flow, 4837 const struct rte_flow_attr *attr, 4838 const struct rte_flow_item items[], 4839 const struct rte_flow_action actions[], 4840 struct mlx5_flow_split_info *flow_split_info, 4841 struct rte_flow_error *error) 4842 { 4843 struct mlx5_flow *dev_flow; 4844 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4845 4846 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4847 flow_split_info->flow_idx, error); 4848 if (!dev_flow) 4849 return -rte_errno; 4850 dev_flow->flow = flow; 4851 dev_flow->external = flow_split_info->external; 4852 dev_flow->skip_scale = flow_split_info->skip_scale; 4853 /* Subflow object was created, we must include one in the list. */ 4854 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4855 dev_flow->handle, next); 4856 /* 4857 * If dev_flow is as one of the suffix flow, some actions in suffix 4858 * flow may need some user defined item layer flags, and pass the 4859 * Metadata rxq mark flag to suffix flow as well. 4860 */ 4861 if (flow_split_info->prefix_layers) 4862 dev_flow->handle->layers = flow_split_info->prefix_layers; 4863 if (flow_split_info->prefix_mark) { 4864 MLX5_ASSERT(wks); 4865 wks->mark = 1; 4866 } 4867 if (sub_flow) 4868 *sub_flow = dev_flow; 4869 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4870 dev_flow->dv.table_id = flow_split_info->table_id; 4871 #endif 4872 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4873 } 4874 4875 /** 4876 * Get the sub policy of a meter. 4877 * 4878 * @param[in] dev 4879 * Pointer to Ethernet device. 4880 * @param[in] flow 4881 * Parent flow structure pointer. 4882 * @param wks 4883 * Pointer to thread flow work space. 4884 * @param[in] attr 4885 * Flow rule attributes. 4886 * @param[in] items 4887 * Pattern specification (list terminated by the END pattern item). 4888 * @param[out] error 4889 * Perform verbose error reporting if not NULL. 4890 * 4891 * @return 4892 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4893 */ 4894 static struct mlx5_flow_meter_sub_policy * 4895 get_meter_sub_policy(struct rte_eth_dev *dev, 4896 struct rte_flow *flow, 4897 struct mlx5_flow_workspace *wks, 4898 const struct rte_flow_attr *attr, 4899 const struct rte_flow_item items[], 4900 struct rte_flow_error *error) 4901 { 4902 struct mlx5_flow_meter_policy *policy; 4903 struct mlx5_flow_meter_policy *final_policy; 4904 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4905 4906 policy = wks->policy; 4907 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4908 if (final_policy->is_rss || final_policy->is_queue) { 4909 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4910 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4911 uint32_t i; 4912 4913 /* 4914 * This is a tmp dev_flow, 4915 * no need to register any matcher for it in translate. 4916 */ 4917 wks->skip_matcher_reg = 1; 4918 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4919 struct mlx5_flow dev_flow = {0}; 4920 struct mlx5_flow_handle dev_handle = { {0} }; 4921 uint8_t fate = final_policy->act_cnt[i].fate_action; 4922 4923 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4924 const struct rte_flow_action_rss *rss_act = 4925 final_policy->act_cnt[i].rss->conf; 4926 struct rte_flow_action rss_actions[2] = { 4927 [0] = { 4928 .type = RTE_FLOW_ACTION_TYPE_RSS, 4929 .conf = rss_act, 4930 }, 4931 [1] = { 4932 .type = RTE_FLOW_ACTION_TYPE_END, 4933 .conf = NULL, 4934 } 4935 }; 4936 4937 dev_flow.handle = &dev_handle; 4938 dev_flow.ingress = attr->ingress; 4939 dev_flow.flow = flow; 4940 dev_flow.external = 0; 4941 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4942 dev_flow.dv.transfer = attr->transfer; 4943 #endif 4944 /** 4945 * Translate RSS action to get rss hash fields. 4946 */ 4947 if (flow_drv_translate(dev, &dev_flow, attr, 4948 items, rss_actions, error)) 4949 goto exit; 4950 rss_desc_v[i] = wks->rss_desc; 4951 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4952 rss_desc_v[i].hash_fields = 4953 dev_flow.hash_fields; 4954 rss_desc_v[i].queue_num = 4955 rss_desc_v[i].hash_fields ? 4956 rss_desc_v[i].queue_num : 1; 4957 rss_desc_v[i].tunnel = 4958 !!(dev_flow.handle->layers & 4959 MLX5_FLOW_LAYER_TUNNEL); 4960 /* Use the RSS queues in the containers. */ 4961 rss_desc_v[i].queue = 4962 (uint16_t *)(uintptr_t)rss_act->queue; 4963 rss_desc[i] = &rss_desc_v[i]; 4964 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4965 /* This is queue action. */ 4966 rss_desc_v[i] = wks->rss_desc; 4967 rss_desc_v[i].key_len = 0; 4968 rss_desc_v[i].hash_fields = 0; 4969 rss_desc_v[i].queue = 4970 &final_policy->act_cnt[i].queue; 4971 rss_desc_v[i].queue_num = 1; 4972 rss_desc[i] = &rss_desc_v[i]; 4973 } else { 4974 rss_desc[i] = NULL; 4975 } 4976 } 4977 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 4978 flow, policy, rss_desc); 4979 } else { 4980 enum mlx5_meter_domain mtr_domain = 4981 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 4982 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 4983 MLX5_MTR_DOMAIN_INGRESS); 4984 sub_policy = policy->sub_policys[mtr_domain][0]; 4985 } 4986 if (!sub_policy) 4987 rte_flow_error_set(error, EINVAL, 4988 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 4989 "Failed to get meter sub-policy."); 4990 exit: 4991 return sub_policy; 4992 } 4993 4994 /** 4995 * Split the meter flow. 4996 * 4997 * As meter flow will split to three sub flow, other than meter 4998 * action, the other actions make sense to only meter accepts 4999 * the packet. If it need to be dropped, no other additional 5000 * actions should be take. 5001 * 5002 * One kind of special action which decapsulates the L3 tunnel 5003 * header will be in the prefix sub flow, as not to take the 5004 * L3 tunnel header into account. 5005 * 5006 * @param[in] dev 5007 * Pointer to Ethernet device. 5008 * @param[in] flow 5009 * Parent flow structure pointer. 5010 * @param wks 5011 * Pointer to thread flow work space. 5012 * @param[in] attr 5013 * Flow rule attributes. 5014 * @param[in] items 5015 * Pattern specification (list terminated by the END pattern item). 5016 * @param[out] sfx_items 5017 * Suffix flow match items (list terminated by the END pattern item). 5018 * @param[in] actions 5019 * Associated actions (list terminated by the END action). 5020 * @param[out] actions_sfx 5021 * Suffix flow actions. 5022 * @param[out] actions_pre 5023 * Prefix flow actions. 5024 * @param[out] mtr_flow_id 5025 * Pointer to meter flow id. 5026 * @param[out] error 5027 * Perform verbose error reporting if not NULL. 5028 * 5029 * @return 5030 * 0 on success, a negative errno value otherwise and rte_errno is set. 5031 */ 5032 static int 5033 flow_meter_split_prep(struct rte_eth_dev *dev, 5034 struct rte_flow *flow, 5035 struct mlx5_flow_workspace *wks, 5036 const struct rte_flow_attr *attr, 5037 const struct rte_flow_item items[], 5038 struct rte_flow_item sfx_items[], 5039 const struct rte_flow_action actions[], 5040 struct rte_flow_action actions_sfx[], 5041 struct rte_flow_action actions_pre[], 5042 uint32_t *mtr_flow_id, 5043 struct rte_flow_error *error) 5044 { 5045 struct mlx5_priv *priv = dev->data->dev_private; 5046 struct mlx5_flow_meter_info *fm = wks->fm; 5047 struct rte_flow_action *tag_action = NULL; 5048 struct rte_flow_item *tag_item; 5049 struct mlx5_rte_flow_action_set_tag *set_tag; 5050 const struct rte_flow_action_raw_encap *raw_encap; 5051 const struct rte_flow_action_raw_decap *raw_decap; 5052 struct mlx5_rte_flow_item_tag *tag_item_spec; 5053 struct mlx5_rte_flow_item_tag *tag_item_mask; 5054 uint32_t tag_id = 0; 5055 struct rte_flow_item *vlan_item_dst = NULL; 5056 const struct rte_flow_item *vlan_item_src = NULL; 5057 const struct rte_flow_item *orig_items = items; 5058 struct rte_flow_action *hw_mtr_action; 5059 struct rte_flow_action *action_pre_head = NULL; 5060 int32_t flow_src_port = priv->representor_id; 5061 bool mtr_first; 5062 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5063 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5064 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5065 uint32_t flow_id = 0; 5066 uint32_t flow_id_reversed = 0; 5067 uint8_t flow_id_bits = 0; 5068 int shift; 5069 5070 /* Prepare the suffix subflow items. */ 5071 tag_item = sfx_items++; 5072 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5073 struct mlx5_priv *port_priv; 5074 const struct rte_flow_item_port_id *pid_v; 5075 int item_type = items->type; 5076 5077 switch (item_type) { 5078 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5079 pid_v = items->spec; 5080 MLX5_ASSERT(pid_v); 5081 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5082 if (!port_priv) 5083 return rte_flow_error_set(error, 5084 rte_errno, 5085 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5086 pid_v, 5087 "Failed to get port info."); 5088 flow_src_port = port_priv->representor_id; 5089 if (!fm->def_policy && wks->policy->is_hierarchy && 5090 flow_src_port != priv->representor_id) { 5091 if (flow_drv_mtr_hierarchy_rule_create(dev, 5092 flow, fm, 5093 flow_src_port, 5094 items, 5095 error)) 5096 return -rte_errno; 5097 } 5098 memcpy(sfx_items, items, sizeof(*sfx_items)); 5099 sfx_items++; 5100 break; 5101 case RTE_FLOW_ITEM_TYPE_VLAN: 5102 /* Determine if copy vlan item below. */ 5103 vlan_item_src = items; 5104 vlan_item_dst = sfx_items++; 5105 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5106 break; 5107 default: 5108 break; 5109 } 5110 } 5111 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5112 sfx_items++; 5113 mtr_first = priv->sh->meter_aso_en && 5114 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5115 /* For ASO meter, meter must be before tag in TX direction. */ 5116 if (mtr_first) { 5117 action_pre_head = actions_pre++; 5118 /* Leave space for tag action. */ 5119 tag_action = actions_pre++; 5120 } 5121 /* Prepare the actions for prefix and suffix flow. */ 5122 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5123 struct rte_flow_action *action_cur = NULL; 5124 5125 switch (actions->type) { 5126 case RTE_FLOW_ACTION_TYPE_METER: 5127 if (mtr_first) { 5128 action_cur = action_pre_head; 5129 } else { 5130 /* Leave space for tag action. */ 5131 tag_action = actions_pre++; 5132 action_cur = actions_pre++; 5133 } 5134 break; 5135 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5136 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5137 action_cur = actions_pre++; 5138 break; 5139 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5140 raw_encap = actions->conf; 5141 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5142 action_cur = actions_pre++; 5143 break; 5144 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5145 raw_decap = actions->conf; 5146 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5147 action_cur = actions_pre++; 5148 break; 5149 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5150 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5151 if (vlan_item_dst && vlan_item_src) { 5152 memcpy(vlan_item_dst, vlan_item_src, 5153 sizeof(*vlan_item_dst)); 5154 /* 5155 * Convert to internal match item, it is used 5156 * for vlan push and set vid. 5157 */ 5158 vlan_item_dst->type = (enum rte_flow_item_type) 5159 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5160 } 5161 break; 5162 default: 5163 break; 5164 } 5165 if (!action_cur) 5166 action_cur = (fm->def_policy) ? 5167 actions_sfx++ : actions_pre++; 5168 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5169 } 5170 /* Add end action to the actions. */ 5171 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5172 if (priv->sh->meter_aso_en) { 5173 /** 5174 * For ASO meter, need to add an extra jump action explicitly, 5175 * to jump from meter to policer table. 5176 */ 5177 struct mlx5_flow_meter_sub_policy *sub_policy; 5178 struct mlx5_flow_tbl_data_entry *tbl_data; 5179 5180 if (!fm->def_policy) { 5181 sub_policy = get_meter_sub_policy(dev, flow, wks, 5182 attr, orig_items, 5183 error); 5184 if (!sub_policy) 5185 return -rte_errno; 5186 } else { 5187 enum mlx5_meter_domain mtr_domain = 5188 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5189 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5190 MLX5_MTR_DOMAIN_INGRESS); 5191 5192 sub_policy = 5193 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5194 } 5195 tbl_data = container_of(sub_policy->tbl_rsc, 5196 struct mlx5_flow_tbl_data_entry, tbl); 5197 hw_mtr_action = actions_pre++; 5198 hw_mtr_action->type = (enum rte_flow_action_type) 5199 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5200 hw_mtr_action->conf = tbl_data->jump.action; 5201 } 5202 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5203 actions_pre++; 5204 if (!tag_action) 5205 return rte_flow_error_set(error, ENOMEM, 5206 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5207 NULL, "No tag action space."); 5208 if (!mtr_flow_id) { 5209 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5210 goto exit; 5211 } 5212 /* Only default-policy Meter creates mtr flow id. */ 5213 if (fm->def_policy) { 5214 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5215 if (!tag_id) 5216 return rte_flow_error_set(error, ENOMEM, 5217 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5218 "Failed to allocate meter flow id."); 5219 flow_id = tag_id - 1; 5220 flow_id_bits = (!flow_id) ? 1 : 5221 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5222 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5223 mtr_reg_bits) { 5224 mlx5_ipool_free(fm->flow_ipool, tag_id); 5225 return rte_flow_error_set(error, EINVAL, 5226 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5227 "Meter flow id exceeds max limit."); 5228 } 5229 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5230 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5231 } 5232 /* Build tag actions and items for meter_id/meter flow_id. */ 5233 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5234 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5235 tag_item_mask = tag_item_spec + 1; 5236 /* Both flow_id and meter_id share the same register. */ 5237 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5238 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5239 0, error), 5240 .offset = mtr_id_offset, 5241 .length = mtr_reg_bits, 5242 .data = flow->meter, 5243 }; 5244 /* 5245 * The color Reg bits used by flow_id are growing from 5246 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5247 */ 5248 for (shift = 0; shift < flow_id_bits; shift++) 5249 flow_id_reversed = (flow_id_reversed << 1) | 5250 ((flow_id >> shift) & 0x1); 5251 set_tag->data |= 5252 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5253 tag_item_spec->id = set_tag->id; 5254 tag_item_spec->data = set_tag->data << mtr_id_offset; 5255 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5256 tag_action->type = (enum rte_flow_action_type) 5257 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5258 tag_action->conf = set_tag; 5259 tag_item->type = (enum rte_flow_item_type) 5260 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5261 tag_item->spec = tag_item_spec; 5262 tag_item->last = NULL; 5263 tag_item->mask = tag_item_mask; 5264 exit: 5265 if (mtr_flow_id) 5266 *mtr_flow_id = tag_id; 5267 return 0; 5268 } 5269 5270 /** 5271 * Split action list having QUEUE/RSS for metadata register copy. 5272 * 5273 * Once Q/RSS action is detected in user's action list, the flow action 5274 * should be split in order to copy metadata registers, which will happen in 5275 * RX_CP_TBL like, 5276 * - CQE->flow_tag := reg_c[1] (MARK) 5277 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5278 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5279 * This is because the last action of each flow must be a terminal action 5280 * (QUEUE, RSS or DROP). 5281 * 5282 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5283 * stored and kept in the mlx5_flow structure per each sub_flow. 5284 * 5285 * The Q/RSS action is replaced with, 5286 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5287 * And the following JUMP action is added at the end, 5288 * - JUMP, to RX_CP_TBL. 5289 * 5290 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5291 * flow_create_split_metadata() routine. The flow will look like, 5292 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5293 * 5294 * @param dev 5295 * Pointer to Ethernet device. 5296 * @param[out] split_actions 5297 * Pointer to store split actions to jump to CP_TBL. 5298 * @param[in] actions 5299 * Pointer to the list of original flow actions. 5300 * @param[in] qrss 5301 * Pointer to the Q/RSS action. 5302 * @param[in] actions_n 5303 * Number of original actions. 5304 * @param[in] mtr_sfx 5305 * Check if it is in meter suffix table. 5306 * @param[out] error 5307 * Perform verbose error reporting if not NULL. 5308 * 5309 * @return 5310 * non-zero unique flow_id on success, otherwise 0 and 5311 * error/rte_error are set. 5312 */ 5313 static uint32_t 5314 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5315 struct rte_flow_action *split_actions, 5316 const struct rte_flow_action *actions, 5317 const struct rte_flow_action *qrss, 5318 int actions_n, int mtr_sfx, 5319 struct rte_flow_error *error) 5320 { 5321 struct mlx5_priv *priv = dev->data->dev_private; 5322 struct mlx5_rte_flow_action_set_tag *set_tag; 5323 struct rte_flow_action_jump *jump; 5324 const int qrss_idx = qrss - actions; 5325 uint32_t flow_id = 0; 5326 int ret = 0; 5327 5328 /* 5329 * Given actions will be split 5330 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5331 * - Add jump to mreg CP_TBL. 5332 * As a result, there will be one more action. 5333 */ 5334 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5335 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5336 ++actions_n; 5337 set_tag = (void *)(split_actions + actions_n); 5338 /* 5339 * If we are not the meter suffix flow, add the tag action. 5340 * Since meter suffix flow already has the tag added. 5341 */ 5342 if (!mtr_sfx) { 5343 /* 5344 * Allocate the new subflow ID. This one is unique within 5345 * device and not shared with representors. Otherwise, 5346 * we would have to resolve multi-thread access synch 5347 * issue. Each flow on the shared device is appended 5348 * with source vport identifier, so the resulting 5349 * flows will be unique in the shared (by master and 5350 * representors) domain even if they have coinciding 5351 * IDs. 5352 */ 5353 mlx5_ipool_malloc(priv->sh->ipool 5354 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5355 if (!flow_id) 5356 return rte_flow_error_set(error, ENOMEM, 5357 RTE_FLOW_ERROR_TYPE_ACTION, 5358 NULL, "can't allocate id " 5359 "for split Q/RSS subflow"); 5360 /* Internal SET_TAG action to set flow ID. */ 5361 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5362 .data = flow_id, 5363 }; 5364 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5365 if (ret < 0) 5366 return ret; 5367 set_tag->id = ret; 5368 /* Construct new actions array. */ 5369 /* Replace QUEUE/RSS action. */ 5370 split_actions[qrss_idx] = (struct rte_flow_action){ 5371 .type = (enum rte_flow_action_type) 5372 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5373 .conf = set_tag, 5374 }; 5375 } else { 5376 /* 5377 * If we are the suffix flow of meter, tag already exist. 5378 * Set the QUEUE/RSS action to void. 5379 */ 5380 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5381 } 5382 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5383 jump = (void *)(set_tag + 1); 5384 *jump = (struct rte_flow_action_jump){ 5385 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5386 }; 5387 split_actions[actions_n - 2] = (struct rte_flow_action){ 5388 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5389 .conf = jump, 5390 }; 5391 split_actions[actions_n - 1] = (struct rte_flow_action){ 5392 .type = RTE_FLOW_ACTION_TYPE_END, 5393 }; 5394 return flow_id; 5395 } 5396 5397 /** 5398 * Extend the given action list for Tx metadata copy. 5399 * 5400 * Copy the given action list to the ext_actions and add flow metadata register 5401 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5402 * 5403 * @param[out] ext_actions 5404 * Pointer to the extended action list. 5405 * @param[in] actions 5406 * Pointer to the list of actions. 5407 * @param[in] actions_n 5408 * Number of actions in the list. 5409 * @param[out] error 5410 * Perform verbose error reporting if not NULL. 5411 * @param[in] encap_idx 5412 * The encap action index. 5413 * 5414 * @return 5415 * 0 on success, negative value otherwise 5416 */ 5417 static int 5418 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5419 struct rte_flow_action *ext_actions, 5420 const struct rte_flow_action *actions, 5421 int actions_n, struct rte_flow_error *error, 5422 int encap_idx) 5423 { 5424 struct mlx5_flow_action_copy_mreg *cp_mreg = 5425 (struct mlx5_flow_action_copy_mreg *) 5426 (ext_actions + actions_n + 1); 5427 int ret; 5428 5429 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5430 if (ret < 0) 5431 return ret; 5432 cp_mreg->dst = ret; 5433 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5434 if (ret < 0) 5435 return ret; 5436 cp_mreg->src = ret; 5437 if (encap_idx != 0) 5438 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5439 if (encap_idx == actions_n - 1) { 5440 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5441 .type = (enum rte_flow_action_type) 5442 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5443 .conf = cp_mreg, 5444 }; 5445 ext_actions[actions_n] = (struct rte_flow_action){ 5446 .type = RTE_FLOW_ACTION_TYPE_END, 5447 }; 5448 } else { 5449 ext_actions[encap_idx] = (struct rte_flow_action){ 5450 .type = (enum rte_flow_action_type) 5451 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5452 .conf = cp_mreg, 5453 }; 5454 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5455 sizeof(*ext_actions) * (actions_n - encap_idx)); 5456 } 5457 return 0; 5458 } 5459 5460 /** 5461 * Check the match action from the action list. 5462 * 5463 * @param[in] actions 5464 * Pointer to the list of actions. 5465 * @param[in] attr 5466 * Flow rule attributes. 5467 * @param[in] action 5468 * The action to be check if exist. 5469 * @param[out] match_action_pos 5470 * Pointer to the position of the matched action if exists, otherwise is -1. 5471 * @param[out] qrss_action_pos 5472 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5473 * @param[out] modify_after_mirror 5474 * Pointer to the flag of modify action after FDB mirroring. 5475 * 5476 * @return 5477 * > 0 the total number of actions. 5478 * 0 if not found match action in action list. 5479 */ 5480 static int 5481 flow_check_match_action(const struct rte_flow_action actions[], 5482 const struct rte_flow_attr *attr, 5483 enum rte_flow_action_type action, 5484 int *match_action_pos, int *qrss_action_pos, 5485 int *modify_after_mirror) 5486 { 5487 const struct rte_flow_action_sample *sample; 5488 const struct rte_flow_action_raw_decap *decap; 5489 int actions_n = 0; 5490 uint32_t ratio = 0; 5491 int sub_type = 0; 5492 int flag = 0; 5493 int fdb_mirror = 0; 5494 5495 *match_action_pos = -1; 5496 *qrss_action_pos = -1; 5497 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5498 if (actions->type == action) { 5499 flag = 1; 5500 *match_action_pos = actions_n; 5501 } 5502 switch (actions->type) { 5503 case RTE_FLOW_ACTION_TYPE_QUEUE: 5504 case RTE_FLOW_ACTION_TYPE_RSS: 5505 *qrss_action_pos = actions_n; 5506 break; 5507 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5508 sample = actions->conf; 5509 ratio = sample->ratio; 5510 sub_type = ((const struct rte_flow_action *) 5511 (sample->actions))->type; 5512 if (ratio == 1 && attr->transfer) 5513 fdb_mirror = 1; 5514 break; 5515 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5516 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5517 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5518 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5519 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5520 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5521 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5522 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5523 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5524 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5525 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5526 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5527 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5528 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5529 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5530 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5531 case RTE_FLOW_ACTION_TYPE_FLAG: 5532 case RTE_FLOW_ACTION_TYPE_MARK: 5533 case RTE_FLOW_ACTION_TYPE_SET_META: 5534 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5535 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5536 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5537 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5538 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5539 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5540 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5541 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5542 case RTE_FLOW_ACTION_TYPE_METER: 5543 if (fdb_mirror) 5544 *modify_after_mirror = 1; 5545 break; 5546 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5547 decap = actions->conf; 5548 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5549 ; 5550 actions_n++; 5551 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5552 const struct rte_flow_action_raw_encap *encap = 5553 actions->conf; 5554 if (decap->size <= 5555 MLX5_ENCAPSULATION_DECISION_SIZE && 5556 encap->size > 5557 MLX5_ENCAPSULATION_DECISION_SIZE) 5558 /* L3 encap. */ 5559 break; 5560 } 5561 if (fdb_mirror) 5562 *modify_after_mirror = 1; 5563 break; 5564 default: 5565 break; 5566 } 5567 actions_n++; 5568 } 5569 if (flag && fdb_mirror && !*modify_after_mirror) { 5570 /* FDB mirroring uses the destination array to implement 5571 * instead of FLOW_SAMPLER object. 5572 */ 5573 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5574 flag = 0; 5575 } 5576 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5577 return flag ? actions_n + 1 : 0; 5578 } 5579 5580 #define SAMPLE_SUFFIX_ITEM 2 5581 5582 /** 5583 * Split the sample flow. 5584 * 5585 * As sample flow will split to two sub flow, sample flow with 5586 * sample action, the other actions will move to new suffix flow. 5587 * 5588 * Also add unique tag id with tag action in the sample flow, 5589 * the same tag id will be as match in the suffix flow. 5590 * 5591 * @param dev 5592 * Pointer to Ethernet device. 5593 * @param[in] add_tag 5594 * Add extra tag action flag. 5595 * @param[out] sfx_items 5596 * Suffix flow match items (list terminated by the END pattern item). 5597 * @param[in] actions 5598 * Associated actions (list terminated by the END action). 5599 * @param[out] actions_sfx 5600 * Suffix flow actions. 5601 * @param[out] actions_pre 5602 * Prefix flow actions. 5603 * @param[in] actions_n 5604 * The total number of actions. 5605 * @param[in] sample_action_pos 5606 * The sample action position. 5607 * @param[in] qrss_action_pos 5608 * The Queue/RSS action position. 5609 * @param[in] jump_table 5610 * Add extra jump action flag. 5611 * @param[out] error 5612 * Perform verbose error reporting if not NULL. 5613 * 5614 * @return 5615 * 0 on success, or unique flow_id, a negative errno value 5616 * otherwise and rte_errno is set. 5617 */ 5618 static int 5619 flow_sample_split_prep(struct rte_eth_dev *dev, 5620 int add_tag, 5621 struct rte_flow_item sfx_items[], 5622 const struct rte_flow_action actions[], 5623 struct rte_flow_action actions_sfx[], 5624 struct rte_flow_action actions_pre[], 5625 int actions_n, 5626 int sample_action_pos, 5627 int qrss_action_pos, 5628 int jump_table, 5629 struct rte_flow_error *error) 5630 { 5631 struct mlx5_priv *priv = dev->data->dev_private; 5632 struct mlx5_rte_flow_action_set_tag *set_tag; 5633 struct mlx5_rte_flow_item_tag *tag_spec; 5634 struct mlx5_rte_flow_item_tag *tag_mask; 5635 struct rte_flow_action_jump *jump_action; 5636 uint32_t tag_id = 0; 5637 int index; 5638 int append_index = 0; 5639 int ret; 5640 5641 if (sample_action_pos < 0) 5642 return rte_flow_error_set(error, EINVAL, 5643 RTE_FLOW_ERROR_TYPE_ACTION, 5644 NULL, "invalid position of sample " 5645 "action in list"); 5646 /* Prepare the actions for prefix and suffix flow. */ 5647 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5648 index = qrss_action_pos; 5649 /* Put the preceding the Queue/RSS action into prefix flow. */ 5650 if (index != 0) 5651 memcpy(actions_pre, actions, 5652 sizeof(struct rte_flow_action) * index); 5653 /* Put others preceding the sample action into prefix flow. */ 5654 if (sample_action_pos > index + 1) 5655 memcpy(actions_pre + index, actions + index + 1, 5656 sizeof(struct rte_flow_action) * 5657 (sample_action_pos - index - 1)); 5658 index = sample_action_pos - 1; 5659 /* Put Queue/RSS action into Suffix flow. */ 5660 memcpy(actions_sfx, actions + qrss_action_pos, 5661 sizeof(struct rte_flow_action)); 5662 actions_sfx++; 5663 } else { 5664 index = sample_action_pos; 5665 if (index != 0) 5666 memcpy(actions_pre, actions, 5667 sizeof(struct rte_flow_action) * index); 5668 } 5669 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5670 * For CX6DX and above, metadata registers Cx preserve their value, 5671 * add an extra tag action for NIC-RX and E-Switch Domain. 5672 */ 5673 if (add_tag) { 5674 /* Prepare the prefix tag action. */ 5675 append_index++; 5676 set_tag = (void *)(actions_pre + actions_n + append_index); 5677 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5678 if (ret < 0) 5679 return ret; 5680 mlx5_ipool_malloc(priv->sh->ipool 5681 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5682 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5683 .id = ret, 5684 .data = tag_id, 5685 }; 5686 /* Prepare the suffix subflow items. */ 5687 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5688 tag_spec->data = tag_id; 5689 tag_spec->id = set_tag->id; 5690 tag_mask = tag_spec + 1; 5691 tag_mask->data = UINT32_MAX; 5692 sfx_items[0] = (struct rte_flow_item){ 5693 .type = (enum rte_flow_item_type) 5694 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5695 .spec = tag_spec, 5696 .last = NULL, 5697 .mask = tag_mask, 5698 }; 5699 sfx_items[1] = (struct rte_flow_item){ 5700 .type = (enum rte_flow_item_type) 5701 RTE_FLOW_ITEM_TYPE_END, 5702 }; 5703 /* Prepare the tag action in prefix subflow. */ 5704 actions_pre[index++] = 5705 (struct rte_flow_action){ 5706 .type = (enum rte_flow_action_type) 5707 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5708 .conf = set_tag, 5709 }; 5710 } 5711 memcpy(actions_pre + index, actions + sample_action_pos, 5712 sizeof(struct rte_flow_action)); 5713 index += 1; 5714 /* For the modify action after the sample action in E-Switch mirroring, 5715 * Add the extra jump action in prefix subflow and jump into the next 5716 * table, then do the modify action in the new table. 5717 */ 5718 if (jump_table) { 5719 /* Prepare the prefix jump action. */ 5720 append_index++; 5721 jump_action = (void *)(actions_pre + actions_n + append_index); 5722 jump_action->group = jump_table; 5723 actions_pre[index++] = 5724 (struct rte_flow_action){ 5725 .type = (enum rte_flow_action_type) 5726 RTE_FLOW_ACTION_TYPE_JUMP, 5727 .conf = jump_action, 5728 }; 5729 } 5730 actions_pre[index] = (struct rte_flow_action){ 5731 .type = (enum rte_flow_action_type) 5732 RTE_FLOW_ACTION_TYPE_END, 5733 }; 5734 /* Put the actions after sample into Suffix flow. */ 5735 memcpy(actions_sfx, actions + sample_action_pos + 1, 5736 sizeof(struct rte_flow_action) * 5737 (actions_n - sample_action_pos - 1)); 5738 return tag_id; 5739 } 5740 5741 /** 5742 * The splitting for metadata feature. 5743 * 5744 * - Q/RSS action on NIC Rx should be split in order to pass by 5745 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5746 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5747 * 5748 * - All the actions on NIC Tx should have a mreg copy action to 5749 * copy reg_a from WQE to reg_c[0]. 5750 * 5751 * @param dev 5752 * Pointer to Ethernet device. 5753 * @param[in] flow 5754 * Parent flow structure pointer. 5755 * @param[in] attr 5756 * Flow rule attributes. 5757 * @param[in] items 5758 * Pattern specification (list terminated by the END pattern item). 5759 * @param[in] actions 5760 * Associated actions (list terminated by the END action). 5761 * @param[in] flow_split_info 5762 * Pointer to flow split info structure. 5763 * @param[out] error 5764 * Perform verbose error reporting if not NULL. 5765 * @return 5766 * 0 on success, negative value otherwise 5767 */ 5768 static int 5769 flow_create_split_metadata(struct rte_eth_dev *dev, 5770 struct rte_flow *flow, 5771 const struct rte_flow_attr *attr, 5772 const struct rte_flow_item items[], 5773 const struct rte_flow_action actions[], 5774 struct mlx5_flow_split_info *flow_split_info, 5775 struct rte_flow_error *error) 5776 { 5777 struct mlx5_priv *priv = dev->data->dev_private; 5778 struct mlx5_sh_config *config = &priv->sh->config; 5779 const struct rte_flow_action *qrss = NULL; 5780 struct rte_flow_action *ext_actions = NULL; 5781 struct mlx5_flow *dev_flow = NULL; 5782 uint32_t qrss_id = 0; 5783 int mtr_sfx = 0; 5784 size_t act_size; 5785 int actions_n; 5786 int encap_idx; 5787 int ret; 5788 5789 /* Check whether extensive metadata feature is engaged. */ 5790 if (!config->dv_flow_en || 5791 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5792 !mlx5_flow_ext_mreg_supported(dev)) 5793 return flow_create_split_inner(dev, flow, NULL, attr, items, 5794 actions, flow_split_info, error); 5795 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5796 &encap_idx); 5797 if (qrss) { 5798 /* Exclude hairpin flows from splitting. */ 5799 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5800 const struct rte_flow_action_queue *queue; 5801 5802 queue = qrss->conf; 5803 if (mlx5_rxq_get_type(dev, queue->index) == 5804 MLX5_RXQ_TYPE_HAIRPIN) 5805 qrss = NULL; 5806 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5807 const struct rte_flow_action_rss *rss; 5808 5809 rss = qrss->conf; 5810 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5811 MLX5_RXQ_TYPE_HAIRPIN) 5812 qrss = NULL; 5813 } 5814 } 5815 if (qrss) { 5816 /* Check if it is in meter suffix table. */ 5817 mtr_sfx = attr->group == (attr->transfer ? 5818 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5819 MLX5_FLOW_TABLE_LEVEL_METER); 5820 /* 5821 * Q/RSS action on NIC Rx should be split in order to pass by 5822 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5823 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5824 */ 5825 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5826 sizeof(struct rte_flow_action_set_tag) + 5827 sizeof(struct rte_flow_action_jump); 5828 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5829 SOCKET_ID_ANY); 5830 if (!ext_actions) 5831 return rte_flow_error_set(error, ENOMEM, 5832 RTE_FLOW_ERROR_TYPE_ACTION, 5833 NULL, "no memory to split " 5834 "metadata flow"); 5835 /* 5836 * Create the new actions list with removed Q/RSS action 5837 * and appended set tag and jump to register copy table 5838 * (RX_CP_TBL). We should preallocate unique tag ID here 5839 * in advance, because it is needed for set tag action. 5840 */ 5841 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5842 qrss, actions_n, 5843 mtr_sfx, error); 5844 if (!mtr_sfx && !qrss_id) { 5845 ret = -rte_errno; 5846 goto exit; 5847 } 5848 } else if (attr->egress && !attr->transfer) { 5849 /* 5850 * All the actions on NIC Tx should have a metadata register 5851 * copy action to copy reg_a from WQE to reg_c[meta] 5852 */ 5853 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5854 sizeof(struct mlx5_flow_action_copy_mreg); 5855 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5856 SOCKET_ID_ANY); 5857 if (!ext_actions) 5858 return rte_flow_error_set(error, ENOMEM, 5859 RTE_FLOW_ERROR_TYPE_ACTION, 5860 NULL, "no memory to split " 5861 "metadata flow"); 5862 /* Create the action list appended with copy register. */ 5863 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5864 actions_n, error, encap_idx); 5865 if (ret < 0) 5866 goto exit; 5867 } 5868 /* Add the unmodified original or prefix subflow. */ 5869 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5870 items, ext_actions ? ext_actions : 5871 actions, flow_split_info, error); 5872 if (ret < 0) 5873 goto exit; 5874 MLX5_ASSERT(dev_flow); 5875 if (qrss) { 5876 const struct rte_flow_attr q_attr = { 5877 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5878 .ingress = 1, 5879 }; 5880 /* Internal PMD action to set register. */ 5881 struct mlx5_rte_flow_item_tag q_tag_spec = { 5882 .data = qrss_id, 5883 .id = REG_NON, 5884 }; 5885 struct rte_flow_item q_items[] = { 5886 { 5887 .type = (enum rte_flow_item_type) 5888 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5889 .spec = &q_tag_spec, 5890 .last = NULL, 5891 .mask = NULL, 5892 }, 5893 { 5894 .type = RTE_FLOW_ITEM_TYPE_END, 5895 }, 5896 }; 5897 struct rte_flow_action q_actions[] = { 5898 { 5899 .type = qrss->type, 5900 .conf = qrss->conf, 5901 }, 5902 { 5903 .type = RTE_FLOW_ACTION_TYPE_END, 5904 }, 5905 }; 5906 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5907 5908 /* 5909 * Configure the tag item only if there is no meter subflow. 5910 * Since tag is already marked in the meter suffix subflow 5911 * we can just use the meter suffix items as is. 5912 */ 5913 if (qrss_id) { 5914 /* Not meter subflow. */ 5915 MLX5_ASSERT(!mtr_sfx); 5916 /* 5917 * Put unique id in prefix flow due to it is destroyed 5918 * after suffix flow and id will be freed after there 5919 * is no actual flows with this id and identifier 5920 * reallocation becomes possible (for example, for 5921 * other flows in other threads). 5922 */ 5923 dev_flow->handle->split_flow_id = qrss_id; 5924 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5925 error); 5926 if (ret < 0) 5927 goto exit; 5928 q_tag_spec.id = ret; 5929 } 5930 dev_flow = NULL; 5931 /* Add suffix subflow to execute Q/RSS. */ 5932 flow_split_info->prefix_layers = layers; 5933 flow_split_info->prefix_mark = 0; 5934 flow_split_info->table_id = 0; 5935 ret = flow_create_split_inner(dev, flow, &dev_flow, 5936 &q_attr, mtr_sfx ? items : 5937 q_items, q_actions, 5938 flow_split_info, error); 5939 if (ret < 0) 5940 goto exit; 5941 /* qrss ID should be freed if failed. */ 5942 qrss_id = 0; 5943 MLX5_ASSERT(dev_flow); 5944 } 5945 5946 exit: 5947 /* 5948 * We do not destroy the partially created sub_flows in case of error. 5949 * These ones are included into parent flow list and will be destroyed 5950 * by flow_drv_destroy. 5951 */ 5952 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5953 qrss_id); 5954 mlx5_free(ext_actions); 5955 return ret; 5956 } 5957 5958 /** 5959 * Create meter internal drop flow with the original pattern. 5960 * 5961 * @param dev 5962 * Pointer to Ethernet device. 5963 * @param[in] flow 5964 * Parent flow structure pointer. 5965 * @param[in] attr 5966 * Flow rule attributes. 5967 * @param[in] items 5968 * Pattern specification (list terminated by the END pattern item). 5969 * @param[in] flow_split_info 5970 * Pointer to flow split info structure. 5971 * @param[in] fm 5972 * Pointer to flow meter structure. 5973 * @param[out] error 5974 * Perform verbose error reporting if not NULL. 5975 * @return 5976 * 0 on success, negative value otherwise 5977 */ 5978 static uint32_t 5979 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 5980 struct rte_flow *flow, 5981 const struct rte_flow_attr *attr, 5982 const struct rte_flow_item items[], 5983 struct mlx5_flow_split_info *flow_split_info, 5984 struct mlx5_flow_meter_info *fm, 5985 struct rte_flow_error *error) 5986 { 5987 struct mlx5_flow *dev_flow = NULL; 5988 struct rte_flow_attr drop_attr = *attr; 5989 struct rte_flow_action drop_actions[3]; 5990 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 5991 5992 MLX5_ASSERT(fm->drop_cnt); 5993 drop_actions[0].type = 5994 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 5995 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 5996 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 5997 drop_actions[1].conf = NULL; 5998 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 5999 drop_actions[2].conf = NULL; 6000 drop_split_info.external = false; 6001 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6002 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6003 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6004 return flow_create_split_inner(dev, flow, &dev_flow, 6005 &drop_attr, items, drop_actions, 6006 &drop_split_info, error); 6007 } 6008 6009 /** 6010 * The splitting for meter feature. 6011 * 6012 * - The meter flow will be split to two flows as prefix and 6013 * suffix flow. The packets make sense only it pass the prefix 6014 * meter action. 6015 * 6016 * - Reg_C_5 is used for the packet to match betweend prefix and 6017 * suffix flow. 6018 * 6019 * @param dev 6020 * Pointer to Ethernet device. 6021 * @param[in] flow 6022 * Parent flow structure pointer. 6023 * @param[in] attr 6024 * Flow rule attributes. 6025 * @param[in] items 6026 * Pattern specification (list terminated by the END pattern item). 6027 * @param[in] actions 6028 * Associated actions (list terminated by the END action). 6029 * @param[in] flow_split_info 6030 * Pointer to flow split info structure. 6031 * @param[out] error 6032 * Perform verbose error reporting if not NULL. 6033 * @return 6034 * 0 on success, negative value otherwise 6035 */ 6036 static int 6037 flow_create_split_meter(struct rte_eth_dev *dev, 6038 struct rte_flow *flow, 6039 const struct rte_flow_attr *attr, 6040 const struct rte_flow_item items[], 6041 const struct rte_flow_action actions[], 6042 struct mlx5_flow_split_info *flow_split_info, 6043 struct rte_flow_error *error) 6044 { 6045 struct mlx5_priv *priv = dev->data->dev_private; 6046 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6047 struct rte_flow_action *sfx_actions = NULL; 6048 struct rte_flow_action *pre_actions = NULL; 6049 struct rte_flow_item *sfx_items = NULL; 6050 struct mlx5_flow *dev_flow = NULL; 6051 struct rte_flow_attr sfx_attr = *attr; 6052 struct mlx5_flow_meter_info *fm = NULL; 6053 uint8_t skip_scale_restore; 6054 bool has_mtr = false; 6055 bool has_modify = false; 6056 bool set_mtr_reg = true; 6057 bool is_mtr_hierarchy = false; 6058 uint32_t meter_id = 0; 6059 uint32_t mtr_idx = 0; 6060 uint32_t mtr_flow_id = 0; 6061 size_t act_size; 6062 size_t item_size; 6063 int actions_n = 0; 6064 int ret = 0; 6065 6066 if (priv->mtr_en) 6067 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6068 &has_modify, &meter_id); 6069 if (has_mtr) { 6070 if (flow->meter) { 6071 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6072 if (!fm) 6073 return rte_flow_error_set(error, EINVAL, 6074 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6075 NULL, "Meter not found."); 6076 } else { 6077 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6078 if (!fm) 6079 return rte_flow_error_set(error, EINVAL, 6080 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6081 NULL, "Meter not found."); 6082 ret = mlx5_flow_meter_attach(priv, fm, 6083 &sfx_attr, error); 6084 if (ret) 6085 return -rte_errno; 6086 flow->meter = mtr_idx; 6087 } 6088 MLX5_ASSERT(wks); 6089 wks->fm = fm; 6090 if (!fm->def_policy) { 6091 wks->policy = mlx5_flow_meter_policy_find(dev, 6092 fm->policy_id, 6093 NULL); 6094 MLX5_ASSERT(wks->policy); 6095 if (wks->policy->is_hierarchy) { 6096 wks->final_policy = 6097 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6098 wks->policy); 6099 if (!wks->final_policy) 6100 return rte_flow_error_set(error, 6101 EINVAL, 6102 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6103 "Failed to find terminal policy of hierarchy."); 6104 is_mtr_hierarchy = true; 6105 } 6106 } 6107 /* 6108 * If it isn't default-policy Meter, and 6109 * 1. There's no action in flow to change 6110 * packet (modify/encap/decap etc.), OR 6111 * 2. No drop count needed for this meter. 6112 * 3. It's not meter hierarchy. 6113 * Then no need to use regC to save meter id anymore. 6114 */ 6115 if (!fm->def_policy && !is_mtr_hierarchy && 6116 (!has_modify || !fm->drop_cnt)) 6117 set_mtr_reg = false; 6118 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6119 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6120 sizeof(struct mlx5_rte_flow_action_set_tag); 6121 /* Suffix items: tag, vlan, port id, end. */ 6122 #define METER_SUFFIX_ITEM 4 6123 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6124 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6125 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6126 0, SOCKET_ID_ANY); 6127 if (!sfx_actions) 6128 return rte_flow_error_set(error, ENOMEM, 6129 RTE_FLOW_ERROR_TYPE_ACTION, 6130 NULL, "no memory to split " 6131 "meter flow"); 6132 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6133 act_size); 6134 /* There's no suffix flow for meter of non-default policy. */ 6135 if (!fm->def_policy) 6136 pre_actions = sfx_actions + 1; 6137 else 6138 pre_actions = sfx_actions + actions_n; 6139 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6140 items, sfx_items, actions, 6141 sfx_actions, pre_actions, 6142 (set_mtr_reg ? &mtr_flow_id : NULL), 6143 error); 6144 if (ret) { 6145 ret = -rte_errno; 6146 goto exit; 6147 } 6148 /* Add the prefix subflow. */ 6149 skip_scale_restore = flow_split_info->skip_scale; 6150 flow_split_info->skip_scale |= 6151 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6152 ret = flow_create_split_inner(dev, flow, &dev_flow, 6153 attr, items, pre_actions, 6154 flow_split_info, error); 6155 flow_split_info->skip_scale = skip_scale_restore; 6156 if (ret) { 6157 if (mtr_flow_id) 6158 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6159 ret = -rte_errno; 6160 goto exit; 6161 } 6162 if (mtr_flow_id) { 6163 dev_flow->handle->split_flow_id = mtr_flow_id; 6164 dev_flow->handle->is_meter_flow_id = 1; 6165 } 6166 if (!fm->def_policy) { 6167 if (!set_mtr_reg && fm->drop_cnt) 6168 ret = 6169 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6170 &sfx_attr, items, 6171 flow_split_info, 6172 fm, error); 6173 goto exit; 6174 } 6175 /* Setting the sfx group atrr. */ 6176 sfx_attr.group = sfx_attr.transfer ? 6177 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6178 MLX5_FLOW_TABLE_LEVEL_METER; 6179 flow_split_info->prefix_layers = 6180 flow_get_prefix_layer_flags(dev_flow); 6181 flow_split_info->prefix_mark |= wks->mark; 6182 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6183 } 6184 /* Add the prefix subflow. */ 6185 ret = flow_create_split_metadata(dev, flow, 6186 &sfx_attr, sfx_items ? 6187 sfx_items : items, 6188 sfx_actions ? sfx_actions : actions, 6189 flow_split_info, error); 6190 exit: 6191 if (sfx_actions) 6192 mlx5_free(sfx_actions); 6193 return ret; 6194 } 6195 6196 /** 6197 * The splitting for sample feature. 6198 * 6199 * Once Sample action is detected in the action list, the flow actions should 6200 * be split into prefix sub flow and suffix sub flow. 6201 * 6202 * The original items remain in the prefix sub flow, all actions preceding the 6203 * sample action and the sample action itself will be copied to the prefix 6204 * sub flow, the actions following the sample action will be copied to the 6205 * suffix sub flow, Queue action always be located in the suffix sub flow. 6206 * 6207 * In order to make the packet from prefix sub flow matches with suffix sub 6208 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6209 * flow uses tag item with the unique flow id. 6210 * 6211 * @param dev 6212 * Pointer to Ethernet device. 6213 * @param[in] flow 6214 * Parent flow structure pointer. 6215 * @param[in] attr 6216 * Flow rule attributes. 6217 * @param[in] items 6218 * Pattern specification (list terminated by the END pattern item). 6219 * @param[in] actions 6220 * Associated actions (list terminated by the END action). 6221 * @param[in] flow_split_info 6222 * Pointer to flow split info structure. 6223 * @param[out] error 6224 * Perform verbose error reporting if not NULL. 6225 * @return 6226 * 0 on success, negative value otherwise 6227 */ 6228 static int 6229 flow_create_split_sample(struct rte_eth_dev *dev, 6230 struct rte_flow *flow, 6231 const struct rte_flow_attr *attr, 6232 const struct rte_flow_item items[], 6233 const struct rte_flow_action actions[], 6234 struct mlx5_flow_split_info *flow_split_info, 6235 struct rte_flow_error *error) 6236 { 6237 struct mlx5_priv *priv = dev->data->dev_private; 6238 struct rte_flow_action *sfx_actions = NULL; 6239 struct rte_flow_action *pre_actions = NULL; 6240 struct rte_flow_item *sfx_items = NULL; 6241 struct mlx5_flow *dev_flow = NULL; 6242 struct rte_flow_attr sfx_attr = *attr; 6243 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6244 struct mlx5_flow_dv_sample_resource *sample_res; 6245 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6246 struct mlx5_flow_tbl_resource *sfx_tbl; 6247 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6248 #endif 6249 size_t act_size; 6250 size_t item_size; 6251 uint32_t fdb_tx = 0; 6252 int32_t tag_id = 0; 6253 int actions_n = 0; 6254 int sample_action_pos; 6255 int qrss_action_pos; 6256 int add_tag = 0; 6257 int modify_after_mirror = 0; 6258 uint16_t jump_table = 0; 6259 const uint32_t next_ft_step = 1; 6260 int ret = 0; 6261 6262 if (priv->sampler_en) 6263 actions_n = flow_check_match_action(actions, attr, 6264 RTE_FLOW_ACTION_TYPE_SAMPLE, 6265 &sample_action_pos, &qrss_action_pos, 6266 &modify_after_mirror); 6267 if (actions_n) { 6268 /* The prefix actions must includes sample, tag, end. */ 6269 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6270 + sizeof(struct mlx5_rte_flow_action_set_tag); 6271 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6272 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6273 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6274 item_size), 0, SOCKET_ID_ANY); 6275 if (!sfx_actions) 6276 return rte_flow_error_set(error, ENOMEM, 6277 RTE_FLOW_ERROR_TYPE_ACTION, 6278 NULL, "no memory to split " 6279 "sample flow"); 6280 /* The representor_id is UINT16_MAX for uplink. */ 6281 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6282 /* 6283 * When reg_c_preserve is set, metadata registers Cx preserve 6284 * their value even through packet duplication. 6285 */ 6286 add_tag = (!fdb_tx || 6287 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6288 if (add_tag) 6289 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6290 + act_size); 6291 if (modify_after_mirror) 6292 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6293 next_ft_step; 6294 pre_actions = sfx_actions + actions_n; 6295 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6296 actions, sfx_actions, 6297 pre_actions, actions_n, 6298 sample_action_pos, 6299 qrss_action_pos, jump_table, 6300 error); 6301 if (tag_id < 0 || (add_tag && !tag_id)) { 6302 ret = -rte_errno; 6303 goto exit; 6304 } 6305 if (modify_after_mirror) 6306 flow_split_info->skip_scale = 6307 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6308 /* Add the prefix subflow. */ 6309 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6310 items, pre_actions, 6311 flow_split_info, error); 6312 if (ret) { 6313 ret = -rte_errno; 6314 goto exit; 6315 } 6316 dev_flow->handle->split_flow_id = tag_id; 6317 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6318 if (!modify_after_mirror) { 6319 /* Set the sfx group attr. */ 6320 sample_res = (struct mlx5_flow_dv_sample_resource *) 6321 dev_flow->dv.sample_res; 6322 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6323 sample_res->normal_path_tbl; 6324 sfx_tbl_data = container_of(sfx_tbl, 6325 struct mlx5_flow_tbl_data_entry, 6326 tbl); 6327 sfx_attr.group = sfx_attr.transfer ? 6328 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6329 } else { 6330 MLX5_ASSERT(attr->transfer); 6331 sfx_attr.group = jump_table; 6332 } 6333 flow_split_info->prefix_layers = 6334 flow_get_prefix_layer_flags(dev_flow); 6335 MLX5_ASSERT(wks); 6336 flow_split_info->prefix_mark |= wks->mark; 6337 /* Suffix group level already be scaled with factor, set 6338 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6339 * again in translation. 6340 */ 6341 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6342 #endif 6343 } 6344 /* Add the suffix subflow. */ 6345 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6346 sfx_items ? sfx_items : items, 6347 sfx_actions ? sfx_actions : actions, 6348 flow_split_info, error); 6349 exit: 6350 if (sfx_actions) 6351 mlx5_free(sfx_actions); 6352 return ret; 6353 } 6354 6355 /** 6356 * Split the flow to subflow set. The splitters might be linked 6357 * in the chain, like this: 6358 * flow_create_split_outer() calls: 6359 * flow_create_split_meter() calls: 6360 * flow_create_split_metadata(meter_subflow_0) calls: 6361 * flow_create_split_inner(metadata_subflow_0) 6362 * flow_create_split_inner(metadata_subflow_1) 6363 * flow_create_split_inner(metadata_subflow_2) 6364 * flow_create_split_metadata(meter_subflow_1) calls: 6365 * flow_create_split_inner(metadata_subflow_0) 6366 * flow_create_split_inner(metadata_subflow_1) 6367 * flow_create_split_inner(metadata_subflow_2) 6368 * 6369 * This provide flexible way to add new levels of flow splitting. 6370 * The all of successfully created subflows are included to the 6371 * parent flow dev_flow list. 6372 * 6373 * @param dev 6374 * Pointer to Ethernet device. 6375 * @param[in] flow 6376 * Parent flow structure pointer. 6377 * @param[in] attr 6378 * Flow rule attributes. 6379 * @param[in] items 6380 * Pattern specification (list terminated by the END pattern item). 6381 * @param[in] actions 6382 * Associated actions (list terminated by the END action). 6383 * @param[in] flow_split_info 6384 * Pointer to flow split info structure. 6385 * @param[out] error 6386 * Perform verbose error reporting if not NULL. 6387 * @return 6388 * 0 on success, negative value otherwise 6389 */ 6390 static int 6391 flow_create_split_outer(struct rte_eth_dev *dev, 6392 struct rte_flow *flow, 6393 const struct rte_flow_attr *attr, 6394 const struct rte_flow_item items[], 6395 const struct rte_flow_action actions[], 6396 struct mlx5_flow_split_info *flow_split_info, 6397 struct rte_flow_error *error) 6398 { 6399 int ret; 6400 6401 ret = flow_create_split_sample(dev, flow, attr, items, 6402 actions, flow_split_info, error); 6403 MLX5_ASSERT(ret <= 0); 6404 return ret; 6405 } 6406 6407 static inline struct mlx5_flow_tunnel * 6408 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6409 { 6410 struct mlx5_flow_tunnel *tunnel; 6411 6412 #pragma GCC diagnostic push 6413 #pragma GCC diagnostic ignored "-Wcast-qual" 6414 tunnel = (typeof(tunnel))flow->tunnel; 6415 #pragma GCC diagnostic pop 6416 6417 return tunnel; 6418 } 6419 6420 /** 6421 * Adjust flow RSS workspace if needed. 6422 * 6423 * @param wks 6424 * Pointer to thread flow work space. 6425 * @param rss_desc 6426 * Pointer to RSS descriptor. 6427 * @param[in] nrssq_num 6428 * New RSS queue number. 6429 * 6430 * @return 6431 * 0 on success, -1 otherwise and rte_errno is set. 6432 */ 6433 static int 6434 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6435 struct mlx5_flow_rss_desc *rss_desc, 6436 uint32_t nrssq_num) 6437 { 6438 if (likely(nrssq_num <= wks->rssq_num)) 6439 return 0; 6440 rss_desc->queue = realloc(rss_desc->queue, 6441 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6442 if (!rss_desc->queue) { 6443 rte_errno = ENOMEM; 6444 return -1; 6445 } 6446 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6447 return 0; 6448 } 6449 6450 /** 6451 * Create a flow and add it to @p list. 6452 * 6453 * @param dev 6454 * Pointer to Ethernet device. 6455 * @param list 6456 * Pointer to a TAILQ flow list. If this parameter NULL, 6457 * no list insertion occurred, flow is just created, 6458 * this is caller's responsibility to track the 6459 * created flow. 6460 * @param[in] attr 6461 * Flow rule attributes. 6462 * @param[in] items 6463 * Pattern specification (list terminated by the END pattern item). 6464 * @param[in] actions 6465 * Associated actions (list terminated by the END action). 6466 * @param[in] external 6467 * This flow rule is created by request external to PMD. 6468 * @param[out] error 6469 * Perform verbose error reporting if not NULL. 6470 * 6471 * @return 6472 * A flow index on success, 0 otherwise and rte_errno is set. 6473 */ 6474 static uint32_t 6475 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6476 const struct rte_flow_attr *attr, 6477 const struct rte_flow_item items[], 6478 const struct rte_flow_action original_actions[], 6479 bool external, struct rte_flow_error *error) 6480 { 6481 struct mlx5_priv *priv = dev->data->dev_private; 6482 struct rte_flow *flow = NULL; 6483 struct mlx5_flow *dev_flow; 6484 const struct rte_flow_action_rss *rss = NULL; 6485 struct mlx5_translated_action_handle 6486 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6487 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6488 union { 6489 struct mlx5_flow_expand_rss buf; 6490 uint8_t buffer[4096]; 6491 } expand_buffer; 6492 union { 6493 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6494 uint8_t buffer[2048]; 6495 } actions_rx; 6496 union { 6497 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6498 uint8_t buffer[2048]; 6499 } actions_hairpin_tx; 6500 union { 6501 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6502 uint8_t buffer[2048]; 6503 } items_tx; 6504 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6505 struct mlx5_flow_rss_desc *rss_desc; 6506 const struct rte_flow_action *p_actions_rx; 6507 uint32_t i; 6508 uint32_t idx = 0; 6509 int hairpin_flow; 6510 struct rte_flow_attr attr_tx = { .priority = 0 }; 6511 const struct rte_flow_action *actions; 6512 struct rte_flow_action *translated_actions = NULL; 6513 struct mlx5_flow_tunnel *tunnel; 6514 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6515 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6516 struct mlx5_flow_split_info flow_split_info = { 6517 .external = !!external, 6518 .skip_scale = 0, 6519 .flow_idx = 0, 6520 .prefix_mark = 0, 6521 .prefix_layers = 0, 6522 .table_id = 0 6523 }; 6524 int ret; 6525 6526 MLX5_ASSERT(wks); 6527 rss_desc = &wks->rss_desc; 6528 ret = flow_action_handles_translate(dev, original_actions, 6529 indir_actions, 6530 &indir_actions_n, 6531 &translated_actions, error); 6532 if (ret < 0) { 6533 MLX5_ASSERT(translated_actions == NULL); 6534 return 0; 6535 } 6536 actions = translated_actions ? translated_actions : original_actions; 6537 p_actions_rx = actions; 6538 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6539 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6540 external, hairpin_flow, error); 6541 if (ret < 0) 6542 goto error_before_hairpin_split; 6543 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6544 if (!flow) { 6545 rte_errno = ENOMEM; 6546 goto error_before_hairpin_split; 6547 } 6548 if (hairpin_flow > 0) { 6549 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6550 rte_errno = EINVAL; 6551 goto error_before_hairpin_split; 6552 } 6553 flow_hairpin_split(dev, actions, actions_rx.actions, 6554 actions_hairpin_tx.actions, items_tx.items, 6555 idx); 6556 p_actions_rx = actions_rx.actions; 6557 } 6558 flow_split_info.flow_idx = idx; 6559 flow->drv_type = flow_get_drv_type(dev, attr); 6560 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6561 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6562 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6563 /* RSS Action only works on NIC RX domain */ 6564 if (attr->ingress && !attr->transfer) 6565 rss = flow_get_rss_action(dev, p_actions_rx); 6566 if (rss) { 6567 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6568 return 0; 6569 /* 6570 * The following information is required by 6571 * mlx5_flow_hashfields_adjust() in advance. 6572 */ 6573 rss_desc->level = rss->level; 6574 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6575 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6576 } 6577 flow->dev_handles = 0; 6578 if (rss && rss->types) { 6579 unsigned int graph_root; 6580 6581 graph_root = find_graph_root(rss->level); 6582 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6583 items, rss->types, 6584 mlx5_support_expansion, graph_root); 6585 MLX5_ASSERT(ret > 0 && 6586 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6587 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6588 for (i = 0; i < buf->entries; ++i) 6589 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6590 } 6591 } else { 6592 buf->entries = 1; 6593 buf->entry[0].pattern = (void *)(uintptr_t)items; 6594 } 6595 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6596 indir_actions_n); 6597 for (i = 0; i < buf->entries; ++i) { 6598 /* Initialize flow split data. */ 6599 flow_split_info.prefix_layers = 0; 6600 flow_split_info.prefix_mark = 0; 6601 flow_split_info.skip_scale = 0; 6602 /* 6603 * The splitter may create multiple dev_flows, 6604 * depending on configuration. In the simplest 6605 * case it just creates unmodified original flow. 6606 */ 6607 ret = flow_create_split_outer(dev, flow, attr, 6608 buf->entry[i].pattern, 6609 p_actions_rx, &flow_split_info, 6610 error); 6611 if (ret < 0) 6612 goto error; 6613 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6614 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6615 p_actions_rx, 6616 idx, 6617 wks->flows[0].tunnel, 6618 &default_miss_ctx, 6619 error); 6620 if (ret < 0) { 6621 mlx5_free(default_miss_ctx.queue); 6622 goto error; 6623 } 6624 } 6625 } 6626 /* Create the tx flow. */ 6627 if (hairpin_flow) { 6628 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6629 attr_tx.ingress = 0; 6630 attr_tx.egress = 1; 6631 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6632 actions_hairpin_tx.actions, 6633 idx, error); 6634 if (!dev_flow) 6635 goto error; 6636 dev_flow->flow = flow; 6637 dev_flow->external = 0; 6638 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6639 dev_flow->handle, next); 6640 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6641 items_tx.items, 6642 actions_hairpin_tx.actions, error); 6643 if (ret < 0) 6644 goto error; 6645 } 6646 /* 6647 * Update the metadata register copy table. If extensive 6648 * metadata feature is enabled and registers are supported 6649 * we might create the extra rte_flow for each unique 6650 * MARK/FLAG action ID. 6651 * 6652 * The table is updated for ingress Flows only, because 6653 * the egress Flows belong to the different device and 6654 * copy table should be updated in peer NIC Rx domain. 6655 */ 6656 if (attr->ingress && 6657 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6658 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6659 if (ret) 6660 goto error; 6661 } 6662 /* 6663 * If the flow is external (from application) OR device is started, 6664 * OR mreg discover, then apply immediately. 6665 */ 6666 if (external || dev->data->dev_started || 6667 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6668 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6669 ret = flow_drv_apply(dev, flow, error); 6670 if (ret < 0) 6671 goto error; 6672 } 6673 flow->type = type; 6674 flow_rxq_flags_set(dev, flow); 6675 rte_free(translated_actions); 6676 tunnel = flow_tunnel_from_rule(wks->flows); 6677 if (tunnel) { 6678 flow->tunnel = 1; 6679 flow->tunnel_id = tunnel->tunnel_id; 6680 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6681 mlx5_free(default_miss_ctx.queue); 6682 } 6683 mlx5_flow_pop_thread_workspace(); 6684 return idx; 6685 error: 6686 MLX5_ASSERT(flow); 6687 ret = rte_errno; /* Save rte_errno before cleanup. */ 6688 flow_mreg_del_copy_action(dev, flow); 6689 flow_drv_destroy(dev, flow); 6690 if (rss_desc->shared_rss) 6691 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6692 mlx5_ipool_get 6693 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6694 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6695 mlx5_ipool_free(priv->flows[type], idx); 6696 rte_errno = ret; /* Restore rte_errno. */ 6697 ret = rte_errno; 6698 rte_errno = ret; 6699 mlx5_flow_pop_thread_workspace(); 6700 error_before_hairpin_split: 6701 rte_free(translated_actions); 6702 return 0; 6703 } 6704 6705 /** 6706 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6707 * incoming packets to table 1. 6708 * 6709 * Other flow rules, requested for group n, will be created in 6710 * e-switch table n+1. 6711 * Jump action to e-switch group n will be created to group n+1. 6712 * 6713 * Used when working in switchdev mode, to utilise advantages of table 1 6714 * and above. 6715 * 6716 * @param dev 6717 * Pointer to Ethernet device. 6718 * 6719 * @return 6720 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6721 */ 6722 struct rte_flow * 6723 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6724 { 6725 const struct rte_flow_attr attr = { 6726 .group = 0, 6727 .priority = 0, 6728 .ingress = 1, 6729 .egress = 0, 6730 .transfer = 1, 6731 }; 6732 const struct rte_flow_item pattern = { 6733 .type = RTE_FLOW_ITEM_TYPE_END, 6734 }; 6735 struct rte_flow_action_jump jump = { 6736 .group = 1, 6737 }; 6738 const struct rte_flow_action actions[] = { 6739 { 6740 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6741 .conf = &jump, 6742 }, 6743 { 6744 .type = RTE_FLOW_ACTION_TYPE_END, 6745 }, 6746 }; 6747 struct rte_flow_error error; 6748 6749 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6750 &attr, &pattern, 6751 actions, false, &error); 6752 } 6753 6754 /** 6755 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6756 * and sq number, directs all packets to peer vport. 6757 * 6758 * @param dev 6759 * Pointer to Ethernet device. 6760 * @param txq 6761 * Txq index. 6762 * 6763 * @return 6764 * Flow ID on success, 0 otherwise and rte_errno is set. 6765 */ 6766 uint32_t 6767 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6768 { 6769 struct rte_flow_attr attr = { 6770 .group = 0, 6771 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6772 .ingress = 1, 6773 .egress = 0, 6774 .transfer = 1, 6775 }; 6776 struct rte_flow_item_port_id port_spec = { 6777 .id = MLX5_PORT_ESW_MGR, 6778 }; 6779 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6780 .queue = txq, 6781 }; 6782 struct rte_flow_item pattern[] = { 6783 { 6784 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6785 .spec = &port_spec, 6786 }, 6787 { 6788 .type = (enum rte_flow_item_type) 6789 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6790 .spec = &txq_spec, 6791 }, 6792 { 6793 .type = RTE_FLOW_ITEM_TYPE_END, 6794 }, 6795 }; 6796 struct rte_flow_action_jump jump = { 6797 .group = 1, 6798 }; 6799 struct rte_flow_action_port_id port = { 6800 .id = dev->data->port_id, 6801 }; 6802 struct rte_flow_action actions[] = { 6803 { 6804 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6805 .conf = &jump, 6806 }, 6807 { 6808 .type = RTE_FLOW_ACTION_TYPE_END, 6809 }, 6810 }; 6811 struct rte_flow_error error; 6812 6813 /* 6814 * Creates group 0, highest priority jump flow. 6815 * Matches txq to bypass kernel packets. 6816 */ 6817 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6818 false, &error) == 0) 6819 return 0; 6820 /* Create group 1, lowest priority redirect flow for txq. */ 6821 attr.group = 1; 6822 actions[0].conf = &port; 6823 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6824 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6825 actions, false, &error); 6826 } 6827 6828 /** 6829 * Validate a flow supported by the NIC. 6830 * 6831 * @see rte_flow_validate() 6832 * @see rte_flow_ops 6833 */ 6834 int 6835 mlx5_flow_validate(struct rte_eth_dev *dev, 6836 const struct rte_flow_attr *attr, 6837 const struct rte_flow_item items[], 6838 const struct rte_flow_action original_actions[], 6839 struct rte_flow_error *error) 6840 { 6841 int hairpin_flow; 6842 struct mlx5_translated_action_handle 6843 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6844 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6845 const struct rte_flow_action *actions; 6846 struct rte_flow_action *translated_actions = NULL; 6847 int ret = flow_action_handles_translate(dev, original_actions, 6848 indir_actions, 6849 &indir_actions_n, 6850 &translated_actions, error); 6851 6852 if (ret) 6853 return ret; 6854 actions = translated_actions ? translated_actions : original_actions; 6855 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6856 ret = flow_drv_validate(dev, attr, items, actions, 6857 true, hairpin_flow, error); 6858 rte_free(translated_actions); 6859 return ret; 6860 } 6861 6862 /** 6863 * Create a flow. 6864 * 6865 * @see rte_flow_create() 6866 * @see rte_flow_ops 6867 */ 6868 struct rte_flow * 6869 mlx5_flow_create(struct rte_eth_dev *dev, 6870 const struct rte_flow_attr *attr, 6871 const struct rte_flow_item items[], 6872 const struct rte_flow_action actions[], 6873 struct rte_flow_error *error) 6874 { 6875 struct mlx5_priv *priv = dev->data->dev_private; 6876 6877 if (priv->sh->config.dv_flow_en == 2) { 6878 rte_flow_error_set(error, ENOTSUP, 6879 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6880 NULL, 6881 "Flow non-Q creation not supported"); 6882 return NULL; 6883 } 6884 /* 6885 * If the device is not started yet, it is not allowed to created a 6886 * flow from application. PMD default flows and traffic control flows 6887 * are not affected. 6888 */ 6889 if (unlikely(!dev->data->dev_started)) { 6890 DRV_LOG(DEBUG, "port %u is not started when " 6891 "inserting a flow", dev->data->port_id); 6892 rte_flow_error_set(error, ENODEV, 6893 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6894 NULL, 6895 "port not started"); 6896 return NULL; 6897 } 6898 6899 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6900 attr, items, actions, 6901 true, error); 6902 } 6903 6904 /** 6905 * Destroy a flow in a list. 6906 * 6907 * @param dev 6908 * Pointer to Ethernet device. 6909 * @param[in] flow_idx 6910 * Index of flow to destroy. 6911 */ 6912 static void 6913 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6914 uint32_t flow_idx) 6915 { 6916 struct mlx5_priv *priv = dev->data->dev_private; 6917 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6918 6919 if (!flow) 6920 return; 6921 MLX5_ASSERT(flow->type == type); 6922 /* 6923 * Update RX queue flags only if port is started, otherwise it is 6924 * already clean. 6925 */ 6926 if (dev->data->dev_started) 6927 flow_rxq_flags_trim(dev, flow); 6928 flow_drv_destroy(dev, flow); 6929 if (flow->tunnel) { 6930 struct mlx5_flow_tunnel *tunnel; 6931 6932 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6933 RTE_VERIFY(tunnel); 6934 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6935 mlx5_flow_tunnel_free(dev, tunnel); 6936 } 6937 flow_mreg_del_copy_action(dev, flow); 6938 mlx5_ipool_free(priv->flows[type], flow_idx); 6939 } 6940 6941 /** 6942 * Destroy all flows. 6943 * 6944 * @param dev 6945 * Pointer to Ethernet device. 6946 * @param type 6947 * Flow type to be flushed. 6948 * @param active 6949 * If flushing is called actively. 6950 */ 6951 void 6952 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6953 bool active) 6954 { 6955 struct mlx5_priv *priv = dev->data->dev_private; 6956 uint32_t num_flushed = 0, fidx = 1; 6957 struct rte_flow *flow; 6958 6959 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6960 flow_list_destroy(dev, type, fidx); 6961 num_flushed++; 6962 } 6963 if (active) { 6964 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6965 dev->data->port_id, num_flushed); 6966 } 6967 } 6968 6969 /** 6970 * Stop all default actions for flows. 6971 * 6972 * @param dev 6973 * Pointer to Ethernet device. 6974 */ 6975 void 6976 mlx5_flow_stop_default(struct rte_eth_dev *dev) 6977 { 6978 flow_mreg_del_default_copy_action(dev); 6979 flow_rxq_flags_clear(dev); 6980 } 6981 6982 /** 6983 * Start all default actions for flows. 6984 * 6985 * @param dev 6986 * Pointer to Ethernet device. 6987 * @return 6988 * 0 on success, a negative errno value otherwise and rte_errno is set. 6989 */ 6990 int 6991 mlx5_flow_start_default(struct rte_eth_dev *dev) 6992 { 6993 struct rte_flow_error error; 6994 6995 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 6996 return flow_mreg_add_default_copy_action(dev, &error); 6997 } 6998 6999 /** 7000 * Release key of thread specific flow workspace data. 7001 */ 7002 void 7003 flow_release_workspace(void *data) 7004 { 7005 struct mlx5_flow_workspace *wks = data; 7006 struct mlx5_flow_workspace *next; 7007 7008 while (wks) { 7009 next = wks->next; 7010 free(wks->rss_desc.queue); 7011 free(wks); 7012 wks = next; 7013 } 7014 } 7015 7016 /** 7017 * Get thread specific current flow workspace. 7018 * 7019 * @return pointer to thread specific flow workspace data, NULL on error. 7020 */ 7021 struct mlx5_flow_workspace* 7022 mlx5_flow_get_thread_workspace(void) 7023 { 7024 struct mlx5_flow_workspace *data; 7025 7026 data = mlx5_flow_os_get_specific_workspace(); 7027 MLX5_ASSERT(data && data->inuse); 7028 if (!data || !data->inuse) 7029 DRV_LOG(ERR, "flow workspace not initialized."); 7030 return data; 7031 } 7032 7033 /** 7034 * Allocate and init new flow workspace. 7035 * 7036 * @return pointer to flow workspace data, NULL on error. 7037 */ 7038 static struct mlx5_flow_workspace* 7039 flow_alloc_thread_workspace(void) 7040 { 7041 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7042 7043 if (!data) { 7044 DRV_LOG(ERR, "Failed to allocate flow workspace " 7045 "memory."); 7046 return NULL; 7047 } 7048 data->rss_desc.queue = calloc(1, 7049 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7050 if (!data->rss_desc.queue) 7051 goto err; 7052 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7053 return data; 7054 err: 7055 free(data->rss_desc.queue); 7056 free(data); 7057 return NULL; 7058 } 7059 7060 /** 7061 * Get new thread specific flow workspace. 7062 * 7063 * If current workspace inuse, create new one and set as current. 7064 * 7065 * @return pointer to thread specific flow workspace data, NULL on error. 7066 */ 7067 static struct mlx5_flow_workspace* 7068 mlx5_flow_push_thread_workspace(void) 7069 { 7070 struct mlx5_flow_workspace *curr; 7071 struct mlx5_flow_workspace *data; 7072 7073 curr = mlx5_flow_os_get_specific_workspace(); 7074 if (!curr) { 7075 data = flow_alloc_thread_workspace(); 7076 if (!data) 7077 return NULL; 7078 } else if (!curr->inuse) { 7079 data = curr; 7080 } else if (curr->next) { 7081 data = curr->next; 7082 } else { 7083 data = flow_alloc_thread_workspace(); 7084 if (!data) 7085 return NULL; 7086 curr->next = data; 7087 data->prev = curr; 7088 } 7089 data->inuse = 1; 7090 data->flow_idx = 0; 7091 /* Set as current workspace */ 7092 if (mlx5_flow_os_set_specific_workspace(data)) 7093 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7094 return data; 7095 } 7096 7097 /** 7098 * Close current thread specific flow workspace. 7099 * 7100 * If previous workspace available, set it as current. 7101 * 7102 * @return pointer to thread specific flow workspace data, NULL on error. 7103 */ 7104 static void 7105 mlx5_flow_pop_thread_workspace(void) 7106 { 7107 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7108 7109 if (!data) 7110 return; 7111 if (!data->inuse) { 7112 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7113 return; 7114 } 7115 data->inuse = 0; 7116 if (!data->prev) 7117 return; 7118 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7119 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7120 } 7121 7122 /** 7123 * Verify the flow list is empty 7124 * 7125 * @param dev 7126 * Pointer to Ethernet device. 7127 * 7128 * @return the number of flows not released. 7129 */ 7130 int 7131 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7132 { 7133 struct mlx5_priv *priv = dev->data->dev_private; 7134 struct rte_flow *flow; 7135 uint32_t idx = 0; 7136 int ret = 0, i; 7137 7138 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7139 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7140 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7141 dev->data->port_id, (void *)flow); 7142 ret++; 7143 } 7144 } 7145 return ret; 7146 } 7147 7148 /** 7149 * Enable default hairpin egress flow. 7150 * 7151 * @param dev 7152 * Pointer to Ethernet device. 7153 * @param queue 7154 * The queue index. 7155 * 7156 * @return 7157 * 0 on success, a negative errno value otherwise and rte_errno is set. 7158 */ 7159 int 7160 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7161 uint32_t queue) 7162 { 7163 const struct rte_flow_attr attr = { 7164 .egress = 1, 7165 .priority = 0, 7166 }; 7167 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7168 .queue = queue, 7169 }; 7170 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7171 .queue = UINT32_MAX, 7172 }; 7173 struct rte_flow_item items[] = { 7174 { 7175 .type = (enum rte_flow_item_type) 7176 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7177 .spec = &queue_spec, 7178 .last = NULL, 7179 .mask = &queue_mask, 7180 }, 7181 { 7182 .type = RTE_FLOW_ITEM_TYPE_END, 7183 }, 7184 }; 7185 struct rte_flow_action_jump jump = { 7186 .group = MLX5_HAIRPIN_TX_TABLE, 7187 }; 7188 struct rte_flow_action actions[2]; 7189 uint32_t flow_idx; 7190 struct rte_flow_error error; 7191 7192 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7193 actions[0].conf = &jump; 7194 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7195 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7196 &attr, items, actions, false, &error); 7197 if (!flow_idx) { 7198 DRV_LOG(DEBUG, 7199 "Failed to create ctrl flow: rte_errno(%d)," 7200 " type(%d), message(%s)", 7201 rte_errno, error.type, 7202 error.message ? error.message : " (no stated reason)"); 7203 return -rte_errno; 7204 } 7205 return 0; 7206 } 7207 7208 /** 7209 * Enable a control flow configured from the control plane. 7210 * 7211 * @param dev 7212 * Pointer to Ethernet device. 7213 * @param eth_spec 7214 * An Ethernet flow spec to apply. 7215 * @param eth_mask 7216 * An Ethernet flow mask to apply. 7217 * @param vlan_spec 7218 * A VLAN flow spec to apply. 7219 * @param vlan_mask 7220 * A VLAN flow mask to apply. 7221 * 7222 * @return 7223 * 0 on success, a negative errno value otherwise and rte_errno is set. 7224 */ 7225 int 7226 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7227 struct rte_flow_item_eth *eth_spec, 7228 struct rte_flow_item_eth *eth_mask, 7229 struct rte_flow_item_vlan *vlan_spec, 7230 struct rte_flow_item_vlan *vlan_mask) 7231 { 7232 struct mlx5_priv *priv = dev->data->dev_private; 7233 const struct rte_flow_attr attr = { 7234 .ingress = 1, 7235 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7236 }; 7237 struct rte_flow_item items[] = { 7238 { 7239 .type = RTE_FLOW_ITEM_TYPE_ETH, 7240 .spec = eth_spec, 7241 .last = NULL, 7242 .mask = eth_mask, 7243 }, 7244 { 7245 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7246 RTE_FLOW_ITEM_TYPE_END, 7247 .spec = vlan_spec, 7248 .last = NULL, 7249 .mask = vlan_mask, 7250 }, 7251 { 7252 .type = RTE_FLOW_ITEM_TYPE_END, 7253 }, 7254 }; 7255 uint16_t queue[priv->reta_idx_n]; 7256 struct rte_flow_action_rss action_rss = { 7257 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7258 .level = 0, 7259 .types = priv->rss_conf.rss_hf, 7260 .key_len = priv->rss_conf.rss_key_len, 7261 .queue_num = priv->reta_idx_n, 7262 .key = priv->rss_conf.rss_key, 7263 .queue = queue, 7264 }; 7265 struct rte_flow_action actions[] = { 7266 { 7267 .type = RTE_FLOW_ACTION_TYPE_RSS, 7268 .conf = &action_rss, 7269 }, 7270 { 7271 .type = RTE_FLOW_ACTION_TYPE_END, 7272 }, 7273 }; 7274 uint32_t flow_idx; 7275 struct rte_flow_error error; 7276 unsigned int i; 7277 7278 if (!priv->reta_idx_n || !priv->rxqs_n) { 7279 return 0; 7280 } 7281 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7282 action_rss.types = 0; 7283 for (i = 0; i != priv->reta_idx_n; ++i) 7284 queue[i] = (*priv->reta_idx)[i]; 7285 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7286 &attr, items, actions, false, &error); 7287 if (!flow_idx) 7288 return -rte_errno; 7289 return 0; 7290 } 7291 7292 /** 7293 * Enable a flow control configured from the control plane. 7294 * 7295 * @param dev 7296 * Pointer to Ethernet device. 7297 * @param eth_spec 7298 * An Ethernet flow spec to apply. 7299 * @param eth_mask 7300 * An Ethernet flow mask to apply. 7301 * 7302 * @return 7303 * 0 on success, a negative errno value otherwise and rte_errno is set. 7304 */ 7305 int 7306 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7307 struct rte_flow_item_eth *eth_spec, 7308 struct rte_flow_item_eth *eth_mask) 7309 { 7310 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7311 } 7312 7313 /** 7314 * Create default miss flow rule matching lacp traffic 7315 * 7316 * @param dev 7317 * Pointer to Ethernet device. 7318 * @param eth_spec 7319 * An Ethernet flow spec to apply. 7320 * 7321 * @return 7322 * 0 on success, a negative errno value otherwise and rte_errno is set. 7323 */ 7324 int 7325 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7326 { 7327 /* 7328 * The LACP matching is done by only using ether type since using 7329 * a multicast dst mac causes kernel to give low priority to this flow. 7330 */ 7331 static const struct rte_flow_item_eth lacp_spec = { 7332 .type = RTE_BE16(0x8809), 7333 }; 7334 static const struct rte_flow_item_eth lacp_mask = { 7335 .type = 0xffff, 7336 }; 7337 const struct rte_flow_attr attr = { 7338 .ingress = 1, 7339 }; 7340 struct rte_flow_item items[] = { 7341 { 7342 .type = RTE_FLOW_ITEM_TYPE_ETH, 7343 .spec = &lacp_spec, 7344 .mask = &lacp_mask, 7345 }, 7346 { 7347 .type = RTE_FLOW_ITEM_TYPE_END, 7348 }, 7349 }; 7350 struct rte_flow_action actions[] = { 7351 { 7352 .type = (enum rte_flow_action_type) 7353 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7354 }, 7355 { 7356 .type = RTE_FLOW_ACTION_TYPE_END, 7357 }, 7358 }; 7359 struct rte_flow_error error; 7360 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7361 &attr, items, actions, 7362 false, &error); 7363 7364 if (!flow_idx) 7365 return -rte_errno; 7366 return 0; 7367 } 7368 7369 /** 7370 * Destroy a flow. 7371 * 7372 * @see rte_flow_destroy() 7373 * @see rte_flow_ops 7374 */ 7375 int 7376 mlx5_flow_destroy(struct rte_eth_dev *dev, 7377 struct rte_flow *flow, 7378 struct rte_flow_error *error __rte_unused) 7379 { 7380 struct mlx5_priv *priv = dev->data->dev_private; 7381 7382 if (priv->sh->config.dv_flow_en == 2) 7383 return rte_flow_error_set(error, ENOTSUP, 7384 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7385 NULL, 7386 "Flow non-Q destruction not supported"); 7387 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7388 (uintptr_t)(void *)flow); 7389 return 0; 7390 } 7391 7392 /** 7393 * Destroy all flows. 7394 * 7395 * @see rte_flow_flush() 7396 * @see rte_flow_ops 7397 */ 7398 int 7399 mlx5_flow_flush(struct rte_eth_dev *dev, 7400 struct rte_flow_error *error __rte_unused) 7401 { 7402 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7403 return 0; 7404 } 7405 7406 /** 7407 * Isolated mode. 7408 * 7409 * @see rte_flow_isolate() 7410 * @see rte_flow_ops 7411 */ 7412 int 7413 mlx5_flow_isolate(struct rte_eth_dev *dev, 7414 int enable, 7415 struct rte_flow_error *error) 7416 { 7417 struct mlx5_priv *priv = dev->data->dev_private; 7418 7419 if (dev->data->dev_started) { 7420 rte_flow_error_set(error, EBUSY, 7421 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7422 NULL, 7423 "port must be stopped first"); 7424 return -rte_errno; 7425 } 7426 priv->isolated = !!enable; 7427 if (enable) 7428 dev->dev_ops = &mlx5_dev_ops_isolate; 7429 else 7430 dev->dev_ops = &mlx5_dev_ops; 7431 7432 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7433 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7434 7435 return 0; 7436 } 7437 7438 /** 7439 * Query a flow. 7440 * 7441 * @see rte_flow_query() 7442 * @see rte_flow_ops 7443 */ 7444 static int 7445 flow_drv_query(struct rte_eth_dev *dev, 7446 uint32_t flow_idx, 7447 const struct rte_flow_action *actions, 7448 void *data, 7449 struct rte_flow_error *error) 7450 { 7451 struct mlx5_priv *priv = dev->data->dev_private; 7452 const struct mlx5_flow_driver_ops *fops; 7453 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7454 flow_idx); 7455 enum mlx5_flow_drv_type ftype; 7456 7457 if (!flow) { 7458 return rte_flow_error_set(error, ENOENT, 7459 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7460 NULL, 7461 "invalid flow handle"); 7462 } 7463 ftype = flow->drv_type; 7464 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7465 fops = flow_get_drv_ops(ftype); 7466 7467 return fops->query(dev, flow, actions, data, error); 7468 } 7469 7470 /** 7471 * Query a flow. 7472 * 7473 * @see rte_flow_query() 7474 * @see rte_flow_ops 7475 */ 7476 int 7477 mlx5_flow_query(struct rte_eth_dev *dev, 7478 struct rte_flow *flow, 7479 const struct rte_flow_action *actions, 7480 void *data, 7481 struct rte_flow_error *error) 7482 { 7483 int ret; 7484 struct mlx5_priv *priv = dev->data->dev_private; 7485 7486 if (priv->sh->config.dv_flow_en == 2) 7487 return rte_flow_error_set(error, ENOTSUP, 7488 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7489 NULL, 7490 "Flow non-Q query not supported"); 7491 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7492 error); 7493 if (ret < 0) 7494 return ret; 7495 return 0; 7496 } 7497 7498 /** 7499 * Get rte_flow callbacks. 7500 * 7501 * @param dev 7502 * Pointer to Ethernet device structure. 7503 * @param ops 7504 * Pointer to operation-specific structure. 7505 * 7506 * @return 0 7507 */ 7508 int 7509 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7510 const struct rte_flow_ops **ops) 7511 { 7512 *ops = &mlx5_flow_ops; 7513 return 0; 7514 } 7515 7516 /** 7517 * Validate meter policy actions. 7518 * Dispatcher for action type specific validation. 7519 * 7520 * @param[in] dev 7521 * Pointer to the Ethernet device structure. 7522 * @param[in] action 7523 * The meter policy action object to validate. 7524 * @param[in] attr 7525 * Attributes of flow to determine steering domain. 7526 * @param[out] is_rss 7527 * Is RSS or not. 7528 * @param[out] domain_bitmap 7529 * Domain bitmap. 7530 * @param[out] is_def_policy 7531 * Is default policy or not. 7532 * @param[out] error 7533 * Perform verbose error reporting if not NULL. Initialized in case of 7534 * error only. 7535 * 7536 * @return 7537 * 0 on success, otherwise negative errno value. 7538 */ 7539 int 7540 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7541 const struct rte_flow_action *actions[RTE_COLORS], 7542 struct rte_flow_attr *attr, 7543 bool *is_rss, 7544 uint8_t *domain_bitmap, 7545 uint8_t *policy_mode, 7546 struct rte_mtr_error *error) 7547 { 7548 const struct mlx5_flow_driver_ops *fops; 7549 7550 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7551 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7552 domain_bitmap, policy_mode, error); 7553 } 7554 7555 /** 7556 * Destroy the meter table set. 7557 * 7558 * @param[in] dev 7559 * Pointer to Ethernet device. 7560 * @param[in] mtr_policy 7561 * Meter policy struct. 7562 */ 7563 void 7564 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7565 struct mlx5_flow_meter_policy *mtr_policy) 7566 { 7567 const struct mlx5_flow_driver_ops *fops; 7568 7569 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7570 fops->destroy_mtr_acts(dev, mtr_policy); 7571 } 7572 7573 /** 7574 * Create policy action, lock free, 7575 * (mutex should be acquired by caller). 7576 * Dispatcher for action type specific call. 7577 * 7578 * @param[in] dev 7579 * Pointer to the Ethernet device structure. 7580 * @param[in] mtr_policy 7581 * Meter policy struct. 7582 * @param[in] action 7583 * Action specification used to create meter actions. 7584 * @param[out] error 7585 * Perform verbose error reporting if not NULL. Initialized in case of 7586 * error only. 7587 * 7588 * @return 7589 * 0 on success, otherwise negative errno value. 7590 */ 7591 int 7592 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7593 struct mlx5_flow_meter_policy *mtr_policy, 7594 const struct rte_flow_action *actions[RTE_COLORS], 7595 struct rte_mtr_error *error) 7596 { 7597 const struct mlx5_flow_driver_ops *fops; 7598 7599 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7600 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7601 } 7602 7603 /** 7604 * Create policy rules, lock free, 7605 * (mutex should be acquired by caller). 7606 * Dispatcher for action type specific call. 7607 * 7608 * @param[in] dev 7609 * Pointer to the Ethernet device structure. 7610 * @param[in] mtr_policy 7611 * Meter policy struct. 7612 * 7613 * @return 7614 * 0 on success, -1 otherwise. 7615 */ 7616 int 7617 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7618 struct mlx5_flow_meter_policy *mtr_policy) 7619 { 7620 const struct mlx5_flow_driver_ops *fops; 7621 7622 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7623 return fops->create_policy_rules(dev, mtr_policy); 7624 } 7625 7626 /** 7627 * Destroy policy rules, lock free, 7628 * (mutex should be acquired by caller). 7629 * Dispatcher for action type specific call. 7630 * 7631 * @param[in] dev 7632 * Pointer to the Ethernet device structure. 7633 * @param[in] mtr_policy 7634 * Meter policy struct. 7635 */ 7636 void 7637 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7638 struct mlx5_flow_meter_policy *mtr_policy) 7639 { 7640 const struct mlx5_flow_driver_ops *fops; 7641 7642 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7643 fops->destroy_policy_rules(dev, mtr_policy); 7644 } 7645 7646 /** 7647 * Destroy the default policy table set. 7648 * 7649 * @param[in] dev 7650 * Pointer to Ethernet device. 7651 */ 7652 void 7653 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7654 { 7655 const struct mlx5_flow_driver_ops *fops; 7656 7657 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7658 fops->destroy_def_policy(dev); 7659 } 7660 7661 /** 7662 * Destroy the default policy table set. 7663 * 7664 * @param[in] dev 7665 * Pointer to Ethernet device. 7666 * 7667 * @return 7668 * 0 on success, -1 otherwise. 7669 */ 7670 int 7671 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7672 { 7673 const struct mlx5_flow_driver_ops *fops; 7674 7675 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7676 return fops->create_def_policy(dev); 7677 } 7678 7679 /** 7680 * Create the needed meter and suffix tables. 7681 * 7682 * @param[in] dev 7683 * Pointer to Ethernet device. 7684 * 7685 * @return 7686 * 0 on success, -1 otherwise. 7687 */ 7688 int 7689 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7690 struct mlx5_flow_meter_info *fm, 7691 uint32_t mtr_idx, 7692 uint8_t domain_bitmap) 7693 { 7694 const struct mlx5_flow_driver_ops *fops; 7695 7696 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7697 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7698 } 7699 7700 /** 7701 * Destroy the meter table set. 7702 * 7703 * @param[in] dev 7704 * Pointer to Ethernet device. 7705 * @param[in] tbl 7706 * Pointer to the meter table set. 7707 */ 7708 void 7709 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7710 struct mlx5_flow_meter_info *fm) 7711 { 7712 const struct mlx5_flow_driver_ops *fops; 7713 7714 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7715 fops->destroy_mtr_tbls(dev, fm); 7716 } 7717 7718 /** 7719 * Destroy the global meter drop table. 7720 * 7721 * @param[in] dev 7722 * Pointer to Ethernet device. 7723 */ 7724 void 7725 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7726 { 7727 const struct mlx5_flow_driver_ops *fops; 7728 7729 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7730 fops->destroy_mtr_drop_tbls(dev); 7731 } 7732 7733 /** 7734 * Destroy the sub policy table with RX queue. 7735 * 7736 * @param[in] dev 7737 * Pointer to Ethernet device. 7738 * @param[in] mtr_policy 7739 * Pointer to meter policy table. 7740 */ 7741 void 7742 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7743 struct mlx5_flow_meter_policy *mtr_policy) 7744 { 7745 const struct mlx5_flow_driver_ops *fops; 7746 7747 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7748 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7749 } 7750 7751 /** 7752 * Allocate the needed aso flow meter id. 7753 * 7754 * @param[in] dev 7755 * Pointer to Ethernet device. 7756 * 7757 * @return 7758 * Index to aso flow meter on success, NULL otherwise. 7759 */ 7760 uint32_t 7761 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7762 { 7763 const struct mlx5_flow_driver_ops *fops; 7764 7765 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7766 return fops->create_meter(dev); 7767 } 7768 7769 /** 7770 * Free the aso flow meter id. 7771 * 7772 * @param[in] dev 7773 * Pointer to Ethernet device. 7774 * @param[in] mtr_idx 7775 * Index to aso flow meter to be free. 7776 * 7777 * @return 7778 * 0 on success. 7779 */ 7780 void 7781 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7782 { 7783 const struct mlx5_flow_driver_ops *fops; 7784 7785 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7786 fops->free_meter(dev, mtr_idx); 7787 } 7788 7789 /** 7790 * Allocate a counter. 7791 * 7792 * @param[in] dev 7793 * Pointer to Ethernet device structure. 7794 * 7795 * @return 7796 * Index to allocated counter on success, 0 otherwise. 7797 */ 7798 uint32_t 7799 mlx5_counter_alloc(struct rte_eth_dev *dev) 7800 { 7801 const struct mlx5_flow_driver_ops *fops; 7802 struct rte_flow_attr attr = { .transfer = 0 }; 7803 7804 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7805 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7806 return fops->counter_alloc(dev); 7807 } 7808 DRV_LOG(ERR, 7809 "port %u counter allocate is not supported.", 7810 dev->data->port_id); 7811 return 0; 7812 } 7813 7814 /** 7815 * Free a counter. 7816 * 7817 * @param[in] dev 7818 * Pointer to Ethernet device structure. 7819 * @param[in] cnt 7820 * Index to counter to be free. 7821 */ 7822 void 7823 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7824 { 7825 const struct mlx5_flow_driver_ops *fops; 7826 struct rte_flow_attr attr = { .transfer = 0 }; 7827 7828 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7829 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7830 fops->counter_free(dev, cnt); 7831 return; 7832 } 7833 DRV_LOG(ERR, 7834 "port %u counter free is not supported.", 7835 dev->data->port_id); 7836 } 7837 7838 /** 7839 * Query counter statistics. 7840 * 7841 * @param[in] dev 7842 * Pointer to Ethernet device structure. 7843 * @param[in] cnt 7844 * Index to counter to query. 7845 * @param[in] clear 7846 * Set to clear counter statistics. 7847 * @param[out] pkts 7848 * The counter hits packets number to save. 7849 * @param[out] bytes 7850 * The counter hits bytes number to save. 7851 * 7852 * @return 7853 * 0 on success, a negative errno value otherwise. 7854 */ 7855 int 7856 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7857 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 7858 { 7859 const struct mlx5_flow_driver_ops *fops; 7860 struct rte_flow_attr attr = { .transfer = 0 }; 7861 7862 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7863 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7864 return fops->counter_query(dev, cnt, clear, pkts, 7865 bytes, action); 7866 } 7867 DRV_LOG(ERR, 7868 "port %u counter query is not supported.", 7869 dev->data->port_id); 7870 return -ENOTSUP; 7871 } 7872 7873 /** 7874 * Get information about HWS pre-configurable resources. 7875 * 7876 * @param[in] dev 7877 * Pointer to the rte_eth_dev structure. 7878 * @param[out] port_info 7879 * Pointer to port information. 7880 * @param[out] queue_info 7881 * Pointer to queue information. 7882 * @param[out] error 7883 * Pointer to error structure. 7884 * 7885 * @return 7886 * 0 on success, a negative errno value otherwise and rte_errno is set. 7887 */ 7888 static int 7889 mlx5_flow_info_get(struct rte_eth_dev *dev, 7890 struct rte_flow_port_info *port_info, 7891 struct rte_flow_queue_info *queue_info, 7892 struct rte_flow_error *error) 7893 { 7894 const struct mlx5_flow_driver_ops *fops; 7895 7896 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7897 return rte_flow_error_set(error, ENOTSUP, 7898 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7899 NULL, 7900 "info get with incorrect steering mode"); 7901 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7902 return fops->info_get(dev, port_info, queue_info, error); 7903 } 7904 7905 /** 7906 * Configure port HWS resources. 7907 * 7908 * @param[in] dev 7909 * Pointer to the rte_eth_dev structure. 7910 * @param[in] port_attr 7911 * Port configuration attributes. 7912 * @param[in] nb_queue 7913 * Number of queue. 7914 * @param[in] queue_attr 7915 * Array that holds attributes for each flow queue. 7916 * @param[out] error 7917 * Pointer to error structure. 7918 * 7919 * @return 7920 * 0 on success, a negative errno value otherwise and rte_errno is set. 7921 */ 7922 static int 7923 mlx5_flow_port_configure(struct rte_eth_dev *dev, 7924 const struct rte_flow_port_attr *port_attr, 7925 uint16_t nb_queue, 7926 const struct rte_flow_queue_attr *queue_attr[], 7927 struct rte_flow_error *error) 7928 { 7929 const struct mlx5_flow_driver_ops *fops; 7930 7931 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7932 return rte_flow_error_set(error, ENOTSUP, 7933 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7934 NULL, 7935 "port configure with incorrect steering mode"); 7936 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7937 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 7938 } 7939 7940 /** 7941 * Create flow item template. 7942 * 7943 * @param[in] dev 7944 * Pointer to the rte_eth_dev structure. 7945 * @param[in] attr 7946 * Pointer to the item template attributes. 7947 * @param[in] items 7948 * The template item pattern. 7949 * @param[out] error 7950 * Pointer to error structure. 7951 * 7952 * @return 7953 * 0 on success, a negative errno value otherwise and rte_errno is set. 7954 */ 7955 static struct rte_flow_pattern_template * 7956 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 7957 const struct rte_flow_pattern_template_attr *attr, 7958 const struct rte_flow_item items[], 7959 struct rte_flow_error *error) 7960 { 7961 const struct mlx5_flow_driver_ops *fops; 7962 7963 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 7964 rte_flow_error_set(error, ENOTSUP, 7965 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7966 NULL, 7967 "pattern create with incorrect steering mode"); 7968 return NULL; 7969 } 7970 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7971 return fops->pattern_template_create(dev, attr, items, error); 7972 } 7973 7974 /** 7975 * Destroy flow item template. 7976 * 7977 * @param[in] dev 7978 * Pointer to the rte_eth_dev structure. 7979 * @param[in] template 7980 * Pointer to the item template to be destroyed. 7981 * @param[out] error 7982 * Pointer to error structure. 7983 * 7984 * @return 7985 * 0 on success, a negative errno value otherwise and rte_errno is set. 7986 */ 7987 static int 7988 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 7989 struct rte_flow_pattern_template *template, 7990 struct rte_flow_error *error) 7991 { 7992 const struct mlx5_flow_driver_ops *fops; 7993 7994 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7995 return rte_flow_error_set(error, ENOTSUP, 7996 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7997 NULL, 7998 "pattern destroy with incorrect steering mode"); 7999 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8000 return fops->pattern_template_destroy(dev, template, error); 8001 } 8002 8003 /** 8004 * Allocate a new memory for the counter values wrapped by all the needed 8005 * management. 8006 * 8007 * @param[in] sh 8008 * Pointer to mlx5_dev_ctx_shared object. 8009 * 8010 * @return 8011 * 0 on success, a negative errno value otherwise. 8012 */ 8013 static int 8014 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8015 { 8016 struct mlx5_counter_stats_mem_mng *mem_mng; 8017 volatile struct flow_counter_stats *raw_data; 8018 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8019 int size = (sizeof(struct flow_counter_stats) * 8020 MLX5_COUNTERS_PER_POOL + 8021 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8022 sizeof(struct mlx5_counter_stats_mem_mng); 8023 size_t pgsize = rte_mem_page_size(); 8024 uint8_t *mem; 8025 int ret; 8026 int i; 8027 8028 if (pgsize == (size_t)-1) { 8029 DRV_LOG(ERR, "Failed to get mem page size"); 8030 rte_errno = ENOMEM; 8031 return -ENOMEM; 8032 } 8033 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8034 if (!mem) { 8035 rte_errno = ENOMEM; 8036 return -ENOMEM; 8037 } 8038 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8039 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8040 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8041 sh->cdev->pdn, mem, size, 8042 &mem_mng->wm); 8043 if (ret) { 8044 rte_errno = errno; 8045 mlx5_free(mem); 8046 return -rte_errno; 8047 } 8048 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8049 raw_data = (volatile struct flow_counter_stats *)mem; 8050 for (i = 0; i < raws_n; ++i) { 8051 mem_mng->raws[i].mem_mng = mem_mng; 8052 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8053 } 8054 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8055 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8056 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8057 next); 8058 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8059 sh->cmng.mem_mng = mem_mng; 8060 return 0; 8061 } 8062 8063 /** 8064 * Set the statistic memory to the new counter pool. 8065 * 8066 * @param[in] sh 8067 * Pointer to mlx5_dev_ctx_shared object. 8068 * @param[in] pool 8069 * Pointer to the pool to set the statistic memory. 8070 * 8071 * @return 8072 * 0 on success, a negative errno value otherwise. 8073 */ 8074 static int 8075 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8076 struct mlx5_flow_counter_pool *pool) 8077 { 8078 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8079 /* Resize statistic memory once used out. */ 8080 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8081 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8082 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8083 return -1; 8084 } 8085 rte_spinlock_lock(&pool->sl); 8086 pool->raw = cmng->mem_mng->raws + pool->index % 8087 MLX5_CNT_CONTAINER_RESIZE; 8088 rte_spinlock_unlock(&pool->sl); 8089 pool->raw_hw = NULL; 8090 return 0; 8091 } 8092 8093 #define MLX5_POOL_QUERY_FREQ_US 1000000 8094 8095 /** 8096 * Set the periodic procedure for triggering asynchronous batch queries for all 8097 * the counter pools. 8098 * 8099 * @param[in] sh 8100 * Pointer to mlx5_dev_ctx_shared object. 8101 */ 8102 void 8103 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8104 { 8105 uint32_t pools_n, us; 8106 8107 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8108 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8109 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8110 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8111 sh->cmng.query_thread_on = 0; 8112 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8113 } else { 8114 sh->cmng.query_thread_on = 1; 8115 } 8116 } 8117 8118 /** 8119 * The periodic procedure for triggering asynchronous batch queries for all the 8120 * counter pools. This function is probably called by the host thread. 8121 * 8122 * @param[in] arg 8123 * The parameter for the alarm process. 8124 */ 8125 void 8126 mlx5_flow_query_alarm(void *arg) 8127 { 8128 struct mlx5_dev_ctx_shared *sh = arg; 8129 int ret; 8130 uint16_t pool_index = sh->cmng.pool_index; 8131 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8132 struct mlx5_flow_counter_pool *pool; 8133 uint16_t n_valid; 8134 8135 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8136 goto set_alarm; 8137 rte_spinlock_lock(&cmng->pool_update_sl); 8138 pool = cmng->pools[pool_index]; 8139 n_valid = cmng->n_valid; 8140 rte_spinlock_unlock(&cmng->pool_update_sl); 8141 /* Set the statistic memory to the new created pool. */ 8142 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8143 goto set_alarm; 8144 if (pool->raw_hw) 8145 /* There is a pool query in progress. */ 8146 goto set_alarm; 8147 pool->raw_hw = 8148 LIST_FIRST(&sh->cmng.free_stat_raws); 8149 if (!pool->raw_hw) 8150 /* No free counter statistics raw memory. */ 8151 goto set_alarm; 8152 /* 8153 * Identify the counters released between query trigger and query 8154 * handle more efficiently. The counter released in this gap period 8155 * should wait for a new round of query as the new arrived packets 8156 * will not be taken into account. 8157 */ 8158 pool->query_gen++; 8159 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8160 MLX5_COUNTERS_PER_POOL, 8161 NULL, NULL, 8162 pool->raw_hw->mem_mng->wm.lkey, 8163 (void *)(uintptr_t) 8164 pool->raw_hw->data, 8165 sh->devx_comp, 8166 (uint64_t)(uintptr_t)pool); 8167 if (ret) { 8168 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8169 " %d", pool->min_dcs->id); 8170 pool->raw_hw = NULL; 8171 goto set_alarm; 8172 } 8173 LIST_REMOVE(pool->raw_hw, next); 8174 sh->cmng.pending_queries++; 8175 pool_index++; 8176 if (pool_index >= n_valid) 8177 pool_index = 0; 8178 set_alarm: 8179 sh->cmng.pool_index = pool_index; 8180 mlx5_set_query_alarm(sh); 8181 } 8182 8183 /** 8184 * Check and callback event for new aged flow in the counter pool 8185 * 8186 * @param[in] sh 8187 * Pointer to mlx5_dev_ctx_shared object. 8188 * @param[in] pool 8189 * Pointer to Current counter pool. 8190 */ 8191 static void 8192 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8193 struct mlx5_flow_counter_pool *pool) 8194 { 8195 struct mlx5_priv *priv; 8196 struct mlx5_flow_counter *cnt; 8197 struct mlx5_age_info *age_info; 8198 struct mlx5_age_param *age_param; 8199 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8200 struct mlx5_counter_stats_raw *prev = pool->raw; 8201 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8202 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8203 uint16_t expected = AGE_CANDIDATE; 8204 uint32_t i; 8205 8206 pool->time_of_last_age_check = curr_time; 8207 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8208 cnt = MLX5_POOL_GET_CNT(pool, i); 8209 age_param = MLX5_CNT_TO_AGE(cnt); 8210 if (__atomic_load_n(&age_param->state, 8211 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8212 continue; 8213 if (cur->data[i].hits != prev->data[i].hits) { 8214 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8215 __ATOMIC_RELAXED); 8216 continue; 8217 } 8218 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8219 time_delta, 8220 __ATOMIC_RELAXED) <= age_param->timeout) 8221 continue; 8222 /** 8223 * Hold the lock first, or if between the 8224 * state AGE_TMOUT and tailq operation the 8225 * release happened, the release procedure 8226 * may delete a non-existent tailq node. 8227 */ 8228 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8229 age_info = GET_PORT_AGE_INFO(priv); 8230 rte_spinlock_lock(&age_info->aged_sl); 8231 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8232 AGE_TMOUT, false, 8233 __ATOMIC_RELAXED, 8234 __ATOMIC_RELAXED)) { 8235 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8236 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8237 } 8238 rte_spinlock_unlock(&age_info->aged_sl); 8239 } 8240 mlx5_age_event_prepare(sh); 8241 } 8242 8243 /** 8244 * Handler for the HW respond about ready values from an asynchronous batch 8245 * query. This function is probably called by the host thread. 8246 * 8247 * @param[in] sh 8248 * The pointer to the shared device context. 8249 * @param[in] async_id 8250 * The Devx async ID. 8251 * @param[in] status 8252 * The status of the completion. 8253 */ 8254 void 8255 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8256 uint64_t async_id, int status) 8257 { 8258 struct mlx5_flow_counter_pool *pool = 8259 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8260 struct mlx5_counter_stats_raw *raw_to_free; 8261 uint8_t query_gen = pool->query_gen ^ 1; 8262 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8263 enum mlx5_counter_type cnt_type = 8264 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8265 MLX5_COUNTER_TYPE_ORIGIN; 8266 8267 if (unlikely(status)) { 8268 raw_to_free = pool->raw_hw; 8269 } else { 8270 raw_to_free = pool->raw; 8271 if (pool->is_aged) 8272 mlx5_flow_aging_check(sh, pool); 8273 rte_spinlock_lock(&pool->sl); 8274 pool->raw = pool->raw_hw; 8275 rte_spinlock_unlock(&pool->sl); 8276 /* Be sure the new raw counters data is updated in memory. */ 8277 rte_io_wmb(); 8278 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8279 rte_spinlock_lock(&cmng->csl[cnt_type]); 8280 TAILQ_CONCAT(&cmng->counters[cnt_type], 8281 &pool->counters[query_gen], next); 8282 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8283 } 8284 } 8285 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8286 pool->raw_hw = NULL; 8287 sh->cmng.pending_queries--; 8288 } 8289 8290 static int 8291 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8292 const struct flow_grp_info *grp_info, 8293 struct rte_flow_error *error) 8294 { 8295 if (grp_info->transfer && grp_info->external && 8296 grp_info->fdb_def_rule) { 8297 if (group == UINT32_MAX) 8298 return rte_flow_error_set 8299 (error, EINVAL, 8300 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8301 NULL, 8302 "group index not supported"); 8303 *table = group + 1; 8304 } else { 8305 *table = group; 8306 } 8307 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8308 return 0; 8309 } 8310 8311 /** 8312 * Translate the rte_flow group index to HW table value. 8313 * 8314 * If tunnel offload is disabled, all group ids converted to flow table 8315 * id using the standard method. 8316 * If tunnel offload is enabled, group id can be converted using the 8317 * standard or tunnel conversion method. Group conversion method 8318 * selection depends on flags in `grp_info` parameter: 8319 * - Internal (grp_info.external == 0) groups conversion uses the 8320 * standard method. 8321 * - Group ids in JUMP action converted with the tunnel conversion. 8322 * - Group id in rule attribute conversion depends on a rule type and 8323 * group id value: 8324 * ** non zero group attributes converted with the tunnel method 8325 * ** zero group attribute in non-tunnel rule is converted using the 8326 * standard method - there's only one root table 8327 * ** zero group attribute in steer tunnel rule is converted with the 8328 * standard method - single root table 8329 * ** zero group attribute in match tunnel rule is a special OvS 8330 * case: that value is used for portability reasons. That group 8331 * id is converted with the tunnel conversion method. 8332 * 8333 * @param[in] dev 8334 * Port device 8335 * @param[in] tunnel 8336 * PMD tunnel offload object 8337 * @param[in] group 8338 * rte_flow group index value. 8339 * @param[out] table 8340 * HW table value. 8341 * @param[in] grp_info 8342 * flags used for conversion 8343 * @param[out] error 8344 * Pointer to error structure. 8345 * 8346 * @return 8347 * 0 on success, a negative errno value otherwise and rte_errno is set. 8348 */ 8349 int 8350 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8351 const struct mlx5_flow_tunnel *tunnel, 8352 uint32_t group, uint32_t *table, 8353 const struct flow_grp_info *grp_info, 8354 struct rte_flow_error *error) 8355 { 8356 int ret; 8357 bool standard_translation; 8358 8359 if (!grp_info->skip_scale && grp_info->external && 8360 group < MLX5_MAX_TABLES_EXTERNAL) 8361 group *= MLX5_FLOW_TABLE_FACTOR; 8362 if (is_tunnel_offload_active(dev)) { 8363 standard_translation = !grp_info->external || 8364 grp_info->std_tbl_fix; 8365 } else { 8366 standard_translation = true; 8367 } 8368 DRV_LOG(DEBUG, 8369 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8370 dev->data->port_id, group, grp_info->transfer, 8371 grp_info->external, grp_info->fdb_def_rule, 8372 standard_translation ? "STANDARD" : "TUNNEL"); 8373 if (standard_translation) 8374 ret = flow_group_to_table(dev->data->port_id, group, table, 8375 grp_info, error); 8376 else 8377 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8378 table, error); 8379 8380 return ret; 8381 } 8382 8383 /** 8384 * Discover availability of metadata reg_c's. 8385 * 8386 * Iteratively use test flows to check availability. 8387 * 8388 * @param[in] dev 8389 * Pointer to the Ethernet device structure. 8390 * 8391 * @return 8392 * 0 on success, a negative errno value otherwise and rte_errno is set. 8393 */ 8394 int 8395 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8396 { 8397 struct mlx5_priv *priv = dev->data->dev_private; 8398 enum modify_reg idx; 8399 int n = 0; 8400 8401 /* reg_c[0] and reg_c[1] are reserved. */ 8402 priv->sh->flow_mreg_c[n++] = REG_C_0; 8403 priv->sh->flow_mreg_c[n++] = REG_C_1; 8404 /* Discover availability of other reg_c's. */ 8405 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8406 struct rte_flow_attr attr = { 8407 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8408 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8409 .ingress = 1, 8410 }; 8411 struct rte_flow_item items[] = { 8412 [0] = { 8413 .type = RTE_FLOW_ITEM_TYPE_END, 8414 }, 8415 }; 8416 struct rte_flow_action actions[] = { 8417 [0] = { 8418 .type = (enum rte_flow_action_type) 8419 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8420 .conf = &(struct mlx5_flow_action_copy_mreg){ 8421 .src = REG_C_1, 8422 .dst = idx, 8423 }, 8424 }, 8425 [1] = { 8426 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8427 .conf = &(struct rte_flow_action_jump){ 8428 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8429 }, 8430 }, 8431 [2] = { 8432 .type = RTE_FLOW_ACTION_TYPE_END, 8433 }, 8434 }; 8435 uint32_t flow_idx; 8436 struct rte_flow *flow; 8437 struct rte_flow_error error; 8438 8439 if (!priv->sh->config.dv_flow_en) 8440 break; 8441 /* Create internal flow, validation skips copy action. */ 8442 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8443 items, actions, false, &error); 8444 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8445 flow_idx); 8446 if (!flow) 8447 continue; 8448 priv->sh->flow_mreg_c[n++] = idx; 8449 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8450 } 8451 for (; n < MLX5_MREG_C_NUM; ++n) 8452 priv->sh->flow_mreg_c[n] = REG_NON; 8453 priv->sh->metadata_regc_check_flag = 1; 8454 return 0; 8455 } 8456 8457 int 8458 save_dump_file(const uint8_t *data, uint32_t size, 8459 uint32_t type, uint64_t id, void *arg, FILE *file) 8460 { 8461 char line[BUF_SIZE]; 8462 uint32_t out = 0; 8463 uint32_t k; 8464 uint32_t actions_num; 8465 struct rte_flow_query_count *count; 8466 8467 memset(line, 0, BUF_SIZE); 8468 switch (type) { 8469 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8470 actions_num = *(uint32_t *)(arg); 8471 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8472 type, id, actions_num); 8473 break; 8474 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8475 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8476 type, id); 8477 break; 8478 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8479 count = (struct rte_flow_query_count *)arg; 8480 fprintf(file, 8481 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8482 type, id, count->hits, count->bytes); 8483 return 0; 8484 default: 8485 return -1; 8486 } 8487 8488 for (k = 0; k < size; k++) { 8489 /* Make sure we do not overrun the line buffer length. */ 8490 if (out >= BUF_SIZE - 4) { 8491 line[out] = '\0'; 8492 break; 8493 } 8494 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8495 (data[k]) & 0xff); 8496 } 8497 fprintf(file, "%s\n", line); 8498 return 0; 8499 } 8500 8501 int 8502 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8503 struct rte_flow_query_count *count, struct rte_flow_error *error) 8504 { 8505 struct rte_flow_action action[2]; 8506 enum mlx5_flow_drv_type ftype; 8507 const struct mlx5_flow_driver_ops *fops; 8508 8509 if (!flow) { 8510 return rte_flow_error_set(error, ENOENT, 8511 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8512 NULL, 8513 "invalid flow handle"); 8514 } 8515 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8516 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8517 if (flow->counter) { 8518 memset(count, 0, sizeof(struct rte_flow_query_count)); 8519 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8520 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8521 ftype < MLX5_FLOW_TYPE_MAX); 8522 fops = flow_get_drv_ops(ftype); 8523 return fops->query(dev, flow, action, count, error); 8524 } 8525 return -1; 8526 } 8527 8528 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8529 /** 8530 * Dump flow ipool data to file 8531 * 8532 * @param[in] dev 8533 * The pointer to Ethernet device. 8534 * @param[in] file 8535 * A pointer to a file for output. 8536 * @param[out] error 8537 * Perform verbose error reporting if not NULL. PMDs initialize this 8538 * structure in case of error only. 8539 * @return 8540 * 0 on success, a negative value otherwise. 8541 */ 8542 int 8543 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8544 struct rte_flow *flow, FILE *file, 8545 struct rte_flow_error *error) 8546 { 8547 struct mlx5_priv *priv = dev->data->dev_private; 8548 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8549 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8550 uint32_t handle_idx; 8551 struct mlx5_flow_handle *dh; 8552 struct rte_flow_query_count count; 8553 uint32_t actions_num; 8554 const uint8_t *data; 8555 size_t size; 8556 uint64_t id; 8557 uint32_t type; 8558 void *action = NULL; 8559 8560 if (!flow) { 8561 return rte_flow_error_set(error, ENOENT, 8562 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8563 NULL, 8564 "invalid flow handle"); 8565 } 8566 handle_idx = flow->dev_handles; 8567 /* query counter */ 8568 if (flow->counter && 8569 (!mlx5_counter_query(dev, flow->counter, false, 8570 &count.hits, &count.bytes, &action)) && action) { 8571 id = (uint64_t)(uintptr_t)action; 8572 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8573 save_dump_file(NULL, 0, type, 8574 id, (void *)&count, file); 8575 } 8576 8577 while (handle_idx) { 8578 dh = mlx5_ipool_get(priv->sh->ipool 8579 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8580 if (!dh) 8581 continue; 8582 handle_idx = dh->next.next; 8583 8584 /* Get modify_hdr and encap_decap buf from ipools. */ 8585 encap_decap = NULL; 8586 modify_hdr = dh->dvh.modify_hdr; 8587 8588 if (dh->dvh.rix_encap_decap) { 8589 encap_decap = mlx5_ipool_get(priv->sh->ipool 8590 [MLX5_IPOOL_DECAP_ENCAP], 8591 dh->dvh.rix_encap_decap); 8592 } 8593 if (modify_hdr) { 8594 data = (const uint8_t *)modify_hdr->actions; 8595 size = (size_t)(modify_hdr->actions_num) * 8; 8596 id = (uint64_t)(uintptr_t)modify_hdr->action; 8597 actions_num = modify_hdr->actions_num; 8598 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8599 save_dump_file(data, size, type, id, 8600 (void *)(&actions_num), file); 8601 } 8602 if (encap_decap) { 8603 data = encap_decap->buf; 8604 size = encap_decap->size; 8605 id = (uint64_t)(uintptr_t)encap_decap->action; 8606 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8607 save_dump_file(data, size, type, 8608 id, NULL, file); 8609 } 8610 } 8611 return 0; 8612 } 8613 8614 /** 8615 * Dump all flow's encap_decap/modify_hdr/counter data to file 8616 * 8617 * @param[in] dev 8618 * The pointer to Ethernet device. 8619 * @param[in] file 8620 * A pointer to a file for output. 8621 * @param[out] error 8622 * Perform verbose error reporting if not NULL. PMDs initialize this 8623 * structure in case of error only. 8624 * @return 8625 * 0 on success, a negative value otherwise. 8626 */ 8627 static int 8628 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8629 FILE *file, struct rte_flow_error *error __rte_unused) 8630 { 8631 struct mlx5_priv *priv = dev->data->dev_private; 8632 struct mlx5_dev_ctx_shared *sh = priv->sh; 8633 struct mlx5_hlist *h; 8634 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8635 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8636 struct rte_flow_query_count count; 8637 uint32_t actions_num; 8638 const uint8_t *data; 8639 size_t size; 8640 uint64_t id; 8641 uint32_t type; 8642 uint32_t i; 8643 uint32_t j; 8644 struct mlx5_list_inconst *l_inconst; 8645 struct mlx5_list_entry *e; 8646 int lcore_index; 8647 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 8648 uint32_t max; 8649 void *action; 8650 8651 /* encap_decap hlist is lcore_share, get global core cache. */ 8652 i = MLX5_LIST_GLOBAL; 8653 h = sh->encaps_decaps; 8654 if (h) { 8655 for (j = 0; j <= h->mask; j++) { 8656 l_inconst = &h->buckets[j].l; 8657 if (!l_inconst || !l_inconst->cache[i]) 8658 continue; 8659 8660 e = LIST_FIRST(&l_inconst->cache[i]->h); 8661 while (e) { 8662 encap_decap = 8663 (struct mlx5_flow_dv_encap_decap_resource *)e; 8664 data = encap_decap->buf; 8665 size = encap_decap->size; 8666 id = (uint64_t)(uintptr_t)encap_decap->action; 8667 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8668 save_dump_file(data, size, type, 8669 id, NULL, file); 8670 e = LIST_NEXT(e, next); 8671 } 8672 } 8673 } 8674 8675 /* get modify_hdr */ 8676 h = sh->modify_cmds; 8677 if (h) { 8678 lcore_index = rte_lcore_index(rte_lcore_id()); 8679 if (unlikely(lcore_index == -1)) { 8680 lcore_index = MLX5_LIST_NLCORE; 8681 rte_spinlock_lock(&h->l_const.lcore_lock); 8682 } 8683 i = lcore_index; 8684 8685 for (j = 0; j <= h->mask; j++) { 8686 l_inconst = &h->buckets[j].l; 8687 if (!l_inconst || !l_inconst->cache[i]) 8688 continue; 8689 8690 e = LIST_FIRST(&l_inconst->cache[i]->h); 8691 while (e) { 8692 modify_hdr = 8693 (struct mlx5_flow_dv_modify_hdr_resource *)e; 8694 data = (const uint8_t *)modify_hdr->actions; 8695 size = (size_t)(modify_hdr->actions_num) * 8; 8696 actions_num = modify_hdr->actions_num; 8697 id = (uint64_t)(uintptr_t)modify_hdr->action; 8698 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8699 save_dump_file(data, size, type, id, 8700 (void *)(&actions_num), file); 8701 e = LIST_NEXT(e, next); 8702 } 8703 } 8704 8705 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 8706 rte_spinlock_unlock(&h->l_const.lcore_lock); 8707 } 8708 8709 /* get counter */ 8710 MLX5_ASSERT(cmng->n_valid <= cmng->n); 8711 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 8712 for (j = 1; j <= max; j++) { 8713 action = NULL; 8714 if ((!mlx5_counter_query(dev, j, false, &count.hits, 8715 &count.bytes, &action)) && action) { 8716 id = (uint64_t)(uintptr_t)action; 8717 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8718 save_dump_file(NULL, 0, type, 8719 id, (void *)&count, file); 8720 } 8721 } 8722 return 0; 8723 } 8724 #endif 8725 8726 /** 8727 * Dump flow raw hw data to file 8728 * 8729 * @param[in] dev 8730 * The pointer to Ethernet device. 8731 * @param[in] file 8732 * A pointer to a file for output. 8733 * @param[out] error 8734 * Perform verbose error reporting if not NULL. PMDs initialize this 8735 * structure in case of error only. 8736 * @return 8737 * 0 on success, a negative value otherwise. 8738 */ 8739 int 8740 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8741 FILE *file, 8742 struct rte_flow_error *error __rte_unused) 8743 { 8744 struct mlx5_priv *priv = dev->data->dev_private; 8745 struct mlx5_dev_ctx_shared *sh = priv->sh; 8746 uint32_t handle_idx; 8747 int ret; 8748 struct mlx5_flow_handle *dh; 8749 struct rte_flow *flow; 8750 8751 if (!sh->config.dv_flow_en) { 8752 if (fputs("device dv flow disabled\n", file) <= 0) 8753 return -errno; 8754 return -ENOTSUP; 8755 } 8756 8757 /* dump all */ 8758 if (!flow_idx) { 8759 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8760 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 8761 return -EINVAL; 8762 #endif 8763 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8764 sh->rx_domain, 8765 sh->tx_domain, file); 8766 } 8767 /* dump one */ 8768 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8769 (uintptr_t)(void *)flow_idx); 8770 if (!flow) 8771 return -EINVAL; 8772 8773 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8774 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8775 #endif 8776 handle_idx = flow->dev_handles; 8777 while (handle_idx) { 8778 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8779 handle_idx); 8780 if (!dh) 8781 return -ENOENT; 8782 if (dh->drv_flow) { 8783 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8784 file); 8785 if (ret) 8786 return -ENOENT; 8787 } 8788 handle_idx = dh->next.next; 8789 } 8790 return 0; 8791 } 8792 8793 /** 8794 * Get aged-out flows. 8795 * 8796 * @param[in] dev 8797 * Pointer to the Ethernet device structure. 8798 * @param[in] context 8799 * The address of an array of pointers to the aged-out flows contexts. 8800 * @param[in] nb_countexts 8801 * The length of context array pointers. 8802 * @param[out] error 8803 * Perform verbose error reporting if not NULL. Initialized in case of 8804 * error only. 8805 * 8806 * @return 8807 * how many contexts get in success, otherwise negative errno value. 8808 * if nb_contexts is 0, return the amount of all aged contexts. 8809 * if nb_contexts is not 0 , return the amount of aged flows reported 8810 * in the context array. 8811 */ 8812 int 8813 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8814 uint32_t nb_contexts, struct rte_flow_error *error) 8815 { 8816 const struct mlx5_flow_driver_ops *fops; 8817 struct rte_flow_attr attr = { .transfer = 0 }; 8818 8819 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8820 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8821 return fops->get_aged_flows(dev, contexts, nb_contexts, 8822 error); 8823 } 8824 DRV_LOG(ERR, 8825 "port %u get aged flows is not supported.", 8826 dev->data->port_id); 8827 return -ENOTSUP; 8828 } 8829 8830 /* Wrapper for driver action_validate op callback */ 8831 static int 8832 flow_drv_action_validate(struct rte_eth_dev *dev, 8833 const struct rte_flow_indir_action_conf *conf, 8834 const struct rte_flow_action *action, 8835 const struct mlx5_flow_driver_ops *fops, 8836 struct rte_flow_error *error) 8837 { 8838 static const char err_msg[] = "indirect action validation unsupported"; 8839 8840 if (!fops->action_validate) { 8841 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8842 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8843 NULL, err_msg); 8844 return -rte_errno; 8845 } 8846 return fops->action_validate(dev, conf, action, error); 8847 } 8848 8849 /** 8850 * Destroys the shared action by handle. 8851 * 8852 * @param dev 8853 * Pointer to Ethernet device structure. 8854 * @param[in] handle 8855 * Handle for the indirect action object to be destroyed. 8856 * @param[out] error 8857 * Perform verbose error reporting if not NULL. PMDs initialize this 8858 * structure in case of error only. 8859 * 8860 * @return 8861 * 0 on success, a negative errno value otherwise and rte_errno is set. 8862 * 8863 * @note: wrapper for driver action_create op callback. 8864 */ 8865 static int 8866 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 8867 struct rte_flow_action_handle *handle, 8868 struct rte_flow_error *error) 8869 { 8870 static const char err_msg[] = "indirect action destruction unsupported"; 8871 struct rte_flow_attr attr = { .transfer = 0 }; 8872 const struct mlx5_flow_driver_ops *fops = 8873 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8874 8875 if (!fops->action_destroy) { 8876 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8877 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8878 NULL, err_msg); 8879 return -rte_errno; 8880 } 8881 return fops->action_destroy(dev, handle, error); 8882 } 8883 8884 /* Wrapper for driver action_destroy op callback */ 8885 static int 8886 flow_drv_action_update(struct rte_eth_dev *dev, 8887 struct rte_flow_action_handle *handle, 8888 const void *update, 8889 const struct mlx5_flow_driver_ops *fops, 8890 struct rte_flow_error *error) 8891 { 8892 static const char err_msg[] = "indirect action update unsupported"; 8893 8894 if (!fops->action_update) { 8895 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8896 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8897 NULL, err_msg); 8898 return -rte_errno; 8899 } 8900 return fops->action_update(dev, handle, update, error); 8901 } 8902 8903 /* Wrapper for driver action_destroy op callback */ 8904 static int 8905 flow_drv_action_query(struct rte_eth_dev *dev, 8906 const struct rte_flow_action_handle *handle, 8907 void *data, 8908 const struct mlx5_flow_driver_ops *fops, 8909 struct rte_flow_error *error) 8910 { 8911 static const char err_msg[] = "indirect action query unsupported"; 8912 8913 if (!fops->action_query) { 8914 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8915 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8916 NULL, err_msg); 8917 return -rte_errno; 8918 } 8919 return fops->action_query(dev, handle, data, error); 8920 } 8921 8922 /** 8923 * Create indirect action for reuse in multiple flow rules. 8924 * 8925 * @param dev 8926 * Pointer to Ethernet device structure. 8927 * @param conf 8928 * Pointer to indirect action object configuration. 8929 * @param[in] action 8930 * Action configuration for indirect action object creation. 8931 * @param[out] error 8932 * Perform verbose error reporting if not NULL. PMDs initialize this 8933 * structure in case of error only. 8934 * @return 8935 * A valid handle in case of success, NULL otherwise and rte_errno is set. 8936 */ 8937 static struct rte_flow_action_handle * 8938 mlx5_action_handle_create(struct rte_eth_dev *dev, 8939 const struct rte_flow_indir_action_conf *conf, 8940 const struct rte_flow_action *action, 8941 struct rte_flow_error *error) 8942 { 8943 static const char err_msg[] = "indirect action creation unsupported"; 8944 struct rte_flow_attr attr = { .transfer = 0 }; 8945 const struct mlx5_flow_driver_ops *fops = 8946 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8947 8948 if (flow_drv_action_validate(dev, conf, action, fops, error)) 8949 return NULL; 8950 if (!fops->action_create) { 8951 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 8952 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 8953 NULL, err_msg); 8954 return NULL; 8955 } 8956 return fops->action_create(dev, conf, action, error); 8957 } 8958 8959 /** 8960 * Updates inplace the indirect action configuration pointed by *handle* 8961 * with the configuration provided as *update* argument. 8962 * The update of the indirect action configuration effects all flow rules 8963 * reusing the action via handle. 8964 * 8965 * @param dev 8966 * Pointer to Ethernet device structure. 8967 * @param[in] handle 8968 * Handle for the indirect action to be updated. 8969 * @param[in] update 8970 * Action specification used to modify the action pointed by handle. 8971 * *update* could be of same type with the action pointed by the *handle* 8972 * handle argument, or some other structures like a wrapper, depending on 8973 * the indirect action type. 8974 * @param[out] error 8975 * Perform verbose error reporting if not NULL. PMDs initialize this 8976 * structure in case of error only. 8977 * 8978 * @return 8979 * 0 on success, a negative errno value otherwise and rte_errno is set. 8980 */ 8981 static int 8982 mlx5_action_handle_update(struct rte_eth_dev *dev, 8983 struct rte_flow_action_handle *handle, 8984 const void *update, 8985 struct rte_flow_error *error) 8986 { 8987 struct rte_flow_attr attr = { .transfer = 0 }; 8988 const struct mlx5_flow_driver_ops *fops = 8989 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 8990 int ret; 8991 8992 ret = flow_drv_action_validate(dev, NULL, 8993 (const struct rte_flow_action *)update, fops, error); 8994 if (ret) 8995 return ret; 8996 return flow_drv_action_update(dev, handle, update, fops, 8997 error); 8998 } 8999 9000 /** 9001 * Query the indirect action by handle. 9002 * 9003 * This function allows retrieving action-specific data such as counters. 9004 * Data is gathered by special action which may be present/referenced in 9005 * more than one flow rule definition. 9006 * 9007 * see @RTE_FLOW_ACTION_TYPE_COUNT 9008 * 9009 * @param dev 9010 * Pointer to Ethernet device structure. 9011 * @param[in] handle 9012 * Handle for the indirect action to query. 9013 * @param[in, out] data 9014 * Pointer to storage for the associated query data type. 9015 * @param[out] error 9016 * Perform verbose error reporting if not NULL. PMDs initialize this 9017 * structure in case of error only. 9018 * 9019 * @return 9020 * 0 on success, a negative errno value otherwise and rte_errno is set. 9021 */ 9022 static int 9023 mlx5_action_handle_query(struct rte_eth_dev *dev, 9024 const struct rte_flow_action_handle *handle, 9025 void *data, 9026 struct rte_flow_error *error) 9027 { 9028 struct rte_flow_attr attr = { .transfer = 0 }; 9029 const struct mlx5_flow_driver_ops *fops = 9030 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9031 9032 return flow_drv_action_query(dev, handle, data, fops, error); 9033 } 9034 9035 /** 9036 * Destroy all indirect actions (shared RSS). 9037 * 9038 * @param dev 9039 * Pointer to Ethernet device. 9040 * 9041 * @return 9042 * 0 on success, a negative errno value otherwise and rte_errno is set. 9043 */ 9044 int 9045 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9046 { 9047 struct rte_flow_error error; 9048 struct mlx5_priv *priv = dev->data->dev_private; 9049 struct mlx5_shared_action_rss *shared_rss; 9050 int ret = 0; 9051 uint32_t idx; 9052 9053 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9054 priv->rss_shared_actions, idx, shared_rss, next) { 9055 ret |= mlx5_action_handle_destroy(dev, 9056 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9057 } 9058 return ret; 9059 } 9060 9061 /** 9062 * Validate existing indirect actions against current device configuration 9063 * and attach them to device resources. 9064 * 9065 * @param dev 9066 * Pointer to Ethernet device. 9067 * 9068 * @return 9069 * 0 on success, a negative errno value otherwise and rte_errno is set. 9070 */ 9071 int 9072 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9073 { 9074 struct mlx5_priv *priv = dev->data->dev_private; 9075 struct mlx5_indexed_pool *ipool = 9076 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 9077 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 9078 int ret = 0; 9079 uint32_t idx; 9080 9081 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9082 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9083 const char *message; 9084 uint32_t queue_idx; 9085 9086 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9087 ind_tbl->queues_n, 9088 &message, &queue_idx); 9089 if (ret != 0) { 9090 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9091 dev->data->port_id, ind_tbl->queues[queue_idx], 9092 message); 9093 break; 9094 } 9095 } 9096 if (ret != 0) 9097 return ret; 9098 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9099 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9100 9101 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9102 if (ret != 0) { 9103 DRV_LOG(ERR, "Port %u could not attach " 9104 "indirection table obj %p", 9105 dev->data->port_id, (void *)ind_tbl); 9106 goto error; 9107 } 9108 } 9109 return 0; 9110 error: 9111 shared_rss_last = shared_rss; 9112 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9113 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9114 9115 if (shared_rss == shared_rss_last) 9116 break; 9117 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9118 DRV_LOG(CRIT, "Port %u could not detach " 9119 "indirection table obj %p on rollback", 9120 dev->data->port_id, (void *)ind_tbl); 9121 } 9122 return ret; 9123 } 9124 9125 /** 9126 * Detach indirect actions of the device from its resources. 9127 * 9128 * @param dev 9129 * Pointer to Ethernet device. 9130 * 9131 * @return 9132 * 0 on success, a negative errno value otherwise and rte_errno is set. 9133 */ 9134 int 9135 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9136 { 9137 struct mlx5_priv *priv = dev->data->dev_private; 9138 struct mlx5_indexed_pool *ipool = 9139 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 9140 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 9141 int ret = 0; 9142 uint32_t idx; 9143 9144 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9145 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9146 9147 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9148 if (ret != 0) { 9149 DRV_LOG(ERR, "Port %u could not detach " 9150 "indirection table obj %p", 9151 dev->data->port_id, (void *)ind_tbl); 9152 goto error; 9153 } 9154 } 9155 return 0; 9156 error: 9157 shared_rss_last = shared_rss; 9158 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9159 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9160 9161 if (shared_rss == shared_rss_last) 9162 break; 9163 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9164 DRV_LOG(CRIT, "Port %u could not attach " 9165 "indirection table obj %p on rollback", 9166 dev->data->port_id, (void *)ind_tbl); 9167 } 9168 return ret; 9169 } 9170 9171 #ifndef HAVE_MLX5DV_DR 9172 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9173 #else 9174 #define MLX5_DOMAIN_SYNC_FLOW \ 9175 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9176 #endif 9177 9178 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9179 { 9180 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9181 const struct mlx5_flow_driver_ops *fops; 9182 int ret; 9183 struct rte_flow_attr attr = { .transfer = 0 }; 9184 9185 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9186 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9187 if (ret > 0) 9188 ret = -ret; 9189 return ret; 9190 } 9191 9192 const struct mlx5_flow_tunnel * 9193 mlx5_get_tof(const struct rte_flow_item *item, 9194 const struct rte_flow_action *action, 9195 enum mlx5_tof_rule_type *rule_type) 9196 { 9197 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9198 if (item->type == (typeof(item->type)) 9199 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9200 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9201 return flow_items_to_tunnel(item); 9202 } 9203 } 9204 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9205 if (action->type == (typeof(action->type)) 9206 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9207 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9208 return flow_actions_to_tunnel(action); 9209 } 9210 } 9211 return NULL; 9212 } 9213 9214 /** 9215 * tunnel offload functionality is defined for DV environment only 9216 */ 9217 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9218 __extension__ 9219 union tunnel_offload_mark { 9220 uint32_t val; 9221 struct { 9222 uint32_t app_reserve:8; 9223 uint32_t table_id:15; 9224 uint32_t transfer:1; 9225 uint32_t _unused_:8; 9226 }; 9227 }; 9228 9229 static bool 9230 mlx5_access_tunnel_offload_db 9231 (struct rte_eth_dev *dev, 9232 bool (*match)(struct rte_eth_dev *, 9233 struct mlx5_flow_tunnel *, const void *), 9234 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9235 void (*miss)(struct rte_eth_dev *, void *), 9236 void *ctx, bool lock_op); 9237 9238 static int 9239 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9240 struct rte_flow *flow, 9241 const struct rte_flow_attr *attr, 9242 const struct rte_flow_action *app_actions, 9243 uint32_t flow_idx, 9244 const struct mlx5_flow_tunnel *tunnel, 9245 struct tunnel_default_miss_ctx *ctx, 9246 struct rte_flow_error *error) 9247 { 9248 struct mlx5_priv *priv = dev->data->dev_private; 9249 struct mlx5_flow *dev_flow; 9250 struct rte_flow_attr miss_attr = *attr; 9251 const struct rte_flow_item miss_items[2] = { 9252 { 9253 .type = RTE_FLOW_ITEM_TYPE_ETH, 9254 .spec = NULL, 9255 .last = NULL, 9256 .mask = NULL 9257 }, 9258 { 9259 .type = RTE_FLOW_ITEM_TYPE_END, 9260 .spec = NULL, 9261 .last = NULL, 9262 .mask = NULL 9263 } 9264 }; 9265 union tunnel_offload_mark mark_id; 9266 struct rte_flow_action_mark miss_mark; 9267 struct rte_flow_action miss_actions[3] = { 9268 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9269 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9270 }; 9271 const struct rte_flow_action_jump *jump_data; 9272 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9273 struct flow_grp_info grp_info = { 9274 .external = 1, 9275 .transfer = attr->transfer, 9276 .fdb_def_rule = !!priv->fdb_def_rule, 9277 .std_tbl_fix = 0, 9278 }; 9279 int ret; 9280 9281 if (!attr->transfer) { 9282 uint32_t q_size; 9283 9284 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9285 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9286 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9287 0, SOCKET_ID_ANY); 9288 if (!ctx->queue) 9289 return rte_flow_error_set 9290 (error, ENOMEM, 9291 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9292 NULL, "invalid default miss RSS"); 9293 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9294 ctx->action_rss.level = 0, 9295 ctx->action_rss.types = priv->rss_conf.rss_hf, 9296 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9297 ctx->action_rss.queue_num = priv->reta_idx_n, 9298 ctx->action_rss.key = priv->rss_conf.rss_key, 9299 ctx->action_rss.queue = ctx->queue; 9300 if (!priv->reta_idx_n || !priv->rxqs_n) 9301 return rte_flow_error_set 9302 (error, EINVAL, 9303 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9304 NULL, "invalid port configuration"); 9305 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9306 ctx->action_rss.types = 0; 9307 for (i = 0; i != priv->reta_idx_n; ++i) 9308 ctx->queue[i] = (*priv->reta_idx)[i]; 9309 } else { 9310 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9311 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9312 } 9313 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9314 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9315 jump_data = app_actions->conf; 9316 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9317 miss_attr.group = jump_data->group; 9318 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9319 &flow_table, &grp_info, error); 9320 if (ret) 9321 return rte_flow_error_set(error, EINVAL, 9322 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9323 NULL, "invalid tunnel id"); 9324 mark_id.app_reserve = 0; 9325 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9326 mark_id.transfer = !!attr->transfer; 9327 mark_id._unused_ = 0; 9328 miss_mark.id = mark_id.val; 9329 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9330 miss_items, miss_actions, flow_idx, error); 9331 if (!dev_flow) 9332 return -rte_errno; 9333 dev_flow->flow = flow; 9334 dev_flow->external = true; 9335 dev_flow->tunnel = tunnel; 9336 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9337 /* Subflow object was created, we must include one in the list. */ 9338 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9339 dev_flow->handle, next); 9340 DRV_LOG(DEBUG, 9341 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9342 dev->data->port_id, tunnel->app_tunnel.type, 9343 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9344 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9345 miss_actions, error); 9346 if (!ret) 9347 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9348 error); 9349 9350 return ret; 9351 } 9352 9353 static const struct mlx5_flow_tbl_data_entry * 9354 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9355 { 9356 struct mlx5_priv *priv = dev->data->dev_private; 9357 struct mlx5_dev_ctx_shared *sh = priv->sh; 9358 struct mlx5_list_entry *he; 9359 union tunnel_offload_mark mbits = { .val = mark }; 9360 union mlx5_flow_tbl_key table_key = { 9361 { 9362 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9363 .id = 0, 9364 .reserved = 0, 9365 .dummy = 0, 9366 .is_fdb = !!mbits.transfer, 9367 .is_egress = 0, 9368 } 9369 }; 9370 struct mlx5_flow_cb_ctx ctx = { 9371 .data = &table_key.v64, 9372 }; 9373 9374 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9375 return he ? 9376 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9377 } 9378 9379 static void 9380 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9381 struct mlx5_list_entry *entry) 9382 { 9383 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9384 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9385 9386 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9387 tunnel_flow_tbl_to_id(tte->flow_table)); 9388 mlx5_free(tte); 9389 } 9390 9391 static int 9392 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9393 struct mlx5_list_entry *entry, void *cb_ctx) 9394 { 9395 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9396 union tunnel_tbl_key tbl = { 9397 .val = *(uint64_t *)(ctx->data), 9398 }; 9399 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9400 9401 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9402 } 9403 9404 static struct mlx5_list_entry * 9405 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9406 { 9407 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9408 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9409 struct tunnel_tbl_entry *tte; 9410 union tunnel_tbl_key tbl = { 9411 .val = *(uint64_t *)(ctx->data), 9412 }; 9413 9414 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9415 sizeof(*tte), 0, 9416 SOCKET_ID_ANY); 9417 if (!tte) 9418 goto err; 9419 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9420 &tte->flow_table); 9421 if (tte->flow_table >= MLX5_MAX_TABLES) { 9422 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9423 tte->flow_table); 9424 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9425 tte->flow_table); 9426 goto err; 9427 } else if (!tte->flow_table) { 9428 goto err; 9429 } 9430 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9431 tte->tunnel_id = tbl.tunnel_id; 9432 tte->group = tbl.group; 9433 return &tte->hash; 9434 err: 9435 if (tte) 9436 mlx5_free(tte); 9437 return NULL; 9438 } 9439 9440 static struct mlx5_list_entry * 9441 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9442 struct mlx5_list_entry *oentry, 9443 void *cb_ctx __rte_unused) 9444 { 9445 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9446 0, SOCKET_ID_ANY); 9447 9448 if (!tte) 9449 return NULL; 9450 memcpy(tte, oentry, sizeof(*tte)); 9451 return &tte->hash; 9452 } 9453 9454 static void 9455 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9456 struct mlx5_list_entry *entry) 9457 { 9458 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9459 9460 mlx5_free(tte); 9461 } 9462 9463 static uint32_t 9464 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9465 const struct mlx5_flow_tunnel *tunnel, 9466 uint32_t group, uint32_t *table, 9467 struct rte_flow_error *error) 9468 { 9469 struct mlx5_list_entry *he; 9470 struct tunnel_tbl_entry *tte; 9471 union tunnel_tbl_key key = { 9472 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9473 .group = group 9474 }; 9475 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9476 struct mlx5_hlist *group_hash; 9477 struct mlx5_flow_cb_ctx ctx = { 9478 .data = &key.val, 9479 }; 9480 9481 group_hash = tunnel ? tunnel->groups : thub->groups; 9482 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9483 if (!he) 9484 return rte_flow_error_set(error, EINVAL, 9485 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9486 NULL, 9487 "tunnel group index not supported"); 9488 tte = container_of(he, typeof(*tte), hash); 9489 *table = tte->flow_table; 9490 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9491 dev->data->port_id, key.tunnel_id, group, *table); 9492 return 0; 9493 } 9494 9495 static void 9496 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9497 struct mlx5_flow_tunnel *tunnel) 9498 { 9499 struct mlx5_priv *priv = dev->data->dev_private; 9500 struct mlx5_indexed_pool *ipool; 9501 9502 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9503 dev->data->port_id, tunnel->tunnel_id); 9504 LIST_REMOVE(tunnel, chain); 9505 mlx5_hlist_destroy(tunnel->groups); 9506 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9507 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9508 } 9509 9510 static bool 9511 mlx5_access_tunnel_offload_db 9512 (struct rte_eth_dev *dev, 9513 bool (*match)(struct rte_eth_dev *, 9514 struct mlx5_flow_tunnel *, const void *), 9515 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9516 void (*miss)(struct rte_eth_dev *, void *), 9517 void *ctx, bool lock_op) 9518 { 9519 bool verdict = false; 9520 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9521 struct mlx5_flow_tunnel *tunnel; 9522 9523 rte_spinlock_lock(&thub->sl); 9524 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9525 verdict = match(dev, tunnel, (const void *)ctx); 9526 if (verdict) 9527 break; 9528 } 9529 if (!lock_op) 9530 rte_spinlock_unlock(&thub->sl); 9531 if (verdict && hit) 9532 hit(dev, tunnel, ctx); 9533 if (!verdict && miss) 9534 miss(dev, ctx); 9535 if (lock_op) 9536 rte_spinlock_unlock(&thub->sl); 9537 9538 return verdict; 9539 } 9540 9541 struct tunnel_db_find_tunnel_id_ctx { 9542 uint32_t tunnel_id; 9543 struct mlx5_flow_tunnel *tunnel; 9544 }; 9545 9546 static bool 9547 find_tunnel_id_match(struct rte_eth_dev *dev, 9548 struct mlx5_flow_tunnel *tunnel, const void *x) 9549 { 9550 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9551 9552 RTE_SET_USED(dev); 9553 return tunnel->tunnel_id == ctx->tunnel_id; 9554 } 9555 9556 static void 9557 find_tunnel_id_hit(struct rte_eth_dev *dev, 9558 struct mlx5_flow_tunnel *tunnel, void *x) 9559 { 9560 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9561 RTE_SET_USED(dev); 9562 ctx->tunnel = tunnel; 9563 } 9564 9565 static struct mlx5_flow_tunnel * 9566 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9567 { 9568 struct tunnel_db_find_tunnel_id_ctx ctx = { 9569 .tunnel_id = id, 9570 }; 9571 9572 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9573 find_tunnel_id_hit, NULL, &ctx, true); 9574 9575 return ctx.tunnel; 9576 } 9577 9578 static struct mlx5_flow_tunnel * 9579 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9580 const struct rte_flow_tunnel *app_tunnel) 9581 { 9582 struct mlx5_priv *priv = dev->data->dev_private; 9583 struct mlx5_indexed_pool *ipool; 9584 struct mlx5_flow_tunnel *tunnel; 9585 uint32_t id; 9586 9587 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9588 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9589 if (!tunnel) 9590 return NULL; 9591 if (id >= MLX5_MAX_TUNNELS) { 9592 mlx5_ipool_free(ipool, id); 9593 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9594 return NULL; 9595 } 9596 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9597 priv->sh, 9598 mlx5_flow_tunnel_grp2tbl_create_cb, 9599 mlx5_flow_tunnel_grp2tbl_match_cb, 9600 mlx5_flow_tunnel_grp2tbl_remove_cb, 9601 mlx5_flow_tunnel_grp2tbl_clone_cb, 9602 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9603 if (!tunnel->groups) { 9604 mlx5_ipool_free(ipool, id); 9605 return NULL; 9606 } 9607 /* initiate new PMD tunnel */ 9608 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9609 tunnel->tunnel_id = id; 9610 tunnel->action.type = (typeof(tunnel->action.type)) 9611 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9612 tunnel->action.conf = tunnel; 9613 tunnel->item.type = (typeof(tunnel->item.type)) 9614 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9615 tunnel->item.spec = tunnel; 9616 tunnel->item.last = NULL; 9617 tunnel->item.mask = NULL; 9618 9619 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9620 dev->data->port_id, tunnel->tunnel_id); 9621 9622 return tunnel; 9623 } 9624 9625 struct tunnel_db_get_tunnel_ctx { 9626 const struct rte_flow_tunnel *app_tunnel; 9627 struct mlx5_flow_tunnel *tunnel; 9628 }; 9629 9630 static bool get_tunnel_match(struct rte_eth_dev *dev, 9631 struct mlx5_flow_tunnel *tunnel, const void *x) 9632 { 9633 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9634 9635 RTE_SET_USED(dev); 9636 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9637 sizeof(*ctx->app_tunnel)); 9638 } 9639 9640 static void get_tunnel_hit(struct rte_eth_dev *dev, 9641 struct mlx5_flow_tunnel *tunnel, void *x) 9642 { 9643 /* called under tunnel spinlock protection */ 9644 struct tunnel_db_get_tunnel_ctx *ctx = x; 9645 9646 RTE_SET_USED(dev); 9647 tunnel->refctn++; 9648 ctx->tunnel = tunnel; 9649 } 9650 9651 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9652 { 9653 /* called under tunnel spinlock protection */ 9654 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9655 struct tunnel_db_get_tunnel_ctx *ctx = x; 9656 9657 rte_spinlock_unlock(&thub->sl); 9658 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9659 rte_spinlock_lock(&thub->sl); 9660 if (ctx->tunnel) { 9661 ctx->tunnel->refctn = 1; 9662 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9663 } 9664 } 9665 9666 9667 static int 9668 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9669 const struct rte_flow_tunnel *app_tunnel, 9670 struct mlx5_flow_tunnel **tunnel) 9671 { 9672 struct tunnel_db_get_tunnel_ctx ctx = { 9673 .app_tunnel = app_tunnel, 9674 }; 9675 9676 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9677 get_tunnel_miss, &ctx, true); 9678 *tunnel = ctx.tunnel; 9679 return ctx.tunnel ? 0 : -ENOMEM; 9680 } 9681 9682 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9683 { 9684 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9685 9686 if (!thub) 9687 return; 9688 if (!LIST_EMPTY(&thub->tunnels)) 9689 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9690 mlx5_hlist_destroy(thub->groups); 9691 mlx5_free(thub); 9692 } 9693 9694 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9695 { 9696 int err; 9697 struct mlx5_flow_tunnel_hub *thub; 9698 9699 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9700 0, SOCKET_ID_ANY); 9701 if (!thub) 9702 return -ENOMEM; 9703 LIST_INIT(&thub->tunnels); 9704 rte_spinlock_init(&thub->sl); 9705 thub->groups = mlx5_hlist_create("flow groups", 64, 9706 false, true, sh, 9707 mlx5_flow_tunnel_grp2tbl_create_cb, 9708 mlx5_flow_tunnel_grp2tbl_match_cb, 9709 mlx5_flow_tunnel_grp2tbl_remove_cb, 9710 mlx5_flow_tunnel_grp2tbl_clone_cb, 9711 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9712 if (!thub->groups) { 9713 err = -rte_errno; 9714 goto err; 9715 } 9716 sh->tunnel_hub = thub; 9717 9718 return 0; 9719 9720 err: 9721 if (thub->groups) 9722 mlx5_hlist_destroy(thub->groups); 9723 if (thub) 9724 mlx5_free(thub); 9725 return err; 9726 } 9727 9728 static inline int 9729 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9730 struct rte_flow_tunnel *tunnel, 9731 struct rte_flow_error *error) 9732 { 9733 struct mlx5_priv *priv = dev->data->dev_private; 9734 9735 if (!priv->sh->config.dv_flow_en) 9736 return rte_flow_error_set(error, ENOTSUP, 9737 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9738 "flow DV interface is off"); 9739 if (!is_tunnel_offload_active(dev)) 9740 return rte_flow_error_set(error, ENOTSUP, 9741 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9742 "tunnel offload was not activated"); 9743 if (!tunnel) 9744 return rte_flow_error_set(error, EINVAL, 9745 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9746 "no application tunnel"); 9747 switch (tunnel->type) { 9748 default: 9749 return rte_flow_error_set(error, EINVAL, 9750 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9751 "unsupported tunnel type"); 9752 case RTE_FLOW_ITEM_TYPE_VXLAN: 9753 case RTE_FLOW_ITEM_TYPE_GRE: 9754 case RTE_FLOW_ITEM_TYPE_NVGRE: 9755 case RTE_FLOW_ITEM_TYPE_GENEVE: 9756 break; 9757 } 9758 return 0; 9759 } 9760 9761 static int 9762 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9763 struct rte_flow_tunnel *app_tunnel, 9764 struct rte_flow_action **actions, 9765 uint32_t *num_of_actions, 9766 struct rte_flow_error *error) 9767 { 9768 struct mlx5_flow_tunnel *tunnel; 9769 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9770 9771 if (ret) 9772 return ret; 9773 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9774 if (ret < 0) { 9775 return rte_flow_error_set(error, ret, 9776 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9777 "failed to initialize pmd tunnel"); 9778 } 9779 *actions = &tunnel->action; 9780 *num_of_actions = 1; 9781 return 0; 9782 } 9783 9784 static int 9785 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9786 struct rte_flow_tunnel *app_tunnel, 9787 struct rte_flow_item **items, 9788 uint32_t *num_of_items, 9789 struct rte_flow_error *error) 9790 { 9791 struct mlx5_flow_tunnel *tunnel; 9792 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9793 9794 if (ret) 9795 return ret; 9796 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9797 if (ret < 0) { 9798 return rte_flow_error_set(error, ret, 9799 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9800 "failed to initialize pmd tunnel"); 9801 } 9802 *items = &tunnel->item; 9803 *num_of_items = 1; 9804 return 0; 9805 } 9806 9807 struct tunnel_db_element_release_ctx { 9808 struct rte_flow_item *items; 9809 struct rte_flow_action *actions; 9810 uint32_t num_elements; 9811 struct rte_flow_error *error; 9812 int ret; 9813 }; 9814 9815 static bool 9816 tunnel_element_release_match(struct rte_eth_dev *dev, 9817 struct mlx5_flow_tunnel *tunnel, const void *x) 9818 { 9819 const struct tunnel_db_element_release_ctx *ctx = x; 9820 9821 RTE_SET_USED(dev); 9822 if (ctx->num_elements != 1) 9823 return false; 9824 else if (ctx->items) 9825 return ctx->items == &tunnel->item; 9826 else if (ctx->actions) 9827 return ctx->actions == &tunnel->action; 9828 9829 return false; 9830 } 9831 9832 static void 9833 tunnel_element_release_hit(struct rte_eth_dev *dev, 9834 struct mlx5_flow_tunnel *tunnel, void *x) 9835 { 9836 struct tunnel_db_element_release_ctx *ctx = x; 9837 ctx->ret = 0; 9838 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 9839 mlx5_flow_tunnel_free(dev, tunnel); 9840 } 9841 9842 static void 9843 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 9844 { 9845 struct tunnel_db_element_release_ctx *ctx = x; 9846 RTE_SET_USED(dev); 9847 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 9848 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9849 "invalid argument"); 9850 } 9851 9852 static int 9853 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 9854 struct rte_flow_item *pmd_items, 9855 uint32_t num_items, struct rte_flow_error *err) 9856 { 9857 struct tunnel_db_element_release_ctx ctx = { 9858 .items = pmd_items, 9859 .actions = NULL, 9860 .num_elements = num_items, 9861 .error = err, 9862 }; 9863 9864 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9865 tunnel_element_release_hit, 9866 tunnel_element_release_miss, &ctx, false); 9867 9868 return ctx.ret; 9869 } 9870 9871 static int 9872 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 9873 struct rte_flow_action *pmd_actions, 9874 uint32_t num_actions, struct rte_flow_error *err) 9875 { 9876 struct tunnel_db_element_release_ctx ctx = { 9877 .items = NULL, 9878 .actions = pmd_actions, 9879 .num_elements = num_actions, 9880 .error = err, 9881 }; 9882 9883 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 9884 tunnel_element_release_hit, 9885 tunnel_element_release_miss, &ctx, false); 9886 9887 return ctx.ret; 9888 } 9889 9890 static int 9891 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 9892 struct rte_mbuf *m, 9893 struct rte_flow_restore_info *info, 9894 struct rte_flow_error *err) 9895 { 9896 uint64_t ol_flags = m->ol_flags; 9897 const struct mlx5_flow_tbl_data_entry *tble; 9898 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 9899 9900 if (!is_tunnel_offload_active(dev)) { 9901 info->flags = 0; 9902 return 0; 9903 } 9904 9905 if ((ol_flags & mask) != mask) 9906 goto err; 9907 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 9908 if (!tble) { 9909 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 9910 dev->data->port_id, m->hash.fdir.hi); 9911 goto err; 9912 } 9913 MLX5_ASSERT(tble->tunnel); 9914 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 9915 info->group_id = tble->group_id; 9916 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 9917 RTE_FLOW_RESTORE_INFO_GROUP_ID | 9918 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 9919 9920 return 0; 9921 9922 err: 9923 return rte_flow_error_set(err, EINVAL, 9924 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9925 "failed to get restore info"); 9926 } 9927 9928 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 9929 static int 9930 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 9931 __rte_unused struct rte_flow_tunnel *app_tunnel, 9932 __rte_unused struct rte_flow_action **actions, 9933 __rte_unused uint32_t *num_of_actions, 9934 __rte_unused struct rte_flow_error *error) 9935 { 9936 return -ENOTSUP; 9937 } 9938 9939 static int 9940 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 9941 __rte_unused struct rte_flow_tunnel *app_tunnel, 9942 __rte_unused struct rte_flow_item **items, 9943 __rte_unused uint32_t *num_of_items, 9944 __rte_unused struct rte_flow_error *error) 9945 { 9946 return -ENOTSUP; 9947 } 9948 9949 static int 9950 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 9951 __rte_unused struct rte_flow_item *pmd_items, 9952 __rte_unused uint32_t num_items, 9953 __rte_unused struct rte_flow_error *err) 9954 { 9955 return -ENOTSUP; 9956 } 9957 9958 static int 9959 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 9960 __rte_unused struct rte_flow_action *pmd_action, 9961 __rte_unused uint32_t num_actions, 9962 __rte_unused struct rte_flow_error *err) 9963 { 9964 return -ENOTSUP; 9965 } 9966 9967 static int 9968 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 9969 __rte_unused struct rte_mbuf *m, 9970 __rte_unused struct rte_flow_restore_info *i, 9971 __rte_unused struct rte_flow_error *err) 9972 { 9973 return -ENOTSUP; 9974 } 9975 9976 static int 9977 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 9978 __rte_unused struct rte_flow *flow, 9979 __rte_unused const struct rte_flow_attr *attr, 9980 __rte_unused const struct rte_flow_action *actions, 9981 __rte_unused uint32_t flow_idx, 9982 __rte_unused const struct mlx5_flow_tunnel *tunnel, 9983 __rte_unused struct tunnel_default_miss_ctx *ctx, 9984 __rte_unused struct rte_flow_error *error) 9985 { 9986 return -ENOTSUP; 9987 } 9988 9989 static struct mlx5_flow_tunnel * 9990 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 9991 __rte_unused uint32_t id) 9992 { 9993 return NULL; 9994 } 9995 9996 static void 9997 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 9998 __rte_unused struct mlx5_flow_tunnel *tunnel) 9999 { 10000 } 10001 10002 static uint32_t 10003 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10004 __rte_unused const struct mlx5_flow_tunnel *t, 10005 __rte_unused uint32_t group, 10006 __rte_unused uint32_t *table, 10007 struct rte_flow_error *error) 10008 { 10009 return rte_flow_error_set(error, ENOTSUP, 10010 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10011 "tunnel offload requires DV support"); 10012 } 10013 10014 void 10015 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10016 __rte_unused uint16_t port_id) 10017 { 10018 } 10019 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10020 10021 /* Flex flow item API */ 10022 static struct rte_flow_item_flex_handle * 10023 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10024 const struct rte_flow_item_flex_conf *conf, 10025 struct rte_flow_error *error) 10026 { 10027 static const char err_msg[] = "flex item creation unsupported"; 10028 struct rte_flow_attr attr = { .transfer = 0 }; 10029 const struct mlx5_flow_driver_ops *fops = 10030 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10031 10032 if (!fops->item_create) { 10033 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10034 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10035 NULL, err_msg); 10036 return NULL; 10037 } 10038 return fops->item_create(dev, conf, error); 10039 } 10040 10041 static int 10042 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10043 const struct rte_flow_item_flex_handle *handle, 10044 struct rte_flow_error *error) 10045 { 10046 static const char err_msg[] = "flex item release unsupported"; 10047 struct rte_flow_attr attr = { .transfer = 0 }; 10048 const struct mlx5_flow_driver_ops *fops = 10049 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10050 10051 if (!fops->item_release) { 10052 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10053 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10054 NULL, err_msg); 10055 return -rte_errno; 10056 } 10057 return fops->item_release(dev, handle, error); 10058 } 10059 10060 static void 10061 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10062 { 10063 int ret; 10064 struct rte_flow_error error; 10065 10066 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10067 char *item_name; 10068 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10069 sizeof(item_name), 10070 (void *)(uintptr_t)item->type, &error); 10071 if (ret > 0) 10072 printf("%s ", item_name); 10073 else 10074 printf("%d\n", (int)item->type); 10075 } 10076 printf("END\n"); 10077 } 10078 10079 static int 10080 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10081 { 10082 const struct rte_flow_item_udp *spec = udp_item->spec; 10083 const struct rte_flow_item_udp *mask = udp_item->mask; 10084 uint16_t udp_dport = 0; 10085 10086 if (spec != NULL) { 10087 if (!mask) 10088 mask = &rte_flow_item_udp_mask; 10089 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10090 mask->hdr.dst_port); 10091 } 10092 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10093 } 10094 10095 static const struct mlx5_flow_expand_node * 10096 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10097 unsigned int item_idx, 10098 const struct mlx5_flow_expand_node graph[], 10099 const struct mlx5_flow_expand_node *node) 10100 { 10101 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10102 10103 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10104 node != NULL && 10105 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10106 /* 10107 * The expansion node is VXLAN and it is also the last 10108 * expandable item in the pattern, so need to continue 10109 * expansion of the inner tunnel. 10110 */ 10111 MLX5_ASSERT(item_idx > 0); 10112 prev_item = pattern + item_idx - 1; 10113 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10114 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10115 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10116 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10117 } 10118 return node; 10119 } 10120 10121 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10122 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10123 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10124 }; 10125 10126 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10127 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10128 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10129 { 9, 10, 11 }, { 12, 13, 14 }, 10130 }; 10131 10132 /** 10133 * Discover the number of available flow priorities. 10134 * 10135 * @param dev 10136 * Ethernet device. 10137 * 10138 * @return 10139 * On success, number of available flow priorities. 10140 * On failure, a negative errno-style code and rte_errno is set. 10141 */ 10142 int 10143 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10144 { 10145 static const uint16_t vprio[] = {8, 16}; 10146 const struct mlx5_priv *priv = dev->data->dev_private; 10147 const struct mlx5_flow_driver_ops *fops; 10148 enum mlx5_flow_drv_type type; 10149 int ret; 10150 10151 type = mlx5_flow_os_get_type(); 10152 if (type == MLX5_FLOW_TYPE_MAX) { 10153 type = MLX5_FLOW_TYPE_VERBS; 10154 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10155 type = MLX5_FLOW_TYPE_DV; 10156 } 10157 fops = flow_get_drv_ops(type); 10158 if (fops->discover_priorities == NULL) { 10159 DRV_LOG(ERR, "Priority discovery not supported"); 10160 rte_errno = ENOTSUP; 10161 return -rte_errno; 10162 } 10163 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10164 if (ret < 0) 10165 return ret; 10166 switch (ret) { 10167 case 8: 10168 ret = RTE_DIM(priority_map_3); 10169 break; 10170 case 16: 10171 ret = RTE_DIM(priority_map_5); 10172 break; 10173 default: 10174 rte_errno = ENOTSUP; 10175 DRV_LOG(ERR, 10176 "port %u maximum priority: %d expected 8/16", 10177 dev->data->port_id, ret); 10178 return -rte_errno; 10179 } 10180 DRV_LOG(INFO, "port %u supported flow priorities:" 10181 " 0-%d for ingress or egress root table," 10182 " 0-%d for non-root table or transfer root table.", 10183 dev->data->port_id, ret - 2, 10184 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10185 return ret; 10186 } 10187 10188 /** 10189 * Adjust flow priority based on the highest layer and the request priority. 10190 * 10191 * @param[in] dev 10192 * Pointer to the Ethernet device structure. 10193 * @param[in] priority 10194 * The rule base priority. 10195 * @param[in] subpriority 10196 * The priority based on the items. 10197 * 10198 * @return 10199 * The new priority. 10200 */ 10201 uint32_t 10202 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10203 uint32_t subpriority) 10204 { 10205 uint32_t res = 0; 10206 struct mlx5_priv *priv = dev->data->dev_private; 10207 10208 switch (priv->sh->flow_max_priority) { 10209 case RTE_DIM(priority_map_3): 10210 res = priority_map_3[priority][subpriority]; 10211 break; 10212 case RTE_DIM(priority_map_5): 10213 res = priority_map_5[priority][subpriority]; 10214 break; 10215 } 10216 return res; 10217 } 10218