1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 struct tunnel_default_miss_ctx { 37 uint16_t *queue; 38 __extension__ 39 union { 40 struct rte_flow_action_rss action_rss; 41 struct rte_flow_action_queue miss_queue; 42 struct rte_flow_action_jump miss_jump; 43 uint8_t raw[0]; 44 }; 45 }; 46 47 static int 48 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 49 struct rte_flow *flow, 50 const struct rte_flow_attr *attr, 51 const struct rte_flow_action *app_actions, 52 uint32_t flow_idx, 53 const struct mlx5_flow_tunnel *tunnel, 54 struct tunnel_default_miss_ctx *ctx, 55 struct rte_flow_error *error); 56 static struct mlx5_flow_tunnel * 57 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 58 static void 59 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 60 static uint32_t 61 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 62 const struct mlx5_flow_tunnel *tunnel, 63 uint32_t group, uint32_t *table, 64 struct rte_flow_error *error); 65 66 static struct mlx5_flow_workspace *mlx5_flow_push_thread_workspace(void); 67 static void mlx5_flow_pop_thread_workspace(void); 68 69 70 /** Device flow drivers. */ 71 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 72 73 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 74 75 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 76 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 77 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 78 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 79 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 80 #endif 81 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 82 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 83 }; 84 85 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 86 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 87 (const int []){ \ 88 __VA_ARGS__, 0, \ 89 } 90 91 /** Node object of input graph for mlx5_flow_expand_rss(). */ 92 struct mlx5_flow_expand_node { 93 const int *const next; 94 /**< 95 * List of next node indexes. Index 0 is interpreted as a terminator. 96 */ 97 const enum rte_flow_item_type type; 98 /**< Pattern item type of current node. */ 99 uint64_t rss_types; 100 /**< 101 * RSS types bit-field associated with this node 102 * (see RTE_ETH_RSS_* definitions). 103 */ 104 uint64_t node_flags; 105 /**< 106 * Bit-fields that define how the node is used in the expansion. 107 * (see MLX5_EXPANSION_NODE_* definitions). 108 */ 109 }; 110 111 /* Optional expand field. The expansion alg will not go deeper. */ 112 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 113 114 /* The node is not added implicitly as expansion to the flow pattern. 115 * If the node type does not match the flow pattern item type, the 116 * expansion alg will go deeper to its next items. 117 * In the current implementation, the list of next nodes indexes can 118 * have up to one node with this flag set and it has to be the last 119 * node index (before the list terminator). 120 */ 121 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 122 123 /** Object returned by mlx5_flow_expand_rss(). */ 124 struct mlx5_flow_expand_rss { 125 uint32_t entries; 126 /**< Number of entries @p patterns and @p priorities. */ 127 struct { 128 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 129 uint32_t priority; /**< Priority offset for each expansion. */ 130 } entry[]; 131 }; 132 133 static void 134 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 135 136 static const struct mlx5_flow_expand_node * 137 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 138 unsigned int item_idx, 139 const struct mlx5_flow_expand_node graph[], 140 const struct mlx5_flow_expand_node *node); 141 142 static bool 143 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 144 { 145 switch (item->type) { 146 case RTE_FLOW_ITEM_TYPE_ETH: 147 case RTE_FLOW_ITEM_TYPE_VLAN: 148 case RTE_FLOW_ITEM_TYPE_IPV4: 149 case RTE_FLOW_ITEM_TYPE_IPV6: 150 case RTE_FLOW_ITEM_TYPE_UDP: 151 case RTE_FLOW_ITEM_TYPE_TCP: 152 case RTE_FLOW_ITEM_TYPE_VXLAN: 153 case RTE_FLOW_ITEM_TYPE_NVGRE: 154 case RTE_FLOW_ITEM_TYPE_GRE: 155 case RTE_FLOW_ITEM_TYPE_GENEVE: 156 case RTE_FLOW_ITEM_TYPE_MPLS: 157 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 158 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 159 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 160 case RTE_FLOW_ITEM_TYPE_GTP: 161 return true; 162 default: 163 break; 164 } 165 return false; 166 } 167 168 /** 169 * Network Service Header (NSH) and its next protocol values 170 * are described in RFC-8393. 171 */ 172 static enum rte_flow_item_type 173 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 174 { 175 enum rte_flow_item_type type; 176 177 switch (proto_mask & proto_spec) { 178 case RTE_VXLAN_GPE_TYPE_IPV4: 179 type = RTE_FLOW_ITEM_TYPE_IPV4; 180 break; 181 case RTE_VXLAN_GPE_TYPE_IPV6: 182 type = RTE_VXLAN_GPE_TYPE_IPV6; 183 break; 184 case RTE_VXLAN_GPE_TYPE_ETH: 185 type = RTE_FLOW_ITEM_TYPE_ETH; 186 break; 187 default: 188 type = RTE_FLOW_ITEM_TYPE_END; 189 } 190 return type; 191 } 192 193 static enum rte_flow_item_type 194 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 195 { 196 enum rte_flow_item_type type; 197 198 switch (proto_mask & proto_spec) { 199 case IPPROTO_UDP: 200 type = RTE_FLOW_ITEM_TYPE_UDP; 201 break; 202 case IPPROTO_TCP: 203 type = RTE_FLOW_ITEM_TYPE_TCP; 204 break; 205 case IPPROTO_IP: 206 type = RTE_FLOW_ITEM_TYPE_IPV4; 207 break; 208 case IPPROTO_IPV6: 209 type = RTE_FLOW_ITEM_TYPE_IPV6; 210 break; 211 default: 212 type = RTE_FLOW_ITEM_TYPE_END; 213 } 214 return type; 215 } 216 217 static enum rte_flow_item_type 218 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 219 rte_be16_t type_mask, bool is_tunnel) 220 { 221 enum rte_flow_item_type type; 222 223 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 224 case RTE_ETHER_TYPE_TEB: 225 type = is_tunnel ? 226 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 227 break; 228 case RTE_ETHER_TYPE_VLAN: 229 type = !is_tunnel ? 230 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 231 break; 232 case RTE_ETHER_TYPE_IPV4: 233 type = RTE_FLOW_ITEM_TYPE_IPV4; 234 break; 235 case RTE_ETHER_TYPE_IPV6: 236 type = RTE_FLOW_ITEM_TYPE_IPV6; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 246 { 247 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 248 do { \ 249 const void *m = item->mask; \ 250 const void *s = item->spec; \ 251 mask = m ? \ 252 ((const struct rte_flow_item_##type *)m)->fld : \ 253 rte_flow_item_##type##_mask.fld; \ 254 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 255 } while (0) 256 257 enum rte_flow_item_type ret; 258 uint16_t spec, mask; 259 260 if (item == NULL || item->spec == NULL) 261 return RTE_FLOW_ITEM_TYPE_VOID; 262 switch (item->type) { 263 case RTE_FLOW_ITEM_TYPE_ETH: 264 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 265 if (!mask) 266 return RTE_FLOW_ITEM_TYPE_VOID; 267 ret = mlx5_ethertype_to_item_type(spec, mask, false); 268 break; 269 case RTE_FLOW_ITEM_TYPE_VLAN: 270 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 271 if (!mask) 272 return RTE_FLOW_ITEM_TYPE_VOID; 273 ret = mlx5_ethertype_to_item_type(spec, mask, false); 274 break; 275 case RTE_FLOW_ITEM_TYPE_IPV4: 276 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 277 if (!mask) 278 return RTE_FLOW_ITEM_TYPE_VOID; 279 ret = mlx5_inet_proto_to_item_type(spec, mask); 280 break; 281 case RTE_FLOW_ITEM_TYPE_IPV6: 282 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 283 if (!mask) 284 return RTE_FLOW_ITEM_TYPE_VOID; 285 ret = mlx5_inet_proto_to_item_type(spec, mask); 286 break; 287 case RTE_FLOW_ITEM_TYPE_GENEVE: 288 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 289 ret = mlx5_ethertype_to_item_type(spec, mask, true); 290 break; 291 case RTE_FLOW_ITEM_TYPE_GRE: 292 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 293 ret = mlx5_ethertype_to_item_type(spec, mask, true); 294 break; 295 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 296 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 297 ret = mlx5_nsh_proto_to_item_type(spec, mask); 298 break; 299 default: 300 ret = RTE_FLOW_ITEM_TYPE_VOID; 301 break; 302 } 303 return ret; 304 #undef MLX5_XSET_ITEM_MASK_SPEC 305 } 306 307 static const int * 308 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 309 const int *next_node) 310 { 311 const struct mlx5_flow_expand_node *node = NULL; 312 const int *next = next_node; 313 314 while (next && *next) { 315 /* 316 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 317 * flag set, because they were not found in the flow pattern. 318 */ 319 node = &graph[*next]; 320 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 321 break; 322 next = node->next; 323 } 324 return next; 325 } 326 327 #define MLX5_RSS_EXP_ELT_N 16 328 329 /** 330 * Expand RSS flows into several possible flows according to the RSS hash 331 * fields requested and the driver capabilities. 332 * 333 * @param[out] buf 334 * Buffer to store the result expansion. 335 * @param[in] size 336 * Buffer size in bytes. If 0, @p buf can be NULL. 337 * @param[in] pattern 338 * User flow pattern. 339 * @param[in] types 340 * RSS types to expand (see RTE_ETH_RSS_* definitions). 341 * @param[in] graph 342 * Input graph to expand @p pattern according to @p types. 343 * @param[in] graph_root_index 344 * Index of root node in @p graph, typically 0. 345 * 346 * @return 347 * A positive value representing the size of @p buf in bytes regardless of 348 * @p size on success, a negative errno value otherwise and rte_errno is 349 * set, the following errors are defined: 350 * 351 * -E2BIG: graph-depth @p graph is too deep. 352 * -EINVAL: @p size has not enough space for expanded pattern. 353 */ 354 static int 355 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 356 const struct rte_flow_item *pattern, uint64_t types, 357 const struct mlx5_flow_expand_node graph[], 358 int graph_root_index) 359 { 360 const struct rte_flow_item *item; 361 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 362 const int *next_node; 363 const int *stack[MLX5_RSS_EXP_ELT_N]; 364 int stack_pos = 0; 365 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 366 unsigned int i, item_idx, last_expand_item_idx = 0; 367 size_t lsize; 368 size_t user_pattern_size = 0; 369 void *addr = NULL; 370 const struct mlx5_flow_expand_node *next = NULL; 371 struct rte_flow_item missed_item; 372 int missed = 0; 373 int elt = 0; 374 const struct rte_flow_item *last_expand_item = NULL; 375 376 memset(&missed_item, 0, sizeof(missed_item)); 377 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 378 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 379 if (lsize > size) 380 return -EINVAL; 381 buf->entry[0].priority = 0; 382 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 383 buf->entries = 0; 384 addr = buf->entry[0].pattern; 385 for (item = pattern, item_idx = 0; 386 item->type != RTE_FLOW_ITEM_TYPE_END; 387 item++, item_idx++) { 388 if (!mlx5_flow_is_rss_expandable_item(item)) { 389 user_pattern_size += sizeof(*item); 390 continue; 391 } 392 last_expand_item = item; 393 last_expand_item_idx = item_idx; 394 i = 0; 395 while (node->next && node->next[i]) { 396 next = &graph[node->next[i]]; 397 if (next->type == item->type) 398 break; 399 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 400 node = next; 401 i = 0; 402 } else { 403 ++i; 404 } 405 } 406 if (next) 407 node = next; 408 user_pattern_size += sizeof(*item); 409 } 410 user_pattern_size += sizeof(*item); /* Handle END item. */ 411 lsize += user_pattern_size; 412 if (lsize > size) 413 return -EINVAL; 414 /* Copy the user pattern in the first entry of the buffer. */ 415 rte_memcpy(addr, pattern, user_pattern_size); 416 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 417 buf->entries = 1; 418 /* Start expanding. */ 419 memset(flow_items, 0, sizeof(flow_items)); 420 user_pattern_size -= sizeof(*item); 421 /* 422 * Check if the last valid item has spec set, need complete pattern, 423 * and the pattern can be used for expansion. 424 */ 425 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 426 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 427 /* Item type END indicates expansion is not required. */ 428 return lsize; 429 } 430 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 431 next = NULL; 432 missed = 1; 433 i = 0; 434 while (node->next && node->next[i]) { 435 next = &graph[node->next[i]]; 436 if (next->type == missed_item.type) { 437 flow_items[0].type = missed_item.type; 438 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 439 break; 440 } 441 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 442 node = next; 443 i = 0; 444 } else { 445 ++i; 446 } 447 next = NULL; 448 } 449 } 450 if (next && missed) { 451 elt = 2; /* missed item + item end. */ 452 node = next; 453 lsize += elt * sizeof(*item) + user_pattern_size; 454 if (lsize > size) 455 return -EINVAL; 456 if (node->rss_types & types) { 457 buf->entry[buf->entries].priority = 1; 458 buf->entry[buf->entries].pattern = addr; 459 buf->entries++; 460 rte_memcpy(addr, buf->entry[0].pattern, 461 user_pattern_size); 462 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 463 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 464 addr = (void *)(((uintptr_t)addr) + 465 elt * sizeof(*item)); 466 } 467 } else if (last_expand_item != NULL) { 468 node = mlx5_flow_expand_rss_adjust_node(pattern, 469 last_expand_item_idx, graph, node); 470 } 471 memset(flow_items, 0, sizeof(flow_items)); 472 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 473 node->next); 474 stack[stack_pos] = next_node; 475 node = next_node ? &graph[*next_node] : NULL; 476 while (node) { 477 flow_items[stack_pos].type = node->type; 478 if (node->rss_types & types) { 479 size_t n; 480 /* 481 * compute the number of items to copy from the 482 * expansion and copy it. 483 * When the stack_pos is 0, there are 1 element in it, 484 * plus the addition END item. 485 */ 486 elt = stack_pos + 2; 487 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 488 lsize += elt * sizeof(*item) + user_pattern_size; 489 if (lsize > size) 490 return -EINVAL; 491 n = elt * sizeof(*item); 492 buf->entry[buf->entries].priority = 493 stack_pos + 1 + missed; 494 buf->entry[buf->entries].pattern = addr; 495 buf->entries++; 496 rte_memcpy(addr, buf->entry[0].pattern, 497 user_pattern_size); 498 addr = (void *)(((uintptr_t)addr) + 499 user_pattern_size); 500 rte_memcpy(addr, &missed_item, 501 missed * sizeof(*item)); 502 addr = (void *)(((uintptr_t)addr) + 503 missed * sizeof(*item)); 504 rte_memcpy(addr, flow_items, n); 505 addr = (void *)(((uintptr_t)addr) + n); 506 } 507 /* Go deeper. */ 508 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 509 node->next) { 510 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 511 node->next); 512 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 513 rte_errno = E2BIG; 514 return -rte_errno; 515 } 516 stack[stack_pos] = next_node; 517 } else if (*(next_node + 1)) { 518 /* Follow up with the next possibility. */ 519 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 520 ++next_node); 521 } else if (!stack_pos) { 522 /* 523 * Completing the traverse over the different paths. 524 * The next_node is advanced to the terminator. 525 */ 526 ++next_node; 527 } else { 528 /* Move to the next path. */ 529 while (stack_pos) { 530 next_node = stack[--stack_pos]; 531 next_node++; 532 if (*next_node) 533 break; 534 } 535 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 536 next_node); 537 stack[stack_pos] = next_node; 538 } 539 node = next_node && *next_node ? &graph[*next_node] : NULL; 540 }; 541 return lsize; 542 } 543 544 enum mlx5_expansion { 545 MLX5_EXPANSION_ROOT, 546 MLX5_EXPANSION_ROOT_OUTER, 547 MLX5_EXPANSION_OUTER_ETH, 548 MLX5_EXPANSION_OUTER_VLAN, 549 MLX5_EXPANSION_OUTER_IPV4, 550 MLX5_EXPANSION_OUTER_IPV4_UDP, 551 MLX5_EXPANSION_OUTER_IPV4_TCP, 552 MLX5_EXPANSION_OUTER_IPV6, 553 MLX5_EXPANSION_OUTER_IPV6_UDP, 554 MLX5_EXPANSION_OUTER_IPV6_TCP, 555 MLX5_EXPANSION_VXLAN, 556 MLX5_EXPANSION_STD_VXLAN, 557 MLX5_EXPANSION_L3_VXLAN, 558 MLX5_EXPANSION_VXLAN_GPE, 559 MLX5_EXPANSION_GRE, 560 MLX5_EXPANSION_NVGRE, 561 MLX5_EXPANSION_GRE_KEY, 562 MLX5_EXPANSION_MPLS, 563 MLX5_EXPANSION_ETH, 564 MLX5_EXPANSION_VLAN, 565 MLX5_EXPANSION_IPV4, 566 MLX5_EXPANSION_IPV4_UDP, 567 MLX5_EXPANSION_IPV4_TCP, 568 MLX5_EXPANSION_IPV6, 569 MLX5_EXPANSION_IPV6_UDP, 570 MLX5_EXPANSION_IPV6_TCP, 571 MLX5_EXPANSION_IPV6_FRAG_EXT, 572 MLX5_EXPANSION_GTP, 573 MLX5_EXPANSION_GENEVE, 574 }; 575 576 /** Supported expansion of items. */ 577 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 578 [MLX5_EXPANSION_ROOT] = { 579 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 580 MLX5_EXPANSION_IPV4, 581 MLX5_EXPANSION_IPV6), 582 .type = RTE_FLOW_ITEM_TYPE_END, 583 }, 584 [MLX5_EXPANSION_ROOT_OUTER] = { 585 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 586 MLX5_EXPANSION_OUTER_IPV4, 587 MLX5_EXPANSION_OUTER_IPV6), 588 .type = RTE_FLOW_ITEM_TYPE_END, 589 }, 590 [MLX5_EXPANSION_OUTER_ETH] = { 591 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 592 .type = RTE_FLOW_ITEM_TYPE_ETH, 593 .rss_types = 0, 594 }, 595 [MLX5_EXPANSION_OUTER_VLAN] = { 596 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 597 MLX5_EXPANSION_OUTER_IPV6), 598 .type = RTE_FLOW_ITEM_TYPE_VLAN, 599 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 600 }, 601 [MLX5_EXPANSION_OUTER_IPV4] = { 602 .next = MLX5_FLOW_EXPAND_RSS_NEXT 603 (MLX5_EXPANSION_OUTER_IPV4_UDP, 604 MLX5_EXPANSION_OUTER_IPV4_TCP, 605 MLX5_EXPANSION_GRE, 606 MLX5_EXPANSION_NVGRE, 607 MLX5_EXPANSION_IPV4, 608 MLX5_EXPANSION_IPV6), 609 .type = RTE_FLOW_ITEM_TYPE_IPV4, 610 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 611 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 612 }, 613 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 614 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 615 MLX5_EXPANSION_VXLAN_GPE, 616 MLX5_EXPANSION_MPLS, 617 MLX5_EXPANSION_GENEVE, 618 MLX5_EXPANSION_GTP), 619 .type = RTE_FLOW_ITEM_TYPE_UDP, 620 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 621 }, 622 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 623 .type = RTE_FLOW_ITEM_TYPE_TCP, 624 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 625 }, 626 [MLX5_EXPANSION_OUTER_IPV6] = { 627 .next = MLX5_FLOW_EXPAND_RSS_NEXT 628 (MLX5_EXPANSION_OUTER_IPV6_UDP, 629 MLX5_EXPANSION_OUTER_IPV6_TCP, 630 MLX5_EXPANSION_IPV4, 631 MLX5_EXPANSION_IPV6, 632 MLX5_EXPANSION_GRE, 633 MLX5_EXPANSION_NVGRE), 634 .type = RTE_FLOW_ITEM_TYPE_IPV6, 635 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 636 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 637 }, 638 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 639 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 640 MLX5_EXPANSION_VXLAN_GPE, 641 MLX5_EXPANSION_MPLS, 642 MLX5_EXPANSION_GENEVE, 643 MLX5_EXPANSION_GTP), 644 .type = RTE_FLOW_ITEM_TYPE_UDP, 645 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 646 }, 647 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 648 .type = RTE_FLOW_ITEM_TYPE_TCP, 649 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 650 }, 651 [MLX5_EXPANSION_VXLAN] = { 652 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 653 MLX5_EXPANSION_IPV4, 654 MLX5_EXPANSION_IPV6), 655 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 656 }, 657 [MLX5_EXPANSION_STD_VXLAN] = { 658 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 659 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 660 }, 661 [MLX5_EXPANSION_L3_VXLAN] = { 662 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 663 MLX5_EXPANSION_IPV6), 664 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 665 }, 666 [MLX5_EXPANSION_VXLAN_GPE] = { 667 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 668 MLX5_EXPANSION_IPV4, 669 MLX5_EXPANSION_IPV6), 670 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 671 }, 672 [MLX5_EXPANSION_GRE] = { 673 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 674 MLX5_EXPANSION_IPV4, 675 MLX5_EXPANSION_IPV6, 676 MLX5_EXPANSION_GRE_KEY, 677 MLX5_EXPANSION_MPLS), 678 .type = RTE_FLOW_ITEM_TYPE_GRE, 679 }, 680 [MLX5_EXPANSION_GRE_KEY] = { 681 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 682 MLX5_EXPANSION_IPV6, 683 MLX5_EXPANSION_MPLS), 684 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 685 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 686 }, 687 [MLX5_EXPANSION_NVGRE] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 689 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 690 }, 691 [MLX5_EXPANSION_MPLS] = { 692 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 693 MLX5_EXPANSION_IPV6, 694 MLX5_EXPANSION_ETH), 695 .type = RTE_FLOW_ITEM_TYPE_MPLS, 696 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 697 }, 698 [MLX5_EXPANSION_ETH] = { 699 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 700 .type = RTE_FLOW_ITEM_TYPE_ETH, 701 }, 702 [MLX5_EXPANSION_VLAN] = { 703 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 704 MLX5_EXPANSION_IPV6), 705 .type = RTE_FLOW_ITEM_TYPE_VLAN, 706 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 707 }, 708 [MLX5_EXPANSION_IPV4] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 710 MLX5_EXPANSION_IPV4_TCP), 711 .type = RTE_FLOW_ITEM_TYPE_IPV4, 712 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 713 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 714 }, 715 [MLX5_EXPANSION_IPV4_UDP] = { 716 .type = RTE_FLOW_ITEM_TYPE_UDP, 717 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 718 }, 719 [MLX5_EXPANSION_IPV4_TCP] = { 720 .type = RTE_FLOW_ITEM_TYPE_TCP, 721 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 722 }, 723 [MLX5_EXPANSION_IPV6] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 725 MLX5_EXPANSION_IPV6_TCP, 726 MLX5_EXPANSION_IPV6_FRAG_EXT), 727 .type = RTE_FLOW_ITEM_TYPE_IPV6, 728 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 729 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 730 }, 731 [MLX5_EXPANSION_IPV6_UDP] = { 732 .type = RTE_FLOW_ITEM_TYPE_UDP, 733 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 734 }, 735 [MLX5_EXPANSION_IPV6_TCP] = { 736 .type = RTE_FLOW_ITEM_TYPE_TCP, 737 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 738 }, 739 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 740 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 741 }, 742 [MLX5_EXPANSION_GTP] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_GTP, 746 }, 747 [MLX5_EXPANSION_GENEVE] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 752 }, 753 }; 754 755 static struct rte_flow_action_handle * 756 mlx5_action_handle_create(struct rte_eth_dev *dev, 757 const struct rte_flow_indir_action_conf *conf, 758 const struct rte_flow_action *action, 759 struct rte_flow_error *error); 760 static int mlx5_action_handle_destroy 761 (struct rte_eth_dev *dev, 762 struct rte_flow_action_handle *handle, 763 struct rte_flow_error *error); 764 static int mlx5_action_handle_update 765 (struct rte_eth_dev *dev, 766 struct rte_flow_action_handle *handle, 767 const void *update, 768 struct rte_flow_error *error); 769 static int mlx5_action_handle_query 770 (struct rte_eth_dev *dev, 771 const struct rte_flow_action_handle *handle, 772 void *data, 773 struct rte_flow_error *error); 774 static int 775 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 776 struct rte_flow_tunnel *app_tunnel, 777 struct rte_flow_action **actions, 778 uint32_t *num_of_actions, 779 struct rte_flow_error *error); 780 static int 781 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 782 struct rte_flow_tunnel *app_tunnel, 783 struct rte_flow_item **items, 784 uint32_t *num_of_items, 785 struct rte_flow_error *error); 786 static int 787 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 788 struct rte_flow_item *pmd_items, 789 uint32_t num_items, struct rte_flow_error *err); 790 static int 791 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 792 struct rte_flow_action *pmd_actions, 793 uint32_t num_actions, 794 struct rte_flow_error *err); 795 static int 796 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 797 struct rte_mbuf *m, 798 struct rte_flow_restore_info *info, 799 struct rte_flow_error *err); 800 static struct rte_flow_item_flex_handle * 801 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 802 const struct rte_flow_item_flex_conf *conf, 803 struct rte_flow_error *error); 804 static int 805 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 806 const struct rte_flow_item_flex_handle *handle, 807 struct rte_flow_error *error); 808 static int 809 mlx5_flow_info_get(struct rte_eth_dev *dev, 810 struct rte_flow_port_info *port_info, 811 struct rte_flow_queue_info *queue_info, 812 struct rte_flow_error *error); 813 static int 814 mlx5_flow_port_configure(struct rte_eth_dev *dev, 815 const struct rte_flow_port_attr *port_attr, 816 uint16_t nb_queue, 817 const struct rte_flow_queue_attr *queue_attr[], 818 struct rte_flow_error *err); 819 820 static struct rte_flow_pattern_template * 821 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 822 const struct rte_flow_pattern_template_attr *attr, 823 const struct rte_flow_item items[], 824 struct rte_flow_error *error); 825 826 static int 827 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 828 struct rte_flow_pattern_template *template, 829 struct rte_flow_error *error); 830 static struct rte_flow_actions_template * 831 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 832 const struct rte_flow_actions_template_attr *attr, 833 const struct rte_flow_action actions[], 834 const struct rte_flow_action masks[], 835 struct rte_flow_error *error); 836 static int 837 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 838 struct rte_flow_actions_template *template, 839 struct rte_flow_error *error); 840 841 static struct rte_flow_template_table * 842 mlx5_flow_table_create(struct rte_eth_dev *dev, 843 const struct rte_flow_template_table_attr *attr, 844 struct rte_flow_pattern_template *item_templates[], 845 uint8_t nb_item_templates, 846 struct rte_flow_actions_template *action_templates[], 847 uint8_t nb_action_templates, 848 struct rte_flow_error *error); 849 static int 850 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 851 struct rte_flow_template_table *table, 852 struct rte_flow_error *error); 853 854 static const struct rte_flow_ops mlx5_flow_ops = { 855 .validate = mlx5_flow_validate, 856 .create = mlx5_flow_create, 857 .destroy = mlx5_flow_destroy, 858 .flush = mlx5_flow_flush, 859 .isolate = mlx5_flow_isolate, 860 .query = mlx5_flow_query, 861 .dev_dump = mlx5_flow_dev_dump, 862 .get_aged_flows = mlx5_flow_get_aged_flows, 863 .action_handle_create = mlx5_action_handle_create, 864 .action_handle_destroy = mlx5_action_handle_destroy, 865 .action_handle_update = mlx5_action_handle_update, 866 .action_handle_query = mlx5_action_handle_query, 867 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 868 .tunnel_match = mlx5_flow_tunnel_match, 869 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 870 .tunnel_item_release = mlx5_flow_tunnel_item_release, 871 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 872 .flex_item_create = mlx5_flow_flex_item_create, 873 .flex_item_release = mlx5_flow_flex_item_release, 874 .info_get = mlx5_flow_info_get, 875 .configure = mlx5_flow_port_configure, 876 .pattern_template_create = mlx5_flow_pattern_template_create, 877 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 878 .actions_template_create = mlx5_flow_actions_template_create, 879 .actions_template_destroy = mlx5_flow_actions_template_destroy, 880 .template_table_create = mlx5_flow_table_create, 881 .template_table_destroy = mlx5_flow_table_destroy, 882 }; 883 884 /* Tunnel information. */ 885 struct mlx5_flow_tunnel_info { 886 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 887 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 888 }; 889 890 static struct mlx5_flow_tunnel_info tunnels_info[] = { 891 { 892 .tunnel = MLX5_FLOW_LAYER_VXLAN, 893 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 894 }, 895 { 896 .tunnel = MLX5_FLOW_LAYER_GENEVE, 897 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 898 }, 899 { 900 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 901 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 902 }, 903 { 904 .tunnel = MLX5_FLOW_LAYER_GRE, 905 .ptype = RTE_PTYPE_TUNNEL_GRE, 906 }, 907 { 908 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 909 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 910 }, 911 { 912 .tunnel = MLX5_FLOW_LAYER_MPLS, 913 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 914 }, 915 { 916 .tunnel = MLX5_FLOW_LAYER_NVGRE, 917 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 918 }, 919 { 920 .tunnel = MLX5_FLOW_LAYER_IPIP, 921 .ptype = RTE_PTYPE_TUNNEL_IP, 922 }, 923 { 924 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 925 .ptype = RTE_PTYPE_TUNNEL_IP, 926 }, 927 { 928 .tunnel = MLX5_FLOW_LAYER_GTP, 929 .ptype = RTE_PTYPE_TUNNEL_GTPU, 930 }, 931 }; 932 933 934 935 /** 936 * Translate tag ID to register. 937 * 938 * @param[in] dev 939 * Pointer to the Ethernet device structure. 940 * @param[in] feature 941 * The feature that request the register. 942 * @param[in] id 943 * The request register ID. 944 * @param[out] error 945 * Error description in case of any. 946 * 947 * @return 948 * The request register on success, a negative errno 949 * value otherwise and rte_errno is set. 950 */ 951 int 952 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 953 enum mlx5_feature_name feature, 954 uint32_t id, 955 struct rte_flow_error *error) 956 { 957 struct mlx5_priv *priv = dev->data->dev_private; 958 struct mlx5_sh_config *config = &priv->sh->config; 959 enum modify_reg start_reg; 960 bool skip_mtr_reg = false; 961 962 switch (feature) { 963 case MLX5_HAIRPIN_RX: 964 return REG_B; 965 case MLX5_HAIRPIN_TX: 966 return REG_A; 967 case MLX5_METADATA_RX: 968 switch (config->dv_xmeta_en) { 969 case MLX5_XMETA_MODE_LEGACY: 970 return REG_B; 971 case MLX5_XMETA_MODE_META16: 972 return REG_C_0; 973 case MLX5_XMETA_MODE_META32: 974 return REG_C_1; 975 } 976 break; 977 case MLX5_METADATA_TX: 978 return REG_A; 979 case MLX5_METADATA_FDB: 980 switch (config->dv_xmeta_en) { 981 case MLX5_XMETA_MODE_LEGACY: 982 return REG_NON; 983 case MLX5_XMETA_MODE_META16: 984 return REG_C_0; 985 case MLX5_XMETA_MODE_META32: 986 return REG_C_1; 987 } 988 break; 989 case MLX5_FLOW_MARK: 990 switch (config->dv_xmeta_en) { 991 case MLX5_XMETA_MODE_LEGACY: 992 return REG_NON; 993 case MLX5_XMETA_MODE_META16: 994 return REG_C_1; 995 case MLX5_XMETA_MODE_META32: 996 return REG_C_0; 997 } 998 break; 999 case MLX5_MTR_ID: 1000 /* 1001 * If meter color and meter id share one register, flow match 1002 * should use the meter color register for match. 1003 */ 1004 if (priv->mtr_reg_share) 1005 return priv->mtr_color_reg; 1006 else 1007 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1008 REG_C_3; 1009 case MLX5_MTR_COLOR: 1010 case MLX5_ASO_FLOW_HIT: 1011 case MLX5_ASO_CONNTRACK: 1012 case MLX5_SAMPLE_ID: 1013 /* All features use the same REG_C. */ 1014 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1015 return priv->mtr_color_reg; 1016 case MLX5_COPY_MARK: 1017 /* 1018 * Metadata COPY_MARK register using is in meter suffix sub 1019 * flow while with meter. It's safe to share the same register. 1020 */ 1021 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1022 case MLX5_APP_TAG: 1023 /* 1024 * If meter is enable, it will engage the register for color 1025 * match and flow match. If meter color match is not using the 1026 * REG_C_2, need to skip the REG_C_x be used by meter color 1027 * match. 1028 * If meter is disable, free to use all available registers. 1029 */ 1030 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1031 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1032 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1033 if (id > (uint32_t)(REG_C_7 - start_reg)) 1034 return rte_flow_error_set(error, EINVAL, 1035 RTE_FLOW_ERROR_TYPE_ITEM, 1036 NULL, "invalid tag id"); 1037 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1038 return rte_flow_error_set(error, ENOTSUP, 1039 RTE_FLOW_ERROR_TYPE_ITEM, 1040 NULL, "unsupported tag id"); 1041 /* 1042 * This case means meter is using the REG_C_x great than 2. 1043 * Take care not to conflict with meter color REG_C_x. 1044 * If the available index REG_C_y >= REG_C_x, skip the 1045 * color register. 1046 */ 1047 if (skip_mtr_reg && priv->sh->flow_mreg_c 1048 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1049 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1050 return rte_flow_error_set(error, EINVAL, 1051 RTE_FLOW_ERROR_TYPE_ITEM, 1052 NULL, "invalid tag id"); 1053 if (priv->sh->flow_mreg_c 1054 [id + 1 + start_reg - REG_C_0] != REG_NON) 1055 return priv->sh->flow_mreg_c 1056 [id + 1 + start_reg - REG_C_0]; 1057 return rte_flow_error_set(error, ENOTSUP, 1058 RTE_FLOW_ERROR_TYPE_ITEM, 1059 NULL, "unsupported tag id"); 1060 } 1061 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1062 } 1063 MLX5_ASSERT(false); 1064 return rte_flow_error_set(error, EINVAL, 1065 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1066 NULL, "invalid feature name"); 1067 } 1068 1069 /** 1070 * Check extensive flow metadata register support. 1071 * 1072 * @param dev 1073 * Pointer to rte_eth_dev structure. 1074 * 1075 * @return 1076 * True if device supports extensive flow metadata register, otherwise false. 1077 */ 1078 bool 1079 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1080 { 1081 struct mlx5_priv *priv = dev->data->dev_private; 1082 1083 /* 1084 * Having available reg_c can be regarded inclusively as supporting 1085 * extensive flow metadata register, which could mean, 1086 * - metadata register copy action by modify header. 1087 * - 16 modify header actions is supported. 1088 * - reg_c's are preserved across different domain (FDB and NIC) on 1089 * packet loopback by flow lookup miss. 1090 */ 1091 return priv->sh->flow_mreg_c[2] != REG_NON; 1092 } 1093 1094 /** 1095 * Get the lowest priority. 1096 * 1097 * @param[in] dev 1098 * Pointer to the Ethernet device structure. 1099 * @param[in] attributes 1100 * Pointer to device flow rule attributes. 1101 * 1102 * @return 1103 * The value of lowest priority of flow. 1104 */ 1105 uint32_t 1106 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1107 const struct rte_flow_attr *attr) 1108 { 1109 struct mlx5_priv *priv = dev->data->dev_private; 1110 1111 if (!attr->group && !attr->transfer) 1112 return priv->sh->flow_max_priority - 2; 1113 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1114 } 1115 1116 /** 1117 * Calculate matcher priority of the flow. 1118 * 1119 * @param[in] dev 1120 * Pointer to the Ethernet device structure. 1121 * @param[in] attr 1122 * Pointer to device flow rule attributes. 1123 * @param[in] subpriority 1124 * The priority based on the items. 1125 * @param[in] external 1126 * Flow is user flow. 1127 * @return 1128 * The matcher priority of the flow. 1129 */ 1130 uint16_t 1131 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1132 const struct rte_flow_attr *attr, 1133 uint32_t subpriority, bool external) 1134 { 1135 uint16_t priority = (uint16_t)attr->priority; 1136 struct mlx5_priv *priv = dev->data->dev_private; 1137 1138 if (!attr->group && !attr->transfer) { 1139 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1140 priority = priv->sh->flow_max_priority - 1; 1141 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1142 } else if (!external && attr->transfer && attr->group == 0 && 1143 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1144 return (priv->sh->flow_max_priority - 1) * 3; 1145 } 1146 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1147 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1148 return priority * 3 + subpriority; 1149 } 1150 1151 /** 1152 * Verify the @p item specifications (spec, last, mask) are compatible with the 1153 * NIC capabilities. 1154 * 1155 * @param[in] item 1156 * Item specification. 1157 * @param[in] mask 1158 * @p item->mask or flow default bit-masks. 1159 * @param[in] nic_mask 1160 * Bit-masks covering supported fields by the NIC to compare with user mask. 1161 * @param[in] size 1162 * Bit-masks size in bytes. 1163 * @param[in] range_accepted 1164 * True if range of values is accepted for specific fields, false otherwise. 1165 * @param[out] error 1166 * Pointer to error structure. 1167 * 1168 * @return 1169 * 0 on success, a negative errno value otherwise and rte_errno is set. 1170 */ 1171 int 1172 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1173 const uint8_t *mask, 1174 const uint8_t *nic_mask, 1175 unsigned int size, 1176 bool range_accepted, 1177 struct rte_flow_error *error) 1178 { 1179 unsigned int i; 1180 1181 MLX5_ASSERT(nic_mask); 1182 for (i = 0; i < size; ++i) 1183 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1184 return rte_flow_error_set(error, ENOTSUP, 1185 RTE_FLOW_ERROR_TYPE_ITEM, 1186 item, 1187 "mask enables non supported" 1188 " bits"); 1189 if (!item->spec && (item->mask || item->last)) 1190 return rte_flow_error_set(error, EINVAL, 1191 RTE_FLOW_ERROR_TYPE_ITEM, item, 1192 "mask/last without a spec is not" 1193 " supported"); 1194 if (item->spec && item->last && !range_accepted) { 1195 uint8_t spec[size]; 1196 uint8_t last[size]; 1197 unsigned int i; 1198 int ret; 1199 1200 for (i = 0; i < size; ++i) { 1201 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1202 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1203 } 1204 ret = memcmp(spec, last, size); 1205 if (ret != 0) 1206 return rte_flow_error_set(error, EINVAL, 1207 RTE_FLOW_ERROR_TYPE_ITEM, 1208 item, 1209 "range is not valid"); 1210 } 1211 return 0; 1212 } 1213 1214 /** 1215 * Adjust the hash fields according to the @p flow information. 1216 * 1217 * @param[in] dev_flow. 1218 * Pointer to the mlx5_flow. 1219 * @param[in] tunnel 1220 * 1 when the hash field is for a tunnel item. 1221 * @param[in] layer_types 1222 * RTE_ETH_RSS_* types. 1223 * @param[in] hash_fields 1224 * Item hash fields. 1225 * 1226 * @return 1227 * The hash fields that should be used. 1228 */ 1229 uint64_t 1230 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1231 int tunnel __rte_unused, uint64_t layer_types, 1232 uint64_t hash_fields) 1233 { 1234 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1235 int rss_request_inner = rss_desc->level >= 2; 1236 1237 /* Check RSS hash level for tunnel. */ 1238 if (tunnel && rss_request_inner) 1239 hash_fields |= IBV_RX_HASH_INNER; 1240 else if (tunnel || rss_request_inner) 1241 return 0; 1242 #endif 1243 /* Check if requested layer matches RSS hash fields. */ 1244 if (!(rss_desc->types & layer_types)) 1245 return 0; 1246 return hash_fields; 1247 } 1248 1249 /** 1250 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1251 * if several tunnel rules are used on this queue, the tunnel ptype will be 1252 * cleared. 1253 * 1254 * @param rxq_ctrl 1255 * Rx queue to update. 1256 */ 1257 static void 1258 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1259 { 1260 unsigned int i; 1261 uint32_t tunnel_ptype = 0; 1262 1263 /* Look up for the ptype to use. */ 1264 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1265 if (!rxq_ctrl->flow_tunnels_n[i]) 1266 continue; 1267 if (!tunnel_ptype) { 1268 tunnel_ptype = tunnels_info[i].ptype; 1269 } else { 1270 tunnel_ptype = 0; 1271 break; 1272 } 1273 } 1274 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1275 } 1276 1277 /** 1278 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1279 * flow. 1280 * 1281 * @param[in] dev 1282 * Pointer to the Ethernet device structure. 1283 * @param[in] dev_handle 1284 * Pointer to device flow handle structure. 1285 */ 1286 void 1287 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1288 struct mlx5_flow_handle *dev_handle) 1289 { 1290 struct mlx5_priv *priv = dev->data->dev_private; 1291 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1292 struct mlx5_ind_table_obj *ind_tbl = NULL; 1293 unsigned int i; 1294 1295 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1296 struct mlx5_hrxq *hrxq; 1297 1298 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1299 dev_handle->rix_hrxq); 1300 if (hrxq) 1301 ind_tbl = hrxq->ind_table; 1302 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1303 struct mlx5_shared_action_rss *shared_rss; 1304 1305 shared_rss = mlx5_ipool_get 1306 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1307 dev_handle->rix_srss); 1308 if (shared_rss) 1309 ind_tbl = shared_rss->ind_tbl; 1310 } 1311 if (!ind_tbl) 1312 return; 1313 for (i = 0; i != ind_tbl->queues_n; ++i) { 1314 int idx = ind_tbl->queues[i]; 1315 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1316 1317 MLX5_ASSERT(rxq_ctrl != NULL); 1318 if (rxq_ctrl == NULL) 1319 continue; 1320 /* 1321 * To support metadata register copy on Tx loopback, 1322 * this must be always enabled (metadata may arive 1323 * from other port - not from local flows only. 1324 */ 1325 if (tunnel) { 1326 unsigned int j; 1327 1328 /* Increase the counter matching the flow. */ 1329 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1330 if ((tunnels_info[j].tunnel & 1331 dev_handle->layers) == 1332 tunnels_info[j].tunnel) { 1333 rxq_ctrl->flow_tunnels_n[j]++; 1334 break; 1335 } 1336 } 1337 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1338 } 1339 } 1340 } 1341 1342 static void 1343 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1344 { 1345 struct mlx5_priv *priv = dev->data->dev_private; 1346 struct mlx5_rxq_ctrl *rxq_ctrl; 1347 1348 if (priv->mark_enabled) 1349 return; 1350 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1351 rxq_ctrl->rxq.mark = 1; 1352 } 1353 priv->mark_enabled = 1; 1354 } 1355 1356 /** 1357 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1358 * 1359 * @param[in] dev 1360 * Pointer to the Ethernet device structure. 1361 * @param[in] flow 1362 * Pointer to flow structure. 1363 */ 1364 static void 1365 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1366 { 1367 struct mlx5_priv *priv = dev->data->dev_private; 1368 uint32_t handle_idx; 1369 struct mlx5_flow_handle *dev_handle; 1370 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1371 1372 MLX5_ASSERT(wks); 1373 if (wks->mark) 1374 flow_rxq_mark_flag_set(dev); 1375 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1376 handle_idx, dev_handle, next) 1377 flow_drv_rxq_flags_set(dev, dev_handle); 1378 } 1379 1380 /** 1381 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1382 * device flow if no other flow uses it with the same kind of request. 1383 * 1384 * @param dev 1385 * Pointer to Ethernet device. 1386 * @param[in] dev_handle 1387 * Pointer to the device flow handle structure. 1388 */ 1389 static void 1390 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1391 struct mlx5_flow_handle *dev_handle) 1392 { 1393 struct mlx5_priv *priv = dev->data->dev_private; 1394 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1395 struct mlx5_ind_table_obj *ind_tbl = NULL; 1396 unsigned int i; 1397 1398 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1399 struct mlx5_hrxq *hrxq; 1400 1401 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1402 dev_handle->rix_hrxq); 1403 if (hrxq) 1404 ind_tbl = hrxq->ind_table; 1405 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1406 struct mlx5_shared_action_rss *shared_rss; 1407 1408 shared_rss = mlx5_ipool_get 1409 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1410 dev_handle->rix_srss); 1411 if (shared_rss) 1412 ind_tbl = shared_rss->ind_tbl; 1413 } 1414 if (!ind_tbl) 1415 return; 1416 MLX5_ASSERT(dev->data->dev_started); 1417 for (i = 0; i != ind_tbl->queues_n; ++i) { 1418 int idx = ind_tbl->queues[i]; 1419 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1420 1421 MLX5_ASSERT(rxq_ctrl != NULL); 1422 if (rxq_ctrl == NULL) 1423 continue; 1424 if (tunnel) { 1425 unsigned int j; 1426 1427 /* Decrease the counter matching the flow. */ 1428 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1429 if ((tunnels_info[j].tunnel & 1430 dev_handle->layers) == 1431 tunnels_info[j].tunnel) { 1432 rxq_ctrl->flow_tunnels_n[j]--; 1433 break; 1434 } 1435 } 1436 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1437 } 1438 } 1439 } 1440 1441 /** 1442 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1443 * @p flow if no other flow uses it with the same kind of request. 1444 * 1445 * @param dev 1446 * Pointer to Ethernet device. 1447 * @param[in] flow 1448 * Pointer to the flow. 1449 */ 1450 static void 1451 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1452 { 1453 struct mlx5_priv *priv = dev->data->dev_private; 1454 uint32_t handle_idx; 1455 struct mlx5_flow_handle *dev_handle; 1456 1457 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1458 handle_idx, dev_handle, next) 1459 flow_drv_rxq_flags_trim(dev, dev_handle); 1460 } 1461 1462 /** 1463 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1464 * 1465 * @param dev 1466 * Pointer to Ethernet device. 1467 */ 1468 static void 1469 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1470 { 1471 struct mlx5_priv *priv = dev->data->dev_private; 1472 unsigned int i; 1473 1474 for (i = 0; i != priv->rxqs_n; ++i) { 1475 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1476 unsigned int j; 1477 1478 if (rxq == NULL || rxq->ctrl == NULL) 1479 continue; 1480 rxq->ctrl->rxq.mark = 0; 1481 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1482 rxq->ctrl->flow_tunnels_n[j] = 0; 1483 rxq->ctrl->rxq.tunnel = 0; 1484 } 1485 priv->mark_enabled = 0; 1486 } 1487 1488 /** 1489 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1490 * 1491 * @param[in] dev 1492 * Pointer to the Ethernet device structure. 1493 */ 1494 void 1495 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1496 { 1497 struct mlx5_priv *priv = dev->data->dev_private; 1498 unsigned int i; 1499 1500 for (i = 0; i != priv->rxqs_n; ++i) { 1501 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1502 struct mlx5_rxq_data *data; 1503 1504 if (rxq == NULL || rxq->ctrl == NULL) 1505 continue; 1506 data = &rxq->ctrl->rxq; 1507 if (!rte_flow_dynf_metadata_avail()) { 1508 data->dynf_meta = 0; 1509 data->flow_meta_mask = 0; 1510 data->flow_meta_offset = -1; 1511 data->flow_meta_port_mask = 0; 1512 } else { 1513 data->dynf_meta = 1; 1514 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1515 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1516 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1517 } 1518 } 1519 } 1520 1521 /* 1522 * return a pointer to the desired action in the list of actions. 1523 * 1524 * @param[in] actions 1525 * The list of actions to search the action in. 1526 * @param[in] action 1527 * The action to find. 1528 * 1529 * @return 1530 * Pointer to the action in the list, if found. NULL otherwise. 1531 */ 1532 const struct rte_flow_action * 1533 mlx5_flow_find_action(const struct rte_flow_action *actions, 1534 enum rte_flow_action_type action) 1535 { 1536 if (actions == NULL) 1537 return NULL; 1538 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1539 if (actions->type == action) 1540 return actions; 1541 return NULL; 1542 } 1543 1544 /* 1545 * Validate the flag action. 1546 * 1547 * @param[in] action_flags 1548 * Bit-fields that holds the actions detected until now. 1549 * @param[in] attr 1550 * Attributes of flow that includes this action. 1551 * @param[out] error 1552 * Pointer to error structure. 1553 * 1554 * @return 1555 * 0 on success, a negative errno value otherwise and rte_errno is set. 1556 */ 1557 int 1558 mlx5_flow_validate_action_flag(uint64_t action_flags, 1559 const struct rte_flow_attr *attr, 1560 struct rte_flow_error *error) 1561 { 1562 if (action_flags & MLX5_FLOW_ACTION_MARK) 1563 return rte_flow_error_set(error, EINVAL, 1564 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1565 "can't mark and flag in same flow"); 1566 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1567 return rte_flow_error_set(error, EINVAL, 1568 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1569 "can't have 2 flag" 1570 " actions in same flow"); 1571 if (attr->egress) 1572 return rte_flow_error_set(error, ENOTSUP, 1573 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1574 "flag action not supported for " 1575 "egress"); 1576 return 0; 1577 } 1578 1579 /* 1580 * Validate the mark action. 1581 * 1582 * @param[in] action 1583 * Pointer to the queue action. 1584 * @param[in] action_flags 1585 * Bit-fields that holds the actions detected until now. 1586 * @param[in] attr 1587 * Attributes of flow that includes this action. 1588 * @param[out] error 1589 * Pointer to error structure. 1590 * 1591 * @return 1592 * 0 on success, a negative errno value otherwise and rte_errno is set. 1593 */ 1594 int 1595 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1596 uint64_t action_flags, 1597 const struct rte_flow_attr *attr, 1598 struct rte_flow_error *error) 1599 { 1600 const struct rte_flow_action_mark *mark = action->conf; 1601 1602 if (!mark) 1603 return rte_flow_error_set(error, EINVAL, 1604 RTE_FLOW_ERROR_TYPE_ACTION, 1605 action, 1606 "configuration cannot be null"); 1607 if (mark->id >= MLX5_FLOW_MARK_MAX) 1608 return rte_flow_error_set(error, EINVAL, 1609 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1610 &mark->id, 1611 "mark id must in 0 <= id < " 1612 RTE_STR(MLX5_FLOW_MARK_MAX)); 1613 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1614 return rte_flow_error_set(error, EINVAL, 1615 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1616 "can't flag and mark in same flow"); 1617 if (action_flags & MLX5_FLOW_ACTION_MARK) 1618 return rte_flow_error_set(error, EINVAL, 1619 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1620 "can't have 2 mark actions in same" 1621 " flow"); 1622 if (attr->egress) 1623 return rte_flow_error_set(error, ENOTSUP, 1624 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1625 "mark action not supported for " 1626 "egress"); 1627 return 0; 1628 } 1629 1630 /* 1631 * Validate the drop action. 1632 * 1633 * @param[in] action_flags 1634 * Bit-fields that holds the actions detected until now. 1635 * @param[in] attr 1636 * Attributes of flow that includes this action. 1637 * @param[out] error 1638 * Pointer to error structure. 1639 * 1640 * @return 1641 * 0 on success, a negative errno value otherwise and rte_errno is set. 1642 */ 1643 int 1644 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1645 const struct rte_flow_attr *attr, 1646 struct rte_flow_error *error) 1647 { 1648 if (attr->egress) 1649 return rte_flow_error_set(error, ENOTSUP, 1650 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1651 "drop action not supported for " 1652 "egress"); 1653 return 0; 1654 } 1655 1656 /* 1657 * Validate the queue action. 1658 * 1659 * @param[in] action 1660 * Pointer to the queue action. 1661 * @param[in] action_flags 1662 * Bit-fields that holds the actions detected until now. 1663 * @param[in] dev 1664 * Pointer to the Ethernet device structure. 1665 * @param[in] attr 1666 * Attributes of flow that includes this action. 1667 * @param[out] error 1668 * Pointer to error structure. 1669 * 1670 * @return 1671 * 0 on success, a negative errno value otherwise and rte_errno is set. 1672 */ 1673 int 1674 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1675 uint64_t action_flags, 1676 struct rte_eth_dev *dev, 1677 const struct rte_flow_attr *attr, 1678 struct rte_flow_error *error) 1679 { 1680 struct mlx5_priv *priv = dev->data->dev_private; 1681 const struct rte_flow_action_queue *queue = action->conf; 1682 1683 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1684 return rte_flow_error_set(error, EINVAL, 1685 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1686 "can't have 2 fate actions in" 1687 " same flow"); 1688 if (!priv->rxqs_n) 1689 return rte_flow_error_set(error, EINVAL, 1690 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1691 NULL, "No Rx queues configured"); 1692 if (queue->index >= priv->rxqs_n) 1693 return rte_flow_error_set(error, EINVAL, 1694 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1695 &queue->index, 1696 "queue index out of range"); 1697 if (mlx5_rxq_get(dev, queue->index) == NULL) 1698 return rte_flow_error_set(error, EINVAL, 1699 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1700 &queue->index, 1701 "queue is not configured"); 1702 if (attr->egress) 1703 return rte_flow_error_set(error, ENOTSUP, 1704 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1705 "queue action not supported for " 1706 "egress"); 1707 return 0; 1708 } 1709 1710 /** 1711 * Validate queue numbers for device RSS. 1712 * 1713 * @param[in] dev 1714 * Configured device. 1715 * @param[in] queues 1716 * Array of queue numbers. 1717 * @param[in] queues_n 1718 * Size of the @p queues array. 1719 * @param[out] error 1720 * On error, filled with a textual error description. 1721 * @param[out] queue 1722 * On error, filled with an offending queue index in @p queues array. 1723 * 1724 * @return 1725 * 0 on success, a negative errno code on error. 1726 */ 1727 static int 1728 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1729 const uint16_t *queues, uint32_t queues_n, 1730 const char **error, uint32_t *queue_idx) 1731 { 1732 const struct mlx5_priv *priv = dev->data->dev_private; 1733 enum mlx5_rxq_type rxq_type = MLX5_RXQ_TYPE_UNDEFINED; 1734 uint32_t i; 1735 1736 for (i = 0; i != queues_n; ++i) { 1737 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, 1738 queues[i]); 1739 1740 if (queues[i] >= priv->rxqs_n) { 1741 *error = "queue index out of range"; 1742 *queue_idx = i; 1743 return -EINVAL; 1744 } 1745 if (rxq_ctrl == NULL) { 1746 *error = "queue is not configured"; 1747 *queue_idx = i; 1748 return -EINVAL; 1749 } 1750 if (i == 0) 1751 rxq_type = rxq_ctrl->type; 1752 if (rxq_type != rxq_ctrl->type) { 1753 *error = "combining hairpin and regular RSS queues is not supported"; 1754 *queue_idx = i; 1755 return -ENOTSUP; 1756 } 1757 } 1758 return 0; 1759 } 1760 1761 /* 1762 * Validate the rss action. 1763 * 1764 * @param[in] dev 1765 * Pointer to the Ethernet device structure. 1766 * @param[in] action 1767 * Pointer to the queue action. 1768 * @param[out] error 1769 * Pointer to error structure. 1770 * 1771 * @return 1772 * 0 on success, a negative errno value otherwise and rte_errno is set. 1773 */ 1774 int 1775 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1776 const struct rte_flow_action *action, 1777 struct rte_flow_error *error) 1778 { 1779 struct mlx5_priv *priv = dev->data->dev_private; 1780 const struct rte_flow_action_rss *rss = action->conf; 1781 int ret; 1782 const char *message; 1783 uint32_t queue_idx; 1784 1785 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1786 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1787 return rte_flow_error_set(error, ENOTSUP, 1788 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1789 &rss->func, 1790 "RSS hash function not supported"); 1791 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1792 if (rss->level > 2) 1793 #else 1794 if (rss->level > 1) 1795 #endif 1796 return rte_flow_error_set(error, ENOTSUP, 1797 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1798 &rss->level, 1799 "tunnel RSS is not supported"); 1800 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1801 if (rss->key_len == 0 && rss->key != NULL) 1802 return rte_flow_error_set(error, ENOTSUP, 1803 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1804 &rss->key_len, 1805 "RSS hash key length 0"); 1806 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1807 return rte_flow_error_set(error, ENOTSUP, 1808 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1809 &rss->key_len, 1810 "RSS hash key too small"); 1811 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1812 return rte_flow_error_set(error, ENOTSUP, 1813 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1814 &rss->key_len, 1815 "RSS hash key too large"); 1816 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1817 return rte_flow_error_set(error, ENOTSUP, 1818 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1819 &rss->queue_num, 1820 "number of queues too large"); 1821 if (rss->types & MLX5_RSS_HF_MASK) 1822 return rte_flow_error_set(error, ENOTSUP, 1823 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1824 &rss->types, 1825 "some RSS protocols are not" 1826 " supported"); 1827 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 1828 !(rss->types & RTE_ETH_RSS_IP)) 1829 return rte_flow_error_set(error, EINVAL, 1830 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1831 "L3 partial RSS requested but L3 RSS" 1832 " type not specified"); 1833 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 1834 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 1835 return rte_flow_error_set(error, EINVAL, 1836 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1837 "L4 partial RSS requested but L4 RSS" 1838 " type not specified"); 1839 if (!priv->rxqs_n) 1840 return rte_flow_error_set(error, EINVAL, 1841 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1842 NULL, "No Rx queues configured"); 1843 if (!rss->queue_num) 1844 return rte_flow_error_set(error, EINVAL, 1845 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1846 NULL, "No queues configured"); 1847 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 1848 &message, &queue_idx); 1849 if (ret != 0) { 1850 return rte_flow_error_set(error, -ret, 1851 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1852 &rss->queue[queue_idx], message); 1853 } 1854 return 0; 1855 } 1856 1857 /* 1858 * Validate the rss action. 1859 * 1860 * @param[in] action 1861 * Pointer to the queue action. 1862 * @param[in] action_flags 1863 * Bit-fields that holds the actions detected until now. 1864 * @param[in] dev 1865 * Pointer to the Ethernet device structure. 1866 * @param[in] attr 1867 * Attributes of flow that includes this action. 1868 * @param[in] item_flags 1869 * Items that were detected. 1870 * @param[out] error 1871 * Pointer to error structure. 1872 * 1873 * @return 1874 * 0 on success, a negative errno value otherwise and rte_errno is set. 1875 */ 1876 int 1877 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 1878 uint64_t action_flags, 1879 struct rte_eth_dev *dev, 1880 const struct rte_flow_attr *attr, 1881 uint64_t item_flags, 1882 struct rte_flow_error *error) 1883 { 1884 const struct rte_flow_action_rss *rss = action->conf; 1885 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 1886 int ret; 1887 1888 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1889 return rte_flow_error_set(error, EINVAL, 1890 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1891 "can't have 2 fate actions" 1892 " in same flow"); 1893 ret = mlx5_validate_action_rss(dev, action, error); 1894 if (ret) 1895 return ret; 1896 if (attr->egress) 1897 return rte_flow_error_set(error, ENOTSUP, 1898 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1899 "rss action not supported for " 1900 "egress"); 1901 if (rss->level > 1 && !tunnel) 1902 return rte_flow_error_set(error, EINVAL, 1903 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1904 "inner RSS is not supported for " 1905 "non-tunnel flows"); 1906 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 1907 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 1908 return rte_flow_error_set(error, EINVAL, 1909 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 1910 "RSS on eCPRI is not supported now"); 1911 } 1912 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 1913 !(item_flags & 1914 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 1915 rss->level > 1) 1916 return rte_flow_error_set(error, EINVAL, 1917 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 1918 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 1919 return 0; 1920 } 1921 1922 /* 1923 * Validate the default miss action. 1924 * 1925 * @param[in] action_flags 1926 * Bit-fields that holds the actions detected until now. 1927 * @param[out] error 1928 * Pointer to error structure. 1929 * 1930 * @return 1931 * 0 on success, a negative errno value otherwise and rte_errno is set. 1932 */ 1933 int 1934 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 1935 const struct rte_flow_attr *attr, 1936 struct rte_flow_error *error) 1937 { 1938 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1939 return rte_flow_error_set(error, EINVAL, 1940 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1941 "can't have 2 fate actions in" 1942 " same flow"); 1943 if (attr->egress) 1944 return rte_flow_error_set(error, ENOTSUP, 1945 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1946 "default miss action not supported " 1947 "for egress"); 1948 if (attr->group) 1949 return rte_flow_error_set(error, ENOTSUP, 1950 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 1951 "only group 0 is supported"); 1952 if (attr->transfer) 1953 return rte_flow_error_set(error, ENOTSUP, 1954 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 1955 NULL, "transfer is not supported"); 1956 return 0; 1957 } 1958 1959 /* 1960 * Validate the count action. 1961 * 1962 * @param[in] dev 1963 * Pointer to the Ethernet device structure. 1964 * @param[in] attr 1965 * Attributes of flow that includes this action. 1966 * @param[out] error 1967 * Pointer to error structure. 1968 * 1969 * @return 1970 * 0 on success, a negative errno value otherwise and rte_errno is set. 1971 */ 1972 int 1973 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 1974 const struct rte_flow_attr *attr, 1975 struct rte_flow_error *error) 1976 { 1977 if (attr->egress) 1978 return rte_flow_error_set(error, ENOTSUP, 1979 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1980 "count action not supported for " 1981 "egress"); 1982 return 0; 1983 } 1984 1985 /* 1986 * Validate the ASO CT action. 1987 * 1988 * @param[in] dev 1989 * Pointer to the Ethernet device structure. 1990 * @param[in] conntrack 1991 * Pointer to the CT action profile. 1992 * @param[out] error 1993 * Pointer to error structure. 1994 * 1995 * @return 1996 * 0 on success, a negative errno value otherwise and rte_errno is set. 1997 */ 1998 int 1999 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2000 const struct rte_flow_action_conntrack *conntrack, 2001 struct rte_flow_error *error) 2002 { 2003 RTE_SET_USED(dev); 2004 2005 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2006 return rte_flow_error_set(error, EINVAL, 2007 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2008 "Invalid CT state"); 2009 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2010 return rte_flow_error_set(error, EINVAL, 2011 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2012 "Invalid last TCP packet flag"); 2013 return 0; 2014 } 2015 2016 /** 2017 * Verify the @p attributes will be correctly understood by the NIC and store 2018 * them in the @p flow if everything is correct. 2019 * 2020 * @param[in] dev 2021 * Pointer to the Ethernet device structure. 2022 * @param[in] attributes 2023 * Pointer to flow attributes 2024 * @param[out] error 2025 * Pointer to error structure. 2026 * 2027 * @return 2028 * 0 on success, a negative errno value otherwise and rte_errno is set. 2029 */ 2030 int 2031 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2032 const struct rte_flow_attr *attributes, 2033 struct rte_flow_error *error) 2034 { 2035 struct mlx5_priv *priv = dev->data->dev_private; 2036 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2037 2038 if (attributes->group) 2039 return rte_flow_error_set(error, ENOTSUP, 2040 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2041 NULL, "groups is not supported"); 2042 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2043 attributes->priority >= priority_max) 2044 return rte_flow_error_set(error, ENOTSUP, 2045 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2046 NULL, "priority out of range"); 2047 if (attributes->egress) 2048 return rte_flow_error_set(error, ENOTSUP, 2049 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2050 "egress is not supported"); 2051 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2052 return rte_flow_error_set(error, ENOTSUP, 2053 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2054 NULL, "transfer is not supported"); 2055 if (!attributes->ingress) 2056 return rte_flow_error_set(error, EINVAL, 2057 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2058 NULL, 2059 "ingress attribute is mandatory"); 2060 return 0; 2061 } 2062 2063 /** 2064 * Validate ICMP6 item. 2065 * 2066 * @param[in] item 2067 * Item specification. 2068 * @param[in] item_flags 2069 * Bit-fields that holds the items detected until now. 2070 * @param[in] ext_vlan_sup 2071 * Whether extended VLAN features are supported or not. 2072 * @param[out] error 2073 * Pointer to error structure. 2074 * 2075 * @return 2076 * 0 on success, a negative errno value otherwise and rte_errno is set. 2077 */ 2078 int 2079 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2080 uint64_t item_flags, 2081 uint8_t target_protocol, 2082 struct rte_flow_error *error) 2083 { 2084 const struct rte_flow_item_icmp6 *mask = item->mask; 2085 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2086 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2087 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2088 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2089 MLX5_FLOW_LAYER_OUTER_L4; 2090 int ret; 2091 2092 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2093 return rte_flow_error_set(error, EINVAL, 2094 RTE_FLOW_ERROR_TYPE_ITEM, item, 2095 "protocol filtering not compatible" 2096 " with ICMP6 layer"); 2097 if (!(item_flags & l3m)) 2098 return rte_flow_error_set(error, EINVAL, 2099 RTE_FLOW_ERROR_TYPE_ITEM, item, 2100 "IPv6 is mandatory to filter on" 2101 " ICMP6"); 2102 if (item_flags & l4m) 2103 return rte_flow_error_set(error, EINVAL, 2104 RTE_FLOW_ERROR_TYPE_ITEM, item, 2105 "multiple L4 layers not supported"); 2106 if (!mask) 2107 mask = &rte_flow_item_icmp6_mask; 2108 ret = mlx5_flow_item_acceptable 2109 (item, (const uint8_t *)mask, 2110 (const uint8_t *)&rte_flow_item_icmp6_mask, 2111 sizeof(struct rte_flow_item_icmp6), 2112 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2113 if (ret < 0) 2114 return ret; 2115 return 0; 2116 } 2117 2118 /** 2119 * Validate ICMP item. 2120 * 2121 * @param[in] item 2122 * Item specification. 2123 * @param[in] item_flags 2124 * Bit-fields that holds the items detected until now. 2125 * @param[out] error 2126 * Pointer to error structure. 2127 * 2128 * @return 2129 * 0 on success, a negative errno value otherwise and rte_errno is set. 2130 */ 2131 int 2132 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2133 uint64_t item_flags, 2134 uint8_t target_protocol, 2135 struct rte_flow_error *error) 2136 { 2137 const struct rte_flow_item_icmp *mask = item->mask; 2138 const struct rte_flow_item_icmp nic_mask = { 2139 .hdr.icmp_type = 0xff, 2140 .hdr.icmp_code = 0xff, 2141 .hdr.icmp_ident = RTE_BE16(0xffff), 2142 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2143 }; 2144 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2145 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2146 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2147 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2148 MLX5_FLOW_LAYER_OUTER_L4; 2149 int ret; 2150 2151 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2152 return rte_flow_error_set(error, EINVAL, 2153 RTE_FLOW_ERROR_TYPE_ITEM, item, 2154 "protocol filtering not compatible" 2155 " with ICMP layer"); 2156 if (!(item_flags & l3m)) 2157 return rte_flow_error_set(error, EINVAL, 2158 RTE_FLOW_ERROR_TYPE_ITEM, item, 2159 "IPv4 is mandatory to filter" 2160 " on ICMP"); 2161 if (item_flags & l4m) 2162 return rte_flow_error_set(error, EINVAL, 2163 RTE_FLOW_ERROR_TYPE_ITEM, item, 2164 "multiple L4 layers not supported"); 2165 if (!mask) 2166 mask = &nic_mask; 2167 ret = mlx5_flow_item_acceptable 2168 (item, (const uint8_t *)mask, 2169 (const uint8_t *)&nic_mask, 2170 sizeof(struct rte_flow_item_icmp), 2171 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2172 if (ret < 0) 2173 return ret; 2174 return 0; 2175 } 2176 2177 /** 2178 * Validate Ethernet item. 2179 * 2180 * @param[in] item 2181 * Item specification. 2182 * @param[in] item_flags 2183 * Bit-fields that holds the items detected until now. 2184 * @param[out] error 2185 * Pointer to error structure. 2186 * 2187 * @return 2188 * 0 on success, a negative errno value otherwise and rte_errno is set. 2189 */ 2190 int 2191 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2192 uint64_t item_flags, bool ext_vlan_sup, 2193 struct rte_flow_error *error) 2194 { 2195 const struct rte_flow_item_eth *mask = item->mask; 2196 const struct rte_flow_item_eth nic_mask = { 2197 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2198 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2199 .type = RTE_BE16(0xffff), 2200 .has_vlan = ext_vlan_sup ? 1 : 0, 2201 }; 2202 int ret; 2203 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2204 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2205 MLX5_FLOW_LAYER_OUTER_L2; 2206 2207 if (item_flags & ethm) 2208 return rte_flow_error_set(error, ENOTSUP, 2209 RTE_FLOW_ERROR_TYPE_ITEM, item, 2210 "multiple L2 layers not supported"); 2211 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2212 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2213 return rte_flow_error_set(error, EINVAL, 2214 RTE_FLOW_ERROR_TYPE_ITEM, item, 2215 "L2 layer should not follow " 2216 "L3 layers"); 2217 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2218 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2219 return rte_flow_error_set(error, EINVAL, 2220 RTE_FLOW_ERROR_TYPE_ITEM, item, 2221 "L2 layer should not follow VLAN"); 2222 if (item_flags & MLX5_FLOW_LAYER_GTP) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ITEM, item, 2225 "L2 layer should not follow GTP"); 2226 if (!mask) 2227 mask = &rte_flow_item_eth_mask; 2228 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2229 (const uint8_t *)&nic_mask, 2230 sizeof(struct rte_flow_item_eth), 2231 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2232 return ret; 2233 } 2234 2235 /** 2236 * Validate VLAN item. 2237 * 2238 * @param[in] item 2239 * Item specification. 2240 * @param[in] item_flags 2241 * Bit-fields that holds the items detected until now. 2242 * @param[in] dev 2243 * Ethernet device flow is being created on. 2244 * @param[out] error 2245 * Pointer to error structure. 2246 * 2247 * @return 2248 * 0 on success, a negative errno value otherwise and rte_errno is set. 2249 */ 2250 int 2251 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2252 uint64_t item_flags, 2253 struct rte_eth_dev *dev, 2254 struct rte_flow_error *error) 2255 { 2256 const struct rte_flow_item_vlan *spec = item->spec; 2257 const struct rte_flow_item_vlan *mask = item->mask; 2258 const struct rte_flow_item_vlan nic_mask = { 2259 .tci = RTE_BE16(UINT16_MAX), 2260 .inner_type = RTE_BE16(UINT16_MAX), 2261 }; 2262 uint16_t vlan_tag = 0; 2263 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2264 int ret; 2265 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2266 MLX5_FLOW_LAYER_INNER_L4) : 2267 (MLX5_FLOW_LAYER_OUTER_L3 | 2268 MLX5_FLOW_LAYER_OUTER_L4); 2269 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2270 MLX5_FLOW_LAYER_OUTER_VLAN; 2271 2272 if (item_flags & vlanm) 2273 return rte_flow_error_set(error, EINVAL, 2274 RTE_FLOW_ERROR_TYPE_ITEM, item, 2275 "multiple VLAN layers not supported"); 2276 else if ((item_flags & l34m) != 0) 2277 return rte_flow_error_set(error, EINVAL, 2278 RTE_FLOW_ERROR_TYPE_ITEM, item, 2279 "VLAN cannot follow L3/L4 layer"); 2280 if (!mask) 2281 mask = &rte_flow_item_vlan_mask; 2282 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2283 (const uint8_t *)&nic_mask, 2284 sizeof(struct rte_flow_item_vlan), 2285 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2286 if (ret) 2287 return ret; 2288 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2289 struct mlx5_priv *priv = dev->data->dev_private; 2290 2291 if (priv->vmwa_context) { 2292 /* 2293 * Non-NULL context means we have a virtual machine 2294 * and SR-IOV enabled, we have to create VLAN interface 2295 * to make hypervisor to setup E-Switch vport 2296 * context correctly. We avoid creating the multiple 2297 * VLAN interfaces, so we cannot support VLAN tag mask. 2298 */ 2299 return rte_flow_error_set(error, EINVAL, 2300 RTE_FLOW_ERROR_TYPE_ITEM, 2301 item, 2302 "VLAN tag mask is not" 2303 " supported in virtual" 2304 " environment"); 2305 } 2306 } 2307 if (spec) { 2308 vlan_tag = spec->tci; 2309 vlan_tag &= mask->tci; 2310 } 2311 /* 2312 * From verbs perspective an empty VLAN is equivalent 2313 * to a packet without VLAN layer. 2314 */ 2315 if (!vlan_tag) 2316 return rte_flow_error_set(error, EINVAL, 2317 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2318 item->spec, 2319 "VLAN cannot be empty"); 2320 return 0; 2321 } 2322 2323 /** 2324 * Validate IPV4 item. 2325 * 2326 * @param[in] item 2327 * Item specification. 2328 * @param[in] item_flags 2329 * Bit-fields that holds the items detected until now. 2330 * @param[in] last_item 2331 * Previous validated item in the pattern items. 2332 * @param[in] ether_type 2333 * Type in the ethernet layer header (including dot1q). 2334 * @param[in] acc_mask 2335 * Acceptable mask, if NULL default internal default mask 2336 * will be used to check whether item fields are supported. 2337 * @param[in] range_accepted 2338 * True if range of values is accepted for specific fields, false otherwise. 2339 * @param[out] error 2340 * Pointer to error structure. 2341 * 2342 * @return 2343 * 0 on success, a negative errno value otherwise and rte_errno is set. 2344 */ 2345 int 2346 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2347 uint64_t item_flags, 2348 uint64_t last_item, 2349 uint16_t ether_type, 2350 const struct rte_flow_item_ipv4 *acc_mask, 2351 bool range_accepted, 2352 struct rte_flow_error *error) 2353 { 2354 const struct rte_flow_item_ipv4 *mask = item->mask; 2355 const struct rte_flow_item_ipv4 *spec = item->spec; 2356 const struct rte_flow_item_ipv4 nic_mask = { 2357 .hdr = { 2358 .src_addr = RTE_BE32(0xffffffff), 2359 .dst_addr = RTE_BE32(0xffffffff), 2360 .type_of_service = 0xff, 2361 .next_proto_id = 0xff, 2362 }, 2363 }; 2364 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2365 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2366 MLX5_FLOW_LAYER_OUTER_L3; 2367 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2368 MLX5_FLOW_LAYER_OUTER_L4; 2369 int ret; 2370 uint8_t next_proto = 0xFF; 2371 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2372 MLX5_FLOW_LAYER_OUTER_VLAN | 2373 MLX5_FLOW_LAYER_INNER_VLAN); 2374 2375 if ((last_item & l2_vlan) && ether_type && 2376 ether_type != RTE_ETHER_TYPE_IPV4) 2377 return rte_flow_error_set(error, EINVAL, 2378 RTE_FLOW_ERROR_TYPE_ITEM, item, 2379 "IPv4 cannot follow L2/VLAN layer " 2380 "which ether type is not IPv4"); 2381 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2382 if (mask && spec) 2383 next_proto = mask->hdr.next_proto_id & 2384 spec->hdr.next_proto_id; 2385 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2386 return rte_flow_error_set(error, EINVAL, 2387 RTE_FLOW_ERROR_TYPE_ITEM, 2388 item, 2389 "multiple tunnel " 2390 "not supported"); 2391 } 2392 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2393 return rte_flow_error_set(error, EINVAL, 2394 RTE_FLOW_ERROR_TYPE_ITEM, item, 2395 "wrong tunnel type - IPv6 specified " 2396 "but IPv4 item provided"); 2397 if (item_flags & l3m) 2398 return rte_flow_error_set(error, ENOTSUP, 2399 RTE_FLOW_ERROR_TYPE_ITEM, item, 2400 "multiple L3 layers not supported"); 2401 else if (item_flags & l4m) 2402 return rte_flow_error_set(error, EINVAL, 2403 RTE_FLOW_ERROR_TYPE_ITEM, item, 2404 "L3 cannot follow an L4 layer."); 2405 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2406 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2407 return rte_flow_error_set(error, EINVAL, 2408 RTE_FLOW_ERROR_TYPE_ITEM, item, 2409 "L3 cannot follow an NVGRE layer."); 2410 if (!mask) 2411 mask = &rte_flow_item_ipv4_mask; 2412 else if (mask->hdr.next_proto_id != 0 && 2413 mask->hdr.next_proto_id != 0xff) 2414 return rte_flow_error_set(error, EINVAL, 2415 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2416 "partial mask is not supported" 2417 " for protocol"); 2418 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2419 acc_mask ? (const uint8_t *)acc_mask 2420 : (const uint8_t *)&nic_mask, 2421 sizeof(struct rte_flow_item_ipv4), 2422 range_accepted, error); 2423 if (ret < 0) 2424 return ret; 2425 return 0; 2426 } 2427 2428 /** 2429 * Validate IPV6 item. 2430 * 2431 * @param[in] item 2432 * Item specification. 2433 * @param[in] item_flags 2434 * Bit-fields that holds the items detected until now. 2435 * @param[in] last_item 2436 * Previous validated item in the pattern items. 2437 * @param[in] ether_type 2438 * Type in the ethernet layer header (including dot1q). 2439 * @param[in] acc_mask 2440 * Acceptable mask, if NULL default internal default mask 2441 * will be used to check whether item fields are supported. 2442 * @param[out] error 2443 * Pointer to error structure. 2444 * 2445 * @return 2446 * 0 on success, a negative errno value otherwise and rte_errno is set. 2447 */ 2448 int 2449 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2450 uint64_t item_flags, 2451 uint64_t last_item, 2452 uint16_t ether_type, 2453 const struct rte_flow_item_ipv6 *acc_mask, 2454 struct rte_flow_error *error) 2455 { 2456 const struct rte_flow_item_ipv6 *mask = item->mask; 2457 const struct rte_flow_item_ipv6 *spec = item->spec; 2458 const struct rte_flow_item_ipv6 nic_mask = { 2459 .hdr = { 2460 .src_addr = 2461 "\xff\xff\xff\xff\xff\xff\xff\xff" 2462 "\xff\xff\xff\xff\xff\xff\xff\xff", 2463 .dst_addr = 2464 "\xff\xff\xff\xff\xff\xff\xff\xff" 2465 "\xff\xff\xff\xff\xff\xff\xff\xff", 2466 .vtc_flow = RTE_BE32(0xffffffff), 2467 .proto = 0xff, 2468 }, 2469 }; 2470 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2471 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2472 MLX5_FLOW_LAYER_OUTER_L3; 2473 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2474 MLX5_FLOW_LAYER_OUTER_L4; 2475 int ret; 2476 uint8_t next_proto = 0xFF; 2477 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2478 MLX5_FLOW_LAYER_OUTER_VLAN | 2479 MLX5_FLOW_LAYER_INNER_VLAN); 2480 2481 if ((last_item & l2_vlan) && ether_type && 2482 ether_type != RTE_ETHER_TYPE_IPV6) 2483 return rte_flow_error_set(error, EINVAL, 2484 RTE_FLOW_ERROR_TYPE_ITEM, item, 2485 "IPv6 cannot follow L2/VLAN layer " 2486 "which ether type is not IPv6"); 2487 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2488 next_proto = spec->hdr.proto; 2489 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2490 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2491 return rte_flow_error_set(error, EINVAL, 2492 RTE_FLOW_ERROR_TYPE_ITEM, 2493 item, 2494 "multiple tunnel " 2495 "not supported"); 2496 } 2497 if (next_proto == IPPROTO_HOPOPTS || 2498 next_proto == IPPROTO_ROUTING || 2499 next_proto == IPPROTO_FRAGMENT || 2500 next_proto == IPPROTO_ESP || 2501 next_proto == IPPROTO_AH || 2502 next_proto == IPPROTO_DSTOPTS) 2503 return rte_flow_error_set(error, EINVAL, 2504 RTE_FLOW_ERROR_TYPE_ITEM, item, 2505 "IPv6 proto (next header) should " 2506 "not be set as extension header"); 2507 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2508 return rte_flow_error_set(error, EINVAL, 2509 RTE_FLOW_ERROR_TYPE_ITEM, item, 2510 "wrong tunnel type - IPv4 specified " 2511 "but IPv6 item provided"); 2512 if (item_flags & l3m) 2513 return rte_flow_error_set(error, ENOTSUP, 2514 RTE_FLOW_ERROR_TYPE_ITEM, item, 2515 "multiple L3 layers not supported"); 2516 else if (item_flags & l4m) 2517 return rte_flow_error_set(error, EINVAL, 2518 RTE_FLOW_ERROR_TYPE_ITEM, item, 2519 "L3 cannot follow an L4 layer."); 2520 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2521 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2522 return rte_flow_error_set(error, EINVAL, 2523 RTE_FLOW_ERROR_TYPE_ITEM, item, 2524 "L3 cannot follow an NVGRE layer."); 2525 if (!mask) 2526 mask = &rte_flow_item_ipv6_mask; 2527 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2528 acc_mask ? (const uint8_t *)acc_mask 2529 : (const uint8_t *)&nic_mask, 2530 sizeof(struct rte_flow_item_ipv6), 2531 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2532 if (ret < 0) 2533 return ret; 2534 return 0; 2535 } 2536 2537 /** 2538 * Validate UDP item. 2539 * 2540 * @param[in] item 2541 * Item specification. 2542 * @param[in] item_flags 2543 * Bit-fields that holds the items detected until now. 2544 * @param[in] target_protocol 2545 * The next protocol in the previous item. 2546 * @param[in] flow_mask 2547 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2548 * @param[out] error 2549 * Pointer to error structure. 2550 * 2551 * @return 2552 * 0 on success, a negative errno value otherwise and rte_errno is set. 2553 */ 2554 int 2555 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2556 uint64_t item_flags, 2557 uint8_t target_protocol, 2558 struct rte_flow_error *error) 2559 { 2560 const struct rte_flow_item_udp *mask = item->mask; 2561 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2562 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2563 MLX5_FLOW_LAYER_OUTER_L3; 2564 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2565 MLX5_FLOW_LAYER_OUTER_L4; 2566 int ret; 2567 2568 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2569 return rte_flow_error_set(error, EINVAL, 2570 RTE_FLOW_ERROR_TYPE_ITEM, item, 2571 "protocol filtering not compatible" 2572 " with UDP layer"); 2573 if (!(item_flags & l3m)) 2574 return rte_flow_error_set(error, EINVAL, 2575 RTE_FLOW_ERROR_TYPE_ITEM, item, 2576 "L3 is mandatory to filter on L4"); 2577 if (item_flags & l4m) 2578 return rte_flow_error_set(error, EINVAL, 2579 RTE_FLOW_ERROR_TYPE_ITEM, item, 2580 "multiple L4 layers not supported"); 2581 if (!mask) 2582 mask = &rte_flow_item_udp_mask; 2583 ret = mlx5_flow_item_acceptable 2584 (item, (const uint8_t *)mask, 2585 (const uint8_t *)&rte_flow_item_udp_mask, 2586 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2587 error); 2588 if (ret < 0) 2589 return ret; 2590 return 0; 2591 } 2592 2593 /** 2594 * Validate TCP item. 2595 * 2596 * @param[in] item 2597 * Item specification. 2598 * @param[in] item_flags 2599 * Bit-fields that holds the items detected until now. 2600 * @param[in] target_protocol 2601 * The next protocol in the previous item. 2602 * @param[out] error 2603 * Pointer to error structure. 2604 * 2605 * @return 2606 * 0 on success, a negative errno value otherwise and rte_errno is set. 2607 */ 2608 int 2609 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2610 uint64_t item_flags, 2611 uint8_t target_protocol, 2612 const struct rte_flow_item_tcp *flow_mask, 2613 struct rte_flow_error *error) 2614 { 2615 const struct rte_flow_item_tcp *mask = item->mask; 2616 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2617 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2618 MLX5_FLOW_LAYER_OUTER_L3; 2619 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2620 MLX5_FLOW_LAYER_OUTER_L4; 2621 int ret; 2622 2623 MLX5_ASSERT(flow_mask); 2624 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2625 return rte_flow_error_set(error, EINVAL, 2626 RTE_FLOW_ERROR_TYPE_ITEM, item, 2627 "protocol filtering not compatible" 2628 " with TCP layer"); 2629 if (!(item_flags & l3m)) 2630 return rte_flow_error_set(error, EINVAL, 2631 RTE_FLOW_ERROR_TYPE_ITEM, item, 2632 "L3 is mandatory to filter on L4"); 2633 if (item_flags & l4m) 2634 return rte_flow_error_set(error, EINVAL, 2635 RTE_FLOW_ERROR_TYPE_ITEM, item, 2636 "multiple L4 layers not supported"); 2637 if (!mask) 2638 mask = &rte_flow_item_tcp_mask; 2639 ret = mlx5_flow_item_acceptable 2640 (item, (const uint8_t *)mask, 2641 (const uint8_t *)flow_mask, 2642 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2643 error); 2644 if (ret < 0) 2645 return ret; 2646 return 0; 2647 } 2648 2649 /** 2650 * Validate VXLAN item. 2651 * 2652 * @param[in] dev 2653 * Pointer to the Ethernet device structure. 2654 * @param[in] udp_dport 2655 * UDP destination port 2656 * @param[in] item 2657 * Item specification. 2658 * @param[in] item_flags 2659 * Bit-fields that holds the items detected until now. 2660 * @param[in] attr 2661 * Flow rule attributes. 2662 * @param[out] error 2663 * Pointer to error structure. 2664 * 2665 * @return 2666 * 0 on success, a negative errno value otherwise and rte_errno is set. 2667 */ 2668 int 2669 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2670 uint16_t udp_dport, 2671 const struct rte_flow_item *item, 2672 uint64_t item_flags, 2673 const struct rte_flow_attr *attr, 2674 struct rte_flow_error *error) 2675 { 2676 const struct rte_flow_item_vxlan *spec = item->spec; 2677 const struct rte_flow_item_vxlan *mask = item->mask; 2678 int ret; 2679 struct mlx5_priv *priv = dev->data->dev_private; 2680 union vni { 2681 uint32_t vlan_id; 2682 uint8_t vni[4]; 2683 } id = { .vlan_id = 0, }; 2684 const struct rte_flow_item_vxlan nic_mask = { 2685 .vni = "\xff\xff\xff", 2686 .rsvd1 = 0xff, 2687 }; 2688 const struct rte_flow_item_vxlan *valid_mask; 2689 2690 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2691 return rte_flow_error_set(error, ENOTSUP, 2692 RTE_FLOW_ERROR_TYPE_ITEM, item, 2693 "multiple tunnel layers not" 2694 " supported"); 2695 valid_mask = &rte_flow_item_vxlan_mask; 2696 /* 2697 * Verify only UDPv4 is present as defined in 2698 * https://tools.ietf.org/html/rfc7348 2699 */ 2700 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2701 return rte_flow_error_set(error, EINVAL, 2702 RTE_FLOW_ERROR_TYPE_ITEM, item, 2703 "no outer UDP layer found"); 2704 if (!mask) 2705 mask = &rte_flow_item_vxlan_mask; 2706 2707 if (priv->sh->steering_format_version != 2708 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2709 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2710 /* FDB domain & NIC domain non-zero group */ 2711 if ((attr->transfer || attr->group) && priv->sh->misc5_cap) 2712 valid_mask = &nic_mask; 2713 /* Group zero in NIC domain */ 2714 if (!attr->group && !attr->transfer && 2715 priv->sh->tunnel_header_0_1) 2716 valid_mask = &nic_mask; 2717 } 2718 ret = mlx5_flow_item_acceptable 2719 (item, (const uint8_t *)mask, 2720 (const uint8_t *)valid_mask, 2721 sizeof(struct rte_flow_item_vxlan), 2722 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2723 if (ret < 0) 2724 return ret; 2725 if (spec) { 2726 memcpy(&id.vni[1], spec->vni, 3); 2727 memcpy(&id.vni[1], mask->vni, 3); 2728 } 2729 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2730 return rte_flow_error_set(error, ENOTSUP, 2731 RTE_FLOW_ERROR_TYPE_ITEM, item, 2732 "VXLAN tunnel must be fully defined"); 2733 return 0; 2734 } 2735 2736 /** 2737 * Validate VXLAN_GPE item. 2738 * 2739 * @param[in] item 2740 * Item specification. 2741 * @param[in] item_flags 2742 * Bit-fields that holds the items detected until now. 2743 * @param[in] priv 2744 * Pointer to the private data structure. 2745 * @param[in] target_protocol 2746 * The next protocol in the previous item. 2747 * @param[out] error 2748 * Pointer to error structure. 2749 * 2750 * @return 2751 * 0 on success, a negative errno value otherwise and rte_errno is set. 2752 */ 2753 int 2754 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2755 uint64_t item_flags, 2756 struct rte_eth_dev *dev, 2757 struct rte_flow_error *error) 2758 { 2759 struct mlx5_priv *priv = dev->data->dev_private; 2760 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2761 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2762 int ret; 2763 union vni { 2764 uint32_t vlan_id; 2765 uint8_t vni[4]; 2766 } id = { .vlan_id = 0, }; 2767 2768 if (!priv->sh->config.l3_vxlan_en) 2769 return rte_flow_error_set(error, ENOTSUP, 2770 RTE_FLOW_ERROR_TYPE_ITEM, item, 2771 "L3 VXLAN is not enabled by device" 2772 " parameter and/or not configured in" 2773 " firmware"); 2774 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2775 return rte_flow_error_set(error, ENOTSUP, 2776 RTE_FLOW_ERROR_TYPE_ITEM, item, 2777 "multiple tunnel layers not" 2778 " supported"); 2779 /* 2780 * Verify only UDPv4 is present as defined in 2781 * https://tools.ietf.org/html/rfc7348 2782 */ 2783 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2784 return rte_flow_error_set(error, EINVAL, 2785 RTE_FLOW_ERROR_TYPE_ITEM, item, 2786 "no outer UDP layer found"); 2787 if (!mask) 2788 mask = &rte_flow_item_vxlan_gpe_mask; 2789 ret = mlx5_flow_item_acceptable 2790 (item, (const uint8_t *)mask, 2791 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2792 sizeof(struct rte_flow_item_vxlan_gpe), 2793 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2794 if (ret < 0) 2795 return ret; 2796 if (spec) { 2797 if (spec->protocol) 2798 return rte_flow_error_set(error, ENOTSUP, 2799 RTE_FLOW_ERROR_TYPE_ITEM, 2800 item, 2801 "VxLAN-GPE protocol" 2802 " not supported"); 2803 memcpy(&id.vni[1], spec->vni, 3); 2804 memcpy(&id.vni[1], mask->vni, 3); 2805 } 2806 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2807 return rte_flow_error_set(error, ENOTSUP, 2808 RTE_FLOW_ERROR_TYPE_ITEM, item, 2809 "VXLAN-GPE tunnel must be fully" 2810 " defined"); 2811 return 0; 2812 } 2813 /** 2814 * Validate GRE Key item. 2815 * 2816 * @param[in] item 2817 * Item specification. 2818 * @param[in] item_flags 2819 * Bit flags to mark detected items. 2820 * @param[in] gre_item 2821 * Pointer to gre_item 2822 * @param[out] error 2823 * Pointer to error structure. 2824 * 2825 * @return 2826 * 0 on success, a negative errno value otherwise and rte_errno is set. 2827 */ 2828 int 2829 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 2830 uint64_t item_flags, 2831 const struct rte_flow_item *gre_item, 2832 struct rte_flow_error *error) 2833 { 2834 const rte_be32_t *mask = item->mask; 2835 int ret = 0; 2836 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 2837 const struct rte_flow_item_gre *gre_spec; 2838 const struct rte_flow_item_gre *gre_mask; 2839 2840 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 2841 return rte_flow_error_set(error, ENOTSUP, 2842 RTE_FLOW_ERROR_TYPE_ITEM, item, 2843 "Multiple GRE key not support"); 2844 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 2845 return rte_flow_error_set(error, ENOTSUP, 2846 RTE_FLOW_ERROR_TYPE_ITEM, item, 2847 "No preceding GRE header"); 2848 if (item_flags & MLX5_FLOW_LAYER_INNER) 2849 return rte_flow_error_set(error, ENOTSUP, 2850 RTE_FLOW_ERROR_TYPE_ITEM, item, 2851 "GRE key following a wrong item"); 2852 gre_mask = gre_item->mask; 2853 if (!gre_mask) 2854 gre_mask = &rte_flow_item_gre_mask; 2855 gre_spec = gre_item->spec; 2856 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 2857 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 2858 return rte_flow_error_set(error, EINVAL, 2859 RTE_FLOW_ERROR_TYPE_ITEM, item, 2860 "Key bit must be on"); 2861 2862 if (!mask) 2863 mask = &gre_key_default_mask; 2864 ret = mlx5_flow_item_acceptable 2865 (item, (const uint8_t *)mask, 2866 (const uint8_t *)&gre_key_default_mask, 2867 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2868 return ret; 2869 } 2870 2871 /** 2872 * Validate GRE item. 2873 * 2874 * @param[in] item 2875 * Item specification. 2876 * @param[in] item_flags 2877 * Bit flags to mark detected items. 2878 * @param[in] target_protocol 2879 * The next protocol in the previous item. 2880 * @param[out] error 2881 * Pointer to error structure. 2882 * 2883 * @return 2884 * 0 on success, a negative errno value otherwise and rte_errno is set. 2885 */ 2886 int 2887 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 2888 uint64_t item_flags, 2889 uint8_t target_protocol, 2890 struct rte_flow_error *error) 2891 { 2892 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 2893 const struct rte_flow_item_gre *mask = item->mask; 2894 int ret; 2895 const struct rte_flow_item_gre nic_mask = { 2896 .c_rsvd0_ver = RTE_BE16(0xB000), 2897 .protocol = RTE_BE16(UINT16_MAX), 2898 }; 2899 2900 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 2901 return rte_flow_error_set(error, EINVAL, 2902 RTE_FLOW_ERROR_TYPE_ITEM, item, 2903 "protocol filtering not compatible" 2904 " with this GRE layer"); 2905 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2906 return rte_flow_error_set(error, ENOTSUP, 2907 RTE_FLOW_ERROR_TYPE_ITEM, item, 2908 "multiple tunnel layers not" 2909 " supported"); 2910 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 2911 return rte_flow_error_set(error, ENOTSUP, 2912 RTE_FLOW_ERROR_TYPE_ITEM, item, 2913 "L3 Layer is missing"); 2914 if (!mask) 2915 mask = &rte_flow_item_gre_mask; 2916 ret = mlx5_flow_item_acceptable 2917 (item, (const uint8_t *)mask, 2918 (const uint8_t *)&nic_mask, 2919 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2920 error); 2921 if (ret < 0) 2922 return ret; 2923 #ifndef HAVE_MLX5DV_DR 2924 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 2925 if (spec && (spec->protocol & mask->protocol)) 2926 return rte_flow_error_set(error, ENOTSUP, 2927 RTE_FLOW_ERROR_TYPE_ITEM, item, 2928 "without MPLS support the" 2929 " specification cannot be used for" 2930 " filtering"); 2931 #endif 2932 #endif 2933 return 0; 2934 } 2935 2936 /** 2937 * Validate Geneve item. 2938 * 2939 * @param[in] item 2940 * Item specification. 2941 * @param[in] itemFlags 2942 * Bit-fields that holds the items detected until now. 2943 * @param[in] enPriv 2944 * Pointer to the private data structure. 2945 * @param[out] error 2946 * Pointer to error structure. 2947 * 2948 * @return 2949 * 0 on success, a negative errno value otherwise and rte_errno is set. 2950 */ 2951 2952 int 2953 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 2954 uint64_t item_flags, 2955 struct rte_eth_dev *dev, 2956 struct rte_flow_error *error) 2957 { 2958 struct mlx5_priv *priv = dev->data->dev_private; 2959 const struct rte_flow_item_geneve *spec = item->spec; 2960 const struct rte_flow_item_geneve *mask = item->mask; 2961 int ret; 2962 uint16_t gbhdr; 2963 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 2964 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 2965 const struct rte_flow_item_geneve nic_mask = { 2966 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 2967 .vni = "\xff\xff\xff", 2968 .protocol = RTE_BE16(UINT16_MAX), 2969 }; 2970 2971 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 2972 return rte_flow_error_set(error, ENOTSUP, 2973 RTE_FLOW_ERROR_TYPE_ITEM, item, 2974 "L3 Geneve is not enabled by device" 2975 " parameter and/or not configured in" 2976 " firmware"); 2977 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2978 return rte_flow_error_set(error, ENOTSUP, 2979 RTE_FLOW_ERROR_TYPE_ITEM, item, 2980 "multiple tunnel layers not" 2981 " supported"); 2982 /* 2983 * Verify only UDPv4 is present as defined in 2984 * https://tools.ietf.org/html/rfc7348 2985 */ 2986 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2987 return rte_flow_error_set(error, EINVAL, 2988 RTE_FLOW_ERROR_TYPE_ITEM, item, 2989 "no outer UDP layer found"); 2990 if (!mask) 2991 mask = &rte_flow_item_geneve_mask; 2992 ret = mlx5_flow_item_acceptable 2993 (item, (const uint8_t *)mask, 2994 (const uint8_t *)&nic_mask, 2995 sizeof(struct rte_flow_item_geneve), 2996 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2997 if (ret) 2998 return ret; 2999 if (spec) { 3000 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3001 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3002 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3003 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3004 return rte_flow_error_set(error, ENOTSUP, 3005 RTE_FLOW_ERROR_TYPE_ITEM, 3006 item, 3007 "Geneve protocol unsupported" 3008 " fields are being used"); 3009 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3010 return rte_flow_error_set 3011 (error, ENOTSUP, 3012 RTE_FLOW_ERROR_TYPE_ITEM, 3013 item, 3014 "Unsupported Geneve options length"); 3015 } 3016 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3017 return rte_flow_error_set 3018 (error, ENOTSUP, 3019 RTE_FLOW_ERROR_TYPE_ITEM, item, 3020 "Geneve tunnel must be fully defined"); 3021 return 0; 3022 } 3023 3024 /** 3025 * Validate Geneve TLV option item. 3026 * 3027 * @param[in] item 3028 * Item specification. 3029 * @param[in] last_item 3030 * Previous validated item in the pattern items. 3031 * @param[in] geneve_item 3032 * Previous GENEVE item specification. 3033 * @param[in] dev 3034 * Pointer to the rte_eth_dev structure. 3035 * @param[out] error 3036 * Pointer to error structure. 3037 * 3038 * @return 3039 * 0 on success, a negative errno value otherwise and rte_errno is set. 3040 */ 3041 int 3042 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3043 uint64_t last_item, 3044 const struct rte_flow_item *geneve_item, 3045 struct rte_eth_dev *dev, 3046 struct rte_flow_error *error) 3047 { 3048 struct mlx5_priv *priv = dev->data->dev_private; 3049 struct mlx5_dev_ctx_shared *sh = priv->sh; 3050 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3051 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3052 uint8_t data_max_supported = 3053 hca_attr->max_geneve_tlv_option_data_len * 4; 3054 const struct rte_flow_item_geneve *geneve_spec; 3055 const struct rte_flow_item_geneve *geneve_mask; 3056 const struct rte_flow_item_geneve_opt *spec = item->spec; 3057 const struct rte_flow_item_geneve_opt *mask = item->mask; 3058 unsigned int i; 3059 unsigned int data_len; 3060 uint8_t tlv_option_len; 3061 uint16_t optlen_m, optlen_v; 3062 const struct rte_flow_item_geneve_opt full_mask = { 3063 .option_class = RTE_BE16(0xffff), 3064 .option_type = 0xff, 3065 .option_len = 0x1f, 3066 }; 3067 3068 if (!mask) 3069 mask = &rte_flow_item_geneve_opt_mask; 3070 if (!spec) 3071 return rte_flow_error_set 3072 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3073 "Geneve TLV opt class/type/length must be specified"); 3074 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3075 return rte_flow_error_set 3076 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3077 "Geneve TLV opt length exceeds the limit (31)"); 3078 /* Check if class type and length masks are full. */ 3079 if (full_mask.option_class != mask->option_class || 3080 full_mask.option_type != mask->option_type || 3081 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3082 return rte_flow_error_set 3083 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3084 "Geneve TLV opt class/type/length masks must be full"); 3085 /* Check if length is supported */ 3086 if ((uint32_t)spec->option_len > 3087 hca_attr->max_geneve_tlv_option_data_len) 3088 return rte_flow_error_set 3089 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3090 "Geneve TLV opt length not supported"); 3091 if (hca_attr->max_geneve_tlv_options > 1) 3092 DRV_LOG(DEBUG, 3093 "max_geneve_tlv_options supports more than 1 option"); 3094 /* Check GENEVE item preceding. */ 3095 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3096 return rte_flow_error_set 3097 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3098 "Geneve opt item must be preceded with Geneve item"); 3099 geneve_spec = geneve_item->spec; 3100 geneve_mask = geneve_item->mask ? geneve_item->mask : 3101 &rte_flow_item_geneve_mask; 3102 /* Check if GENEVE TLV option size doesn't exceed option length */ 3103 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3104 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3105 tlv_option_len = spec->option_len & mask->option_len; 3106 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3107 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3108 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3109 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3110 if ((optlen_v & optlen_m) <= tlv_option_len) 3111 return rte_flow_error_set 3112 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3113 "GENEVE TLV option length exceeds optlen"); 3114 } 3115 /* Check if length is 0 or data is 0. */ 3116 if (spec->data == NULL || spec->option_len == 0) 3117 return rte_flow_error_set 3118 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3119 "Geneve TLV opt with zero data/length not supported"); 3120 /* Check not all data & mask are 0. */ 3121 data_len = spec->option_len * 4; 3122 if (mask->data == NULL) { 3123 for (i = 0; i < data_len; i++) 3124 if (spec->data[i]) 3125 break; 3126 if (i == data_len) 3127 return rte_flow_error_set(error, ENOTSUP, 3128 RTE_FLOW_ERROR_TYPE_ITEM, item, 3129 "Can't match on Geneve option data 0"); 3130 } else { 3131 for (i = 0; i < data_len; i++) 3132 if (spec->data[i] & mask->data[i]) 3133 break; 3134 if (i == data_len) 3135 return rte_flow_error_set(error, ENOTSUP, 3136 RTE_FLOW_ERROR_TYPE_ITEM, item, 3137 "Can't match on Geneve option data and mask 0"); 3138 /* Check data mask supported. */ 3139 for (i = data_max_supported; i < data_len ; i++) 3140 if (mask->data[i]) 3141 return rte_flow_error_set(error, ENOTSUP, 3142 RTE_FLOW_ERROR_TYPE_ITEM, item, 3143 "Data mask is of unsupported size"); 3144 } 3145 /* Check GENEVE option is supported in NIC. */ 3146 if (!hca_attr->geneve_tlv_opt) 3147 return rte_flow_error_set 3148 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3149 "Geneve TLV opt not supported"); 3150 /* Check if we already have geneve option with different type/class. */ 3151 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3152 geneve_opt_resource = sh->geneve_tlv_option_resource; 3153 if (geneve_opt_resource != NULL) 3154 if (geneve_opt_resource->option_class != spec->option_class || 3155 geneve_opt_resource->option_type != spec->option_type || 3156 geneve_opt_resource->length != spec->option_len) { 3157 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3158 return rte_flow_error_set(error, ENOTSUP, 3159 RTE_FLOW_ERROR_TYPE_ITEM, item, 3160 "Only one Geneve TLV option supported"); 3161 } 3162 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3163 return 0; 3164 } 3165 3166 /** 3167 * Validate MPLS item. 3168 * 3169 * @param[in] dev 3170 * Pointer to the rte_eth_dev structure. 3171 * @param[in] item 3172 * Item specification. 3173 * @param[in] item_flags 3174 * Bit-fields that holds the items detected until now. 3175 * @param[in] prev_layer 3176 * The protocol layer indicated in previous item. 3177 * @param[out] error 3178 * Pointer to error structure. 3179 * 3180 * @return 3181 * 0 on success, a negative errno value otherwise and rte_errno is set. 3182 */ 3183 int 3184 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3185 const struct rte_flow_item *item __rte_unused, 3186 uint64_t item_flags __rte_unused, 3187 uint64_t prev_layer __rte_unused, 3188 struct rte_flow_error *error) 3189 { 3190 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3191 const struct rte_flow_item_mpls *mask = item->mask; 3192 struct mlx5_priv *priv = dev->data->dev_private; 3193 int ret; 3194 3195 if (!priv->sh->dev_cap.mpls_en) 3196 return rte_flow_error_set(error, ENOTSUP, 3197 RTE_FLOW_ERROR_TYPE_ITEM, item, 3198 "MPLS not supported or" 3199 " disabled in firmware" 3200 " configuration."); 3201 /* MPLS over UDP, GRE is allowed */ 3202 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3203 MLX5_FLOW_LAYER_GRE | 3204 MLX5_FLOW_LAYER_GRE_KEY))) 3205 return rte_flow_error_set(error, EINVAL, 3206 RTE_FLOW_ERROR_TYPE_ITEM, item, 3207 "protocol filtering not compatible" 3208 " with MPLS layer"); 3209 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3210 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3211 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3212 return rte_flow_error_set(error, ENOTSUP, 3213 RTE_FLOW_ERROR_TYPE_ITEM, item, 3214 "multiple tunnel layers not" 3215 " supported"); 3216 if (!mask) 3217 mask = &rte_flow_item_mpls_mask; 3218 ret = mlx5_flow_item_acceptable 3219 (item, (const uint8_t *)mask, 3220 (const uint8_t *)&rte_flow_item_mpls_mask, 3221 sizeof(struct rte_flow_item_mpls), 3222 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3223 if (ret < 0) 3224 return ret; 3225 return 0; 3226 #else 3227 return rte_flow_error_set(error, ENOTSUP, 3228 RTE_FLOW_ERROR_TYPE_ITEM, item, 3229 "MPLS is not supported by Verbs, please" 3230 " update."); 3231 #endif 3232 } 3233 3234 /** 3235 * Validate NVGRE item. 3236 * 3237 * @param[in] item 3238 * Item specification. 3239 * @param[in] item_flags 3240 * Bit flags to mark detected items. 3241 * @param[in] target_protocol 3242 * The next protocol in the previous item. 3243 * @param[out] error 3244 * Pointer to error structure. 3245 * 3246 * @return 3247 * 0 on success, a negative errno value otherwise and rte_errno is set. 3248 */ 3249 int 3250 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3251 uint64_t item_flags, 3252 uint8_t target_protocol, 3253 struct rte_flow_error *error) 3254 { 3255 const struct rte_flow_item_nvgre *mask = item->mask; 3256 int ret; 3257 3258 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3259 return rte_flow_error_set(error, EINVAL, 3260 RTE_FLOW_ERROR_TYPE_ITEM, item, 3261 "protocol filtering not compatible" 3262 " with this GRE layer"); 3263 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3264 return rte_flow_error_set(error, ENOTSUP, 3265 RTE_FLOW_ERROR_TYPE_ITEM, item, 3266 "multiple tunnel layers not" 3267 " supported"); 3268 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3269 return rte_flow_error_set(error, ENOTSUP, 3270 RTE_FLOW_ERROR_TYPE_ITEM, item, 3271 "L3 Layer is missing"); 3272 if (!mask) 3273 mask = &rte_flow_item_nvgre_mask; 3274 ret = mlx5_flow_item_acceptable 3275 (item, (const uint8_t *)mask, 3276 (const uint8_t *)&rte_flow_item_nvgre_mask, 3277 sizeof(struct rte_flow_item_nvgre), 3278 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3279 if (ret < 0) 3280 return ret; 3281 return 0; 3282 } 3283 3284 /** 3285 * Validate eCPRI item. 3286 * 3287 * @param[in] item 3288 * Item specification. 3289 * @param[in] item_flags 3290 * Bit-fields that holds the items detected until now. 3291 * @param[in] last_item 3292 * Previous validated item in the pattern items. 3293 * @param[in] ether_type 3294 * Type in the ethernet layer header (including dot1q). 3295 * @param[in] acc_mask 3296 * Acceptable mask, if NULL default internal default mask 3297 * will be used to check whether item fields are supported. 3298 * @param[out] error 3299 * Pointer to error structure. 3300 * 3301 * @return 3302 * 0 on success, a negative errno value otherwise and rte_errno is set. 3303 */ 3304 int 3305 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3306 uint64_t item_flags, 3307 uint64_t last_item, 3308 uint16_t ether_type, 3309 const struct rte_flow_item_ecpri *acc_mask, 3310 struct rte_flow_error *error) 3311 { 3312 const struct rte_flow_item_ecpri *mask = item->mask; 3313 const struct rte_flow_item_ecpri nic_mask = { 3314 .hdr = { 3315 .common = { 3316 .u32 = 3317 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3318 .type = 0xFF, 3319 }).u32), 3320 }, 3321 .dummy[0] = 0xFFFFFFFF, 3322 }, 3323 }; 3324 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3325 MLX5_FLOW_LAYER_OUTER_VLAN); 3326 struct rte_flow_item_ecpri mask_lo; 3327 3328 if (!(last_item & outer_l2_vlan) && 3329 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3330 return rte_flow_error_set(error, EINVAL, 3331 RTE_FLOW_ERROR_TYPE_ITEM, item, 3332 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3333 if ((last_item & outer_l2_vlan) && ether_type && 3334 ether_type != RTE_ETHER_TYPE_ECPRI) 3335 return rte_flow_error_set(error, EINVAL, 3336 RTE_FLOW_ERROR_TYPE_ITEM, item, 3337 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3338 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3339 return rte_flow_error_set(error, EINVAL, 3340 RTE_FLOW_ERROR_TYPE_ITEM, item, 3341 "eCPRI with tunnel is not supported right now"); 3342 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3343 return rte_flow_error_set(error, ENOTSUP, 3344 RTE_FLOW_ERROR_TYPE_ITEM, item, 3345 "multiple L3 layers not supported"); 3346 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3347 return rte_flow_error_set(error, EINVAL, 3348 RTE_FLOW_ERROR_TYPE_ITEM, item, 3349 "eCPRI cannot coexist with a TCP layer"); 3350 /* In specification, eCPRI could be over UDP layer. */ 3351 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3352 return rte_flow_error_set(error, EINVAL, 3353 RTE_FLOW_ERROR_TYPE_ITEM, item, 3354 "eCPRI over UDP layer is not yet supported right now"); 3355 /* Mask for type field in common header could be zero. */ 3356 if (!mask) 3357 mask = &rte_flow_item_ecpri_mask; 3358 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3359 /* Input mask is in big-endian format. */ 3360 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3361 return rte_flow_error_set(error, EINVAL, 3362 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3363 "partial mask is not supported for protocol"); 3364 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3365 return rte_flow_error_set(error, EINVAL, 3366 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3367 "message header mask must be after a type mask"); 3368 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3369 acc_mask ? (const uint8_t *)acc_mask 3370 : (const uint8_t *)&nic_mask, 3371 sizeof(struct rte_flow_item_ecpri), 3372 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3373 } 3374 3375 static int 3376 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3377 const struct rte_flow_attr *attr __rte_unused, 3378 const struct rte_flow_item items[] __rte_unused, 3379 const struct rte_flow_action actions[] __rte_unused, 3380 bool external __rte_unused, 3381 int hairpin __rte_unused, 3382 struct rte_flow_error *error) 3383 { 3384 return rte_flow_error_set(error, ENOTSUP, 3385 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3386 } 3387 3388 static struct mlx5_flow * 3389 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3390 const struct rte_flow_attr *attr __rte_unused, 3391 const struct rte_flow_item items[] __rte_unused, 3392 const struct rte_flow_action actions[] __rte_unused, 3393 struct rte_flow_error *error) 3394 { 3395 rte_flow_error_set(error, ENOTSUP, 3396 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3397 return NULL; 3398 } 3399 3400 static int 3401 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3402 struct mlx5_flow *dev_flow __rte_unused, 3403 const struct rte_flow_attr *attr __rte_unused, 3404 const struct rte_flow_item items[] __rte_unused, 3405 const struct rte_flow_action actions[] __rte_unused, 3406 struct rte_flow_error *error) 3407 { 3408 return rte_flow_error_set(error, ENOTSUP, 3409 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3410 } 3411 3412 static int 3413 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3414 struct rte_flow *flow __rte_unused, 3415 struct rte_flow_error *error) 3416 { 3417 return rte_flow_error_set(error, ENOTSUP, 3418 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3419 } 3420 3421 static void 3422 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3423 struct rte_flow *flow __rte_unused) 3424 { 3425 } 3426 3427 static void 3428 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3429 struct rte_flow *flow __rte_unused) 3430 { 3431 } 3432 3433 static int 3434 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3435 struct rte_flow *flow __rte_unused, 3436 const struct rte_flow_action *actions __rte_unused, 3437 void *data __rte_unused, 3438 struct rte_flow_error *error) 3439 { 3440 return rte_flow_error_set(error, ENOTSUP, 3441 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3442 } 3443 3444 static int 3445 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3446 uint32_t domains __rte_unused, 3447 uint32_t flags __rte_unused) 3448 { 3449 return 0; 3450 } 3451 3452 /* Void driver to protect from null pointer reference. */ 3453 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3454 .validate = flow_null_validate, 3455 .prepare = flow_null_prepare, 3456 .translate = flow_null_translate, 3457 .apply = flow_null_apply, 3458 .remove = flow_null_remove, 3459 .destroy = flow_null_destroy, 3460 .query = flow_null_query, 3461 .sync_domain = flow_null_sync_domain, 3462 }; 3463 3464 /** 3465 * Select flow driver type according to flow attributes and device 3466 * configuration. 3467 * 3468 * @param[in] dev 3469 * Pointer to the dev structure. 3470 * @param[in] attr 3471 * Pointer to the flow attributes. 3472 * 3473 * @return 3474 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3475 */ 3476 static enum mlx5_flow_drv_type 3477 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3478 { 3479 struct mlx5_priv *priv = dev->data->dev_private; 3480 /* The OS can determine first a specific flow type (DV, VERBS) */ 3481 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3482 3483 if (type != MLX5_FLOW_TYPE_MAX) 3484 return type; 3485 /* 3486 * Currently when dv_flow_en == 2, only HW steering engine is 3487 * supported. New engines can also be chosen here if ready. 3488 */ 3489 if (priv->sh->config.dv_flow_en == 2) 3490 return MLX5_FLOW_TYPE_HW; 3491 /* If no OS specific type - continue with DV/VERBS selection */ 3492 if (attr->transfer && priv->sh->config.dv_esw_en) 3493 type = MLX5_FLOW_TYPE_DV; 3494 if (!attr->transfer) 3495 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3496 MLX5_FLOW_TYPE_VERBS; 3497 return type; 3498 } 3499 3500 #define flow_get_drv_ops(type) flow_drv_ops[type] 3501 3502 /** 3503 * Flow driver validation API. This abstracts calling driver specific functions. 3504 * The type of flow driver is determined according to flow attributes. 3505 * 3506 * @param[in] dev 3507 * Pointer to the dev structure. 3508 * @param[in] attr 3509 * Pointer to the flow attributes. 3510 * @param[in] items 3511 * Pointer to the list of items. 3512 * @param[in] actions 3513 * Pointer to the list of actions. 3514 * @param[in] external 3515 * This flow rule is created by request external to PMD. 3516 * @param[in] hairpin 3517 * Number of hairpin TX actions, 0 means classic flow. 3518 * @param[out] error 3519 * Pointer to the error structure. 3520 * 3521 * @return 3522 * 0 on success, a negative errno value otherwise and rte_errno is set. 3523 */ 3524 static inline int 3525 flow_drv_validate(struct rte_eth_dev *dev, 3526 const struct rte_flow_attr *attr, 3527 const struct rte_flow_item items[], 3528 const struct rte_flow_action actions[], 3529 bool external, int hairpin, struct rte_flow_error *error) 3530 { 3531 const struct mlx5_flow_driver_ops *fops; 3532 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3533 3534 fops = flow_get_drv_ops(type); 3535 return fops->validate(dev, attr, items, actions, external, 3536 hairpin, error); 3537 } 3538 3539 /** 3540 * Flow driver preparation API. This abstracts calling driver specific 3541 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3542 * calculates the size of memory required for device flow, allocates the memory, 3543 * initializes the device flow and returns the pointer. 3544 * 3545 * @note 3546 * This function initializes device flow structure such as dv or verbs in 3547 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3548 * rest. For example, adding returning device flow to flow->dev_flow list and 3549 * setting backward reference to the flow should be done out of this function. 3550 * layers field is not filled either. 3551 * 3552 * @param[in] dev 3553 * Pointer to the dev structure. 3554 * @param[in] attr 3555 * Pointer to the flow attributes. 3556 * @param[in] items 3557 * Pointer to the list of items. 3558 * @param[in] actions 3559 * Pointer to the list of actions. 3560 * @param[in] flow_idx 3561 * This memory pool index to the flow. 3562 * @param[out] error 3563 * Pointer to the error structure. 3564 * 3565 * @return 3566 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3567 */ 3568 static inline struct mlx5_flow * 3569 flow_drv_prepare(struct rte_eth_dev *dev, 3570 const struct rte_flow *flow, 3571 const struct rte_flow_attr *attr, 3572 const struct rte_flow_item items[], 3573 const struct rte_flow_action actions[], 3574 uint32_t flow_idx, 3575 struct rte_flow_error *error) 3576 { 3577 const struct mlx5_flow_driver_ops *fops; 3578 enum mlx5_flow_drv_type type = flow->drv_type; 3579 struct mlx5_flow *mlx5_flow = NULL; 3580 3581 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3582 fops = flow_get_drv_ops(type); 3583 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3584 if (mlx5_flow) 3585 mlx5_flow->flow_idx = flow_idx; 3586 return mlx5_flow; 3587 } 3588 3589 /** 3590 * Flow driver translation API. This abstracts calling driver specific 3591 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3592 * translates a generic flow into a driver flow. flow_drv_prepare() must 3593 * precede. 3594 * 3595 * @note 3596 * dev_flow->layers could be filled as a result of parsing during translation 3597 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3598 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3599 * flow->actions could be overwritten even though all the expanded dev_flows 3600 * have the same actions. 3601 * 3602 * @param[in] dev 3603 * Pointer to the rte dev structure. 3604 * @param[in, out] dev_flow 3605 * Pointer to the mlx5 flow. 3606 * @param[in] attr 3607 * Pointer to the flow attributes. 3608 * @param[in] items 3609 * Pointer to the list of items. 3610 * @param[in] actions 3611 * Pointer to the list of actions. 3612 * @param[out] error 3613 * Pointer to the error structure. 3614 * 3615 * @return 3616 * 0 on success, a negative errno value otherwise and rte_errno is set. 3617 */ 3618 static inline int 3619 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3620 const struct rte_flow_attr *attr, 3621 const struct rte_flow_item items[], 3622 const struct rte_flow_action actions[], 3623 struct rte_flow_error *error) 3624 { 3625 const struct mlx5_flow_driver_ops *fops; 3626 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3627 3628 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3629 fops = flow_get_drv_ops(type); 3630 return fops->translate(dev, dev_flow, attr, items, actions, error); 3631 } 3632 3633 /** 3634 * Flow driver apply API. This abstracts calling driver specific functions. 3635 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3636 * translated driver flows on to device. flow_drv_translate() must precede. 3637 * 3638 * @param[in] dev 3639 * Pointer to Ethernet device structure. 3640 * @param[in, out] flow 3641 * Pointer to flow structure. 3642 * @param[out] error 3643 * Pointer to error structure. 3644 * 3645 * @return 3646 * 0 on success, a negative errno value otherwise and rte_errno is set. 3647 */ 3648 static inline int 3649 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3650 struct rte_flow_error *error) 3651 { 3652 const struct mlx5_flow_driver_ops *fops; 3653 enum mlx5_flow_drv_type type = flow->drv_type; 3654 3655 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3656 fops = flow_get_drv_ops(type); 3657 return fops->apply(dev, flow, error); 3658 } 3659 3660 /** 3661 * Flow driver destroy API. This abstracts calling driver specific functions. 3662 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3663 * on device and releases resources of the flow. 3664 * 3665 * @param[in] dev 3666 * Pointer to Ethernet device. 3667 * @param[in, out] flow 3668 * Pointer to flow structure. 3669 */ 3670 static inline void 3671 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3672 { 3673 const struct mlx5_flow_driver_ops *fops; 3674 enum mlx5_flow_drv_type type = flow->drv_type; 3675 3676 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3677 fops = flow_get_drv_ops(type); 3678 fops->destroy(dev, flow); 3679 } 3680 3681 /** 3682 * Flow driver find RSS policy tbl API. This abstracts calling driver 3683 * specific functions. Parent flow (rte_flow) should have driver 3684 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3685 * 3686 * @param[in] dev 3687 * Pointer to Ethernet device. 3688 * @param[in, out] flow 3689 * Pointer to flow structure. 3690 * @param[in] policy 3691 * Pointer to meter policy table. 3692 * @param[in] rss_desc 3693 * Pointer to rss_desc 3694 */ 3695 static struct mlx5_flow_meter_sub_policy * 3696 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3697 struct rte_flow *flow, 3698 struct mlx5_flow_meter_policy *policy, 3699 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3700 { 3701 const struct mlx5_flow_driver_ops *fops; 3702 enum mlx5_flow_drv_type type = flow->drv_type; 3703 3704 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3705 fops = flow_get_drv_ops(type); 3706 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3707 } 3708 3709 /** 3710 * Flow driver color tag rule API. This abstracts calling driver 3711 * specific functions. Parent flow (rte_flow) should have driver 3712 * type (drv_type). It will create the color tag rules in hierarchy meter. 3713 * 3714 * @param[in] dev 3715 * Pointer to Ethernet device. 3716 * @param[in, out] flow 3717 * Pointer to flow structure. 3718 * @param[in] fm 3719 * Pointer to flow meter structure. 3720 * @param[in] src_port 3721 * The src port this extra rule should use. 3722 * @param[in] item 3723 * The src port id match item. 3724 * @param[out] error 3725 * Pointer to error structure. 3726 */ 3727 static int 3728 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 3729 struct rte_flow *flow, 3730 struct mlx5_flow_meter_info *fm, 3731 int32_t src_port, 3732 const struct rte_flow_item *item, 3733 struct rte_flow_error *error) 3734 { 3735 const struct mlx5_flow_driver_ops *fops; 3736 enum mlx5_flow_drv_type type = flow->drv_type; 3737 3738 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3739 fops = flow_get_drv_ops(type); 3740 return fops->meter_hierarchy_rule_create(dev, fm, 3741 src_port, item, error); 3742 } 3743 3744 /** 3745 * Get RSS action from the action list. 3746 * 3747 * @param[in] dev 3748 * Pointer to Ethernet device. 3749 * @param[in] actions 3750 * Pointer to the list of actions. 3751 * @param[in] flow 3752 * Parent flow structure pointer. 3753 * 3754 * @return 3755 * Pointer to the RSS action if exist, else return NULL. 3756 */ 3757 static const struct rte_flow_action_rss* 3758 flow_get_rss_action(struct rte_eth_dev *dev, 3759 const struct rte_flow_action actions[]) 3760 { 3761 struct mlx5_priv *priv = dev->data->dev_private; 3762 const struct rte_flow_action_rss *rss = NULL; 3763 struct mlx5_meter_policy_action_container *acg; 3764 struct mlx5_meter_policy_action_container *acy; 3765 3766 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 3767 switch (actions->type) { 3768 case RTE_FLOW_ACTION_TYPE_RSS: 3769 rss = actions->conf; 3770 break; 3771 case RTE_FLOW_ACTION_TYPE_SAMPLE: 3772 { 3773 const struct rte_flow_action_sample *sample = 3774 actions->conf; 3775 const struct rte_flow_action *act = sample->actions; 3776 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 3777 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 3778 rss = act->conf; 3779 break; 3780 } 3781 case RTE_FLOW_ACTION_TYPE_METER: 3782 { 3783 uint32_t mtr_idx; 3784 struct mlx5_flow_meter_info *fm; 3785 struct mlx5_flow_meter_policy *policy; 3786 const struct rte_flow_action_meter *mtr = actions->conf; 3787 3788 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 3789 if (fm && !fm->def_policy) { 3790 policy = mlx5_flow_meter_policy_find(dev, 3791 fm->policy_id, NULL); 3792 MLX5_ASSERT(policy); 3793 if (policy->is_hierarchy) { 3794 policy = 3795 mlx5_flow_meter_hierarchy_get_final_policy(dev, 3796 policy); 3797 if (!policy) 3798 return NULL; 3799 } 3800 if (policy->is_rss) { 3801 acg = 3802 &policy->act_cnt[RTE_COLOR_GREEN]; 3803 acy = 3804 &policy->act_cnt[RTE_COLOR_YELLOW]; 3805 if (acg->fate_action == 3806 MLX5_FLOW_FATE_SHARED_RSS) 3807 rss = acg->rss->conf; 3808 else if (acy->fate_action == 3809 MLX5_FLOW_FATE_SHARED_RSS) 3810 rss = acy->rss->conf; 3811 } 3812 } 3813 break; 3814 } 3815 default: 3816 break; 3817 } 3818 } 3819 return rss; 3820 } 3821 3822 /** 3823 * Get ASO age action by index. 3824 * 3825 * @param[in] dev 3826 * Pointer to the Ethernet device structure. 3827 * @param[in] age_idx 3828 * Index to the ASO age action. 3829 * 3830 * @return 3831 * The specified ASO age action. 3832 */ 3833 struct mlx5_aso_age_action* 3834 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 3835 { 3836 uint16_t pool_idx = age_idx & UINT16_MAX; 3837 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 3838 struct mlx5_priv *priv = dev->data->dev_private; 3839 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 3840 struct mlx5_aso_age_pool *pool; 3841 3842 rte_rwlock_read_lock(&mng->resize_rwl); 3843 pool = mng->pools[pool_idx]; 3844 rte_rwlock_read_unlock(&mng->resize_rwl); 3845 return &pool->actions[offset - 1]; 3846 } 3847 3848 /* maps indirect action to translated direct in some actions array */ 3849 struct mlx5_translated_action_handle { 3850 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 3851 int index; /**< Index in related array of rte_flow_action. */ 3852 }; 3853 3854 /** 3855 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 3856 * direct action if translation possible. 3857 * This functionality used to run same execution path for both direct and 3858 * indirect actions on flow create. All necessary preparations for indirect 3859 * action handling should be performed on *handle* actions list returned 3860 * from this call. 3861 * 3862 * @param[in] dev 3863 * Pointer to Ethernet device. 3864 * @param[in] actions 3865 * List of actions to translate. 3866 * @param[out] handle 3867 * List to store translated indirect action object handles. 3868 * @param[in, out] indir_n 3869 * Size of *handle* array. On return should be updated with number of 3870 * indirect actions retrieved from the *actions* list. 3871 * @param[out] translated_actions 3872 * List of actions where all indirect actions were translated to direct 3873 * if possible. NULL if no translation took place. 3874 * @param[out] error 3875 * Pointer to the error structure. 3876 * 3877 * @return 3878 * 0 on success, a negative errno value otherwise and rte_errno is set. 3879 */ 3880 static int 3881 flow_action_handles_translate(struct rte_eth_dev *dev, 3882 const struct rte_flow_action actions[], 3883 struct mlx5_translated_action_handle *handle, 3884 int *indir_n, 3885 struct rte_flow_action **translated_actions, 3886 struct rte_flow_error *error) 3887 { 3888 struct mlx5_priv *priv = dev->data->dev_private; 3889 struct rte_flow_action *translated = NULL; 3890 size_t actions_size; 3891 int n; 3892 int copied_n = 0; 3893 struct mlx5_translated_action_handle *handle_end = NULL; 3894 3895 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 3896 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 3897 continue; 3898 if (copied_n == *indir_n) { 3899 return rte_flow_error_set 3900 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 3901 NULL, "too many shared actions"); 3902 } 3903 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 3904 sizeof(actions[n].conf)); 3905 handle[copied_n].index = n; 3906 copied_n++; 3907 } 3908 n++; 3909 *indir_n = copied_n; 3910 if (!copied_n) 3911 return 0; 3912 actions_size = sizeof(struct rte_flow_action) * n; 3913 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 3914 if (!translated) { 3915 rte_errno = ENOMEM; 3916 return -ENOMEM; 3917 } 3918 memcpy(translated, actions, actions_size); 3919 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 3920 struct mlx5_shared_action_rss *shared_rss; 3921 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3922 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3923 uint32_t idx = act_idx & 3924 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3925 3926 switch (type) { 3927 case MLX5_INDIRECT_ACTION_TYPE_RSS: 3928 shared_rss = mlx5_ipool_get 3929 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 3930 translated[handle->index].type = 3931 RTE_FLOW_ACTION_TYPE_RSS; 3932 translated[handle->index].conf = 3933 &shared_rss->origin; 3934 break; 3935 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 3936 translated[handle->index].type = 3937 (enum rte_flow_action_type) 3938 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 3939 translated[handle->index].conf = (void *)(uintptr_t)idx; 3940 break; 3941 case MLX5_INDIRECT_ACTION_TYPE_AGE: 3942 if (priv->sh->flow_hit_aso_en) { 3943 translated[handle->index].type = 3944 (enum rte_flow_action_type) 3945 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 3946 translated[handle->index].conf = 3947 (void *)(uintptr_t)idx; 3948 break; 3949 } 3950 /* Fall-through */ 3951 case MLX5_INDIRECT_ACTION_TYPE_CT: 3952 if (priv->sh->ct_aso_en) { 3953 translated[handle->index].type = 3954 RTE_FLOW_ACTION_TYPE_CONNTRACK; 3955 translated[handle->index].conf = 3956 (void *)(uintptr_t)idx; 3957 break; 3958 } 3959 /* Fall-through */ 3960 default: 3961 mlx5_free(translated); 3962 return rte_flow_error_set 3963 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 3964 NULL, "invalid indirect action type"); 3965 } 3966 } 3967 *translated_actions = translated; 3968 return 0; 3969 } 3970 3971 /** 3972 * Get Shared RSS action from the action list. 3973 * 3974 * @param[in] dev 3975 * Pointer to Ethernet device. 3976 * @param[in] shared 3977 * Pointer to the list of actions. 3978 * @param[in] shared_n 3979 * Actions list length. 3980 * 3981 * @return 3982 * The MLX5 RSS action ID if exists, otherwise return 0. 3983 */ 3984 static uint32_t 3985 flow_get_shared_rss_action(struct rte_eth_dev *dev, 3986 struct mlx5_translated_action_handle *handle, 3987 int shared_n) 3988 { 3989 struct mlx5_translated_action_handle *handle_end; 3990 struct mlx5_priv *priv = dev->data->dev_private; 3991 struct mlx5_shared_action_rss *shared_rss; 3992 3993 3994 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 3995 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 3996 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 3997 uint32_t idx = act_idx & 3998 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 3999 switch (type) { 4000 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4001 shared_rss = mlx5_ipool_get 4002 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4003 idx); 4004 __atomic_add_fetch(&shared_rss->refcnt, 1, 4005 __ATOMIC_RELAXED); 4006 return idx; 4007 default: 4008 break; 4009 } 4010 } 4011 return 0; 4012 } 4013 4014 static unsigned int 4015 find_graph_root(uint32_t rss_level) 4016 { 4017 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4018 MLX5_EXPANSION_ROOT_OUTER; 4019 } 4020 4021 /** 4022 * Get layer flags from the prefix flow. 4023 * 4024 * Some flows may be split to several subflows, the prefix subflow gets the 4025 * match items and the suffix sub flow gets the actions. 4026 * Some actions need the user defined match item flags to get the detail for 4027 * the action. 4028 * This function helps the suffix flow to get the item layer flags from prefix 4029 * subflow. 4030 * 4031 * @param[in] dev_flow 4032 * Pointer the created prefix subflow. 4033 * 4034 * @return 4035 * The layers get from prefix subflow. 4036 */ 4037 static inline uint64_t 4038 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4039 { 4040 uint64_t layers = 0; 4041 4042 /* 4043 * Layers bits could be localization, but usually the compiler will 4044 * help to do the optimization work for source code. 4045 * If no decap actions, use the layers directly. 4046 */ 4047 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4048 return dev_flow->handle->layers; 4049 /* Convert L3 layers with decap action. */ 4050 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4051 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4052 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4053 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4054 /* Convert L4 layers with decap action. */ 4055 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4056 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4057 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4058 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4059 return layers; 4060 } 4061 4062 /** 4063 * Get metadata split action information. 4064 * 4065 * @param[in] actions 4066 * Pointer to the list of actions. 4067 * @param[out] qrss 4068 * Pointer to the return pointer. 4069 * @param[out] qrss_type 4070 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4071 * if no QUEUE/RSS is found. 4072 * @param[out] encap_idx 4073 * Pointer to the index of the encap action if exists, otherwise the last 4074 * action index. 4075 * 4076 * @return 4077 * Total number of actions. 4078 */ 4079 static int 4080 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4081 const struct rte_flow_action **qrss, 4082 int *encap_idx) 4083 { 4084 const struct rte_flow_action_raw_encap *raw_encap; 4085 int actions_n = 0; 4086 int raw_decap_idx = -1; 4087 4088 *encap_idx = -1; 4089 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4090 switch (actions->type) { 4091 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4092 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4093 *encap_idx = actions_n; 4094 break; 4095 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4096 raw_decap_idx = actions_n; 4097 break; 4098 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4099 raw_encap = actions->conf; 4100 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4101 *encap_idx = raw_decap_idx != -1 ? 4102 raw_decap_idx : actions_n; 4103 break; 4104 case RTE_FLOW_ACTION_TYPE_QUEUE: 4105 case RTE_FLOW_ACTION_TYPE_RSS: 4106 *qrss = actions; 4107 break; 4108 default: 4109 break; 4110 } 4111 actions_n++; 4112 } 4113 if (*encap_idx == -1) 4114 *encap_idx = actions_n; 4115 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4116 return actions_n + 1; 4117 } 4118 4119 /** 4120 * Check if the action will change packet. 4121 * 4122 * @param dev 4123 * Pointer to Ethernet device. 4124 * @param[in] type 4125 * action type. 4126 * 4127 * @return 4128 * true if action will change packet, false otherwise. 4129 */ 4130 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4131 enum rte_flow_action_type type) 4132 { 4133 struct mlx5_priv *priv = dev->data->dev_private; 4134 4135 switch (type) { 4136 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4137 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4138 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4139 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4140 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4141 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4142 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4143 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4144 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4145 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4146 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4147 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4148 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4149 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4150 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4151 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4152 case RTE_FLOW_ACTION_TYPE_SET_META: 4153 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4154 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4155 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4156 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4157 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4158 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4159 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4160 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4161 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4162 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4163 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4164 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4165 return true; 4166 case RTE_FLOW_ACTION_TYPE_FLAG: 4167 case RTE_FLOW_ACTION_TYPE_MARK: 4168 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) 4169 return true; 4170 else 4171 return false; 4172 default: 4173 return false; 4174 } 4175 } 4176 4177 /** 4178 * Check meter action from the action list. 4179 * 4180 * @param dev 4181 * Pointer to Ethernet device. 4182 * @param[in] actions 4183 * Pointer to the list of actions. 4184 * @param[out] has_mtr 4185 * Pointer to the meter exist flag. 4186 * @param[out] has_modify 4187 * Pointer to the flag showing there's packet change action. 4188 * @param[out] meter_id 4189 * Pointer to the meter id. 4190 * 4191 * @return 4192 * Total number of actions. 4193 */ 4194 static int 4195 flow_check_meter_action(struct rte_eth_dev *dev, 4196 const struct rte_flow_action actions[], 4197 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4198 { 4199 const struct rte_flow_action_meter *mtr = NULL; 4200 int actions_n = 0; 4201 4202 MLX5_ASSERT(has_mtr); 4203 *has_mtr = false; 4204 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4205 switch (actions->type) { 4206 case RTE_FLOW_ACTION_TYPE_METER: 4207 mtr = actions->conf; 4208 *meter_id = mtr->mtr_id; 4209 *has_mtr = true; 4210 break; 4211 default: 4212 break; 4213 } 4214 if (!*has_mtr) 4215 *has_modify |= flow_check_modify_action_type(dev, 4216 actions->type); 4217 actions_n++; 4218 } 4219 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4220 return actions_n + 1; 4221 } 4222 4223 /** 4224 * Check if the flow should be split due to hairpin. 4225 * The reason for the split is that in current HW we can't 4226 * support encap and push-vlan on Rx, so if a flow contains 4227 * these actions we move it to Tx. 4228 * 4229 * @param dev 4230 * Pointer to Ethernet device. 4231 * @param[in] attr 4232 * Flow rule attributes. 4233 * @param[in] actions 4234 * Associated actions (list terminated by the END action). 4235 * 4236 * @return 4237 * > 0 the number of actions and the flow should be split, 4238 * 0 when no split required. 4239 */ 4240 static int 4241 flow_check_hairpin_split(struct rte_eth_dev *dev, 4242 const struct rte_flow_attr *attr, 4243 const struct rte_flow_action actions[]) 4244 { 4245 int queue_action = 0; 4246 int action_n = 0; 4247 int split = 0; 4248 const struct rte_flow_action_queue *queue; 4249 const struct rte_flow_action_rss *rss; 4250 const struct rte_flow_action_raw_encap *raw_encap; 4251 const struct rte_eth_hairpin_conf *conf; 4252 4253 if (!attr->ingress) 4254 return 0; 4255 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4256 switch (actions->type) { 4257 case RTE_FLOW_ACTION_TYPE_QUEUE: 4258 queue = actions->conf; 4259 if (queue == NULL) 4260 return 0; 4261 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4262 if (conf == NULL || conf->tx_explicit != 0) 4263 return 0; 4264 queue_action = 1; 4265 action_n++; 4266 break; 4267 case RTE_FLOW_ACTION_TYPE_RSS: 4268 rss = actions->conf; 4269 if (rss == NULL || rss->queue_num == 0) 4270 return 0; 4271 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4272 if (conf == NULL || conf->tx_explicit != 0) 4273 return 0; 4274 queue_action = 1; 4275 action_n++; 4276 break; 4277 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4278 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4279 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4280 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4281 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4282 split++; 4283 action_n++; 4284 break; 4285 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4286 raw_encap = actions->conf; 4287 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4288 split++; 4289 action_n++; 4290 break; 4291 default: 4292 action_n++; 4293 break; 4294 } 4295 } 4296 if (split && queue_action) 4297 return action_n; 4298 return 0; 4299 } 4300 4301 /* Declare flow create/destroy prototype in advance. */ 4302 static uint32_t 4303 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4304 const struct rte_flow_attr *attr, 4305 const struct rte_flow_item items[], 4306 const struct rte_flow_action actions[], 4307 bool external, struct rte_flow_error *error); 4308 4309 static void 4310 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4311 uint32_t flow_idx); 4312 4313 int 4314 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4315 struct mlx5_list_entry *entry, void *cb_ctx) 4316 { 4317 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4318 struct mlx5_flow_mreg_copy_resource *mcp_res = 4319 container_of(entry, typeof(*mcp_res), hlist_ent); 4320 4321 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4322 } 4323 4324 struct mlx5_list_entry * 4325 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4326 { 4327 struct rte_eth_dev *dev = tool_ctx; 4328 struct mlx5_priv *priv = dev->data->dev_private; 4329 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4330 struct mlx5_flow_mreg_copy_resource *mcp_res; 4331 struct rte_flow_error *error = ctx->error; 4332 uint32_t idx = 0; 4333 int ret; 4334 uint32_t mark_id = *(uint32_t *)(ctx->data); 4335 struct rte_flow_attr attr = { 4336 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4337 .ingress = 1, 4338 }; 4339 struct mlx5_rte_flow_item_tag tag_spec = { 4340 .data = mark_id, 4341 }; 4342 struct rte_flow_item items[] = { 4343 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4344 }; 4345 struct rte_flow_action_mark ftag = { 4346 .id = mark_id, 4347 }; 4348 struct mlx5_flow_action_copy_mreg cp_mreg = { 4349 .dst = REG_B, 4350 .src = REG_NON, 4351 }; 4352 struct rte_flow_action_jump jump = { 4353 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4354 }; 4355 struct rte_flow_action actions[] = { 4356 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4357 }; 4358 4359 /* Fill the register fields in the flow. */ 4360 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4361 if (ret < 0) 4362 return NULL; 4363 tag_spec.id = ret; 4364 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4365 if (ret < 0) 4366 return NULL; 4367 cp_mreg.src = ret; 4368 /* Provide the full width of FLAG specific value. */ 4369 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4370 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4371 /* Build a new flow. */ 4372 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4373 items[0] = (struct rte_flow_item){ 4374 .type = (enum rte_flow_item_type) 4375 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4376 .spec = &tag_spec, 4377 }; 4378 items[1] = (struct rte_flow_item){ 4379 .type = RTE_FLOW_ITEM_TYPE_END, 4380 }; 4381 actions[0] = (struct rte_flow_action){ 4382 .type = (enum rte_flow_action_type) 4383 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4384 .conf = &ftag, 4385 }; 4386 actions[1] = (struct rte_flow_action){ 4387 .type = (enum rte_flow_action_type) 4388 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4389 .conf = &cp_mreg, 4390 }; 4391 actions[2] = (struct rte_flow_action){ 4392 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4393 .conf = &jump, 4394 }; 4395 actions[3] = (struct rte_flow_action){ 4396 .type = RTE_FLOW_ACTION_TYPE_END, 4397 }; 4398 } else { 4399 /* Default rule, wildcard match. */ 4400 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4401 items[0] = (struct rte_flow_item){ 4402 .type = RTE_FLOW_ITEM_TYPE_END, 4403 }; 4404 actions[0] = (struct rte_flow_action){ 4405 .type = (enum rte_flow_action_type) 4406 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4407 .conf = &cp_mreg, 4408 }; 4409 actions[1] = (struct rte_flow_action){ 4410 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4411 .conf = &jump, 4412 }; 4413 actions[2] = (struct rte_flow_action){ 4414 .type = RTE_FLOW_ACTION_TYPE_END, 4415 }; 4416 } 4417 /* Build a new entry. */ 4418 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4419 if (!mcp_res) { 4420 rte_errno = ENOMEM; 4421 return NULL; 4422 } 4423 mcp_res->idx = idx; 4424 mcp_res->mark_id = mark_id; 4425 /* 4426 * The copy Flows are not included in any list. There 4427 * ones are referenced from other Flows and can not 4428 * be applied, removed, deleted in arbitrary order 4429 * by list traversing. 4430 */ 4431 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4432 &attr, items, actions, false, error); 4433 if (!mcp_res->rix_flow) { 4434 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4435 return NULL; 4436 } 4437 return &mcp_res->hlist_ent; 4438 } 4439 4440 struct mlx5_list_entry * 4441 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4442 void *cb_ctx __rte_unused) 4443 { 4444 struct rte_eth_dev *dev = tool_ctx; 4445 struct mlx5_priv *priv = dev->data->dev_private; 4446 struct mlx5_flow_mreg_copy_resource *mcp_res; 4447 uint32_t idx = 0; 4448 4449 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4450 if (!mcp_res) { 4451 rte_errno = ENOMEM; 4452 return NULL; 4453 } 4454 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4455 mcp_res->idx = idx; 4456 return &mcp_res->hlist_ent; 4457 } 4458 4459 void 4460 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4461 { 4462 struct mlx5_flow_mreg_copy_resource *mcp_res = 4463 container_of(entry, typeof(*mcp_res), hlist_ent); 4464 struct rte_eth_dev *dev = tool_ctx; 4465 struct mlx5_priv *priv = dev->data->dev_private; 4466 4467 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4468 } 4469 4470 /** 4471 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4472 * 4473 * As mark_id is unique, if there's already a registered flow for the mark_id, 4474 * return by increasing the reference counter of the resource. Otherwise, create 4475 * the resource (mcp_res) and flow. 4476 * 4477 * Flow looks like, 4478 * - If ingress port is ANY and reg_c[1] is mark_id, 4479 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4480 * 4481 * For default flow (zero mark_id), flow is like, 4482 * - If ingress port is ANY, 4483 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4484 * 4485 * @param dev 4486 * Pointer to Ethernet device. 4487 * @param mark_id 4488 * ID of MARK action, zero means default flow for META. 4489 * @param[out] error 4490 * Perform verbose error reporting if not NULL. 4491 * 4492 * @return 4493 * Associated resource on success, NULL otherwise and rte_errno is set. 4494 */ 4495 static struct mlx5_flow_mreg_copy_resource * 4496 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4497 struct rte_flow_error *error) 4498 { 4499 struct mlx5_priv *priv = dev->data->dev_private; 4500 struct mlx5_list_entry *entry; 4501 struct mlx5_flow_cb_ctx ctx = { 4502 .dev = dev, 4503 .error = error, 4504 .data = &mark_id, 4505 }; 4506 4507 /* Check if already registered. */ 4508 MLX5_ASSERT(priv->mreg_cp_tbl); 4509 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4510 if (!entry) 4511 return NULL; 4512 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4513 hlist_ent); 4514 } 4515 4516 void 4517 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4518 { 4519 struct mlx5_flow_mreg_copy_resource *mcp_res = 4520 container_of(entry, typeof(*mcp_res), hlist_ent); 4521 struct rte_eth_dev *dev = tool_ctx; 4522 struct mlx5_priv *priv = dev->data->dev_private; 4523 4524 MLX5_ASSERT(mcp_res->rix_flow); 4525 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4526 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4527 } 4528 4529 /** 4530 * Release flow in RX_CP_TBL. 4531 * 4532 * @param dev 4533 * Pointer to Ethernet device. 4534 * @flow 4535 * Parent flow for wich copying is provided. 4536 */ 4537 static void 4538 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4539 struct rte_flow *flow) 4540 { 4541 struct mlx5_flow_mreg_copy_resource *mcp_res; 4542 struct mlx5_priv *priv = dev->data->dev_private; 4543 4544 if (!flow->rix_mreg_copy) 4545 return; 4546 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4547 flow->rix_mreg_copy); 4548 if (!mcp_res || !priv->mreg_cp_tbl) 4549 return; 4550 MLX5_ASSERT(mcp_res->rix_flow); 4551 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4552 flow->rix_mreg_copy = 0; 4553 } 4554 4555 /** 4556 * Remove the default copy action from RX_CP_TBL. 4557 * 4558 * This functions is called in the mlx5_dev_start(). No thread safe 4559 * is guaranteed. 4560 * 4561 * @param dev 4562 * Pointer to Ethernet device. 4563 */ 4564 static void 4565 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4566 { 4567 struct mlx5_list_entry *entry; 4568 struct mlx5_priv *priv = dev->data->dev_private; 4569 struct mlx5_flow_cb_ctx ctx; 4570 uint32_t mark_id; 4571 4572 /* Check if default flow is registered. */ 4573 if (!priv->mreg_cp_tbl) 4574 return; 4575 mark_id = MLX5_DEFAULT_COPY_ID; 4576 ctx.data = &mark_id; 4577 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4578 if (!entry) 4579 return; 4580 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4581 } 4582 4583 /** 4584 * Add the default copy action in in RX_CP_TBL. 4585 * 4586 * This functions is called in the mlx5_dev_start(). No thread safe 4587 * is guaranteed. 4588 * 4589 * @param dev 4590 * Pointer to Ethernet device. 4591 * @param[out] error 4592 * Perform verbose error reporting if not NULL. 4593 * 4594 * @return 4595 * 0 for success, negative value otherwise and rte_errno is set. 4596 */ 4597 static int 4598 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4599 struct rte_flow_error *error) 4600 { 4601 struct mlx5_priv *priv = dev->data->dev_private; 4602 struct mlx5_flow_mreg_copy_resource *mcp_res; 4603 struct mlx5_flow_cb_ctx ctx; 4604 uint32_t mark_id; 4605 4606 /* Check whether extensive metadata feature is engaged. */ 4607 if (!priv->sh->config.dv_flow_en || 4608 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4609 !mlx5_flow_ext_mreg_supported(dev) || 4610 !priv->sh->dv_regc0_mask) 4611 return 0; 4612 /* 4613 * Add default mreg copy flow may be called multiple time, but 4614 * only be called once in stop. Avoid register it twice. 4615 */ 4616 mark_id = MLX5_DEFAULT_COPY_ID; 4617 ctx.data = &mark_id; 4618 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4619 return 0; 4620 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4621 if (!mcp_res) 4622 return -rte_errno; 4623 return 0; 4624 } 4625 4626 /** 4627 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4628 * 4629 * All the flow having Q/RSS action should be split by 4630 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4631 * performs the following, 4632 * - CQE->flow_tag := reg_c[1] (MARK) 4633 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4634 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4635 * but there should be a flow per each MARK ID set by MARK action. 4636 * 4637 * For the aforementioned reason, if there's a MARK action in flow's action 4638 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4639 * the MARK ID to CQE's flow_tag like, 4640 * - If reg_c[1] is mark_id, 4641 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4642 * 4643 * For SET_META action which stores value in reg_c[0], as the destination is 4644 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4645 * MARK ID means the default flow. The default flow looks like, 4646 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4647 * 4648 * @param dev 4649 * Pointer to Ethernet device. 4650 * @param flow 4651 * Pointer to flow structure. 4652 * @param[in] actions 4653 * Pointer to the list of actions. 4654 * @param[out] error 4655 * Perform verbose error reporting if not NULL. 4656 * 4657 * @return 4658 * 0 on success, negative value otherwise and rte_errno is set. 4659 */ 4660 static int 4661 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4662 struct rte_flow *flow, 4663 const struct rte_flow_action *actions, 4664 struct rte_flow_error *error) 4665 { 4666 struct mlx5_priv *priv = dev->data->dev_private; 4667 struct mlx5_sh_config *config = &priv->sh->config; 4668 struct mlx5_flow_mreg_copy_resource *mcp_res; 4669 const struct rte_flow_action_mark *mark; 4670 4671 /* Check whether extensive metadata feature is engaged. */ 4672 if (!config->dv_flow_en || 4673 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4674 !mlx5_flow_ext_mreg_supported(dev) || 4675 !priv->sh->dv_regc0_mask) 4676 return 0; 4677 /* Find MARK action. */ 4678 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4679 switch (actions->type) { 4680 case RTE_FLOW_ACTION_TYPE_FLAG: 4681 mcp_res = flow_mreg_add_copy_action 4682 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4683 if (!mcp_res) 4684 return -rte_errno; 4685 flow->rix_mreg_copy = mcp_res->idx; 4686 return 0; 4687 case RTE_FLOW_ACTION_TYPE_MARK: 4688 mark = (const struct rte_flow_action_mark *) 4689 actions->conf; 4690 mcp_res = 4691 flow_mreg_add_copy_action(dev, mark->id, error); 4692 if (!mcp_res) 4693 return -rte_errno; 4694 flow->rix_mreg_copy = mcp_res->idx; 4695 return 0; 4696 default: 4697 break; 4698 } 4699 } 4700 return 0; 4701 } 4702 4703 #define MLX5_MAX_SPLIT_ACTIONS 24 4704 #define MLX5_MAX_SPLIT_ITEMS 24 4705 4706 /** 4707 * Split the hairpin flow. 4708 * Since HW can't support encap and push-vlan on Rx, we move these 4709 * actions to Tx. 4710 * If the count action is after the encap then we also 4711 * move the count action. in this case the count will also measure 4712 * the outer bytes. 4713 * 4714 * @param dev 4715 * Pointer to Ethernet device. 4716 * @param[in] actions 4717 * Associated actions (list terminated by the END action). 4718 * @param[out] actions_rx 4719 * Rx flow actions. 4720 * @param[out] actions_tx 4721 * Tx flow actions.. 4722 * @param[out] pattern_tx 4723 * The pattern items for the Tx flow. 4724 * @param[out] flow_id 4725 * The flow ID connected to this flow. 4726 * 4727 * @return 4728 * 0 on success. 4729 */ 4730 static int 4731 flow_hairpin_split(struct rte_eth_dev *dev, 4732 const struct rte_flow_action actions[], 4733 struct rte_flow_action actions_rx[], 4734 struct rte_flow_action actions_tx[], 4735 struct rte_flow_item pattern_tx[], 4736 uint32_t flow_id) 4737 { 4738 const struct rte_flow_action_raw_encap *raw_encap; 4739 const struct rte_flow_action_raw_decap *raw_decap; 4740 struct mlx5_rte_flow_action_set_tag *set_tag; 4741 struct rte_flow_action *tag_action; 4742 struct mlx5_rte_flow_item_tag *tag_item; 4743 struct rte_flow_item *item; 4744 char *addr; 4745 int encap = 0; 4746 4747 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4748 switch (actions->type) { 4749 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4750 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4751 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4752 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4753 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4754 rte_memcpy(actions_tx, actions, 4755 sizeof(struct rte_flow_action)); 4756 actions_tx++; 4757 break; 4758 case RTE_FLOW_ACTION_TYPE_COUNT: 4759 if (encap) { 4760 rte_memcpy(actions_tx, actions, 4761 sizeof(struct rte_flow_action)); 4762 actions_tx++; 4763 } else { 4764 rte_memcpy(actions_rx, actions, 4765 sizeof(struct rte_flow_action)); 4766 actions_rx++; 4767 } 4768 break; 4769 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4770 raw_encap = actions->conf; 4771 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 4772 memcpy(actions_tx, actions, 4773 sizeof(struct rte_flow_action)); 4774 actions_tx++; 4775 encap = 1; 4776 } else { 4777 rte_memcpy(actions_rx, actions, 4778 sizeof(struct rte_flow_action)); 4779 actions_rx++; 4780 } 4781 break; 4782 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4783 raw_decap = actions->conf; 4784 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 4785 memcpy(actions_tx, actions, 4786 sizeof(struct rte_flow_action)); 4787 actions_tx++; 4788 } else { 4789 rte_memcpy(actions_rx, actions, 4790 sizeof(struct rte_flow_action)); 4791 actions_rx++; 4792 } 4793 break; 4794 default: 4795 rte_memcpy(actions_rx, actions, 4796 sizeof(struct rte_flow_action)); 4797 actions_rx++; 4798 break; 4799 } 4800 } 4801 /* Add set meta action and end action for the Rx flow. */ 4802 tag_action = actions_rx; 4803 tag_action->type = (enum rte_flow_action_type) 4804 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 4805 actions_rx++; 4806 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 4807 actions_rx++; 4808 set_tag = (void *)actions_rx; 4809 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 4810 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 4811 .data = flow_id, 4812 }; 4813 MLX5_ASSERT(set_tag->id > REG_NON); 4814 tag_action->conf = set_tag; 4815 /* Create Tx item list. */ 4816 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 4817 addr = (void *)&pattern_tx[2]; 4818 item = pattern_tx; 4819 item->type = (enum rte_flow_item_type) 4820 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 4821 tag_item = (void *)addr; 4822 tag_item->data = flow_id; 4823 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 4824 MLX5_ASSERT(set_tag->id > REG_NON); 4825 item->spec = tag_item; 4826 addr += sizeof(struct mlx5_rte_flow_item_tag); 4827 tag_item = (void *)addr; 4828 tag_item->data = UINT32_MAX; 4829 tag_item->id = UINT16_MAX; 4830 item->mask = tag_item; 4831 item->last = NULL; 4832 item++; 4833 item->type = RTE_FLOW_ITEM_TYPE_END; 4834 return 0; 4835 } 4836 4837 /** 4838 * The last stage of splitting chain, just creates the subflow 4839 * without any modification. 4840 * 4841 * @param[in] dev 4842 * Pointer to Ethernet device. 4843 * @param[in] flow 4844 * Parent flow structure pointer. 4845 * @param[in, out] sub_flow 4846 * Pointer to return the created subflow, may be NULL. 4847 * @param[in] attr 4848 * Flow rule attributes. 4849 * @param[in] items 4850 * Pattern specification (list terminated by the END pattern item). 4851 * @param[in] actions 4852 * Associated actions (list terminated by the END action). 4853 * @param[in] flow_split_info 4854 * Pointer to flow split info structure. 4855 * @param[out] error 4856 * Perform verbose error reporting if not NULL. 4857 * @return 4858 * 0 on success, negative value otherwise 4859 */ 4860 static int 4861 flow_create_split_inner(struct rte_eth_dev *dev, 4862 struct rte_flow *flow, 4863 struct mlx5_flow **sub_flow, 4864 const struct rte_flow_attr *attr, 4865 const struct rte_flow_item items[], 4866 const struct rte_flow_action actions[], 4867 struct mlx5_flow_split_info *flow_split_info, 4868 struct rte_flow_error *error) 4869 { 4870 struct mlx5_flow *dev_flow; 4871 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 4872 4873 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 4874 flow_split_info->flow_idx, error); 4875 if (!dev_flow) 4876 return -rte_errno; 4877 dev_flow->flow = flow; 4878 dev_flow->external = flow_split_info->external; 4879 dev_flow->skip_scale = flow_split_info->skip_scale; 4880 /* Subflow object was created, we must include one in the list. */ 4881 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 4882 dev_flow->handle, next); 4883 /* 4884 * If dev_flow is as one of the suffix flow, some actions in suffix 4885 * flow may need some user defined item layer flags, and pass the 4886 * Metadata rxq mark flag to suffix flow as well. 4887 */ 4888 if (flow_split_info->prefix_layers) 4889 dev_flow->handle->layers = flow_split_info->prefix_layers; 4890 if (flow_split_info->prefix_mark) { 4891 MLX5_ASSERT(wks); 4892 wks->mark = 1; 4893 } 4894 if (sub_flow) 4895 *sub_flow = dev_flow; 4896 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4897 dev_flow->dv.table_id = flow_split_info->table_id; 4898 #endif 4899 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 4900 } 4901 4902 /** 4903 * Get the sub policy of a meter. 4904 * 4905 * @param[in] dev 4906 * Pointer to Ethernet device. 4907 * @param[in] flow 4908 * Parent flow structure pointer. 4909 * @param wks 4910 * Pointer to thread flow work space. 4911 * @param[in] attr 4912 * Flow rule attributes. 4913 * @param[in] items 4914 * Pattern specification (list terminated by the END pattern item). 4915 * @param[out] error 4916 * Perform verbose error reporting if not NULL. 4917 * 4918 * @return 4919 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 4920 */ 4921 static struct mlx5_flow_meter_sub_policy * 4922 get_meter_sub_policy(struct rte_eth_dev *dev, 4923 struct rte_flow *flow, 4924 struct mlx5_flow_workspace *wks, 4925 const struct rte_flow_attr *attr, 4926 const struct rte_flow_item items[], 4927 struct rte_flow_error *error) 4928 { 4929 struct mlx5_flow_meter_policy *policy; 4930 struct mlx5_flow_meter_policy *final_policy; 4931 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 4932 4933 policy = wks->policy; 4934 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 4935 if (final_policy->is_rss || final_policy->is_queue) { 4936 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 4937 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 4938 uint32_t i; 4939 4940 /* 4941 * This is a tmp dev_flow, 4942 * no need to register any matcher for it in translate. 4943 */ 4944 wks->skip_matcher_reg = 1; 4945 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 4946 struct mlx5_flow dev_flow = {0}; 4947 struct mlx5_flow_handle dev_handle = { {0} }; 4948 uint8_t fate = final_policy->act_cnt[i].fate_action; 4949 4950 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 4951 const struct rte_flow_action_rss *rss_act = 4952 final_policy->act_cnt[i].rss->conf; 4953 struct rte_flow_action rss_actions[2] = { 4954 [0] = { 4955 .type = RTE_FLOW_ACTION_TYPE_RSS, 4956 .conf = rss_act, 4957 }, 4958 [1] = { 4959 .type = RTE_FLOW_ACTION_TYPE_END, 4960 .conf = NULL, 4961 } 4962 }; 4963 4964 dev_flow.handle = &dev_handle; 4965 dev_flow.ingress = attr->ingress; 4966 dev_flow.flow = flow; 4967 dev_flow.external = 0; 4968 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 4969 dev_flow.dv.transfer = attr->transfer; 4970 #endif 4971 /** 4972 * Translate RSS action to get rss hash fields. 4973 */ 4974 if (flow_drv_translate(dev, &dev_flow, attr, 4975 items, rss_actions, error)) 4976 goto exit; 4977 rss_desc_v[i] = wks->rss_desc; 4978 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 4979 rss_desc_v[i].hash_fields = 4980 dev_flow.hash_fields; 4981 rss_desc_v[i].queue_num = 4982 rss_desc_v[i].hash_fields ? 4983 rss_desc_v[i].queue_num : 1; 4984 rss_desc_v[i].tunnel = 4985 !!(dev_flow.handle->layers & 4986 MLX5_FLOW_LAYER_TUNNEL); 4987 /* Use the RSS queues in the containers. */ 4988 rss_desc_v[i].queue = 4989 (uint16_t *)(uintptr_t)rss_act->queue; 4990 rss_desc[i] = &rss_desc_v[i]; 4991 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 4992 /* This is queue action. */ 4993 rss_desc_v[i] = wks->rss_desc; 4994 rss_desc_v[i].key_len = 0; 4995 rss_desc_v[i].hash_fields = 0; 4996 rss_desc_v[i].queue = 4997 &final_policy->act_cnt[i].queue; 4998 rss_desc_v[i].queue_num = 1; 4999 rss_desc[i] = &rss_desc_v[i]; 5000 } else { 5001 rss_desc[i] = NULL; 5002 } 5003 } 5004 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5005 flow, policy, rss_desc); 5006 } else { 5007 enum mlx5_meter_domain mtr_domain = 5008 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5009 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5010 MLX5_MTR_DOMAIN_INGRESS); 5011 sub_policy = policy->sub_policys[mtr_domain][0]; 5012 } 5013 if (!sub_policy) 5014 rte_flow_error_set(error, EINVAL, 5015 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5016 "Failed to get meter sub-policy."); 5017 exit: 5018 return sub_policy; 5019 } 5020 5021 /** 5022 * Split the meter flow. 5023 * 5024 * As meter flow will split to three sub flow, other than meter 5025 * action, the other actions make sense to only meter accepts 5026 * the packet. If it need to be dropped, no other additional 5027 * actions should be take. 5028 * 5029 * One kind of special action which decapsulates the L3 tunnel 5030 * header will be in the prefix sub flow, as not to take the 5031 * L3 tunnel header into account. 5032 * 5033 * @param[in] dev 5034 * Pointer to Ethernet device. 5035 * @param[in] flow 5036 * Parent flow structure pointer. 5037 * @param wks 5038 * Pointer to thread flow work space. 5039 * @param[in] attr 5040 * Flow rule attributes. 5041 * @param[in] items 5042 * Pattern specification (list terminated by the END pattern item). 5043 * @param[out] sfx_items 5044 * Suffix flow match items (list terminated by the END pattern item). 5045 * @param[in] actions 5046 * Associated actions (list terminated by the END action). 5047 * @param[out] actions_sfx 5048 * Suffix flow actions. 5049 * @param[out] actions_pre 5050 * Prefix flow actions. 5051 * @param[out] mtr_flow_id 5052 * Pointer to meter flow id. 5053 * @param[out] error 5054 * Perform verbose error reporting if not NULL. 5055 * 5056 * @return 5057 * 0 on success, a negative errno value otherwise and rte_errno is set. 5058 */ 5059 static int 5060 flow_meter_split_prep(struct rte_eth_dev *dev, 5061 struct rte_flow *flow, 5062 struct mlx5_flow_workspace *wks, 5063 const struct rte_flow_attr *attr, 5064 const struct rte_flow_item items[], 5065 struct rte_flow_item sfx_items[], 5066 const struct rte_flow_action actions[], 5067 struct rte_flow_action actions_sfx[], 5068 struct rte_flow_action actions_pre[], 5069 uint32_t *mtr_flow_id, 5070 struct rte_flow_error *error) 5071 { 5072 struct mlx5_priv *priv = dev->data->dev_private; 5073 struct mlx5_flow_meter_info *fm = wks->fm; 5074 struct rte_flow_action *tag_action = NULL; 5075 struct rte_flow_item *tag_item; 5076 struct mlx5_rte_flow_action_set_tag *set_tag; 5077 const struct rte_flow_action_raw_encap *raw_encap; 5078 const struct rte_flow_action_raw_decap *raw_decap; 5079 struct mlx5_rte_flow_item_tag *tag_item_spec; 5080 struct mlx5_rte_flow_item_tag *tag_item_mask; 5081 uint32_t tag_id = 0; 5082 struct rte_flow_item *vlan_item_dst = NULL; 5083 const struct rte_flow_item *vlan_item_src = NULL; 5084 const struct rte_flow_item *orig_items = items; 5085 struct rte_flow_action *hw_mtr_action; 5086 struct rte_flow_action *action_pre_head = NULL; 5087 int32_t flow_src_port = priv->representor_id; 5088 bool mtr_first; 5089 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5090 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5091 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5092 uint32_t flow_id = 0; 5093 uint32_t flow_id_reversed = 0; 5094 uint8_t flow_id_bits = 0; 5095 int shift; 5096 5097 /* Prepare the suffix subflow items. */ 5098 tag_item = sfx_items++; 5099 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5100 struct mlx5_priv *port_priv; 5101 const struct rte_flow_item_port_id *pid_v; 5102 int item_type = items->type; 5103 5104 switch (item_type) { 5105 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5106 pid_v = items->spec; 5107 MLX5_ASSERT(pid_v); 5108 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 5109 if (!port_priv) 5110 return rte_flow_error_set(error, 5111 rte_errno, 5112 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 5113 pid_v, 5114 "Failed to get port info."); 5115 flow_src_port = port_priv->representor_id; 5116 if (!fm->def_policy && wks->policy->is_hierarchy && 5117 flow_src_port != priv->representor_id) { 5118 if (flow_drv_mtr_hierarchy_rule_create(dev, 5119 flow, fm, 5120 flow_src_port, 5121 items, 5122 error)) 5123 return -rte_errno; 5124 } 5125 memcpy(sfx_items, items, sizeof(*sfx_items)); 5126 sfx_items++; 5127 break; 5128 case RTE_FLOW_ITEM_TYPE_VLAN: 5129 /* Determine if copy vlan item below. */ 5130 vlan_item_src = items; 5131 vlan_item_dst = sfx_items++; 5132 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5133 break; 5134 default: 5135 break; 5136 } 5137 } 5138 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5139 sfx_items++; 5140 mtr_first = priv->sh->meter_aso_en && 5141 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5142 /* For ASO meter, meter must be before tag in TX direction. */ 5143 if (mtr_first) { 5144 action_pre_head = actions_pre++; 5145 /* Leave space for tag action. */ 5146 tag_action = actions_pre++; 5147 } 5148 /* Prepare the actions for prefix and suffix flow. */ 5149 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5150 struct rte_flow_action *action_cur = NULL; 5151 5152 switch (actions->type) { 5153 case RTE_FLOW_ACTION_TYPE_METER: 5154 if (mtr_first) { 5155 action_cur = action_pre_head; 5156 } else { 5157 /* Leave space for tag action. */ 5158 tag_action = actions_pre++; 5159 action_cur = actions_pre++; 5160 } 5161 break; 5162 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5163 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5164 action_cur = actions_pre++; 5165 break; 5166 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5167 raw_encap = actions->conf; 5168 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5169 action_cur = actions_pre++; 5170 break; 5171 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5172 raw_decap = actions->conf; 5173 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5174 action_cur = actions_pre++; 5175 break; 5176 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5177 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5178 if (vlan_item_dst && vlan_item_src) { 5179 memcpy(vlan_item_dst, vlan_item_src, 5180 sizeof(*vlan_item_dst)); 5181 /* 5182 * Convert to internal match item, it is used 5183 * for vlan push and set vid. 5184 */ 5185 vlan_item_dst->type = (enum rte_flow_item_type) 5186 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5187 } 5188 break; 5189 default: 5190 break; 5191 } 5192 if (!action_cur) 5193 action_cur = (fm->def_policy) ? 5194 actions_sfx++ : actions_pre++; 5195 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5196 } 5197 /* Add end action to the actions. */ 5198 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5199 if (priv->sh->meter_aso_en) { 5200 /** 5201 * For ASO meter, need to add an extra jump action explicitly, 5202 * to jump from meter to policer table. 5203 */ 5204 struct mlx5_flow_meter_sub_policy *sub_policy; 5205 struct mlx5_flow_tbl_data_entry *tbl_data; 5206 5207 if (!fm->def_policy) { 5208 sub_policy = get_meter_sub_policy(dev, flow, wks, 5209 attr, orig_items, 5210 error); 5211 if (!sub_policy) 5212 return -rte_errno; 5213 } else { 5214 enum mlx5_meter_domain mtr_domain = 5215 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5216 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5217 MLX5_MTR_DOMAIN_INGRESS); 5218 5219 sub_policy = 5220 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5221 } 5222 tbl_data = container_of(sub_policy->tbl_rsc, 5223 struct mlx5_flow_tbl_data_entry, tbl); 5224 hw_mtr_action = actions_pre++; 5225 hw_mtr_action->type = (enum rte_flow_action_type) 5226 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5227 hw_mtr_action->conf = tbl_data->jump.action; 5228 } 5229 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5230 actions_pre++; 5231 if (!tag_action) 5232 return rte_flow_error_set(error, ENOMEM, 5233 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5234 NULL, "No tag action space."); 5235 if (!mtr_flow_id) { 5236 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5237 goto exit; 5238 } 5239 /* Only default-policy Meter creates mtr flow id. */ 5240 if (fm->def_policy) { 5241 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5242 if (!tag_id) 5243 return rte_flow_error_set(error, ENOMEM, 5244 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5245 "Failed to allocate meter flow id."); 5246 flow_id = tag_id - 1; 5247 flow_id_bits = (!flow_id) ? 1 : 5248 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5249 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5250 mtr_reg_bits) { 5251 mlx5_ipool_free(fm->flow_ipool, tag_id); 5252 return rte_flow_error_set(error, EINVAL, 5253 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5254 "Meter flow id exceeds max limit."); 5255 } 5256 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5257 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5258 } 5259 /* Build tag actions and items for meter_id/meter flow_id. */ 5260 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5261 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5262 tag_item_mask = tag_item_spec + 1; 5263 /* Both flow_id and meter_id share the same register. */ 5264 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5265 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5266 0, error), 5267 .offset = mtr_id_offset, 5268 .length = mtr_reg_bits, 5269 .data = flow->meter, 5270 }; 5271 /* 5272 * The color Reg bits used by flow_id are growing from 5273 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5274 */ 5275 for (shift = 0; shift < flow_id_bits; shift++) 5276 flow_id_reversed = (flow_id_reversed << 1) | 5277 ((flow_id >> shift) & 0x1); 5278 set_tag->data |= 5279 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5280 tag_item_spec->id = set_tag->id; 5281 tag_item_spec->data = set_tag->data << mtr_id_offset; 5282 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5283 tag_action->type = (enum rte_flow_action_type) 5284 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5285 tag_action->conf = set_tag; 5286 tag_item->type = (enum rte_flow_item_type) 5287 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5288 tag_item->spec = tag_item_spec; 5289 tag_item->last = NULL; 5290 tag_item->mask = tag_item_mask; 5291 exit: 5292 if (mtr_flow_id) 5293 *mtr_flow_id = tag_id; 5294 return 0; 5295 } 5296 5297 /** 5298 * Split action list having QUEUE/RSS for metadata register copy. 5299 * 5300 * Once Q/RSS action is detected in user's action list, the flow action 5301 * should be split in order to copy metadata registers, which will happen in 5302 * RX_CP_TBL like, 5303 * - CQE->flow_tag := reg_c[1] (MARK) 5304 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5305 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5306 * This is because the last action of each flow must be a terminal action 5307 * (QUEUE, RSS or DROP). 5308 * 5309 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5310 * stored and kept in the mlx5_flow structure per each sub_flow. 5311 * 5312 * The Q/RSS action is replaced with, 5313 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5314 * And the following JUMP action is added at the end, 5315 * - JUMP, to RX_CP_TBL. 5316 * 5317 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5318 * flow_create_split_metadata() routine. The flow will look like, 5319 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5320 * 5321 * @param dev 5322 * Pointer to Ethernet device. 5323 * @param[out] split_actions 5324 * Pointer to store split actions to jump to CP_TBL. 5325 * @param[in] actions 5326 * Pointer to the list of original flow actions. 5327 * @param[in] qrss 5328 * Pointer to the Q/RSS action. 5329 * @param[in] actions_n 5330 * Number of original actions. 5331 * @param[in] mtr_sfx 5332 * Check if it is in meter suffix table. 5333 * @param[out] error 5334 * Perform verbose error reporting if not NULL. 5335 * 5336 * @return 5337 * non-zero unique flow_id on success, otherwise 0 and 5338 * error/rte_error are set. 5339 */ 5340 static uint32_t 5341 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5342 struct rte_flow_action *split_actions, 5343 const struct rte_flow_action *actions, 5344 const struct rte_flow_action *qrss, 5345 int actions_n, int mtr_sfx, 5346 struct rte_flow_error *error) 5347 { 5348 struct mlx5_priv *priv = dev->data->dev_private; 5349 struct mlx5_rte_flow_action_set_tag *set_tag; 5350 struct rte_flow_action_jump *jump; 5351 const int qrss_idx = qrss - actions; 5352 uint32_t flow_id = 0; 5353 int ret = 0; 5354 5355 /* 5356 * Given actions will be split 5357 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5358 * - Add jump to mreg CP_TBL. 5359 * As a result, there will be one more action. 5360 */ 5361 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5362 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5363 ++actions_n; 5364 set_tag = (void *)(split_actions + actions_n); 5365 /* 5366 * If we are not the meter suffix flow, add the tag action. 5367 * Since meter suffix flow already has the tag added. 5368 */ 5369 if (!mtr_sfx) { 5370 /* 5371 * Allocate the new subflow ID. This one is unique within 5372 * device and not shared with representors. Otherwise, 5373 * we would have to resolve multi-thread access synch 5374 * issue. Each flow on the shared device is appended 5375 * with source vport identifier, so the resulting 5376 * flows will be unique in the shared (by master and 5377 * representors) domain even if they have coinciding 5378 * IDs. 5379 */ 5380 mlx5_ipool_malloc(priv->sh->ipool 5381 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5382 if (!flow_id) 5383 return rte_flow_error_set(error, ENOMEM, 5384 RTE_FLOW_ERROR_TYPE_ACTION, 5385 NULL, "can't allocate id " 5386 "for split Q/RSS subflow"); 5387 /* Internal SET_TAG action to set flow ID. */ 5388 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5389 .data = flow_id, 5390 }; 5391 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5392 if (ret < 0) 5393 return ret; 5394 set_tag->id = ret; 5395 /* Construct new actions array. */ 5396 /* Replace QUEUE/RSS action. */ 5397 split_actions[qrss_idx] = (struct rte_flow_action){ 5398 .type = (enum rte_flow_action_type) 5399 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5400 .conf = set_tag, 5401 }; 5402 } else { 5403 /* 5404 * If we are the suffix flow of meter, tag already exist. 5405 * Set the QUEUE/RSS action to void. 5406 */ 5407 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5408 } 5409 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5410 jump = (void *)(set_tag + 1); 5411 *jump = (struct rte_flow_action_jump){ 5412 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5413 }; 5414 split_actions[actions_n - 2] = (struct rte_flow_action){ 5415 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5416 .conf = jump, 5417 }; 5418 split_actions[actions_n - 1] = (struct rte_flow_action){ 5419 .type = RTE_FLOW_ACTION_TYPE_END, 5420 }; 5421 return flow_id; 5422 } 5423 5424 /** 5425 * Extend the given action list for Tx metadata copy. 5426 * 5427 * Copy the given action list to the ext_actions and add flow metadata register 5428 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5429 * 5430 * @param[out] ext_actions 5431 * Pointer to the extended action list. 5432 * @param[in] actions 5433 * Pointer to the list of actions. 5434 * @param[in] actions_n 5435 * Number of actions in the list. 5436 * @param[out] error 5437 * Perform verbose error reporting if not NULL. 5438 * @param[in] encap_idx 5439 * The encap action index. 5440 * 5441 * @return 5442 * 0 on success, negative value otherwise 5443 */ 5444 static int 5445 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5446 struct rte_flow_action *ext_actions, 5447 const struct rte_flow_action *actions, 5448 int actions_n, struct rte_flow_error *error, 5449 int encap_idx) 5450 { 5451 struct mlx5_flow_action_copy_mreg *cp_mreg = 5452 (struct mlx5_flow_action_copy_mreg *) 5453 (ext_actions + actions_n + 1); 5454 int ret; 5455 5456 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5457 if (ret < 0) 5458 return ret; 5459 cp_mreg->dst = ret; 5460 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5461 if (ret < 0) 5462 return ret; 5463 cp_mreg->src = ret; 5464 if (encap_idx != 0) 5465 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5466 if (encap_idx == actions_n - 1) { 5467 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5468 .type = (enum rte_flow_action_type) 5469 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5470 .conf = cp_mreg, 5471 }; 5472 ext_actions[actions_n] = (struct rte_flow_action){ 5473 .type = RTE_FLOW_ACTION_TYPE_END, 5474 }; 5475 } else { 5476 ext_actions[encap_idx] = (struct rte_flow_action){ 5477 .type = (enum rte_flow_action_type) 5478 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5479 .conf = cp_mreg, 5480 }; 5481 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5482 sizeof(*ext_actions) * (actions_n - encap_idx)); 5483 } 5484 return 0; 5485 } 5486 5487 /** 5488 * Check the match action from the action list. 5489 * 5490 * @param[in] actions 5491 * Pointer to the list of actions. 5492 * @param[in] attr 5493 * Flow rule attributes. 5494 * @param[in] action 5495 * The action to be check if exist. 5496 * @param[out] match_action_pos 5497 * Pointer to the position of the matched action if exists, otherwise is -1. 5498 * @param[out] qrss_action_pos 5499 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5500 * @param[out] modify_after_mirror 5501 * Pointer to the flag of modify action after FDB mirroring. 5502 * 5503 * @return 5504 * > 0 the total number of actions. 5505 * 0 if not found match action in action list. 5506 */ 5507 static int 5508 flow_check_match_action(const struct rte_flow_action actions[], 5509 const struct rte_flow_attr *attr, 5510 enum rte_flow_action_type action, 5511 int *match_action_pos, int *qrss_action_pos, 5512 int *modify_after_mirror) 5513 { 5514 const struct rte_flow_action_sample *sample; 5515 const struct rte_flow_action_raw_decap *decap; 5516 int actions_n = 0; 5517 uint32_t ratio = 0; 5518 int sub_type = 0; 5519 int flag = 0; 5520 int fdb_mirror = 0; 5521 5522 *match_action_pos = -1; 5523 *qrss_action_pos = -1; 5524 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5525 if (actions->type == action) { 5526 flag = 1; 5527 *match_action_pos = actions_n; 5528 } 5529 switch (actions->type) { 5530 case RTE_FLOW_ACTION_TYPE_QUEUE: 5531 case RTE_FLOW_ACTION_TYPE_RSS: 5532 *qrss_action_pos = actions_n; 5533 break; 5534 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5535 sample = actions->conf; 5536 ratio = sample->ratio; 5537 sub_type = ((const struct rte_flow_action *) 5538 (sample->actions))->type; 5539 if (ratio == 1 && attr->transfer) 5540 fdb_mirror = 1; 5541 break; 5542 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5543 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5544 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5545 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5546 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5547 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5548 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5549 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5550 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5551 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5552 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5553 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5554 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5555 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5556 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5557 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5558 case RTE_FLOW_ACTION_TYPE_FLAG: 5559 case RTE_FLOW_ACTION_TYPE_MARK: 5560 case RTE_FLOW_ACTION_TYPE_SET_META: 5561 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5562 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5563 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5564 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5565 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5566 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5567 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5568 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5569 case RTE_FLOW_ACTION_TYPE_METER: 5570 if (fdb_mirror) 5571 *modify_after_mirror = 1; 5572 break; 5573 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5574 decap = actions->conf; 5575 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5576 ; 5577 actions_n++; 5578 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5579 const struct rte_flow_action_raw_encap *encap = 5580 actions->conf; 5581 if (decap->size <= 5582 MLX5_ENCAPSULATION_DECISION_SIZE && 5583 encap->size > 5584 MLX5_ENCAPSULATION_DECISION_SIZE) 5585 /* L3 encap. */ 5586 break; 5587 } 5588 if (fdb_mirror) 5589 *modify_after_mirror = 1; 5590 break; 5591 default: 5592 break; 5593 } 5594 actions_n++; 5595 } 5596 if (flag && fdb_mirror && !*modify_after_mirror) { 5597 /* FDB mirroring uses the destination array to implement 5598 * instead of FLOW_SAMPLER object. 5599 */ 5600 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5601 flag = 0; 5602 } 5603 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5604 return flag ? actions_n + 1 : 0; 5605 } 5606 5607 #define SAMPLE_SUFFIX_ITEM 2 5608 5609 /** 5610 * Split the sample flow. 5611 * 5612 * As sample flow will split to two sub flow, sample flow with 5613 * sample action, the other actions will move to new suffix flow. 5614 * 5615 * Also add unique tag id with tag action in the sample flow, 5616 * the same tag id will be as match in the suffix flow. 5617 * 5618 * @param dev 5619 * Pointer to Ethernet device. 5620 * @param[in] add_tag 5621 * Add extra tag action flag. 5622 * @param[out] sfx_items 5623 * Suffix flow match items (list terminated by the END pattern item). 5624 * @param[in] actions 5625 * Associated actions (list terminated by the END action). 5626 * @param[out] actions_sfx 5627 * Suffix flow actions. 5628 * @param[out] actions_pre 5629 * Prefix flow actions. 5630 * @param[in] actions_n 5631 * The total number of actions. 5632 * @param[in] sample_action_pos 5633 * The sample action position. 5634 * @param[in] qrss_action_pos 5635 * The Queue/RSS action position. 5636 * @param[in] jump_table 5637 * Add extra jump action flag. 5638 * @param[out] error 5639 * Perform verbose error reporting if not NULL. 5640 * 5641 * @return 5642 * 0 on success, or unique flow_id, a negative errno value 5643 * otherwise and rte_errno is set. 5644 */ 5645 static int 5646 flow_sample_split_prep(struct rte_eth_dev *dev, 5647 int add_tag, 5648 struct rte_flow_item sfx_items[], 5649 const struct rte_flow_action actions[], 5650 struct rte_flow_action actions_sfx[], 5651 struct rte_flow_action actions_pre[], 5652 int actions_n, 5653 int sample_action_pos, 5654 int qrss_action_pos, 5655 int jump_table, 5656 struct rte_flow_error *error) 5657 { 5658 struct mlx5_priv *priv = dev->data->dev_private; 5659 struct mlx5_rte_flow_action_set_tag *set_tag; 5660 struct mlx5_rte_flow_item_tag *tag_spec; 5661 struct mlx5_rte_flow_item_tag *tag_mask; 5662 struct rte_flow_action_jump *jump_action; 5663 uint32_t tag_id = 0; 5664 int index; 5665 int append_index = 0; 5666 int ret; 5667 5668 if (sample_action_pos < 0) 5669 return rte_flow_error_set(error, EINVAL, 5670 RTE_FLOW_ERROR_TYPE_ACTION, 5671 NULL, "invalid position of sample " 5672 "action in list"); 5673 /* Prepare the actions for prefix and suffix flow. */ 5674 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 5675 index = qrss_action_pos; 5676 /* Put the preceding the Queue/RSS action into prefix flow. */ 5677 if (index != 0) 5678 memcpy(actions_pre, actions, 5679 sizeof(struct rte_flow_action) * index); 5680 /* Put others preceding the sample action into prefix flow. */ 5681 if (sample_action_pos > index + 1) 5682 memcpy(actions_pre + index, actions + index + 1, 5683 sizeof(struct rte_flow_action) * 5684 (sample_action_pos - index - 1)); 5685 index = sample_action_pos - 1; 5686 /* Put Queue/RSS action into Suffix flow. */ 5687 memcpy(actions_sfx, actions + qrss_action_pos, 5688 sizeof(struct rte_flow_action)); 5689 actions_sfx++; 5690 } else { 5691 index = sample_action_pos; 5692 if (index != 0) 5693 memcpy(actions_pre, actions, 5694 sizeof(struct rte_flow_action) * index); 5695 } 5696 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 5697 * For CX6DX and above, metadata registers Cx preserve their value, 5698 * add an extra tag action for NIC-RX and E-Switch Domain. 5699 */ 5700 if (add_tag) { 5701 /* Prepare the prefix tag action. */ 5702 append_index++; 5703 set_tag = (void *)(actions_pre + actions_n + append_index); 5704 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 5705 if (ret < 0) 5706 return ret; 5707 mlx5_ipool_malloc(priv->sh->ipool 5708 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 5709 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5710 .id = ret, 5711 .data = tag_id, 5712 }; 5713 /* Prepare the suffix subflow items. */ 5714 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 5715 tag_spec->data = tag_id; 5716 tag_spec->id = set_tag->id; 5717 tag_mask = tag_spec + 1; 5718 tag_mask->data = UINT32_MAX; 5719 sfx_items[0] = (struct rte_flow_item){ 5720 .type = (enum rte_flow_item_type) 5721 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5722 .spec = tag_spec, 5723 .last = NULL, 5724 .mask = tag_mask, 5725 }; 5726 sfx_items[1] = (struct rte_flow_item){ 5727 .type = (enum rte_flow_item_type) 5728 RTE_FLOW_ITEM_TYPE_END, 5729 }; 5730 /* Prepare the tag action in prefix subflow. */ 5731 actions_pre[index++] = 5732 (struct rte_flow_action){ 5733 .type = (enum rte_flow_action_type) 5734 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5735 .conf = set_tag, 5736 }; 5737 } 5738 memcpy(actions_pre + index, actions + sample_action_pos, 5739 sizeof(struct rte_flow_action)); 5740 index += 1; 5741 /* For the modify action after the sample action in E-Switch mirroring, 5742 * Add the extra jump action in prefix subflow and jump into the next 5743 * table, then do the modify action in the new table. 5744 */ 5745 if (jump_table) { 5746 /* Prepare the prefix jump action. */ 5747 append_index++; 5748 jump_action = (void *)(actions_pre + actions_n + append_index); 5749 jump_action->group = jump_table; 5750 actions_pre[index++] = 5751 (struct rte_flow_action){ 5752 .type = (enum rte_flow_action_type) 5753 RTE_FLOW_ACTION_TYPE_JUMP, 5754 .conf = jump_action, 5755 }; 5756 } 5757 actions_pre[index] = (struct rte_flow_action){ 5758 .type = (enum rte_flow_action_type) 5759 RTE_FLOW_ACTION_TYPE_END, 5760 }; 5761 /* Put the actions after sample into Suffix flow. */ 5762 memcpy(actions_sfx, actions + sample_action_pos + 1, 5763 sizeof(struct rte_flow_action) * 5764 (actions_n - sample_action_pos - 1)); 5765 return tag_id; 5766 } 5767 5768 /** 5769 * The splitting for metadata feature. 5770 * 5771 * - Q/RSS action on NIC Rx should be split in order to pass by 5772 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5773 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5774 * 5775 * - All the actions on NIC Tx should have a mreg copy action to 5776 * copy reg_a from WQE to reg_c[0]. 5777 * 5778 * @param dev 5779 * Pointer to Ethernet device. 5780 * @param[in] flow 5781 * Parent flow structure pointer. 5782 * @param[in] attr 5783 * Flow rule attributes. 5784 * @param[in] items 5785 * Pattern specification (list terminated by the END pattern item). 5786 * @param[in] actions 5787 * Associated actions (list terminated by the END action). 5788 * @param[in] flow_split_info 5789 * Pointer to flow split info structure. 5790 * @param[out] error 5791 * Perform verbose error reporting if not NULL. 5792 * @return 5793 * 0 on success, negative value otherwise 5794 */ 5795 static int 5796 flow_create_split_metadata(struct rte_eth_dev *dev, 5797 struct rte_flow *flow, 5798 const struct rte_flow_attr *attr, 5799 const struct rte_flow_item items[], 5800 const struct rte_flow_action actions[], 5801 struct mlx5_flow_split_info *flow_split_info, 5802 struct rte_flow_error *error) 5803 { 5804 struct mlx5_priv *priv = dev->data->dev_private; 5805 struct mlx5_sh_config *config = &priv->sh->config; 5806 const struct rte_flow_action *qrss = NULL; 5807 struct rte_flow_action *ext_actions = NULL; 5808 struct mlx5_flow *dev_flow = NULL; 5809 uint32_t qrss_id = 0; 5810 int mtr_sfx = 0; 5811 size_t act_size; 5812 int actions_n; 5813 int encap_idx; 5814 int ret; 5815 5816 /* Check whether extensive metadata feature is engaged. */ 5817 if (!config->dv_flow_en || 5818 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5819 !mlx5_flow_ext_mreg_supported(dev)) 5820 return flow_create_split_inner(dev, flow, NULL, attr, items, 5821 actions, flow_split_info, error); 5822 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 5823 &encap_idx); 5824 if (qrss) { 5825 /* Exclude hairpin flows from splitting. */ 5826 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 5827 const struct rte_flow_action_queue *queue; 5828 5829 queue = qrss->conf; 5830 if (mlx5_rxq_get_type(dev, queue->index) == 5831 MLX5_RXQ_TYPE_HAIRPIN) 5832 qrss = NULL; 5833 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 5834 const struct rte_flow_action_rss *rss; 5835 5836 rss = qrss->conf; 5837 if (mlx5_rxq_get_type(dev, rss->queue[0]) == 5838 MLX5_RXQ_TYPE_HAIRPIN) 5839 qrss = NULL; 5840 } 5841 } 5842 if (qrss) { 5843 /* Check if it is in meter suffix table. */ 5844 mtr_sfx = attr->group == (attr->transfer ? 5845 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 5846 MLX5_FLOW_TABLE_LEVEL_METER); 5847 /* 5848 * Q/RSS action on NIC Rx should be split in order to pass by 5849 * the mreg copy table (RX_CP_TBL) and then it jumps to the 5850 * action table (RX_ACT_TBL) which has the split Q/RSS action. 5851 */ 5852 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5853 sizeof(struct rte_flow_action_set_tag) + 5854 sizeof(struct rte_flow_action_jump); 5855 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5856 SOCKET_ID_ANY); 5857 if (!ext_actions) 5858 return rte_flow_error_set(error, ENOMEM, 5859 RTE_FLOW_ERROR_TYPE_ACTION, 5860 NULL, "no memory to split " 5861 "metadata flow"); 5862 /* 5863 * Create the new actions list with removed Q/RSS action 5864 * and appended set tag and jump to register copy table 5865 * (RX_CP_TBL). We should preallocate unique tag ID here 5866 * in advance, because it is needed for set tag action. 5867 */ 5868 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 5869 qrss, actions_n, 5870 mtr_sfx, error); 5871 if (!mtr_sfx && !qrss_id) { 5872 ret = -rte_errno; 5873 goto exit; 5874 } 5875 } else if (attr->egress && !attr->transfer) { 5876 /* 5877 * All the actions on NIC Tx should have a metadata register 5878 * copy action to copy reg_a from WQE to reg_c[meta] 5879 */ 5880 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 5881 sizeof(struct mlx5_flow_action_copy_mreg); 5882 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 5883 SOCKET_ID_ANY); 5884 if (!ext_actions) 5885 return rte_flow_error_set(error, ENOMEM, 5886 RTE_FLOW_ERROR_TYPE_ACTION, 5887 NULL, "no memory to split " 5888 "metadata flow"); 5889 /* Create the action list appended with copy register. */ 5890 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 5891 actions_n, error, encap_idx); 5892 if (ret < 0) 5893 goto exit; 5894 } 5895 /* Add the unmodified original or prefix subflow. */ 5896 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 5897 items, ext_actions ? ext_actions : 5898 actions, flow_split_info, error); 5899 if (ret < 0) 5900 goto exit; 5901 MLX5_ASSERT(dev_flow); 5902 if (qrss) { 5903 const struct rte_flow_attr q_attr = { 5904 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5905 .ingress = 1, 5906 }; 5907 /* Internal PMD action to set register. */ 5908 struct mlx5_rte_flow_item_tag q_tag_spec = { 5909 .data = qrss_id, 5910 .id = REG_NON, 5911 }; 5912 struct rte_flow_item q_items[] = { 5913 { 5914 .type = (enum rte_flow_item_type) 5915 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5916 .spec = &q_tag_spec, 5917 .last = NULL, 5918 .mask = NULL, 5919 }, 5920 { 5921 .type = RTE_FLOW_ITEM_TYPE_END, 5922 }, 5923 }; 5924 struct rte_flow_action q_actions[] = { 5925 { 5926 .type = qrss->type, 5927 .conf = qrss->conf, 5928 }, 5929 { 5930 .type = RTE_FLOW_ACTION_TYPE_END, 5931 }, 5932 }; 5933 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 5934 5935 /* 5936 * Configure the tag item only if there is no meter subflow. 5937 * Since tag is already marked in the meter suffix subflow 5938 * we can just use the meter suffix items as is. 5939 */ 5940 if (qrss_id) { 5941 /* Not meter subflow. */ 5942 MLX5_ASSERT(!mtr_sfx); 5943 /* 5944 * Put unique id in prefix flow due to it is destroyed 5945 * after suffix flow and id will be freed after there 5946 * is no actual flows with this id and identifier 5947 * reallocation becomes possible (for example, for 5948 * other flows in other threads). 5949 */ 5950 dev_flow->handle->split_flow_id = qrss_id; 5951 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 5952 error); 5953 if (ret < 0) 5954 goto exit; 5955 q_tag_spec.id = ret; 5956 } 5957 dev_flow = NULL; 5958 /* Add suffix subflow to execute Q/RSS. */ 5959 flow_split_info->prefix_layers = layers; 5960 flow_split_info->prefix_mark = 0; 5961 flow_split_info->table_id = 0; 5962 ret = flow_create_split_inner(dev, flow, &dev_flow, 5963 &q_attr, mtr_sfx ? items : 5964 q_items, q_actions, 5965 flow_split_info, error); 5966 if (ret < 0) 5967 goto exit; 5968 /* qrss ID should be freed if failed. */ 5969 qrss_id = 0; 5970 MLX5_ASSERT(dev_flow); 5971 } 5972 5973 exit: 5974 /* 5975 * We do not destroy the partially created sub_flows in case of error. 5976 * These ones are included into parent flow list and will be destroyed 5977 * by flow_drv_destroy. 5978 */ 5979 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 5980 qrss_id); 5981 mlx5_free(ext_actions); 5982 return ret; 5983 } 5984 5985 /** 5986 * Create meter internal drop flow with the original pattern. 5987 * 5988 * @param dev 5989 * Pointer to Ethernet device. 5990 * @param[in] flow 5991 * Parent flow structure pointer. 5992 * @param[in] attr 5993 * Flow rule attributes. 5994 * @param[in] items 5995 * Pattern specification (list terminated by the END pattern item). 5996 * @param[in] flow_split_info 5997 * Pointer to flow split info structure. 5998 * @param[in] fm 5999 * Pointer to flow meter structure. 6000 * @param[out] error 6001 * Perform verbose error reporting if not NULL. 6002 * @return 6003 * 0 on success, negative value otherwise 6004 */ 6005 static uint32_t 6006 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6007 struct rte_flow *flow, 6008 const struct rte_flow_attr *attr, 6009 const struct rte_flow_item items[], 6010 struct mlx5_flow_split_info *flow_split_info, 6011 struct mlx5_flow_meter_info *fm, 6012 struct rte_flow_error *error) 6013 { 6014 struct mlx5_flow *dev_flow = NULL; 6015 struct rte_flow_attr drop_attr = *attr; 6016 struct rte_flow_action drop_actions[3]; 6017 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6018 6019 MLX5_ASSERT(fm->drop_cnt); 6020 drop_actions[0].type = 6021 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6022 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6023 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6024 drop_actions[1].conf = NULL; 6025 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6026 drop_actions[2].conf = NULL; 6027 drop_split_info.external = false; 6028 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6029 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6030 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6031 return flow_create_split_inner(dev, flow, &dev_flow, 6032 &drop_attr, items, drop_actions, 6033 &drop_split_info, error); 6034 } 6035 6036 /** 6037 * The splitting for meter feature. 6038 * 6039 * - The meter flow will be split to two flows as prefix and 6040 * suffix flow. The packets make sense only it pass the prefix 6041 * meter action. 6042 * 6043 * - Reg_C_5 is used for the packet to match betweend prefix and 6044 * suffix flow. 6045 * 6046 * @param dev 6047 * Pointer to Ethernet device. 6048 * @param[in] flow 6049 * Parent flow structure pointer. 6050 * @param[in] attr 6051 * Flow rule attributes. 6052 * @param[in] items 6053 * Pattern specification (list terminated by the END pattern item). 6054 * @param[in] actions 6055 * Associated actions (list terminated by the END action). 6056 * @param[in] flow_split_info 6057 * Pointer to flow split info structure. 6058 * @param[out] error 6059 * Perform verbose error reporting if not NULL. 6060 * @return 6061 * 0 on success, negative value otherwise 6062 */ 6063 static int 6064 flow_create_split_meter(struct rte_eth_dev *dev, 6065 struct rte_flow *flow, 6066 const struct rte_flow_attr *attr, 6067 const struct rte_flow_item items[], 6068 const struct rte_flow_action actions[], 6069 struct mlx5_flow_split_info *flow_split_info, 6070 struct rte_flow_error *error) 6071 { 6072 struct mlx5_priv *priv = dev->data->dev_private; 6073 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6074 struct rte_flow_action *sfx_actions = NULL; 6075 struct rte_flow_action *pre_actions = NULL; 6076 struct rte_flow_item *sfx_items = NULL; 6077 struct mlx5_flow *dev_flow = NULL; 6078 struct rte_flow_attr sfx_attr = *attr; 6079 struct mlx5_flow_meter_info *fm = NULL; 6080 uint8_t skip_scale_restore; 6081 bool has_mtr = false; 6082 bool has_modify = false; 6083 bool set_mtr_reg = true; 6084 bool is_mtr_hierarchy = false; 6085 uint32_t meter_id = 0; 6086 uint32_t mtr_idx = 0; 6087 uint32_t mtr_flow_id = 0; 6088 size_t act_size; 6089 size_t item_size; 6090 int actions_n = 0; 6091 int ret = 0; 6092 6093 if (priv->mtr_en) 6094 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6095 &has_modify, &meter_id); 6096 if (has_mtr) { 6097 if (flow->meter) { 6098 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6099 if (!fm) 6100 return rte_flow_error_set(error, EINVAL, 6101 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6102 NULL, "Meter not found."); 6103 } else { 6104 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6105 if (!fm) 6106 return rte_flow_error_set(error, EINVAL, 6107 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6108 NULL, "Meter not found."); 6109 ret = mlx5_flow_meter_attach(priv, fm, 6110 &sfx_attr, error); 6111 if (ret) 6112 return -rte_errno; 6113 flow->meter = mtr_idx; 6114 } 6115 MLX5_ASSERT(wks); 6116 wks->fm = fm; 6117 if (!fm->def_policy) { 6118 wks->policy = mlx5_flow_meter_policy_find(dev, 6119 fm->policy_id, 6120 NULL); 6121 MLX5_ASSERT(wks->policy); 6122 if (wks->policy->is_hierarchy) { 6123 wks->final_policy = 6124 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6125 wks->policy); 6126 if (!wks->final_policy) 6127 return rte_flow_error_set(error, 6128 EINVAL, 6129 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6130 "Failed to find terminal policy of hierarchy."); 6131 is_mtr_hierarchy = true; 6132 } 6133 } 6134 /* 6135 * If it isn't default-policy Meter, and 6136 * 1. There's no action in flow to change 6137 * packet (modify/encap/decap etc.), OR 6138 * 2. No drop count needed for this meter. 6139 * 3. It's not meter hierarchy. 6140 * Then no need to use regC to save meter id anymore. 6141 */ 6142 if (!fm->def_policy && !is_mtr_hierarchy && 6143 (!has_modify || !fm->drop_cnt)) 6144 set_mtr_reg = false; 6145 /* Prefix actions: meter, decap, encap, tag, jump, end. */ 6146 act_size = sizeof(struct rte_flow_action) * (actions_n + 6) + 6147 sizeof(struct mlx5_rte_flow_action_set_tag); 6148 /* Suffix items: tag, vlan, port id, end. */ 6149 #define METER_SUFFIX_ITEM 4 6150 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6151 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6152 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6153 0, SOCKET_ID_ANY); 6154 if (!sfx_actions) 6155 return rte_flow_error_set(error, ENOMEM, 6156 RTE_FLOW_ERROR_TYPE_ACTION, 6157 NULL, "no memory to split " 6158 "meter flow"); 6159 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6160 act_size); 6161 /* There's no suffix flow for meter of non-default policy. */ 6162 if (!fm->def_policy) 6163 pre_actions = sfx_actions + 1; 6164 else 6165 pre_actions = sfx_actions + actions_n; 6166 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6167 items, sfx_items, actions, 6168 sfx_actions, pre_actions, 6169 (set_mtr_reg ? &mtr_flow_id : NULL), 6170 error); 6171 if (ret) { 6172 ret = -rte_errno; 6173 goto exit; 6174 } 6175 /* Add the prefix subflow. */ 6176 skip_scale_restore = flow_split_info->skip_scale; 6177 flow_split_info->skip_scale |= 6178 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6179 ret = flow_create_split_inner(dev, flow, &dev_flow, 6180 attr, items, pre_actions, 6181 flow_split_info, error); 6182 flow_split_info->skip_scale = skip_scale_restore; 6183 if (ret) { 6184 if (mtr_flow_id) 6185 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6186 ret = -rte_errno; 6187 goto exit; 6188 } 6189 if (mtr_flow_id) { 6190 dev_flow->handle->split_flow_id = mtr_flow_id; 6191 dev_flow->handle->is_meter_flow_id = 1; 6192 } 6193 if (!fm->def_policy) { 6194 if (!set_mtr_reg && fm->drop_cnt) 6195 ret = 6196 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6197 &sfx_attr, items, 6198 flow_split_info, 6199 fm, error); 6200 goto exit; 6201 } 6202 /* Setting the sfx group atrr. */ 6203 sfx_attr.group = sfx_attr.transfer ? 6204 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6205 MLX5_FLOW_TABLE_LEVEL_METER; 6206 flow_split_info->prefix_layers = 6207 flow_get_prefix_layer_flags(dev_flow); 6208 flow_split_info->prefix_mark |= wks->mark; 6209 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6210 } 6211 /* Add the prefix subflow. */ 6212 ret = flow_create_split_metadata(dev, flow, 6213 &sfx_attr, sfx_items ? 6214 sfx_items : items, 6215 sfx_actions ? sfx_actions : actions, 6216 flow_split_info, error); 6217 exit: 6218 if (sfx_actions) 6219 mlx5_free(sfx_actions); 6220 return ret; 6221 } 6222 6223 /** 6224 * The splitting for sample feature. 6225 * 6226 * Once Sample action is detected in the action list, the flow actions should 6227 * be split into prefix sub flow and suffix sub flow. 6228 * 6229 * The original items remain in the prefix sub flow, all actions preceding the 6230 * sample action and the sample action itself will be copied to the prefix 6231 * sub flow, the actions following the sample action will be copied to the 6232 * suffix sub flow, Queue action always be located in the suffix sub flow. 6233 * 6234 * In order to make the packet from prefix sub flow matches with suffix sub 6235 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6236 * flow uses tag item with the unique flow id. 6237 * 6238 * @param dev 6239 * Pointer to Ethernet device. 6240 * @param[in] flow 6241 * Parent flow structure pointer. 6242 * @param[in] attr 6243 * Flow rule attributes. 6244 * @param[in] items 6245 * Pattern specification (list terminated by the END pattern item). 6246 * @param[in] actions 6247 * Associated actions (list terminated by the END action). 6248 * @param[in] flow_split_info 6249 * Pointer to flow split info structure. 6250 * @param[out] error 6251 * Perform verbose error reporting if not NULL. 6252 * @return 6253 * 0 on success, negative value otherwise 6254 */ 6255 static int 6256 flow_create_split_sample(struct rte_eth_dev *dev, 6257 struct rte_flow *flow, 6258 const struct rte_flow_attr *attr, 6259 const struct rte_flow_item items[], 6260 const struct rte_flow_action actions[], 6261 struct mlx5_flow_split_info *flow_split_info, 6262 struct rte_flow_error *error) 6263 { 6264 struct mlx5_priv *priv = dev->data->dev_private; 6265 struct rte_flow_action *sfx_actions = NULL; 6266 struct rte_flow_action *pre_actions = NULL; 6267 struct rte_flow_item *sfx_items = NULL; 6268 struct mlx5_flow *dev_flow = NULL; 6269 struct rte_flow_attr sfx_attr = *attr; 6270 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6271 struct mlx5_flow_dv_sample_resource *sample_res; 6272 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6273 struct mlx5_flow_tbl_resource *sfx_tbl; 6274 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6275 #endif 6276 size_t act_size; 6277 size_t item_size; 6278 uint32_t fdb_tx = 0; 6279 int32_t tag_id = 0; 6280 int actions_n = 0; 6281 int sample_action_pos; 6282 int qrss_action_pos; 6283 int add_tag = 0; 6284 int modify_after_mirror = 0; 6285 uint16_t jump_table = 0; 6286 const uint32_t next_ft_step = 1; 6287 int ret = 0; 6288 6289 if (priv->sampler_en) 6290 actions_n = flow_check_match_action(actions, attr, 6291 RTE_FLOW_ACTION_TYPE_SAMPLE, 6292 &sample_action_pos, &qrss_action_pos, 6293 &modify_after_mirror); 6294 if (actions_n) { 6295 /* The prefix actions must includes sample, tag, end. */ 6296 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6297 + sizeof(struct mlx5_rte_flow_action_set_tag); 6298 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6299 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6300 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6301 item_size), 0, SOCKET_ID_ANY); 6302 if (!sfx_actions) 6303 return rte_flow_error_set(error, ENOMEM, 6304 RTE_FLOW_ERROR_TYPE_ACTION, 6305 NULL, "no memory to split " 6306 "sample flow"); 6307 /* The representor_id is UINT16_MAX for uplink. */ 6308 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6309 /* 6310 * When reg_c_preserve is set, metadata registers Cx preserve 6311 * their value even through packet duplication. 6312 */ 6313 add_tag = (!fdb_tx || 6314 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6315 if (add_tag) 6316 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6317 + act_size); 6318 if (modify_after_mirror) 6319 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6320 next_ft_step; 6321 pre_actions = sfx_actions + actions_n; 6322 tag_id = flow_sample_split_prep(dev, add_tag, sfx_items, 6323 actions, sfx_actions, 6324 pre_actions, actions_n, 6325 sample_action_pos, 6326 qrss_action_pos, jump_table, 6327 error); 6328 if (tag_id < 0 || (add_tag && !tag_id)) { 6329 ret = -rte_errno; 6330 goto exit; 6331 } 6332 if (modify_after_mirror) 6333 flow_split_info->skip_scale = 6334 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6335 /* Add the prefix subflow. */ 6336 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6337 items, pre_actions, 6338 flow_split_info, error); 6339 if (ret) { 6340 ret = -rte_errno; 6341 goto exit; 6342 } 6343 dev_flow->handle->split_flow_id = tag_id; 6344 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6345 if (!modify_after_mirror) { 6346 /* Set the sfx group attr. */ 6347 sample_res = (struct mlx5_flow_dv_sample_resource *) 6348 dev_flow->dv.sample_res; 6349 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6350 sample_res->normal_path_tbl; 6351 sfx_tbl_data = container_of(sfx_tbl, 6352 struct mlx5_flow_tbl_data_entry, 6353 tbl); 6354 sfx_attr.group = sfx_attr.transfer ? 6355 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6356 } else { 6357 MLX5_ASSERT(attr->transfer); 6358 sfx_attr.group = jump_table; 6359 } 6360 flow_split_info->prefix_layers = 6361 flow_get_prefix_layer_flags(dev_flow); 6362 MLX5_ASSERT(wks); 6363 flow_split_info->prefix_mark |= wks->mark; 6364 /* Suffix group level already be scaled with factor, set 6365 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6366 * again in translation. 6367 */ 6368 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6369 #endif 6370 } 6371 /* Add the suffix subflow. */ 6372 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6373 sfx_items ? sfx_items : items, 6374 sfx_actions ? sfx_actions : actions, 6375 flow_split_info, error); 6376 exit: 6377 if (sfx_actions) 6378 mlx5_free(sfx_actions); 6379 return ret; 6380 } 6381 6382 /** 6383 * Split the flow to subflow set. The splitters might be linked 6384 * in the chain, like this: 6385 * flow_create_split_outer() calls: 6386 * flow_create_split_meter() calls: 6387 * flow_create_split_metadata(meter_subflow_0) calls: 6388 * flow_create_split_inner(metadata_subflow_0) 6389 * flow_create_split_inner(metadata_subflow_1) 6390 * flow_create_split_inner(metadata_subflow_2) 6391 * flow_create_split_metadata(meter_subflow_1) calls: 6392 * flow_create_split_inner(metadata_subflow_0) 6393 * flow_create_split_inner(metadata_subflow_1) 6394 * flow_create_split_inner(metadata_subflow_2) 6395 * 6396 * This provide flexible way to add new levels of flow splitting. 6397 * The all of successfully created subflows are included to the 6398 * parent flow dev_flow list. 6399 * 6400 * @param dev 6401 * Pointer to Ethernet device. 6402 * @param[in] flow 6403 * Parent flow structure pointer. 6404 * @param[in] attr 6405 * Flow rule attributes. 6406 * @param[in] items 6407 * Pattern specification (list terminated by the END pattern item). 6408 * @param[in] actions 6409 * Associated actions (list terminated by the END action). 6410 * @param[in] flow_split_info 6411 * Pointer to flow split info structure. 6412 * @param[out] error 6413 * Perform verbose error reporting if not NULL. 6414 * @return 6415 * 0 on success, negative value otherwise 6416 */ 6417 static int 6418 flow_create_split_outer(struct rte_eth_dev *dev, 6419 struct rte_flow *flow, 6420 const struct rte_flow_attr *attr, 6421 const struct rte_flow_item items[], 6422 const struct rte_flow_action actions[], 6423 struct mlx5_flow_split_info *flow_split_info, 6424 struct rte_flow_error *error) 6425 { 6426 int ret; 6427 6428 ret = flow_create_split_sample(dev, flow, attr, items, 6429 actions, flow_split_info, error); 6430 MLX5_ASSERT(ret <= 0); 6431 return ret; 6432 } 6433 6434 static inline struct mlx5_flow_tunnel * 6435 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6436 { 6437 struct mlx5_flow_tunnel *tunnel; 6438 6439 #pragma GCC diagnostic push 6440 #pragma GCC diagnostic ignored "-Wcast-qual" 6441 tunnel = (typeof(tunnel))flow->tunnel; 6442 #pragma GCC diagnostic pop 6443 6444 return tunnel; 6445 } 6446 6447 /** 6448 * Adjust flow RSS workspace if needed. 6449 * 6450 * @param wks 6451 * Pointer to thread flow work space. 6452 * @param rss_desc 6453 * Pointer to RSS descriptor. 6454 * @param[in] nrssq_num 6455 * New RSS queue number. 6456 * 6457 * @return 6458 * 0 on success, -1 otherwise and rte_errno is set. 6459 */ 6460 static int 6461 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6462 struct mlx5_flow_rss_desc *rss_desc, 6463 uint32_t nrssq_num) 6464 { 6465 if (likely(nrssq_num <= wks->rssq_num)) 6466 return 0; 6467 rss_desc->queue = realloc(rss_desc->queue, 6468 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6469 if (!rss_desc->queue) { 6470 rte_errno = ENOMEM; 6471 return -1; 6472 } 6473 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6474 return 0; 6475 } 6476 6477 /** 6478 * Create a flow and add it to @p list. 6479 * 6480 * @param dev 6481 * Pointer to Ethernet device. 6482 * @param list 6483 * Pointer to a TAILQ flow list. If this parameter NULL, 6484 * no list insertion occurred, flow is just created, 6485 * this is caller's responsibility to track the 6486 * created flow. 6487 * @param[in] attr 6488 * Flow rule attributes. 6489 * @param[in] items 6490 * Pattern specification (list terminated by the END pattern item). 6491 * @param[in] actions 6492 * Associated actions (list terminated by the END action). 6493 * @param[in] external 6494 * This flow rule is created by request external to PMD. 6495 * @param[out] error 6496 * Perform verbose error reporting if not NULL. 6497 * 6498 * @return 6499 * A flow index on success, 0 otherwise and rte_errno is set. 6500 */ 6501 static uint32_t 6502 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6503 const struct rte_flow_attr *attr, 6504 const struct rte_flow_item items[], 6505 const struct rte_flow_action original_actions[], 6506 bool external, struct rte_flow_error *error) 6507 { 6508 struct mlx5_priv *priv = dev->data->dev_private; 6509 struct rte_flow *flow = NULL; 6510 struct mlx5_flow *dev_flow; 6511 const struct rte_flow_action_rss *rss = NULL; 6512 struct mlx5_translated_action_handle 6513 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6514 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6515 union { 6516 struct mlx5_flow_expand_rss buf; 6517 uint8_t buffer[4096]; 6518 } expand_buffer; 6519 union { 6520 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6521 uint8_t buffer[2048]; 6522 } actions_rx; 6523 union { 6524 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6525 uint8_t buffer[2048]; 6526 } actions_hairpin_tx; 6527 union { 6528 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6529 uint8_t buffer[2048]; 6530 } items_tx; 6531 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6532 struct mlx5_flow_rss_desc *rss_desc; 6533 const struct rte_flow_action *p_actions_rx; 6534 uint32_t i; 6535 uint32_t idx = 0; 6536 int hairpin_flow; 6537 struct rte_flow_attr attr_tx = { .priority = 0 }; 6538 const struct rte_flow_action *actions; 6539 struct rte_flow_action *translated_actions = NULL; 6540 struct mlx5_flow_tunnel *tunnel; 6541 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6542 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6543 struct mlx5_flow_split_info flow_split_info = { 6544 .external = !!external, 6545 .skip_scale = 0, 6546 .flow_idx = 0, 6547 .prefix_mark = 0, 6548 .prefix_layers = 0, 6549 .table_id = 0 6550 }; 6551 int ret; 6552 6553 MLX5_ASSERT(wks); 6554 rss_desc = &wks->rss_desc; 6555 ret = flow_action_handles_translate(dev, original_actions, 6556 indir_actions, 6557 &indir_actions_n, 6558 &translated_actions, error); 6559 if (ret < 0) { 6560 MLX5_ASSERT(translated_actions == NULL); 6561 return 0; 6562 } 6563 actions = translated_actions ? translated_actions : original_actions; 6564 p_actions_rx = actions; 6565 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6566 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6567 external, hairpin_flow, error); 6568 if (ret < 0) 6569 goto error_before_hairpin_split; 6570 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6571 if (!flow) { 6572 rte_errno = ENOMEM; 6573 goto error_before_hairpin_split; 6574 } 6575 if (hairpin_flow > 0) { 6576 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6577 rte_errno = EINVAL; 6578 goto error_before_hairpin_split; 6579 } 6580 flow_hairpin_split(dev, actions, actions_rx.actions, 6581 actions_hairpin_tx.actions, items_tx.items, 6582 idx); 6583 p_actions_rx = actions_rx.actions; 6584 } 6585 flow_split_info.flow_idx = idx; 6586 flow->drv_type = flow_get_drv_type(dev, attr); 6587 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6588 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6589 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6590 /* RSS Action only works on NIC RX domain */ 6591 if (attr->ingress && !attr->transfer) 6592 rss = flow_get_rss_action(dev, p_actions_rx); 6593 if (rss) { 6594 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6595 return 0; 6596 /* 6597 * The following information is required by 6598 * mlx5_flow_hashfields_adjust() in advance. 6599 */ 6600 rss_desc->level = rss->level; 6601 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6602 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6603 } 6604 flow->dev_handles = 0; 6605 if (rss && rss->types) { 6606 unsigned int graph_root; 6607 6608 graph_root = find_graph_root(rss->level); 6609 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6610 items, rss->types, 6611 mlx5_support_expansion, graph_root); 6612 MLX5_ASSERT(ret > 0 && 6613 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6614 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6615 for (i = 0; i < buf->entries; ++i) 6616 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6617 } 6618 } else { 6619 buf->entries = 1; 6620 buf->entry[0].pattern = (void *)(uintptr_t)items; 6621 } 6622 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6623 indir_actions_n); 6624 for (i = 0; i < buf->entries; ++i) { 6625 /* Initialize flow split data. */ 6626 flow_split_info.prefix_layers = 0; 6627 flow_split_info.prefix_mark = 0; 6628 flow_split_info.skip_scale = 0; 6629 /* 6630 * The splitter may create multiple dev_flows, 6631 * depending on configuration. In the simplest 6632 * case it just creates unmodified original flow. 6633 */ 6634 ret = flow_create_split_outer(dev, flow, attr, 6635 buf->entry[i].pattern, 6636 p_actions_rx, &flow_split_info, 6637 error); 6638 if (ret < 0) 6639 goto error; 6640 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6641 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6642 p_actions_rx, 6643 idx, 6644 wks->flows[0].tunnel, 6645 &default_miss_ctx, 6646 error); 6647 if (ret < 0) { 6648 mlx5_free(default_miss_ctx.queue); 6649 goto error; 6650 } 6651 } 6652 } 6653 /* Create the tx flow. */ 6654 if (hairpin_flow) { 6655 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 6656 attr_tx.ingress = 0; 6657 attr_tx.egress = 1; 6658 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 6659 actions_hairpin_tx.actions, 6660 idx, error); 6661 if (!dev_flow) 6662 goto error; 6663 dev_flow->flow = flow; 6664 dev_flow->external = 0; 6665 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 6666 dev_flow->handle, next); 6667 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 6668 items_tx.items, 6669 actions_hairpin_tx.actions, error); 6670 if (ret < 0) 6671 goto error; 6672 } 6673 /* 6674 * Update the metadata register copy table. If extensive 6675 * metadata feature is enabled and registers are supported 6676 * we might create the extra rte_flow for each unique 6677 * MARK/FLAG action ID. 6678 * 6679 * The table is updated for ingress Flows only, because 6680 * the egress Flows belong to the different device and 6681 * copy table should be updated in peer NIC Rx domain. 6682 */ 6683 if (attr->ingress && 6684 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 6685 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 6686 if (ret) 6687 goto error; 6688 } 6689 /* 6690 * If the flow is external (from application) OR device is started, 6691 * OR mreg discover, then apply immediately. 6692 */ 6693 if (external || dev->data->dev_started || 6694 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 6695 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 6696 ret = flow_drv_apply(dev, flow, error); 6697 if (ret < 0) 6698 goto error; 6699 } 6700 flow->type = type; 6701 flow_rxq_flags_set(dev, flow); 6702 rte_free(translated_actions); 6703 tunnel = flow_tunnel_from_rule(wks->flows); 6704 if (tunnel) { 6705 flow->tunnel = 1; 6706 flow->tunnel_id = tunnel->tunnel_id; 6707 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 6708 mlx5_free(default_miss_ctx.queue); 6709 } 6710 mlx5_flow_pop_thread_workspace(); 6711 return idx; 6712 error: 6713 MLX5_ASSERT(flow); 6714 ret = rte_errno; /* Save rte_errno before cleanup. */ 6715 flow_mreg_del_copy_action(dev, flow); 6716 flow_drv_destroy(dev, flow); 6717 if (rss_desc->shared_rss) 6718 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 6719 mlx5_ipool_get 6720 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 6721 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 6722 mlx5_ipool_free(priv->flows[type], idx); 6723 rte_errno = ret; /* Restore rte_errno. */ 6724 ret = rte_errno; 6725 rte_errno = ret; 6726 mlx5_flow_pop_thread_workspace(); 6727 error_before_hairpin_split: 6728 rte_free(translated_actions); 6729 return 0; 6730 } 6731 6732 /** 6733 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 6734 * incoming packets to table 1. 6735 * 6736 * Other flow rules, requested for group n, will be created in 6737 * e-switch table n+1. 6738 * Jump action to e-switch group n will be created to group n+1. 6739 * 6740 * Used when working in switchdev mode, to utilise advantages of table 1 6741 * and above. 6742 * 6743 * @param dev 6744 * Pointer to Ethernet device. 6745 * 6746 * @return 6747 * Pointer to flow on success, NULL otherwise and rte_errno is set. 6748 */ 6749 struct rte_flow * 6750 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 6751 { 6752 const struct rte_flow_attr attr = { 6753 .group = 0, 6754 .priority = 0, 6755 .ingress = 1, 6756 .egress = 0, 6757 .transfer = 1, 6758 }; 6759 const struct rte_flow_item pattern = { 6760 .type = RTE_FLOW_ITEM_TYPE_END, 6761 }; 6762 struct rte_flow_action_jump jump = { 6763 .group = 1, 6764 }; 6765 const struct rte_flow_action actions[] = { 6766 { 6767 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6768 .conf = &jump, 6769 }, 6770 { 6771 .type = RTE_FLOW_ACTION_TYPE_END, 6772 }, 6773 }; 6774 struct rte_flow_error error; 6775 6776 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 6777 &attr, &pattern, 6778 actions, false, &error); 6779 } 6780 6781 /** 6782 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 6783 * and sq number, directs all packets to peer vport. 6784 * 6785 * @param dev 6786 * Pointer to Ethernet device. 6787 * @param txq 6788 * Txq index. 6789 * 6790 * @return 6791 * Flow ID on success, 0 otherwise and rte_errno is set. 6792 */ 6793 uint32_t 6794 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 6795 { 6796 struct rte_flow_attr attr = { 6797 .group = 0, 6798 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 6799 .ingress = 1, 6800 .egress = 0, 6801 .transfer = 1, 6802 }; 6803 struct rte_flow_item_port_id port_spec = { 6804 .id = MLX5_PORT_ESW_MGR, 6805 }; 6806 struct mlx5_rte_flow_item_tx_queue txq_spec = { 6807 .queue = txq, 6808 }; 6809 struct rte_flow_item pattern[] = { 6810 { 6811 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 6812 .spec = &port_spec, 6813 }, 6814 { 6815 .type = (enum rte_flow_item_type) 6816 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 6817 .spec = &txq_spec, 6818 }, 6819 { 6820 .type = RTE_FLOW_ITEM_TYPE_END, 6821 }, 6822 }; 6823 struct rte_flow_action_jump jump = { 6824 .group = 1, 6825 }; 6826 struct rte_flow_action_port_id port = { 6827 .id = dev->data->port_id, 6828 }; 6829 struct rte_flow_action actions[] = { 6830 { 6831 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6832 .conf = &jump, 6833 }, 6834 { 6835 .type = RTE_FLOW_ACTION_TYPE_END, 6836 }, 6837 }; 6838 struct rte_flow_error error; 6839 6840 /* 6841 * Creates group 0, highest priority jump flow. 6842 * Matches txq to bypass kernel packets. 6843 */ 6844 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 6845 false, &error) == 0) 6846 return 0; 6847 /* Create group 1, lowest priority redirect flow for txq. */ 6848 attr.group = 1; 6849 actions[0].conf = &port; 6850 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 6851 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 6852 actions, false, &error); 6853 } 6854 6855 /** 6856 * Validate a flow supported by the NIC. 6857 * 6858 * @see rte_flow_validate() 6859 * @see rte_flow_ops 6860 */ 6861 int 6862 mlx5_flow_validate(struct rte_eth_dev *dev, 6863 const struct rte_flow_attr *attr, 6864 const struct rte_flow_item items[], 6865 const struct rte_flow_action original_actions[], 6866 struct rte_flow_error *error) 6867 { 6868 int hairpin_flow; 6869 struct mlx5_translated_action_handle 6870 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6871 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6872 const struct rte_flow_action *actions; 6873 struct rte_flow_action *translated_actions = NULL; 6874 int ret = flow_action_handles_translate(dev, original_actions, 6875 indir_actions, 6876 &indir_actions_n, 6877 &translated_actions, error); 6878 6879 if (ret) 6880 return ret; 6881 actions = translated_actions ? translated_actions : original_actions; 6882 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6883 ret = flow_drv_validate(dev, attr, items, actions, 6884 true, hairpin_flow, error); 6885 rte_free(translated_actions); 6886 return ret; 6887 } 6888 6889 /** 6890 * Create a flow. 6891 * 6892 * @see rte_flow_create() 6893 * @see rte_flow_ops 6894 */ 6895 struct rte_flow * 6896 mlx5_flow_create(struct rte_eth_dev *dev, 6897 const struct rte_flow_attr *attr, 6898 const struct rte_flow_item items[], 6899 const struct rte_flow_action actions[], 6900 struct rte_flow_error *error) 6901 { 6902 struct mlx5_priv *priv = dev->data->dev_private; 6903 6904 if (priv->sh->config.dv_flow_en == 2) { 6905 rte_flow_error_set(error, ENOTSUP, 6906 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6907 NULL, 6908 "Flow non-Q creation not supported"); 6909 return NULL; 6910 } 6911 /* 6912 * If the device is not started yet, it is not allowed to created a 6913 * flow from application. PMD default flows and traffic control flows 6914 * are not affected. 6915 */ 6916 if (unlikely(!dev->data->dev_started)) { 6917 DRV_LOG(DEBUG, "port %u is not started when " 6918 "inserting a flow", dev->data->port_id); 6919 rte_flow_error_set(error, ENODEV, 6920 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6921 NULL, 6922 "port not started"); 6923 return NULL; 6924 } 6925 6926 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 6927 attr, items, actions, 6928 true, error); 6929 } 6930 6931 /** 6932 * Destroy a flow in a list. 6933 * 6934 * @param dev 6935 * Pointer to Ethernet device. 6936 * @param[in] flow_idx 6937 * Index of flow to destroy. 6938 */ 6939 static void 6940 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6941 uint32_t flow_idx) 6942 { 6943 struct mlx5_priv *priv = dev->data->dev_private; 6944 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 6945 6946 if (!flow) 6947 return; 6948 MLX5_ASSERT(flow->type == type); 6949 /* 6950 * Update RX queue flags only if port is started, otherwise it is 6951 * already clean. 6952 */ 6953 if (dev->data->dev_started) 6954 flow_rxq_flags_trim(dev, flow); 6955 flow_drv_destroy(dev, flow); 6956 if (flow->tunnel) { 6957 struct mlx5_flow_tunnel *tunnel; 6958 6959 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 6960 RTE_VERIFY(tunnel); 6961 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 6962 mlx5_flow_tunnel_free(dev, tunnel); 6963 } 6964 flow_mreg_del_copy_action(dev, flow); 6965 mlx5_ipool_free(priv->flows[type], flow_idx); 6966 } 6967 6968 /** 6969 * Destroy all flows. 6970 * 6971 * @param dev 6972 * Pointer to Ethernet device. 6973 * @param type 6974 * Flow type to be flushed. 6975 * @param active 6976 * If flushing is called actively. 6977 */ 6978 void 6979 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6980 bool active) 6981 { 6982 struct mlx5_priv *priv = dev->data->dev_private; 6983 uint32_t num_flushed = 0, fidx = 1; 6984 struct rte_flow *flow; 6985 6986 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 6987 flow_list_destroy(dev, type, fidx); 6988 num_flushed++; 6989 } 6990 if (active) { 6991 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 6992 dev->data->port_id, num_flushed); 6993 } 6994 } 6995 6996 /** 6997 * Stop all default actions for flows. 6998 * 6999 * @param dev 7000 * Pointer to Ethernet device. 7001 */ 7002 void 7003 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7004 { 7005 flow_mreg_del_default_copy_action(dev); 7006 flow_rxq_flags_clear(dev); 7007 } 7008 7009 /** 7010 * Start all default actions for flows. 7011 * 7012 * @param dev 7013 * Pointer to Ethernet device. 7014 * @return 7015 * 0 on success, a negative errno value otherwise and rte_errno is set. 7016 */ 7017 int 7018 mlx5_flow_start_default(struct rte_eth_dev *dev) 7019 { 7020 struct rte_flow_error error; 7021 7022 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7023 return flow_mreg_add_default_copy_action(dev, &error); 7024 } 7025 7026 /** 7027 * Release key of thread specific flow workspace data. 7028 */ 7029 void 7030 flow_release_workspace(void *data) 7031 { 7032 struct mlx5_flow_workspace *wks = data; 7033 struct mlx5_flow_workspace *next; 7034 7035 while (wks) { 7036 next = wks->next; 7037 free(wks->rss_desc.queue); 7038 free(wks); 7039 wks = next; 7040 } 7041 } 7042 7043 /** 7044 * Get thread specific current flow workspace. 7045 * 7046 * @return pointer to thread specific flow workspace data, NULL on error. 7047 */ 7048 struct mlx5_flow_workspace* 7049 mlx5_flow_get_thread_workspace(void) 7050 { 7051 struct mlx5_flow_workspace *data; 7052 7053 data = mlx5_flow_os_get_specific_workspace(); 7054 MLX5_ASSERT(data && data->inuse); 7055 if (!data || !data->inuse) 7056 DRV_LOG(ERR, "flow workspace not initialized."); 7057 return data; 7058 } 7059 7060 /** 7061 * Allocate and init new flow workspace. 7062 * 7063 * @return pointer to flow workspace data, NULL on error. 7064 */ 7065 static struct mlx5_flow_workspace* 7066 flow_alloc_thread_workspace(void) 7067 { 7068 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7069 7070 if (!data) { 7071 DRV_LOG(ERR, "Failed to allocate flow workspace " 7072 "memory."); 7073 return NULL; 7074 } 7075 data->rss_desc.queue = calloc(1, 7076 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7077 if (!data->rss_desc.queue) 7078 goto err; 7079 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7080 return data; 7081 err: 7082 free(data->rss_desc.queue); 7083 free(data); 7084 return NULL; 7085 } 7086 7087 /** 7088 * Get new thread specific flow workspace. 7089 * 7090 * If current workspace inuse, create new one and set as current. 7091 * 7092 * @return pointer to thread specific flow workspace data, NULL on error. 7093 */ 7094 static struct mlx5_flow_workspace* 7095 mlx5_flow_push_thread_workspace(void) 7096 { 7097 struct mlx5_flow_workspace *curr; 7098 struct mlx5_flow_workspace *data; 7099 7100 curr = mlx5_flow_os_get_specific_workspace(); 7101 if (!curr) { 7102 data = flow_alloc_thread_workspace(); 7103 if (!data) 7104 return NULL; 7105 } else if (!curr->inuse) { 7106 data = curr; 7107 } else if (curr->next) { 7108 data = curr->next; 7109 } else { 7110 data = flow_alloc_thread_workspace(); 7111 if (!data) 7112 return NULL; 7113 curr->next = data; 7114 data->prev = curr; 7115 } 7116 data->inuse = 1; 7117 data->flow_idx = 0; 7118 /* Set as current workspace */ 7119 if (mlx5_flow_os_set_specific_workspace(data)) 7120 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7121 return data; 7122 } 7123 7124 /** 7125 * Close current thread specific flow workspace. 7126 * 7127 * If previous workspace available, set it as current. 7128 * 7129 * @return pointer to thread specific flow workspace data, NULL on error. 7130 */ 7131 static void 7132 mlx5_flow_pop_thread_workspace(void) 7133 { 7134 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7135 7136 if (!data) 7137 return; 7138 if (!data->inuse) { 7139 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7140 return; 7141 } 7142 data->inuse = 0; 7143 if (!data->prev) 7144 return; 7145 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7146 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7147 } 7148 7149 /** 7150 * Verify the flow list is empty 7151 * 7152 * @param dev 7153 * Pointer to Ethernet device. 7154 * 7155 * @return the number of flows not released. 7156 */ 7157 int 7158 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7159 { 7160 struct mlx5_priv *priv = dev->data->dev_private; 7161 struct rte_flow *flow; 7162 uint32_t idx = 0; 7163 int ret = 0, i; 7164 7165 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7166 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7167 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7168 dev->data->port_id, (void *)flow); 7169 ret++; 7170 } 7171 } 7172 return ret; 7173 } 7174 7175 /** 7176 * Enable default hairpin egress flow. 7177 * 7178 * @param dev 7179 * Pointer to Ethernet device. 7180 * @param queue 7181 * The queue index. 7182 * 7183 * @return 7184 * 0 on success, a negative errno value otherwise and rte_errno is set. 7185 */ 7186 int 7187 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7188 uint32_t queue) 7189 { 7190 const struct rte_flow_attr attr = { 7191 .egress = 1, 7192 .priority = 0, 7193 }; 7194 struct mlx5_rte_flow_item_tx_queue queue_spec = { 7195 .queue = queue, 7196 }; 7197 struct mlx5_rte_flow_item_tx_queue queue_mask = { 7198 .queue = UINT32_MAX, 7199 }; 7200 struct rte_flow_item items[] = { 7201 { 7202 .type = (enum rte_flow_item_type) 7203 MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE, 7204 .spec = &queue_spec, 7205 .last = NULL, 7206 .mask = &queue_mask, 7207 }, 7208 { 7209 .type = RTE_FLOW_ITEM_TYPE_END, 7210 }, 7211 }; 7212 struct rte_flow_action_jump jump = { 7213 .group = MLX5_HAIRPIN_TX_TABLE, 7214 }; 7215 struct rte_flow_action actions[2]; 7216 uint32_t flow_idx; 7217 struct rte_flow_error error; 7218 7219 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7220 actions[0].conf = &jump; 7221 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7222 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7223 &attr, items, actions, false, &error); 7224 if (!flow_idx) { 7225 DRV_LOG(DEBUG, 7226 "Failed to create ctrl flow: rte_errno(%d)," 7227 " type(%d), message(%s)", 7228 rte_errno, error.type, 7229 error.message ? error.message : " (no stated reason)"); 7230 return -rte_errno; 7231 } 7232 return 0; 7233 } 7234 7235 /** 7236 * Enable a control flow configured from the control plane. 7237 * 7238 * @param dev 7239 * Pointer to Ethernet device. 7240 * @param eth_spec 7241 * An Ethernet flow spec to apply. 7242 * @param eth_mask 7243 * An Ethernet flow mask to apply. 7244 * @param vlan_spec 7245 * A VLAN flow spec to apply. 7246 * @param vlan_mask 7247 * A VLAN flow mask to apply. 7248 * 7249 * @return 7250 * 0 on success, a negative errno value otherwise and rte_errno is set. 7251 */ 7252 int 7253 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7254 struct rte_flow_item_eth *eth_spec, 7255 struct rte_flow_item_eth *eth_mask, 7256 struct rte_flow_item_vlan *vlan_spec, 7257 struct rte_flow_item_vlan *vlan_mask) 7258 { 7259 struct mlx5_priv *priv = dev->data->dev_private; 7260 const struct rte_flow_attr attr = { 7261 .ingress = 1, 7262 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7263 }; 7264 struct rte_flow_item items[] = { 7265 { 7266 .type = RTE_FLOW_ITEM_TYPE_ETH, 7267 .spec = eth_spec, 7268 .last = NULL, 7269 .mask = eth_mask, 7270 }, 7271 { 7272 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7273 RTE_FLOW_ITEM_TYPE_END, 7274 .spec = vlan_spec, 7275 .last = NULL, 7276 .mask = vlan_mask, 7277 }, 7278 { 7279 .type = RTE_FLOW_ITEM_TYPE_END, 7280 }, 7281 }; 7282 uint16_t queue[priv->reta_idx_n]; 7283 struct rte_flow_action_rss action_rss = { 7284 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7285 .level = 0, 7286 .types = priv->rss_conf.rss_hf, 7287 .key_len = priv->rss_conf.rss_key_len, 7288 .queue_num = priv->reta_idx_n, 7289 .key = priv->rss_conf.rss_key, 7290 .queue = queue, 7291 }; 7292 struct rte_flow_action actions[] = { 7293 { 7294 .type = RTE_FLOW_ACTION_TYPE_RSS, 7295 .conf = &action_rss, 7296 }, 7297 { 7298 .type = RTE_FLOW_ACTION_TYPE_END, 7299 }, 7300 }; 7301 uint32_t flow_idx; 7302 struct rte_flow_error error; 7303 unsigned int i; 7304 7305 if (!priv->reta_idx_n || !priv->rxqs_n) { 7306 return 0; 7307 } 7308 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7309 action_rss.types = 0; 7310 for (i = 0; i != priv->reta_idx_n; ++i) 7311 queue[i] = (*priv->reta_idx)[i]; 7312 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7313 &attr, items, actions, false, &error); 7314 if (!flow_idx) 7315 return -rte_errno; 7316 return 0; 7317 } 7318 7319 /** 7320 * Enable a flow control configured from the control plane. 7321 * 7322 * @param dev 7323 * Pointer to Ethernet device. 7324 * @param eth_spec 7325 * An Ethernet flow spec to apply. 7326 * @param eth_mask 7327 * An Ethernet flow mask to apply. 7328 * 7329 * @return 7330 * 0 on success, a negative errno value otherwise and rte_errno is set. 7331 */ 7332 int 7333 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7334 struct rte_flow_item_eth *eth_spec, 7335 struct rte_flow_item_eth *eth_mask) 7336 { 7337 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7338 } 7339 7340 /** 7341 * Create default miss flow rule matching lacp traffic 7342 * 7343 * @param dev 7344 * Pointer to Ethernet device. 7345 * @param eth_spec 7346 * An Ethernet flow spec to apply. 7347 * 7348 * @return 7349 * 0 on success, a negative errno value otherwise and rte_errno is set. 7350 */ 7351 int 7352 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7353 { 7354 /* 7355 * The LACP matching is done by only using ether type since using 7356 * a multicast dst mac causes kernel to give low priority to this flow. 7357 */ 7358 static const struct rte_flow_item_eth lacp_spec = { 7359 .type = RTE_BE16(0x8809), 7360 }; 7361 static const struct rte_flow_item_eth lacp_mask = { 7362 .type = 0xffff, 7363 }; 7364 const struct rte_flow_attr attr = { 7365 .ingress = 1, 7366 }; 7367 struct rte_flow_item items[] = { 7368 { 7369 .type = RTE_FLOW_ITEM_TYPE_ETH, 7370 .spec = &lacp_spec, 7371 .mask = &lacp_mask, 7372 }, 7373 { 7374 .type = RTE_FLOW_ITEM_TYPE_END, 7375 }, 7376 }; 7377 struct rte_flow_action actions[] = { 7378 { 7379 .type = (enum rte_flow_action_type) 7380 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7381 }, 7382 { 7383 .type = RTE_FLOW_ACTION_TYPE_END, 7384 }, 7385 }; 7386 struct rte_flow_error error; 7387 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7388 &attr, items, actions, 7389 false, &error); 7390 7391 if (!flow_idx) 7392 return -rte_errno; 7393 return 0; 7394 } 7395 7396 /** 7397 * Destroy a flow. 7398 * 7399 * @see rte_flow_destroy() 7400 * @see rte_flow_ops 7401 */ 7402 int 7403 mlx5_flow_destroy(struct rte_eth_dev *dev, 7404 struct rte_flow *flow, 7405 struct rte_flow_error *error __rte_unused) 7406 { 7407 struct mlx5_priv *priv = dev->data->dev_private; 7408 7409 if (priv->sh->config.dv_flow_en == 2) 7410 return rte_flow_error_set(error, ENOTSUP, 7411 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7412 NULL, 7413 "Flow non-Q destruction not supported"); 7414 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7415 (uintptr_t)(void *)flow); 7416 return 0; 7417 } 7418 7419 /** 7420 * Destroy all flows. 7421 * 7422 * @see rte_flow_flush() 7423 * @see rte_flow_ops 7424 */ 7425 int 7426 mlx5_flow_flush(struct rte_eth_dev *dev, 7427 struct rte_flow_error *error __rte_unused) 7428 { 7429 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7430 return 0; 7431 } 7432 7433 /** 7434 * Isolated mode. 7435 * 7436 * @see rte_flow_isolate() 7437 * @see rte_flow_ops 7438 */ 7439 int 7440 mlx5_flow_isolate(struct rte_eth_dev *dev, 7441 int enable, 7442 struct rte_flow_error *error) 7443 { 7444 struct mlx5_priv *priv = dev->data->dev_private; 7445 7446 if (dev->data->dev_started) { 7447 rte_flow_error_set(error, EBUSY, 7448 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7449 NULL, 7450 "port must be stopped first"); 7451 return -rte_errno; 7452 } 7453 priv->isolated = !!enable; 7454 if (enable) 7455 dev->dev_ops = &mlx5_dev_ops_isolate; 7456 else 7457 dev->dev_ops = &mlx5_dev_ops; 7458 7459 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7460 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7461 7462 return 0; 7463 } 7464 7465 /** 7466 * Query a flow. 7467 * 7468 * @see rte_flow_query() 7469 * @see rte_flow_ops 7470 */ 7471 static int 7472 flow_drv_query(struct rte_eth_dev *dev, 7473 uint32_t flow_idx, 7474 const struct rte_flow_action *actions, 7475 void *data, 7476 struct rte_flow_error *error) 7477 { 7478 struct mlx5_priv *priv = dev->data->dev_private; 7479 const struct mlx5_flow_driver_ops *fops; 7480 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7481 flow_idx); 7482 enum mlx5_flow_drv_type ftype; 7483 7484 if (!flow) { 7485 return rte_flow_error_set(error, ENOENT, 7486 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7487 NULL, 7488 "invalid flow handle"); 7489 } 7490 ftype = flow->drv_type; 7491 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7492 fops = flow_get_drv_ops(ftype); 7493 7494 return fops->query(dev, flow, actions, data, error); 7495 } 7496 7497 /** 7498 * Query a flow. 7499 * 7500 * @see rte_flow_query() 7501 * @see rte_flow_ops 7502 */ 7503 int 7504 mlx5_flow_query(struct rte_eth_dev *dev, 7505 struct rte_flow *flow, 7506 const struct rte_flow_action *actions, 7507 void *data, 7508 struct rte_flow_error *error) 7509 { 7510 int ret; 7511 struct mlx5_priv *priv = dev->data->dev_private; 7512 7513 if (priv->sh->config.dv_flow_en == 2) 7514 return rte_flow_error_set(error, ENOTSUP, 7515 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7516 NULL, 7517 "Flow non-Q query not supported"); 7518 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7519 error); 7520 if (ret < 0) 7521 return ret; 7522 return 0; 7523 } 7524 7525 /** 7526 * Get rte_flow callbacks. 7527 * 7528 * @param dev 7529 * Pointer to Ethernet device structure. 7530 * @param ops 7531 * Pointer to operation-specific structure. 7532 * 7533 * @return 0 7534 */ 7535 int 7536 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7537 const struct rte_flow_ops **ops) 7538 { 7539 *ops = &mlx5_flow_ops; 7540 return 0; 7541 } 7542 7543 /** 7544 * Validate meter policy actions. 7545 * Dispatcher for action type specific validation. 7546 * 7547 * @param[in] dev 7548 * Pointer to the Ethernet device structure. 7549 * @param[in] action 7550 * The meter policy action object to validate. 7551 * @param[in] attr 7552 * Attributes of flow to determine steering domain. 7553 * @param[out] is_rss 7554 * Is RSS or not. 7555 * @param[out] domain_bitmap 7556 * Domain bitmap. 7557 * @param[out] is_def_policy 7558 * Is default policy or not. 7559 * @param[out] error 7560 * Perform verbose error reporting if not NULL. Initialized in case of 7561 * error only. 7562 * 7563 * @return 7564 * 0 on success, otherwise negative errno value. 7565 */ 7566 int 7567 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7568 const struct rte_flow_action *actions[RTE_COLORS], 7569 struct rte_flow_attr *attr, 7570 bool *is_rss, 7571 uint8_t *domain_bitmap, 7572 uint8_t *policy_mode, 7573 struct rte_mtr_error *error) 7574 { 7575 const struct mlx5_flow_driver_ops *fops; 7576 7577 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7578 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7579 domain_bitmap, policy_mode, error); 7580 } 7581 7582 /** 7583 * Destroy the meter table set. 7584 * 7585 * @param[in] dev 7586 * Pointer to Ethernet device. 7587 * @param[in] mtr_policy 7588 * Meter policy struct. 7589 */ 7590 void 7591 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7592 struct mlx5_flow_meter_policy *mtr_policy) 7593 { 7594 const struct mlx5_flow_driver_ops *fops; 7595 7596 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7597 fops->destroy_mtr_acts(dev, mtr_policy); 7598 } 7599 7600 /** 7601 * Create policy action, lock free, 7602 * (mutex should be acquired by caller). 7603 * Dispatcher for action type specific call. 7604 * 7605 * @param[in] dev 7606 * Pointer to the Ethernet device structure. 7607 * @param[in] mtr_policy 7608 * Meter policy struct. 7609 * @param[in] action 7610 * Action specification used to create meter actions. 7611 * @param[out] error 7612 * Perform verbose error reporting if not NULL. Initialized in case of 7613 * error only. 7614 * 7615 * @return 7616 * 0 on success, otherwise negative errno value. 7617 */ 7618 int 7619 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7620 struct mlx5_flow_meter_policy *mtr_policy, 7621 const struct rte_flow_action *actions[RTE_COLORS], 7622 struct rte_mtr_error *error) 7623 { 7624 const struct mlx5_flow_driver_ops *fops; 7625 7626 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7627 return fops->create_mtr_acts(dev, mtr_policy, actions, error); 7628 } 7629 7630 /** 7631 * Create policy rules, lock free, 7632 * (mutex should be acquired by caller). 7633 * Dispatcher for action type specific call. 7634 * 7635 * @param[in] dev 7636 * Pointer to the Ethernet device structure. 7637 * @param[in] mtr_policy 7638 * Meter policy struct. 7639 * 7640 * @return 7641 * 0 on success, -1 otherwise. 7642 */ 7643 int 7644 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 7645 struct mlx5_flow_meter_policy *mtr_policy) 7646 { 7647 const struct mlx5_flow_driver_ops *fops; 7648 7649 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7650 return fops->create_policy_rules(dev, mtr_policy); 7651 } 7652 7653 /** 7654 * Destroy policy rules, lock free, 7655 * (mutex should be acquired by caller). 7656 * Dispatcher for action type specific call. 7657 * 7658 * @param[in] dev 7659 * Pointer to the Ethernet device structure. 7660 * @param[in] mtr_policy 7661 * Meter policy struct. 7662 */ 7663 void 7664 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 7665 struct mlx5_flow_meter_policy *mtr_policy) 7666 { 7667 const struct mlx5_flow_driver_ops *fops; 7668 7669 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7670 fops->destroy_policy_rules(dev, mtr_policy); 7671 } 7672 7673 /** 7674 * Destroy the default policy table set. 7675 * 7676 * @param[in] dev 7677 * Pointer to Ethernet device. 7678 */ 7679 void 7680 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 7681 { 7682 const struct mlx5_flow_driver_ops *fops; 7683 7684 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7685 fops->destroy_def_policy(dev); 7686 } 7687 7688 /** 7689 * Destroy the default policy table set. 7690 * 7691 * @param[in] dev 7692 * Pointer to Ethernet device. 7693 * 7694 * @return 7695 * 0 on success, -1 otherwise. 7696 */ 7697 int 7698 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 7699 { 7700 const struct mlx5_flow_driver_ops *fops; 7701 7702 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7703 return fops->create_def_policy(dev); 7704 } 7705 7706 /** 7707 * Create the needed meter and suffix tables. 7708 * 7709 * @param[in] dev 7710 * Pointer to Ethernet device. 7711 * 7712 * @return 7713 * 0 on success, -1 otherwise. 7714 */ 7715 int 7716 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 7717 struct mlx5_flow_meter_info *fm, 7718 uint32_t mtr_idx, 7719 uint8_t domain_bitmap) 7720 { 7721 const struct mlx5_flow_driver_ops *fops; 7722 7723 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7724 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 7725 } 7726 7727 /** 7728 * Destroy the meter table set. 7729 * 7730 * @param[in] dev 7731 * Pointer to Ethernet device. 7732 * @param[in] tbl 7733 * Pointer to the meter table set. 7734 */ 7735 void 7736 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 7737 struct mlx5_flow_meter_info *fm) 7738 { 7739 const struct mlx5_flow_driver_ops *fops; 7740 7741 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7742 fops->destroy_mtr_tbls(dev, fm); 7743 } 7744 7745 /** 7746 * Destroy the global meter drop table. 7747 * 7748 * @param[in] dev 7749 * Pointer to Ethernet device. 7750 */ 7751 void 7752 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 7753 { 7754 const struct mlx5_flow_driver_ops *fops; 7755 7756 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7757 fops->destroy_mtr_drop_tbls(dev); 7758 } 7759 7760 /** 7761 * Destroy the sub policy table with RX queue. 7762 * 7763 * @param[in] dev 7764 * Pointer to Ethernet device. 7765 * @param[in] mtr_policy 7766 * Pointer to meter policy table. 7767 */ 7768 void 7769 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 7770 struct mlx5_flow_meter_policy *mtr_policy) 7771 { 7772 const struct mlx5_flow_driver_ops *fops; 7773 7774 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7775 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 7776 } 7777 7778 /** 7779 * Allocate the needed aso flow meter id. 7780 * 7781 * @param[in] dev 7782 * Pointer to Ethernet device. 7783 * 7784 * @return 7785 * Index to aso flow meter on success, NULL otherwise. 7786 */ 7787 uint32_t 7788 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 7789 { 7790 const struct mlx5_flow_driver_ops *fops; 7791 7792 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7793 return fops->create_meter(dev); 7794 } 7795 7796 /** 7797 * Free the aso flow meter id. 7798 * 7799 * @param[in] dev 7800 * Pointer to Ethernet device. 7801 * @param[in] mtr_idx 7802 * Index to aso flow meter to be free. 7803 * 7804 * @return 7805 * 0 on success. 7806 */ 7807 void 7808 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 7809 { 7810 const struct mlx5_flow_driver_ops *fops; 7811 7812 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7813 fops->free_meter(dev, mtr_idx); 7814 } 7815 7816 /** 7817 * Allocate a counter. 7818 * 7819 * @param[in] dev 7820 * Pointer to Ethernet device structure. 7821 * 7822 * @return 7823 * Index to allocated counter on success, 0 otherwise. 7824 */ 7825 uint32_t 7826 mlx5_counter_alloc(struct rte_eth_dev *dev) 7827 { 7828 const struct mlx5_flow_driver_ops *fops; 7829 struct rte_flow_attr attr = { .transfer = 0 }; 7830 7831 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7832 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7833 return fops->counter_alloc(dev); 7834 } 7835 DRV_LOG(ERR, 7836 "port %u counter allocate is not supported.", 7837 dev->data->port_id); 7838 return 0; 7839 } 7840 7841 /** 7842 * Free a counter. 7843 * 7844 * @param[in] dev 7845 * Pointer to Ethernet device structure. 7846 * @param[in] cnt 7847 * Index to counter to be free. 7848 */ 7849 void 7850 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 7851 { 7852 const struct mlx5_flow_driver_ops *fops; 7853 struct rte_flow_attr attr = { .transfer = 0 }; 7854 7855 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7856 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7857 fops->counter_free(dev, cnt); 7858 return; 7859 } 7860 DRV_LOG(ERR, 7861 "port %u counter free is not supported.", 7862 dev->data->port_id); 7863 } 7864 7865 /** 7866 * Query counter statistics. 7867 * 7868 * @param[in] dev 7869 * Pointer to Ethernet device structure. 7870 * @param[in] cnt 7871 * Index to counter to query. 7872 * @param[in] clear 7873 * Set to clear counter statistics. 7874 * @param[out] pkts 7875 * The counter hits packets number to save. 7876 * @param[out] bytes 7877 * The counter hits bytes number to save. 7878 * 7879 * @return 7880 * 0 on success, a negative errno value otherwise. 7881 */ 7882 int 7883 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 7884 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 7885 { 7886 const struct mlx5_flow_driver_ops *fops; 7887 struct rte_flow_attr attr = { .transfer = 0 }; 7888 7889 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 7890 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7891 return fops->counter_query(dev, cnt, clear, pkts, 7892 bytes, action); 7893 } 7894 DRV_LOG(ERR, 7895 "port %u counter query is not supported.", 7896 dev->data->port_id); 7897 return -ENOTSUP; 7898 } 7899 7900 /** 7901 * Get information about HWS pre-configurable resources. 7902 * 7903 * @param[in] dev 7904 * Pointer to the rte_eth_dev structure. 7905 * @param[out] port_info 7906 * Pointer to port information. 7907 * @param[out] queue_info 7908 * Pointer to queue information. 7909 * @param[out] error 7910 * Pointer to error structure. 7911 * 7912 * @return 7913 * 0 on success, a negative errno value otherwise and rte_errno is set. 7914 */ 7915 static int 7916 mlx5_flow_info_get(struct rte_eth_dev *dev, 7917 struct rte_flow_port_info *port_info, 7918 struct rte_flow_queue_info *queue_info, 7919 struct rte_flow_error *error) 7920 { 7921 const struct mlx5_flow_driver_ops *fops; 7922 7923 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7924 return rte_flow_error_set(error, ENOTSUP, 7925 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7926 NULL, 7927 "info get with incorrect steering mode"); 7928 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7929 return fops->info_get(dev, port_info, queue_info, error); 7930 } 7931 7932 /** 7933 * Configure port HWS resources. 7934 * 7935 * @param[in] dev 7936 * Pointer to the rte_eth_dev structure. 7937 * @param[in] port_attr 7938 * Port configuration attributes. 7939 * @param[in] nb_queue 7940 * Number of queue. 7941 * @param[in] queue_attr 7942 * Array that holds attributes for each flow queue. 7943 * @param[out] error 7944 * Pointer to error structure. 7945 * 7946 * @return 7947 * 0 on success, a negative errno value otherwise and rte_errno is set. 7948 */ 7949 static int 7950 mlx5_flow_port_configure(struct rte_eth_dev *dev, 7951 const struct rte_flow_port_attr *port_attr, 7952 uint16_t nb_queue, 7953 const struct rte_flow_queue_attr *queue_attr[], 7954 struct rte_flow_error *error) 7955 { 7956 const struct mlx5_flow_driver_ops *fops; 7957 7958 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 7959 return rte_flow_error_set(error, ENOTSUP, 7960 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7961 NULL, 7962 "port configure with incorrect steering mode"); 7963 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7964 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 7965 } 7966 7967 /** 7968 * Create flow item template. 7969 * 7970 * @param[in] dev 7971 * Pointer to the rte_eth_dev structure. 7972 * @param[in] attr 7973 * Pointer to the item template attributes. 7974 * @param[in] items 7975 * The template item pattern. 7976 * @param[out] error 7977 * Pointer to error structure. 7978 * 7979 * @return 7980 * 0 on success, a negative errno value otherwise and rte_errno is set. 7981 */ 7982 static struct rte_flow_pattern_template * 7983 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 7984 const struct rte_flow_pattern_template_attr *attr, 7985 const struct rte_flow_item items[], 7986 struct rte_flow_error *error) 7987 { 7988 const struct mlx5_flow_driver_ops *fops; 7989 7990 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 7991 rte_flow_error_set(error, ENOTSUP, 7992 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7993 NULL, 7994 "pattern create with incorrect steering mode"); 7995 return NULL; 7996 } 7997 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 7998 return fops->pattern_template_create(dev, attr, items, error); 7999 } 8000 8001 /** 8002 * Destroy flow item template. 8003 * 8004 * @param[in] dev 8005 * Pointer to the rte_eth_dev structure. 8006 * @param[in] template 8007 * Pointer to the item template to be destroyed. 8008 * @param[out] error 8009 * Pointer to error structure. 8010 * 8011 * @return 8012 * 0 on success, a negative errno value otherwise and rte_errno is set. 8013 */ 8014 static int 8015 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8016 struct rte_flow_pattern_template *template, 8017 struct rte_flow_error *error) 8018 { 8019 const struct mlx5_flow_driver_ops *fops; 8020 8021 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8022 return rte_flow_error_set(error, ENOTSUP, 8023 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8024 NULL, 8025 "pattern destroy with incorrect steering mode"); 8026 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8027 return fops->pattern_template_destroy(dev, template, error); 8028 } 8029 8030 /** 8031 * Create flow item template. 8032 * 8033 * @param[in] dev 8034 * Pointer to the rte_eth_dev structure. 8035 * @param[in] attr 8036 * Pointer to the action template attributes. 8037 * @param[in] actions 8038 * Associated actions (list terminated by the END action). 8039 * @param[in] masks 8040 * List of actions that marks which of the action's member is constant. 8041 * @param[out] error 8042 * Pointer to error structure. 8043 * 8044 * @return 8045 * 0 on success, a negative errno value otherwise and rte_errno is set. 8046 */ 8047 static struct rte_flow_actions_template * 8048 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8049 const struct rte_flow_actions_template_attr *attr, 8050 const struct rte_flow_action actions[], 8051 const struct rte_flow_action masks[], 8052 struct rte_flow_error *error) 8053 { 8054 const struct mlx5_flow_driver_ops *fops; 8055 8056 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8057 rte_flow_error_set(error, ENOTSUP, 8058 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8059 NULL, 8060 "action create with incorrect steering mode"); 8061 return NULL; 8062 } 8063 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8064 return fops->actions_template_create(dev, attr, actions, masks, error); 8065 } 8066 8067 /** 8068 * Destroy flow action template. 8069 * 8070 * @param[in] dev 8071 * Pointer to the rte_eth_dev structure. 8072 * @param[in] template 8073 * Pointer to the action template to be destroyed. 8074 * @param[out] error 8075 * Pointer to error structure. 8076 * 8077 * @return 8078 * 0 on success, a negative errno value otherwise and rte_errno is set. 8079 */ 8080 static int 8081 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8082 struct rte_flow_actions_template *template, 8083 struct rte_flow_error *error) 8084 { 8085 const struct mlx5_flow_driver_ops *fops; 8086 8087 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8088 return rte_flow_error_set(error, ENOTSUP, 8089 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8090 NULL, 8091 "action destroy with incorrect steering mode"); 8092 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8093 return fops->actions_template_destroy(dev, template, error); 8094 } 8095 8096 /** 8097 * Create flow table. 8098 * 8099 * @param[in] dev 8100 * Pointer to the rte_eth_dev structure. 8101 * @param[in] attr 8102 * Pointer to the table attributes. 8103 * @param[in] item_templates 8104 * Item template array to be binded to the table. 8105 * @param[in] nb_item_templates 8106 * Number of item template. 8107 * @param[in] action_templates 8108 * Action template array to be binded to the table. 8109 * @param[in] nb_action_templates 8110 * Number of action template. 8111 * @param[out] error 8112 * Pointer to error structure. 8113 * 8114 * @return 8115 * Table on success, NULL otherwise and rte_errno is set. 8116 */ 8117 static struct rte_flow_template_table * 8118 mlx5_flow_table_create(struct rte_eth_dev *dev, 8119 const struct rte_flow_template_table_attr *attr, 8120 struct rte_flow_pattern_template *item_templates[], 8121 uint8_t nb_item_templates, 8122 struct rte_flow_actions_template *action_templates[], 8123 uint8_t nb_action_templates, 8124 struct rte_flow_error *error) 8125 { 8126 const struct mlx5_flow_driver_ops *fops; 8127 8128 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) { 8129 rte_flow_error_set(error, ENOTSUP, 8130 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8131 NULL, 8132 "table create with incorrect steering mode"); 8133 return NULL; 8134 } 8135 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8136 return fops->template_table_create(dev, 8137 attr, 8138 item_templates, 8139 nb_item_templates, 8140 action_templates, 8141 nb_action_templates, 8142 error); 8143 } 8144 8145 /** 8146 * PMD destroy flow table. 8147 * 8148 * @param[in] dev 8149 * Pointer to the rte_eth_dev structure. 8150 * @param[in] table 8151 * Pointer to the table to be destroyed. 8152 * @param[out] error 8153 * Pointer to error structure. 8154 * 8155 * @return 8156 * 0 on success, a negative errno value otherwise and rte_errno is set. 8157 */ 8158 static int 8159 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8160 struct rte_flow_template_table *table, 8161 struct rte_flow_error *error) 8162 { 8163 const struct mlx5_flow_driver_ops *fops; 8164 8165 if (flow_get_drv_type(dev, NULL) != MLX5_FLOW_TYPE_HW) 8166 return rte_flow_error_set(error, ENOTSUP, 8167 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8168 NULL, 8169 "table destroy with incorrect steering mode"); 8170 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8171 return fops->template_table_destroy(dev, table, error); 8172 } 8173 8174 /** 8175 * Allocate a new memory for the counter values wrapped by all the needed 8176 * management. 8177 * 8178 * @param[in] sh 8179 * Pointer to mlx5_dev_ctx_shared object. 8180 * 8181 * @return 8182 * 0 on success, a negative errno value otherwise. 8183 */ 8184 static int 8185 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8186 { 8187 struct mlx5_counter_stats_mem_mng *mem_mng; 8188 volatile struct flow_counter_stats *raw_data; 8189 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8190 int size = (sizeof(struct flow_counter_stats) * 8191 MLX5_COUNTERS_PER_POOL + 8192 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8193 sizeof(struct mlx5_counter_stats_mem_mng); 8194 size_t pgsize = rte_mem_page_size(); 8195 uint8_t *mem; 8196 int ret; 8197 int i; 8198 8199 if (pgsize == (size_t)-1) { 8200 DRV_LOG(ERR, "Failed to get mem page size"); 8201 rte_errno = ENOMEM; 8202 return -ENOMEM; 8203 } 8204 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8205 if (!mem) { 8206 rte_errno = ENOMEM; 8207 return -ENOMEM; 8208 } 8209 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8210 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8211 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8212 sh->cdev->pdn, mem, size, 8213 &mem_mng->wm); 8214 if (ret) { 8215 rte_errno = errno; 8216 mlx5_free(mem); 8217 return -rte_errno; 8218 } 8219 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8220 raw_data = (volatile struct flow_counter_stats *)mem; 8221 for (i = 0; i < raws_n; ++i) { 8222 mem_mng->raws[i].mem_mng = mem_mng; 8223 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8224 } 8225 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8226 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8227 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8228 next); 8229 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8230 sh->cmng.mem_mng = mem_mng; 8231 return 0; 8232 } 8233 8234 /** 8235 * Set the statistic memory to the new counter pool. 8236 * 8237 * @param[in] sh 8238 * Pointer to mlx5_dev_ctx_shared object. 8239 * @param[in] pool 8240 * Pointer to the pool to set the statistic memory. 8241 * 8242 * @return 8243 * 0 on success, a negative errno value otherwise. 8244 */ 8245 static int 8246 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8247 struct mlx5_flow_counter_pool *pool) 8248 { 8249 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8250 /* Resize statistic memory once used out. */ 8251 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8252 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8253 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8254 return -1; 8255 } 8256 rte_spinlock_lock(&pool->sl); 8257 pool->raw = cmng->mem_mng->raws + pool->index % 8258 MLX5_CNT_CONTAINER_RESIZE; 8259 rte_spinlock_unlock(&pool->sl); 8260 pool->raw_hw = NULL; 8261 return 0; 8262 } 8263 8264 #define MLX5_POOL_QUERY_FREQ_US 1000000 8265 8266 /** 8267 * Set the periodic procedure for triggering asynchronous batch queries for all 8268 * the counter pools. 8269 * 8270 * @param[in] sh 8271 * Pointer to mlx5_dev_ctx_shared object. 8272 */ 8273 void 8274 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8275 { 8276 uint32_t pools_n, us; 8277 8278 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8279 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8280 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8281 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8282 sh->cmng.query_thread_on = 0; 8283 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8284 } else { 8285 sh->cmng.query_thread_on = 1; 8286 } 8287 } 8288 8289 /** 8290 * The periodic procedure for triggering asynchronous batch queries for all the 8291 * counter pools. This function is probably called by the host thread. 8292 * 8293 * @param[in] arg 8294 * The parameter for the alarm process. 8295 */ 8296 void 8297 mlx5_flow_query_alarm(void *arg) 8298 { 8299 struct mlx5_dev_ctx_shared *sh = arg; 8300 int ret; 8301 uint16_t pool_index = sh->cmng.pool_index; 8302 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8303 struct mlx5_flow_counter_pool *pool; 8304 uint16_t n_valid; 8305 8306 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 8307 goto set_alarm; 8308 rte_spinlock_lock(&cmng->pool_update_sl); 8309 pool = cmng->pools[pool_index]; 8310 n_valid = cmng->n_valid; 8311 rte_spinlock_unlock(&cmng->pool_update_sl); 8312 /* Set the statistic memory to the new created pool. */ 8313 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 8314 goto set_alarm; 8315 if (pool->raw_hw) 8316 /* There is a pool query in progress. */ 8317 goto set_alarm; 8318 pool->raw_hw = 8319 LIST_FIRST(&sh->cmng.free_stat_raws); 8320 if (!pool->raw_hw) 8321 /* No free counter statistics raw memory. */ 8322 goto set_alarm; 8323 /* 8324 * Identify the counters released between query trigger and query 8325 * handle more efficiently. The counter released in this gap period 8326 * should wait for a new round of query as the new arrived packets 8327 * will not be taken into account. 8328 */ 8329 pool->query_gen++; 8330 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 8331 MLX5_COUNTERS_PER_POOL, 8332 NULL, NULL, 8333 pool->raw_hw->mem_mng->wm.lkey, 8334 (void *)(uintptr_t) 8335 pool->raw_hw->data, 8336 sh->devx_comp, 8337 (uint64_t)(uintptr_t)pool); 8338 if (ret) { 8339 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 8340 " %d", pool->min_dcs->id); 8341 pool->raw_hw = NULL; 8342 goto set_alarm; 8343 } 8344 LIST_REMOVE(pool->raw_hw, next); 8345 sh->cmng.pending_queries++; 8346 pool_index++; 8347 if (pool_index >= n_valid) 8348 pool_index = 0; 8349 set_alarm: 8350 sh->cmng.pool_index = pool_index; 8351 mlx5_set_query_alarm(sh); 8352 } 8353 8354 /** 8355 * Check and callback event for new aged flow in the counter pool 8356 * 8357 * @param[in] sh 8358 * Pointer to mlx5_dev_ctx_shared object. 8359 * @param[in] pool 8360 * Pointer to Current counter pool. 8361 */ 8362 static void 8363 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 8364 struct mlx5_flow_counter_pool *pool) 8365 { 8366 struct mlx5_priv *priv; 8367 struct mlx5_flow_counter *cnt; 8368 struct mlx5_age_info *age_info; 8369 struct mlx5_age_param *age_param; 8370 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 8371 struct mlx5_counter_stats_raw *prev = pool->raw; 8372 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 8373 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 8374 uint16_t expected = AGE_CANDIDATE; 8375 uint32_t i; 8376 8377 pool->time_of_last_age_check = curr_time; 8378 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 8379 cnt = MLX5_POOL_GET_CNT(pool, i); 8380 age_param = MLX5_CNT_TO_AGE(cnt); 8381 if (__atomic_load_n(&age_param->state, 8382 __ATOMIC_RELAXED) != AGE_CANDIDATE) 8383 continue; 8384 if (cur->data[i].hits != prev->data[i].hits) { 8385 __atomic_store_n(&age_param->sec_since_last_hit, 0, 8386 __ATOMIC_RELAXED); 8387 continue; 8388 } 8389 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 8390 time_delta, 8391 __ATOMIC_RELAXED) <= age_param->timeout) 8392 continue; 8393 /** 8394 * Hold the lock first, or if between the 8395 * state AGE_TMOUT and tailq operation the 8396 * release happened, the release procedure 8397 * may delete a non-existent tailq node. 8398 */ 8399 priv = rte_eth_devices[age_param->port_id].data->dev_private; 8400 age_info = GET_PORT_AGE_INFO(priv); 8401 rte_spinlock_lock(&age_info->aged_sl); 8402 if (__atomic_compare_exchange_n(&age_param->state, &expected, 8403 AGE_TMOUT, false, 8404 __ATOMIC_RELAXED, 8405 __ATOMIC_RELAXED)) { 8406 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 8407 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 8408 } 8409 rte_spinlock_unlock(&age_info->aged_sl); 8410 } 8411 mlx5_age_event_prepare(sh); 8412 } 8413 8414 /** 8415 * Handler for the HW respond about ready values from an asynchronous batch 8416 * query. This function is probably called by the host thread. 8417 * 8418 * @param[in] sh 8419 * The pointer to the shared device context. 8420 * @param[in] async_id 8421 * The Devx async ID. 8422 * @param[in] status 8423 * The status of the completion. 8424 */ 8425 void 8426 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 8427 uint64_t async_id, int status) 8428 { 8429 struct mlx5_flow_counter_pool *pool = 8430 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 8431 struct mlx5_counter_stats_raw *raw_to_free; 8432 uint8_t query_gen = pool->query_gen ^ 1; 8433 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8434 enum mlx5_counter_type cnt_type = 8435 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 8436 MLX5_COUNTER_TYPE_ORIGIN; 8437 8438 if (unlikely(status)) { 8439 raw_to_free = pool->raw_hw; 8440 } else { 8441 raw_to_free = pool->raw; 8442 if (pool->is_aged) 8443 mlx5_flow_aging_check(sh, pool); 8444 rte_spinlock_lock(&pool->sl); 8445 pool->raw = pool->raw_hw; 8446 rte_spinlock_unlock(&pool->sl); 8447 /* Be sure the new raw counters data is updated in memory. */ 8448 rte_io_wmb(); 8449 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 8450 rte_spinlock_lock(&cmng->csl[cnt_type]); 8451 TAILQ_CONCAT(&cmng->counters[cnt_type], 8452 &pool->counters[query_gen], next); 8453 rte_spinlock_unlock(&cmng->csl[cnt_type]); 8454 } 8455 } 8456 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 8457 pool->raw_hw = NULL; 8458 sh->cmng.pending_queries--; 8459 } 8460 8461 static int 8462 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 8463 const struct flow_grp_info *grp_info, 8464 struct rte_flow_error *error) 8465 { 8466 if (grp_info->transfer && grp_info->external && 8467 grp_info->fdb_def_rule) { 8468 if (group == UINT32_MAX) 8469 return rte_flow_error_set 8470 (error, EINVAL, 8471 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 8472 NULL, 8473 "group index not supported"); 8474 *table = group + 1; 8475 } else { 8476 *table = group; 8477 } 8478 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 8479 return 0; 8480 } 8481 8482 /** 8483 * Translate the rte_flow group index to HW table value. 8484 * 8485 * If tunnel offload is disabled, all group ids converted to flow table 8486 * id using the standard method. 8487 * If tunnel offload is enabled, group id can be converted using the 8488 * standard or tunnel conversion method. Group conversion method 8489 * selection depends on flags in `grp_info` parameter: 8490 * - Internal (grp_info.external == 0) groups conversion uses the 8491 * standard method. 8492 * - Group ids in JUMP action converted with the tunnel conversion. 8493 * - Group id in rule attribute conversion depends on a rule type and 8494 * group id value: 8495 * ** non zero group attributes converted with the tunnel method 8496 * ** zero group attribute in non-tunnel rule is converted using the 8497 * standard method - there's only one root table 8498 * ** zero group attribute in steer tunnel rule is converted with the 8499 * standard method - single root table 8500 * ** zero group attribute in match tunnel rule is a special OvS 8501 * case: that value is used for portability reasons. That group 8502 * id is converted with the tunnel conversion method. 8503 * 8504 * @param[in] dev 8505 * Port device 8506 * @param[in] tunnel 8507 * PMD tunnel offload object 8508 * @param[in] group 8509 * rte_flow group index value. 8510 * @param[out] table 8511 * HW table value. 8512 * @param[in] grp_info 8513 * flags used for conversion 8514 * @param[out] error 8515 * Pointer to error structure. 8516 * 8517 * @return 8518 * 0 on success, a negative errno value otherwise and rte_errno is set. 8519 */ 8520 int 8521 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 8522 const struct mlx5_flow_tunnel *tunnel, 8523 uint32_t group, uint32_t *table, 8524 const struct flow_grp_info *grp_info, 8525 struct rte_flow_error *error) 8526 { 8527 int ret; 8528 bool standard_translation; 8529 8530 if (!grp_info->skip_scale && grp_info->external && 8531 group < MLX5_MAX_TABLES_EXTERNAL) 8532 group *= MLX5_FLOW_TABLE_FACTOR; 8533 if (is_tunnel_offload_active(dev)) { 8534 standard_translation = !grp_info->external || 8535 grp_info->std_tbl_fix; 8536 } else { 8537 standard_translation = true; 8538 } 8539 DRV_LOG(DEBUG, 8540 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 8541 dev->data->port_id, group, grp_info->transfer, 8542 grp_info->external, grp_info->fdb_def_rule, 8543 standard_translation ? "STANDARD" : "TUNNEL"); 8544 if (standard_translation) 8545 ret = flow_group_to_table(dev->data->port_id, group, table, 8546 grp_info, error); 8547 else 8548 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 8549 table, error); 8550 8551 return ret; 8552 } 8553 8554 /** 8555 * Discover availability of metadata reg_c's. 8556 * 8557 * Iteratively use test flows to check availability. 8558 * 8559 * @param[in] dev 8560 * Pointer to the Ethernet device structure. 8561 * 8562 * @return 8563 * 0 on success, a negative errno value otherwise and rte_errno is set. 8564 */ 8565 int 8566 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 8567 { 8568 struct mlx5_priv *priv = dev->data->dev_private; 8569 enum modify_reg idx; 8570 int n = 0; 8571 8572 /* reg_c[0] and reg_c[1] are reserved. */ 8573 priv->sh->flow_mreg_c[n++] = REG_C_0; 8574 priv->sh->flow_mreg_c[n++] = REG_C_1; 8575 /* Discover availability of other reg_c's. */ 8576 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 8577 struct rte_flow_attr attr = { 8578 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 8579 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8580 .ingress = 1, 8581 }; 8582 struct rte_flow_item items[] = { 8583 [0] = { 8584 .type = RTE_FLOW_ITEM_TYPE_END, 8585 }, 8586 }; 8587 struct rte_flow_action actions[] = { 8588 [0] = { 8589 .type = (enum rte_flow_action_type) 8590 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 8591 .conf = &(struct mlx5_flow_action_copy_mreg){ 8592 .src = REG_C_1, 8593 .dst = idx, 8594 }, 8595 }, 8596 [1] = { 8597 .type = RTE_FLOW_ACTION_TYPE_JUMP, 8598 .conf = &(struct rte_flow_action_jump){ 8599 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 8600 }, 8601 }, 8602 [2] = { 8603 .type = RTE_FLOW_ACTION_TYPE_END, 8604 }, 8605 }; 8606 uint32_t flow_idx; 8607 struct rte_flow *flow; 8608 struct rte_flow_error error; 8609 8610 if (!priv->sh->config.dv_flow_en) 8611 break; 8612 /* Create internal flow, validation skips copy action. */ 8613 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 8614 items, actions, false, &error); 8615 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8616 flow_idx); 8617 if (!flow) 8618 continue; 8619 priv->sh->flow_mreg_c[n++] = idx; 8620 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8621 } 8622 for (; n < MLX5_MREG_C_NUM; ++n) 8623 priv->sh->flow_mreg_c[n] = REG_NON; 8624 priv->sh->metadata_regc_check_flag = 1; 8625 return 0; 8626 } 8627 8628 int 8629 save_dump_file(const uint8_t *data, uint32_t size, 8630 uint32_t type, uint64_t id, void *arg, FILE *file) 8631 { 8632 char line[BUF_SIZE]; 8633 uint32_t out = 0; 8634 uint32_t k; 8635 uint32_t actions_num; 8636 struct rte_flow_query_count *count; 8637 8638 memset(line, 0, BUF_SIZE); 8639 switch (type) { 8640 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 8641 actions_num = *(uint32_t *)(arg); 8642 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 8643 type, id, actions_num); 8644 break; 8645 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 8646 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 8647 type, id); 8648 break; 8649 case DR_DUMP_REC_TYPE_PMD_COUNTER: 8650 count = (struct rte_flow_query_count *)arg; 8651 fprintf(file, 8652 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 8653 type, id, count->hits, count->bytes); 8654 return 0; 8655 default: 8656 return -1; 8657 } 8658 8659 for (k = 0; k < size; k++) { 8660 /* Make sure we do not overrun the line buffer length. */ 8661 if (out >= BUF_SIZE - 4) { 8662 line[out] = '\0'; 8663 break; 8664 } 8665 out += snprintf(line + out, BUF_SIZE - out, "%02x", 8666 (data[k]) & 0xff); 8667 } 8668 fprintf(file, "%s\n", line); 8669 return 0; 8670 } 8671 8672 int 8673 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 8674 struct rte_flow_query_count *count, struct rte_flow_error *error) 8675 { 8676 struct rte_flow_action action[2]; 8677 enum mlx5_flow_drv_type ftype; 8678 const struct mlx5_flow_driver_ops *fops; 8679 8680 if (!flow) { 8681 return rte_flow_error_set(error, ENOENT, 8682 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8683 NULL, 8684 "invalid flow handle"); 8685 } 8686 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 8687 action[1].type = RTE_FLOW_ACTION_TYPE_END; 8688 if (flow->counter) { 8689 memset(count, 0, sizeof(struct rte_flow_query_count)); 8690 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 8691 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 8692 ftype < MLX5_FLOW_TYPE_MAX); 8693 fops = flow_get_drv_ops(ftype); 8694 return fops->query(dev, flow, action, count, error); 8695 } 8696 return -1; 8697 } 8698 8699 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8700 /** 8701 * Dump flow ipool data to file 8702 * 8703 * @param[in] dev 8704 * The pointer to Ethernet device. 8705 * @param[in] file 8706 * A pointer to a file for output. 8707 * @param[out] error 8708 * Perform verbose error reporting if not NULL. PMDs initialize this 8709 * structure in case of error only. 8710 * @return 8711 * 0 on success, a negative value otherwise. 8712 */ 8713 int 8714 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 8715 struct rte_flow *flow, FILE *file, 8716 struct rte_flow_error *error) 8717 { 8718 struct mlx5_priv *priv = dev->data->dev_private; 8719 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8720 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8721 uint32_t handle_idx; 8722 struct mlx5_flow_handle *dh; 8723 struct rte_flow_query_count count; 8724 uint32_t actions_num; 8725 const uint8_t *data; 8726 size_t size; 8727 uint64_t id; 8728 uint32_t type; 8729 void *action = NULL; 8730 8731 if (!flow) { 8732 return rte_flow_error_set(error, ENOENT, 8733 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8734 NULL, 8735 "invalid flow handle"); 8736 } 8737 handle_idx = flow->dev_handles; 8738 /* query counter */ 8739 if (flow->counter && 8740 (!mlx5_counter_query(dev, flow->counter, false, 8741 &count.hits, &count.bytes, &action)) && action) { 8742 id = (uint64_t)(uintptr_t)action; 8743 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8744 save_dump_file(NULL, 0, type, 8745 id, (void *)&count, file); 8746 } 8747 8748 while (handle_idx) { 8749 dh = mlx5_ipool_get(priv->sh->ipool 8750 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 8751 if (!dh) 8752 continue; 8753 handle_idx = dh->next.next; 8754 8755 /* Get modify_hdr and encap_decap buf from ipools. */ 8756 encap_decap = NULL; 8757 modify_hdr = dh->dvh.modify_hdr; 8758 8759 if (dh->dvh.rix_encap_decap) { 8760 encap_decap = mlx5_ipool_get(priv->sh->ipool 8761 [MLX5_IPOOL_DECAP_ENCAP], 8762 dh->dvh.rix_encap_decap); 8763 } 8764 if (modify_hdr) { 8765 data = (const uint8_t *)modify_hdr->actions; 8766 size = (size_t)(modify_hdr->actions_num) * 8; 8767 id = (uint64_t)(uintptr_t)modify_hdr->action; 8768 actions_num = modify_hdr->actions_num; 8769 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8770 save_dump_file(data, size, type, id, 8771 (void *)(&actions_num), file); 8772 } 8773 if (encap_decap) { 8774 data = encap_decap->buf; 8775 size = encap_decap->size; 8776 id = (uint64_t)(uintptr_t)encap_decap->action; 8777 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8778 save_dump_file(data, size, type, 8779 id, NULL, file); 8780 } 8781 } 8782 return 0; 8783 } 8784 8785 /** 8786 * Dump all flow's encap_decap/modify_hdr/counter data to file 8787 * 8788 * @param[in] dev 8789 * The pointer to Ethernet device. 8790 * @param[in] file 8791 * A pointer to a file for output. 8792 * @param[out] error 8793 * Perform verbose error reporting if not NULL. PMDs initialize this 8794 * structure in case of error only. 8795 * @return 8796 * 0 on success, a negative value otherwise. 8797 */ 8798 static int 8799 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 8800 FILE *file, struct rte_flow_error *error __rte_unused) 8801 { 8802 struct mlx5_priv *priv = dev->data->dev_private; 8803 struct mlx5_dev_ctx_shared *sh = priv->sh; 8804 struct mlx5_hlist *h; 8805 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 8806 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 8807 struct rte_flow_query_count count; 8808 uint32_t actions_num; 8809 const uint8_t *data; 8810 size_t size; 8811 uint64_t id; 8812 uint32_t type; 8813 uint32_t i; 8814 uint32_t j; 8815 struct mlx5_list_inconst *l_inconst; 8816 struct mlx5_list_entry *e; 8817 int lcore_index; 8818 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 8819 uint32_t max; 8820 void *action; 8821 8822 /* encap_decap hlist is lcore_share, get global core cache. */ 8823 i = MLX5_LIST_GLOBAL; 8824 h = sh->encaps_decaps; 8825 if (h) { 8826 for (j = 0; j <= h->mask; j++) { 8827 l_inconst = &h->buckets[j].l; 8828 if (!l_inconst || !l_inconst->cache[i]) 8829 continue; 8830 8831 e = LIST_FIRST(&l_inconst->cache[i]->h); 8832 while (e) { 8833 encap_decap = 8834 (struct mlx5_flow_dv_encap_decap_resource *)e; 8835 data = encap_decap->buf; 8836 size = encap_decap->size; 8837 id = (uint64_t)(uintptr_t)encap_decap->action; 8838 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 8839 save_dump_file(data, size, type, 8840 id, NULL, file); 8841 e = LIST_NEXT(e, next); 8842 } 8843 } 8844 } 8845 8846 /* get modify_hdr */ 8847 h = sh->modify_cmds; 8848 if (h) { 8849 lcore_index = rte_lcore_index(rte_lcore_id()); 8850 if (unlikely(lcore_index == -1)) { 8851 lcore_index = MLX5_LIST_NLCORE; 8852 rte_spinlock_lock(&h->l_const.lcore_lock); 8853 } 8854 i = lcore_index; 8855 8856 for (j = 0; j <= h->mask; j++) { 8857 l_inconst = &h->buckets[j].l; 8858 if (!l_inconst || !l_inconst->cache[i]) 8859 continue; 8860 8861 e = LIST_FIRST(&l_inconst->cache[i]->h); 8862 while (e) { 8863 modify_hdr = 8864 (struct mlx5_flow_dv_modify_hdr_resource *)e; 8865 data = (const uint8_t *)modify_hdr->actions; 8866 size = (size_t)(modify_hdr->actions_num) * 8; 8867 actions_num = modify_hdr->actions_num; 8868 id = (uint64_t)(uintptr_t)modify_hdr->action; 8869 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 8870 save_dump_file(data, size, type, id, 8871 (void *)(&actions_num), file); 8872 e = LIST_NEXT(e, next); 8873 } 8874 } 8875 8876 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 8877 rte_spinlock_unlock(&h->l_const.lcore_lock); 8878 } 8879 8880 /* get counter */ 8881 MLX5_ASSERT(cmng->n_valid <= cmng->n); 8882 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 8883 for (j = 1; j <= max; j++) { 8884 action = NULL; 8885 if ((!mlx5_counter_query(dev, j, false, &count.hits, 8886 &count.bytes, &action)) && action) { 8887 id = (uint64_t)(uintptr_t)action; 8888 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 8889 save_dump_file(NULL, 0, type, 8890 id, (void *)&count, file); 8891 } 8892 } 8893 return 0; 8894 } 8895 #endif 8896 8897 /** 8898 * Dump flow raw hw data to file 8899 * 8900 * @param[in] dev 8901 * The pointer to Ethernet device. 8902 * @param[in] file 8903 * A pointer to a file for output. 8904 * @param[out] error 8905 * Perform verbose error reporting if not NULL. PMDs initialize this 8906 * structure in case of error only. 8907 * @return 8908 * 0 on success, a negative value otherwise. 8909 */ 8910 int 8911 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 8912 FILE *file, 8913 struct rte_flow_error *error __rte_unused) 8914 { 8915 struct mlx5_priv *priv = dev->data->dev_private; 8916 struct mlx5_dev_ctx_shared *sh = priv->sh; 8917 uint32_t handle_idx; 8918 int ret; 8919 struct mlx5_flow_handle *dh; 8920 struct rte_flow *flow; 8921 8922 if (!sh->config.dv_flow_en) { 8923 if (fputs("device dv flow disabled\n", file) <= 0) 8924 return -errno; 8925 return -ENOTSUP; 8926 } 8927 8928 /* dump all */ 8929 if (!flow_idx) { 8930 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8931 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 8932 return -EINVAL; 8933 #endif 8934 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 8935 sh->rx_domain, 8936 sh->tx_domain, file); 8937 } 8938 /* dump one */ 8939 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8940 (uintptr_t)(void *)flow_idx); 8941 if (!flow) 8942 return -EINVAL; 8943 8944 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8945 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 8946 #endif 8947 handle_idx = flow->dev_handles; 8948 while (handle_idx) { 8949 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 8950 handle_idx); 8951 if (!dh) 8952 return -ENOENT; 8953 if (dh->drv_flow) { 8954 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 8955 file); 8956 if (ret) 8957 return -ENOENT; 8958 } 8959 handle_idx = dh->next.next; 8960 } 8961 return 0; 8962 } 8963 8964 /** 8965 * Get aged-out flows. 8966 * 8967 * @param[in] dev 8968 * Pointer to the Ethernet device structure. 8969 * @param[in] context 8970 * The address of an array of pointers to the aged-out flows contexts. 8971 * @param[in] nb_countexts 8972 * The length of context array pointers. 8973 * @param[out] error 8974 * Perform verbose error reporting if not NULL. Initialized in case of 8975 * error only. 8976 * 8977 * @return 8978 * how many contexts get in success, otherwise negative errno value. 8979 * if nb_contexts is 0, return the amount of all aged contexts. 8980 * if nb_contexts is not 0 , return the amount of aged flows reported 8981 * in the context array. 8982 */ 8983 int 8984 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 8985 uint32_t nb_contexts, struct rte_flow_error *error) 8986 { 8987 const struct mlx5_flow_driver_ops *fops; 8988 struct rte_flow_attr attr = { .transfer = 0 }; 8989 8990 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8991 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8992 return fops->get_aged_flows(dev, contexts, nb_contexts, 8993 error); 8994 } 8995 DRV_LOG(ERR, 8996 "port %u get aged flows is not supported.", 8997 dev->data->port_id); 8998 return -ENOTSUP; 8999 } 9000 9001 /* Wrapper for driver action_validate op callback */ 9002 static int 9003 flow_drv_action_validate(struct rte_eth_dev *dev, 9004 const struct rte_flow_indir_action_conf *conf, 9005 const struct rte_flow_action *action, 9006 const struct mlx5_flow_driver_ops *fops, 9007 struct rte_flow_error *error) 9008 { 9009 static const char err_msg[] = "indirect action validation unsupported"; 9010 9011 if (!fops->action_validate) { 9012 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9013 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9014 NULL, err_msg); 9015 return -rte_errno; 9016 } 9017 return fops->action_validate(dev, conf, action, error); 9018 } 9019 9020 /** 9021 * Destroys the shared action by handle. 9022 * 9023 * @param dev 9024 * Pointer to Ethernet device structure. 9025 * @param[in] handle 9026 * Handle for the indirect action object to be destroyed. 9027 * @param[out] error 9028 * Perform verbose error reporting if not NULL. PMDs initialize this 9029 * structure in case of error only. 9030 * 9031 * @return 9032 * 0 on success, a negative errno value otherwise and rte_errno is set. 9033 * 9034 * @note: wrapper for driver action_create op callback. 9035 */ 9036 static int 9037 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9038 struct rte_flow_action_handle *handle, 9039 struct rte_flow_error *error) 9040 { 9041 static const char err_msg[] = "indirect action destruction unsupported"; 9042 struct rte_flow_attr attr = { .transfer = 0 }; 9043 const struct mlx5_flow_driver_ops *fops = 9044 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9045 9046 if (!fops->action_destroy) { 9047 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9048 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9049 NULL, err_msg); 9050 return -rte_errno; 9051 } 9052 return fops->action_destroy(dev, handle, error); 9053 } 9054 9055 /* Wrapper for driver action_destroy op callback */ 9056 static int 9057 flow_drv_action_update(struct rte_eth_dev *dev, 9058 struct rte_flow_action_handle *handle, 9059 const void *update, 9060 const struct mlx5_flow_driver_ops *fops, 9061 struct rte_flow_error *error) 9062 { 9063 static const char err_msg[] = "indirect action update unsupported"; 9064 9065 if (!fops->action_update) { 9066 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9067 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9068 NULL, err_msg); 9069 return -rte_errno; 9070 } 9071 return fops->action_update(dev, handle, update, error); 9072 } 9073 9074 /* Wrapper for driver action_destroy op callback */ 9075 static int 9076 flow_drv_action_query(struct rte_eth_dev *dev, 9077 const struct rte_flow_action_handle *handle, 9078 void *data, 9079 const struct mlx5_flow_driver_ops *fops, 9080 struct rte_flow_error *error) 9081 { 9082 static const char err_msg[] = "indirect action query unsupported"; 9083 9084 if (!fops->action_query) { 9085 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9086 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9087 NULL, err_msg); 9088 return -rte_errno; 9089 } 9090 return fops->action_query(dev, handle, data, error); 9091 } 9092 9093 /** 9094 * Create indirect action for reuse in multiple flow rules. 9095 * 9096 * @param dev 9097 * Pointer to Ethernet device structure. 9098 * @param conf 9099 * Pointer to indirect action object configuration. 9100 * @param[in] action 9101 * Action configuration for indirect action object creation. 9102 * @param[out] error 9103 * Perform verbose error reporting if not NULL. PMDs initialize this 9104 * structure in case of error only. 9105 * @return 9106 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9107 */ 9108 static struct rte_flow_action_handle * 9109 mlx5_action_handle_create(struct rte_eth_dev *dev, 9110 const struct rte_flow_indir_action_conf *conf, 9111 const struct rte_flow_action *action, 9112 struct rte_flow_error *error) 9113 { 9114 static const char err_msg[] = "indirect action creation unsupported"; 9115 struct rte_flow_attr attr = { .transfer = 0 }; 9116 const struct mlx5_flow_driver_ops *fops = 9117 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9118 9119 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9120 return NULL; 9121 if (!fops->action_create) { 9122 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9123 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9124 NULL, err_msg); 9125 return NULL; 9126 } 9127 return fops->action_create(dev, conf, action, error); 9128 } 9129 9130 /** 9131 * Updates inplace the indirect action configuration pointed by *handle* 9132 * with the configuration provided as *update* argument. 9133 * The update of the indirect action configuration effects all flow rules 9134 * reusing the action via handle. 9135 * 9136 * @param dev 9137 * Pointer to Ethernet device structure. 9138 * @param[in] handle 9139 * Handle for the indirect action to be updated. 9140 * @param[in] update 9141 * Action specification used to modify the action pointed by handle. 9142 * *update* could be of same type with the action pointed by the *handle* 9143 * handle argument, or some other structures like a wrapper, depending on 9144 * the indirect action type. 9145 * @param[out] error 9146 * Perform verbose error reporting if not NULL. PMDs initialize this 9147 * structure in case of error only. 9148 * 9149 * @return 9150 * 0 on success, a negative errno value otherwise and rte_errno is set. 9151 */ 9152 static int 9153 mlx5_action_handle_update(struct rte_eth_dev *dev, 9154 struct rte_flow_action_handle *handle, 9155 const void *update, 9156 struct rte_flow_error *error) 9157 { 9158 struct rte_flow_attr attr = { .transfer = 0 }; 9159 const struct mlx5_flow_driver_ops *fops = 9160 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9161 int ret; 9162 9163 ret = flow_drv_action_validate(dev, NULL, 9164 (const struct rte_flow_action *)update, fops, error); 9165 if (ret) 9166 return ret; 9167 return flow_drv_action_update(dev, handle, update, fops, 9168 error); 9169 } 9170 9171 /** 9172 * Query the indirect action by handle. 9173 * 9174 * This function allows retrieving action-specific data such as counters. 9175 * Data is gathered by special action which may be present/referenced in 9176 * more than one flow rule definition. 9177 * 9178 * see @RTE_FLOW_ACTION_TYPE_COUNT 9179 * 9180 * @param dev 9181 * Pointer to Ethernet device structure. 9182 * @param[in] handle 9183 * Handle for the indirect action to query. 9184 * @param[in, out] data 9185 * Pointer to storage for the associated query data type. 9186 * @param[out] error 9187 * Perform verbose error reporting if not NULL. PMDs initialize this 9188 * structure in case of error only. 9189 * 9190 * @return 9191 * 0 on success, a negative errno value otherwise and rte_errno is set. 9192 */ 9193 static int 9194 mlx5_action_handle_query(struct rte_eth_dev *dev, 9195 const struct rte_flow_action_handle *handle, 9196 void *data, 9197 struct rte_flow_error *error) 9198 { 9199 struct rte_flow_attr attr = { .transfer = 0 }; 9200 const struct mlx5_flow_driver_ops *fops = 9201 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9202 9203 return flow_drv_action_query(dev, handle, data, fops, error); 9204 } 9205 9206 /** 9207 * Destroy all indirect actions (shared RSS). 9208 * 9209 * @param dev 9210 * Pointer to Ethernet device. 9211 * 9212 * @return 9213 * 0 on success, a negative errno value otherwise and rte_errno is set. 9214 */ 9215 int 9216 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9217 { 9218 struct rte_flow_error error; 9219 struct mlx5_priv *priv = dev->data->dev_private; 9220 struct mlx5_shared_action_rss *shared_rss; 9221 int ret = 0; 9222 uint32_t idx; 9223 9224 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9225 priv->rss_shared_actions, idx, shared_rss, next) { 9226 ret |= mlx5_action_handle_destroy(dev, 9227 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9228 } 9229 return ret; 9230 } 9231 9232 /** 9233 * Validate existing indirect actions against current device configuration 9234 * and attach them to device resources. 9235 * 9236 * @param dev 9237 * Pointer to Ethernet device. 9238 * 9239 * @return 9240 * 0 on success, a negative errno value otherwise and rte_errno is set. 9241 */ 9242 int 9243 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9244 { 9245 struct mlx5_priv *priv = dev->data->dev_private; 9246 struct mlx5_indexed_pool *ipool = 9247 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 9248 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 9249 int ret = 0; 9250 uint32_t idx; 9251 9252 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9253 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9254 const char *message; 9255 uint32_t queue_idx; 9256 9257 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9258 ind_tbl->queues_n, 9259 &message, &queue_idx); 9260 if (ret != 0) { 9261 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9262 dev->data->port_id, ind_tbl->queues[queue_idx], 9263 message); 9264 break; 9265 } 9266 } 9267 if (ret != 0) 9268 return ret; 9269 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9270 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9271 9272 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9273 if (ret != 0) { 9274 DRV_LOG(ERR, "Port %u could not attach " 9275 "indirection table obj %p", 9276 dev->data->port_id, (void *)ind_tbl); 9277 goto error; 9278 } 9279 } 9280 return 0; 9281 error: 9282 shared_rss_last = shared_rss; 9283 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9284 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9285 9286 if (shared_rss == shared_rss_last) 9287 break; 9288 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9289 DRV_LOG(CRIT, "Port %u could not detach " 9290 "indirection table obj %p on rollback", 9291 dev->data->port_id, (void *)ind_tbl); 9292 } 9293 return ret; 9294 } 9295 9296 /** 9297 * Detach indirect actions of the device from its resources. 9298 * 9299 * @param dev 9300 * Pointer to Ethernet device. 9301 * 9302 * @return 9303 * 0 on success, a negative errno value otherwise and rte_errno is set. 9304 */ 9305 int 9306 mlx5_action_handle_detach(struct rte_eth_dev *dev) 9307 { 9308 struct mlx5_priv *priv = dev->data->dev_private; 9309 struct mlx5_indexed_pool *ipool = 9310 priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS]; 9311 struct mlx5_shared_action_rss *shared_rss, *shared_rss_last; 9312 int ret = 0; 9313 uint32_t idx; 9314 9315 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9316 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9317 9318 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 9319 if (ret != 0) { 9320 DRV_LOG(ERR, "Port %u could not detach " 9321 "indirection table obj %p", 9322 dev->data->port_id, (void *)ind_tbl); 9323 goto error; 9324 } 9325 } 9326 return 0; 9327 error: 9328 shared_rss_last = shared_rss; 9329 ILIST_FOREACH(ipool, priv->rss_shared_actions, idx, shared_rss, next) { 9330 struct mlx5_ind_table_obj *ind_tbl = shared_rss->ind_tbl; 9331 9332 if (shared_rss == shared_rss_last) 9333 break; 9334 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 9335 DRV_LOG(CRIT, "Port %u could not attach " 9336 "indirection table obj %p on rollback", 9337 dev->data->port_id, (void *)ind_tbl); 9338 } 9339 return ret; 9340 } 9341 9342 #ifndef HAVE_MLX5DV_DR 9343 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 9344 #else 9345 #define MLX5_DOMAIN_SYNC_FLOW \ 9346 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 9347 #endif 9348 9349 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 9350 { 9351 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 9352 const struct mlx5_flow_driver_ops *fops; 9353 int ret; 9354 struct rte_flow_attr attr = { .transfer = 0 }; 9355 9356 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9357 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 9358 if (ret > 0) 9359 ret = -ret; 9360 return ret; 9361 } 9362 9363 const struct mlx5_flow_tunnel * 9364 mlx5_get_tof(const struct rte_flow_item *item, 9365 const struct rte_flow_action *action, 9366 enum mlx5_tof_rule_type *rule_type) 9367 { 9368 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 9369 if (item->type == (typeof(item->type)) 9370 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 9371 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 9372 return flow_items_to_tunnel(item); 9373 } 9374 } 9375 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 9376 if (action->type == (typeof(action->type)) 9377 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 9378 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 9379 return flow_actions_to_tunnel(action); 9380 } 9381 } 9382 return NULL; 9383 } 9384 9385 /** 9386 * tunnel offload functionality is defined for DV environment only 9387 */ 9388 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9389 __extension__ 9390 union tunnel_offload_mark { 9391 uint32_t val; 9392 struct { 9393 uint32_t app_reserve:8; 9394 uint32_t table_id:15; 9395 uint32_t transfer:1; 9396 uint32_t _unused_:8; 9397 }; 9398 }; 9399 9400 static bool 9401 mlx5_access_tunnel_offload_db 9402 (struct rte_eth_dev *dev, 9403 bool (*match)(struct rte_eth_dev *, 9404 struct mlx5_flow_tunnel *, const void *), 9405 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9406 void (*miss)(struct rte_eth_dev *, void *), 9407 void *ctx, bool lock_op); 9408 9409 static int 9410 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 9411 struct rte_flow *flow, 9412 const struct rte_flow_attr *attr, 9413 const struct rte_flow_action *app_actions, 9414 uint32_t flow_idx, 9415 const struct mlx5_flow_tunnel *tunnel, 9416 struct tunnel_default_miss_ctx *ctx, 9417 struct rte_flow_error *error) 9418 { 9419 struct mlx5_priv *priv = dev->data->dev_private; 9420 struct mlx5_flow *dev_flow; 9421 struct rte_flow_attr miss_attr = *attr; 9422 const struct rte_flow_item miss_items[2] = { 9423 { 9424 .type = RTE_FLOW_ITEM_TYPE_ETH, 9425 .spec = NULL, 9426 .last = NULL, 9427 .mask = NULL 9428 }, 9429 { 9430 .type = RTE_FLOW_ITEM_TYPE_END, 9431 .spec = NULL, 9432 .last = NULL, 9433 .mask = NULL 9434 } 9435 }; 9436 union tunnel_offload_mark mark_id; 9437 struct rte_flow_action_mark miss_mark; 9438 struct rte_flow_action miss_actions[3] = { 9439 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 9440 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 9441 }; 9442 const struct rte_flow_action_jump *jump_data; 9443 uint32_t i, flow_table = 0; /* prevent compilation warning */ 9444 struct flow_grp_info grp_info = { 9445 .external = 1, 9446 .transfer = attr->transfer, 9447 .fdb_def_rule = !!priv->fdb_def_rule, 9448 .std_tbl_fix = 0, 9449 }; 9450 int ret; 9451 9452 if (!attr->transfer) { 9453 uint32_t q_size; 9454 9455 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 9456 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 9457 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 9458 0, SOCKET_ID_ANY); 9459 if (!ctx->queue) 9460 return rte_flow_error_set 9461 (error, ENOMEM, 9462 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9463 NULL, "invalid default miss RSS"); 9464 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 9465 ctx->action_rss.level = 0, 9466 ctx->action_rss.types = priv->rss_conf.rss_hf, 9467 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 9468 ctx->action_rss.queue_num = priv->reta_idx_n, 9469 ctx->action_rss.key = priv->rss_conf.rss_key, 9470 ctx->action_rss.queue = ctx->queue; 9471 if (!priv->reta_idx_n || !priv->rxqs_n) 9472 return rte_flow_error_set 9473 (error, EINVAL, 9474 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9475 NULL, "invalid port configuration"); 9476 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 9477 ctx->action_rss.types = 0; 9478 for (i = 0; i != priv->reta_idx_n; ++i) 9479 ctx->queue[i] = (*priv->reta_idx)[i]; 9480 } else { 9481 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 9482 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 9483 } 9484 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 9485 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 9486 jump_data = app_actions->conf; 9487 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 9488 miss_attr.group = jump_data->group; 9489 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 9490 &flow_table, &grp_info, error); 9491 if (ret) 9492 return rte_flow_error_set(error, EINVAL, 9493 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 9494 NULL, "invalid tunnel id"); 9495 mark_id.app_reserve = 0; 9496 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 9497 mark_id.transfer = !!attr->transfer; 9498 mark_id._unused_ = 0; 9499 miss_mark.id = mark_id.val; 9500 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 9501 miss_items, miss_actions, flow_idx, error); 9502 if (!dev_flow) 9503 return -rte_errno; 9504 dev_flow->flow = flow; 9505 dev_flow->external = true; 9506 dev_flow->tunnel = tunnel; 9507 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 9508 /* Subflow object was created, we must include one in the list. */ 9509 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 9510 dev_flow->handle, next); 9511 DRV_LOG(DEBUG, 9512 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 9513 dev->data->port_id, tunnel->app_tunnel.type, 9514 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 9515 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 9516 miss_actions, error); 9517 if (!ret) 9518 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 9519 error); 9520 9521 return ret; 9522 } 9523 9524 static const struct mlx5_flow_tbl_data_entry * 9525 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 9526 { 9527 struct mlx5_priv *priv = dev->data->dev_private; 9528 struct mlx5_dev_ctx_shared *sh = priv->sh; 9529 struct mlx5_list_entry *he; 9530 union tunnel_offload_mark mbits = { .val = mark }; 9531 union mlx5_flow_tbl_key table_key = { 9532 { 9533 .level = tunnel_id_to_flow_tbl(mbits.table_id), 9534 .id = 0, 9535 .reserved = 0, 9536 .dummy = 0, 9537 .is_fdb = !!mbits.transfer, 9538 .is_egress = 0, 9539 } 9540 }; 9541 struct mlx5_flow_cb_ctx ctx = { 9542 .data = &table_key.v64, 9543 }; 9544 9545 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 9546 return he ? 9547 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 9548 } 9549 9550 static void 9551 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 9552 struct mlx5_list_entry *entry) 9553 { 9554 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9555 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9556 9557 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9558 tunnel_flow_tbl_to_id(tte->flow_table)); 9559 mlx5_free(tte); 9560 } 9561 9562 static int 9563 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 9564 struct mlx5_list_entry *entry, void *cb_ctx) 9565 { 9566 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9567 union tunnel_tbl_key tbl = { 9568 .val = *(uint64_t *)(ctx->data), 9569 }; 9570 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9571 9572 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 9573 } 9574 9575 static struct mlx5_list_entry * 9576 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 9577 { 9578 struct mlx5_dev_ctx_shared *sh = tool_ctx; 9579 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 9580 struct tunnel_tbl_entry *tte; 9581 union tunnel_tbl_key tbl = { 9582 .val = *(uint64_t *)(ctx->data), 9583 }; 9584 9585 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 9586 sizeof(*tte), 0, 9587 SOCKET_ID_ANY); 9588 if (!tte) 9589 goto err; 9590 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9591 &tte->flow_table); 9592 if (tte->flow_table >= MLX5_MAX_TABLES) { 9593 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 9594 tte->flow_table); 9595 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 9596 tte->flow_table); 9597 goto err; 9598 } else if (!tte->flow_table) { 9599 goto err; 9600 } 9601 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 9602 tte->tunnel_id = tbl.tunnel_id; 9603 tte->group = tbl.group; 9604 return &tte->hash; 9605 err: 9606 if (tte) 9607 mlx5_free(tte); 9608 return NULL; 9609 } 9610 9611 static struct mlx5_list_entry * 9612 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 9613 struct mlx5_list_entry *oentry, 9614 void *cb_ctx __rte_unused) 9615 { 9616 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 9617 0, SOCKET_ID_ANY); 9618 9619 if (!tte) 9620 return NULL; 9621 memcpy(tte, oentry, sizeof(*tte)); 9622 return &tte->hash; 9623 } 9624 9625 static void 9626 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 9627 struct mlx5_list_entry *entry) 9628 { 9629 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 9630 9631 mlx5_free(tte); 9632 } 9633 9634 static uint32_t 9635 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 9636 const struct mlx5_flow_tunnel *tunnel, 9637 uint32_t group, uint32_t *table, 9638 struct rte_flow_error *error) 9639 { 9640 struct mlx5_list_entry *he; 9641 struct tunnel_tbl_entry *tte; 9642 union tunnel_tbl_key key = { 9643 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 9644 .group = group 9645 }; 9646 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9647 struct mlx5_hlist *group_hash; 9648 struct mlx5_flow_cb_ctx ctx = { 9649 .data = &key.val, 9650 }; 9651 9652 group_hash = tunnel ? tunnel->groups : thub->groups; 9653 he = mlx5_hlist_register(group_hash, key.val, &ctx); 9654 if (!he) 9655 return rte_flow_error_set(error, EINVAL, 9656 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9657 NULL, 9658 "tunnel group index not supported"); 9659 tte = container_of(he, typeof(*tte), hash); 9660 *table = tte->flow_table; 9661 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 9662 dev->data->port_id, key.tunnel_id, group, *table); 9663 return 0; 9664 } 9665 9666 static void 9667 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 9668 struct mlx5_flow_tunnel *tunnel) 9669 { 9670 struct mlx5_priv *priv = dev->data->dev_private; 9671 struct mlx5_indexed_pool *ipool; 9672 9673 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 9674 dev->data->port_id, tunnel->tunnel_id); 9675 LIST_REMOVE(tunnel, chain); 9676 mlx5_hlist_destroy(tunnel->groups); 9677 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9678 mlx5_ipool_free(ipool, tunnel->tunnel_id); 9679 } 9680 9681 static bool 9682 mlx5_access_tunnel_offload_db 9683 (struct rte_eth_dev *dev, 9684 bool (*match)(struct rte_eth_dev *, 9685 struct mlx5_flow_tunnel *, const void *), 9686 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 9687 void (*miss)(struct rte_eth_dev *, void *), 9688 void *ctx, bool lock_op) 9689 { 9690 bool verdict = false; 9691 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9692 struct mlx5_flow_tunnel *tunnel; 9693 9694 rte_spinlock_lock(&thub->sl); 9695 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 9696 verdict = match(dev, tunnel, (const void *)ctx); 9697 if (verdict) 9698 break; 9699 } 9700 if (!lock_op) 9701 rte_spinlock_unlock(&thub->sl); 9702 if (verdict && hit) 9703 hit(dev, tunnel, ctx); 9704 if (!verdict && miss) 9705 miss(dev, ctx); 9706 if (lock_op) 9707 rte_spinlock_unlock(&thub->sl); 9708 9709 return verdict; 9710 } 9711 9712 struct tunnel_db_find_tunnel_id_ctx { 9713 uint32_t tunnel_id; 9714 struct mlx5_flow_tunnel *tunnel; 9715 }; 9716 9717 static bool 9718 find_tunnel_id_match(struct rte_eth_dev *dev, 9719 struct mlx5_flow_tunnel *tunnel, const void *x) 9720 { 9721 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9722 9723 RTE_SET_USED(dev); 9724 return tunnel->tunnel_id == ctx->tunnel_id; 9725 } 9726 9727 static void 9728 find_tunnel_id_hit(struct rte_eth_dev *dev, 9729 struct mlx5_flow_tunnel *tunnel, void *x) 9730 { 9731 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 9732 RTE_SET_USED(dev); 9733 ctx->tunnel = tunnel; 9734 } 9735 9736 static struct mlx5_flow_tunnel * 9737 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 9738 { 9739 struct tunnel_db_find_tunnel_id_ctx ctx = { 9740 .tunnel_id = id, 9741 }; 9742 9743 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 9744 find_tunnel_id_hit, NULL, &ctx, true); 9745 9746 return ctx.tunnel; 9747 } 9748 9749 static struct mlx5_flow_tunnel * 9750 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 9751 const struct rte_flow_tunnel *app_tunnel) 9752 { 9753 struct mlx5_priv *priv = dev->data->dev_private; 9754 struct mlx5_indexed_pool *ipool; 9755 struct mlx5_flow_tunnel *tunnel; 9756 uint32_t id; 9757 9758 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 9759 tunnel = mlx5_ipool_zmalloc(ipool, &id); 9760 if (!tunnel) 9761 return NULL; 9762 if (id >= MLX5_MAX_TUNNELS) { 9763 mlx5_ipool_free(ipool, id); 9764 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 9765 return NULL; 9766 } 9767 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 9768 priv->sh, 9769 mlx5_flow_tunnel_grp2tbl_create_cb, 9770 mlx5_flow_tunnel_grp2tbl_match_cb, 9771 mlx5_flow_tunnel_grp2tbl_remove_cb, 9772 mlx5_flow_tunnel_grp2tbl_clone_cb, 9773 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9774 if (!tunnel->groups) { 9775 mlx5_ipool_free(ipool, id); 9776 return NULL; 9777 } 9778 /* initiate new PMD tunnel */ 9779 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 9780 tunnel->tunnel_id = id; 9781 tunnel->action.type = (typeof(tunnel->action.type)) 9782 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 9783 tunnel->action.conf = tunnel; 9784 tunnel->item.type = (typeof(tunnel->item.type)) 9785 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 9786 tunnel->item.spec = tunnel; 9787 tunnel->item.last = NULL; 9788 tunnel->item.mask = NULL; 9789 9790 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 9791 dev->data->port_id, tunnel->tunnel_id); 9792 9793 return tunnel; 9794 } 9795 9796 struct tunnel_db_get_tunnel_ctx { 9797 const struct rte_flow_tunnel *app_tunnel; 9798 struct mlx5_flow_tunnel *tunnel; 9799 }; 9800 9801 static bool get_tunnel_match(struct rte_eth_dev *dev, 9802 struct mlx5_flow_tunnel *tunnel, const void *x) 9803 { 9804 const struct tunnel_db_get_tunnel_ctx *ctx = x; 9805 9806 RTE_SET_USED(dev); 9807 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 9808 sizeof(*ctx->app_tunnel)); 9809 } 9810 9811 static void get_tunnel_hit(struct rte_eth_dev *dev, 9812 struct mlx5_flow_tunnel *tunnel, void *x) 9813 { 9814 /* called under tunnel spinlock protection */ 9815 struct tunnel_db_get_tunnel_ctx *ctx = x; 9816 9817 RTE_SET_USED(dev); 9818 tunnel->refctn++; 9819 ctx->tunnel = tunnel; 9820 } 9821 9822 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 9823 { 9824 /* called under tunnel spinlock protection */ 9825 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 9826 struct tunnel_db_get_tunnel_ctx *ctx = x; 9827 9828 rte_spinlock_unlock(&thub->sl); 9829 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 9830 rte_spinlock_lock(&thub->sl); 9831 if (ctx->tunnel) { 9832 ctx->tunnel->refctn = 1; 9833 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 9834 } 9835 } 9836 9837 9838 static int 9839 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 9840 const struct rte_flow_tunnel *app_tunnel, 9841 struct mlx5_flow_tunnel **tunnel) 9842 { 9843 struct tunnel_db_get_tunnel_ctx ctx = { 9844 .app_tunnel = app_tunnel, 9845 }; 9846 9847 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 9848 get_tunnel_miss, &ctx, true); 9849 *tunnel = ctx.tunnel; 9850 return ctx.tunnel ? 0 : -ENOMEM; 9851 } 9852 9853 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 9854 { 9855 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 9856 9857 if (!thub) 9858 return; 9859 if (!LIST_EMPTY(&thub->tunnels)) 9860 DRV_LOG(WARNING, "port %u tunnels present", port_id); 9861 mlx5_hlist_destroy(thub->groups); 9862 mlx5_free(thub); 9863 } 9864 9865 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 9866 { 9867 int err; 9868 struct mlx5_flow_tunnel_hub *thub; 9869 9870 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 9871 0, SOCKET_ID_ANY); 9872 if (!thub) 9873 return -ENOMEM; 9874 LIST_INIT(&thub->tunnels); 9875 rte_spinlock_init(&thub->sl); 9876 thub->groups = mlx5_hlist_create("flow groups", 64, 9877 false, true, sh, 9878 mlx5_flow_tunnel_grp2tbl_create_cb, 9879 mlx5_flow_tunnel_grp2tbl_match_cb, 9880 mlx5_flow_tunnel_grp2tbl_remove_cb, 9881 mlx5_flow_tunnel_grp2tbl_clone_cb, 9882 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 9883 if (!thub->groups) { 9884 err = -rte_errno; 9885 goto err; 9886 } 9887 sh->tunnel_hub = thub; 9888 9889 return 0; 9890 9891 err: 9892 if (thub->groups) 9893 mlx5_hlist_destroy(thub->groups); 9894 if (thub) 9895 mlx5_free(thub); 9896 return err; 9897 } 9898 9899 static inline int 9900 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 9901 struct rte_flow_tunnel *tunnel, 9902 struct rte_flow_error *error) 9903 { 9904 struct mlx5_priv *priv = dev->data->dev_private; 9905 9906 if (!priv->sh->config.dv_flow_en) 9907 return rte_flow_error_set(error, ENOTSUP, 9908 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9909 "flow DV interface is off"); 9910 if (!is_tunnel_offload_active(dev)) 9911 return rte_flow_error_set(error, ENOTSUP, 9912 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9913 "tunnel offload was not activated"); 9914 if (!tunnel) 9915 return rte_flow_error_set(error, EINVAL, 9916 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9917 "no application tunnel"); 9918 switch (tunnel->type) { 9919 default: 9920 return rte_flow_error_set(error, EINVAL, 9921 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9922 "unsupported tunnel type"); 9923 case RTE_FLOW_ITEM_TYPE_VXLAN: 9924 case RTE_FLOW_ITEM_TYPE_GRE: 9925 case RTE_FLOW_ITEM_TYPE_NVGRE: 9926 case RTE_FLOW_ITEM_TYPE_GENEVE: 9927 break; 9928 } 9929 return 0; 9930 } 9931 9932 static int 9933 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 9934 struct rte_flow_tunnel *app_tunnel, 9935 struct rte_flow_action **actions, 9936 uint32_t *num_of_actions, 9937 struct rte_flow_error *error) 9938 { 9939 struct mlx5_flow_tunnel *tunnel; 9940 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9941 9942 if (ret) 9943 return ret; 9944 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9945 if (ret < 0) { 9946 return rte_flow_error_set(error, ret, 9947 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 9948 "failed to initialize pmd tunnel"); 9949 } 9950 *actions = &tunnel->action; 9951 *num_of_actions = 1; 9952 return 0; 9953 } 9954 9955 static int 9956 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 9957 struct rte_flow_tunnel *app_tunnel, 9958 struct rte_flow_item **items, 9959 uint32_t *num_of_items, 9960 struct rte_flow_error *error) 9961 { 9962 struct mlx5_flow_tunnel *tunnel; 9963 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 9964 9965 if (ret) 9966 return ret; 9967 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 9968 if (ret < 0) { 9969 return rte_flow_error_set(error, ret, 9970 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 9971 "failed to initialize pmd tunnel"); 9972 } 9973 *items = &tunnel->item; 9974 *num_of_items = 1; 9975 return 0; 9976 } 9977 9978 struct tunnel_db_element_release_ctx { 9979 struct rte_flow_item *items; 9980 struct rte_flow_action *actions; 9981 uint32_t num_elements; 9982 struct rte_flow_error *error; 9983 int ret; 9984 }; 9985 9986 static bool 9987 tunnel_element_release_match(struct rte_eth_dev *dev, 9988 struct mlx5_flow_tunnel *tunnel, const void *x) 9989 { 9990 const struct tunnel_db_element_release_ctx *ctx = x; 9991 9992 RTE_SET_USED(dev); 9993 if (ctx->num_elements != 1) 9994 return false; 9995 else if (ctx->items) 9996 return ctx->items == &tunnel->item; 9997 else if (ctx->actions) 9998 return ctx->actions == &tunnel->action; 9999 10000 return false; 10001 } 10002 10003 static void 10004 tunnel_element_release_hit(struct rte_eth_dev *dev, 10005 struct mlx5_flow_tunnel *tunnel, void *x) 10006 { 10007 struct tunnel_db_element_release_ctx *ctx = x; 10008 ctx->ret = 0; 10009 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10010 mlx5_flow_tunnel_free(dev, tunnel); 10011 } 10012 10013 static void 10014 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10015 { 10016 struct tunnel_db_element_release_ctx *ctx = x; 10017 RTE_SET_USED(dev); 10018 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10019 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10020 "invalid argument"); 10021 } 10022 10023 static int 10024 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10025 struct rte_flow_item *pmd_items, 10026 uint32_t num_items, struct rte_flow_error *err) 10027 { 10028 struct tunnel_db_element_release_ctx ctx = { 10029 .items = pmd_items, 10030 .actions = NULL, 10031 .num_elements = num_items, 10032 .error = err, 10033 }; 10034 10035 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10036 tunnel_element_release_hit, 10037 tunnel_element_release_miss, &ctx, false); 10038 10039 return ctx.ret; 10040 } 10041 10042 static int 10043 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10044 struct rte_flow_action *pmd_actions, 10045 uint32_t num_actions, struct rte_flow_error *err) 10046 { 10047 struct tunnel_db_element_release_ctx ctx = { 10048 .items = NULL, 10049 .actions = pmd_actions, 10050 .num_elements = num_actions, 10051 .error = err, 10052 }; 10053 10054 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10055 tunnel_element_release_hit, 10056 tunnel_element_release_miss, &ctx, false); 10057 10058 return ctx.ret; 10059 } 10060 10061 static int 10062 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10063 struct rte_mbuf *m, 10064 struct rte_flow_restore_info *info, 10065 struct rte_flow_error *err) 10066 { 10067 uint64_t ol_flags = m->ol_flags; 10068 const struct mlx5_flow_tbl_data_entry *tble; 10069 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10070 10071 if (!is_tunnel_offload_active(dev)) { 10072 info->flags = 0; 10073 return 0; 10074 } 10075 10076 if ((ol_flags & mask) != mask) 10077 goto err; 10078 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10079 if (!tble) { 10080 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10081 dev->data->port_id, m->hash.fdir.hi); 10082 goto err; 10083 } 10084 MLX5_ASSERT(tble->tunnel); 10085 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10086 info->group_id = tble->group_id; 10087 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10088 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10089 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10090 10091 return 0; 10092 10093 err: 10094 return rte_flow_error_set(err, EINVAL, 10095 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10096 "failed to get restore info"); 10097 } 10098 10099 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10100 static int 10101 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10102 __rte_unused struct rte_flow_tunnel *app_tunnel, 10103 __rte_unused struct rte_flow_action **actions, 10104 __rte_unused uint32_t *num_of_actions, 10105 __rte_unused struct rte_flow_error *error) 10106 { 10107 return -ENOTSUP; 10108 } 10109 10110 static int 10111 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10112 __rte_unused struct rte_flow_tunnel *app_tunnel, 10113 __rte_unused struct rte_flow_item **items, 10114 __rte_unused uint32_t *num_of_items, 10115 __rte_unused struct rte_flow_error *error) 10116 { 10117 return -ENOTSUP; 10118 } 10119 10120 static int 10121 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10122 __rte_unused struct rte_flow_item *pmd_items, 10123 __rte_unused uint32_t num_items, 10124 __rte_unused struct rte_flow_error *err) 10125 { 10126 return -ENOTSUP; 10127 } 10128 10129 static int 10130 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10131 __rte_unused struct rte_flow_action *pmd_action, 10132 __rte_unused uint32_t num_actions, 10133 __rte_unused struct rte_flow_error *err) 10134 { 10135 return -ENOTSUP; 10136 } 10137 10138 static int 10139 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10140 __rte_unused struct rte_mbuf *m, 10141 __rte_unused struct rte_flow_restore_info *i, 10142 __rte_unused struct rte_flow_error *err) 10143 { 10144 return -ENOTSUP; 10145 } 10146 10147 static int 10148 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10149 __rte_unused struct rte_flow *flow, 10150 __rte_unused const struct rte_flow_attr *attr, 10151 __rte_unused const struct rte_flow_action *actions, 10152 __rte_unused uint32_t flow_idx, 10153 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10154 __rte_unused struct tunnel_default_miss_ctx *ctx, 10155 __rte_unused struct rte_flow_error *error) 10156 { 10157 return -ENOTSUP; 10158 } 10159 10160 static struct mlx5_flow_tunnel * 10161 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10162 __rte_unused uint32_t id) 10163 { 10164 return NULL; 10165 } 10166 10167 static void 10168 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10169 __rte_unused struct mlx5_flow_tunnel *tunnel) 10170 { 10171 } 10172 10173 static uint32_t 10174 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10175 __rte_unused const struct mlx5_flow_tunnel *t, 10176 __rte_unused uint32_t group, 10177 __rte_unused uint32_t *table, 10178 struct rte_flow_error *error) 10179 { 10180 return rte_flow_error_set(error, ENOTSUP, 10181 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10182 "tunnel offload requires DV support"); 10183 } 10184 10185 void 10186 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10187 __rte_unused uint16_t port_id) 10188 { 10189 } 10190 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10191 10192 /* Flex flow item API */ 10193 static struct rte_flow_item_flex_handle * 10194 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10195 const struct rte_flow_item_flex_conf *conf, 10196 struct rte_flow_error *error) 10197 { 10198 static const char err_msg[] = "flex item creation unsupported"; 10199 struct rte_flow_attr attr = { .transfer = 0 }; 10200 const struct mlx5_flow_driver_ops *fops = 10201 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10202 10203 if (!fops->item_create) { 10204 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10205 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10206 NULL, err_msg); 10207 return NULL; 10208 } 10209 return fops->item_create(dev, conf, error); 10210 } 10211 10212 static int 10213 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10214 const struct rte_flow_item_flex_handle *handle, 10215 struct rte_flow_error *error) 10216 { 10217 static const char err_msg[] = "flex item release unsupported"; 10218 struct rte_flow_attr attr = { .transfer = 0 }; 10219 const struct mlx5_flow_driver_ops *fops = 10220 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10221 10222 if (!fops->item_release) { 10223 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10224 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10225 NULL, err_msg); 10226 return -rte_errno; 10227 } 10228 return fops->item_release(dev, handle, error); 10229 } 10230 10231 static void 10232 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10233 { 10234 int ret; 10235 struct rte_flow_error error; 10236 10237 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10238 char *item_name; 10239 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10240 sizeof(item_name), 10241 (void *)(uintptr_t)item->type, &error); 10242 if (ret > 0) 10243 printf("%s ", item_name); 10244 else 10245 printf("%d\n", (int)item->type); 10246 } 10247 printf("END\n"); 10248 } 10249 10250 static int 10251 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10252 { 10253 const struct rte_flow_item_udp *spec = udp_item->spec; 10254 const struct rte_flow_item_udp *mask = udp_item->mask; 10255 uint16_t udp_dport = 0; 10256 10257 if (spec != NULL) { 10258 if (!mask) 10259 mask = &rte_flow_item_udp_mask; 10260 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10261 mask->hdr.dst_port); 10262 } 10263 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10264 } 10265 10266 static const struct mlx5_flow_expand_node * 10267 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10268 unsigned int item_idx, 10269 const struct mlx5_flow_expand_node graph[], 10270 const struct mlx5_flow_expand_node *node) 10271 { 10272 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10273 10274 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10275 node != NULL && 10276 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10277 /* 10278 * The expansion node is VXLAN and it is also the last 10279 * expandable item in the pattern, so need to continue 10280 * expansion of the inner tunnel. 10281 */ 10282 MLX5_ASSERT(item_idx > 0); 10283 prev_item = pattern + item_idx - 1; 10284 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10285 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10286 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10287 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10288 } 10289 return node; 10290 } 10291 10292 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 10293 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 10294 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 10295 }; 10296 10297 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 10298 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 10299 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 10300 { 9, 10, 11 }, { 12, 13, 14 }, 10301 }; 10302 10303 /** 10304 * Discover the number of available flow priorities. 10305 * 10306 * @param dev 10307 * Ethernet device. 10308 * 10309 * @return 10310 * On success, number of available flow priorities. 10311 * On failure, a negative errno-style code and rte_errno is set. 10312 */ 10313 int 10314 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 10315 { 10316 static const uint16_t vprio[] = {8, 16}; 10317 const struct mlx5_priv *priv = dev->data->dev_private; 10318 const struct mlx5_flow_driver_ops *fops; 10319 enum mlx5_flow_drv_type type; 10320 int ret; 10321 10322 type = mlx5_flow_os_get_type(); 10323 if (type == MLX5_FLOW_TYPE_MAX) { 10324 type = MLX5_FLOW_TYPE_VERBS; 10325 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 10326 type = MLX5_FLOW_TYPE_DV; 10327 } 10328 fops = flow_get_drv_ops(type); 10329 if (fops->discover_priorities == NULL) { 10330 DRV_LOG(ERR, "Priority discovery not supported"); 10331 rte_errno = ENOTSUP; 10332 return -rte_errno; 10333 } 10334 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 10335 if (ret < 0) 10336 return ret; 10337 switch (ret) { 10338 case 8: 10339 ret = RTE_DIM(priority_map_3); 10340 break; 10341 case 16: 10342 ret = RTE_DIM(priority_map_5); 10343 break; 10344 default: 10345 rte_errno = ENOTSUP; 10346 DRV_LOG(ERR, 10347 "port %u maximum priority: %d expected 8/16", 10348 dev->data->port_id, ret); 10349 return -rte_errno; 10350 } 10351 DRV_LOG(INFO, "port %u supported flow priorities:" 10352 " 0-%d for ingress or egress root table," 10353 " 0-%d for non-root table or transfer root table.", 10354 dev->data->port_id, ret - 2, 10355 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 10356 return ret; 10357 } 10358 10359 /** 10360 * Adjust flow priority based on the highest layer and the request priority. 10361 * 10362 * @param[in] dev 10363 * Pointer to the Ethernet device structure. 10364 * @param[in] priority 10365 * The rule base priority. 10366 * @param[in] subpriority 10367 * The priority based on the items. 10368 * 10369 * @return 10370 * The new priority. 10371 */ 10372 uint32_t 10373 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 10374 uint32_t subpriority) 10375 { 10376 uint32_t res = 0; 10377 struct mlx5_priv *priv = dev->data->dev_private; 10378 10379 switch (priv->sh->flow_max_priority) { 10380 case RTE_DIM(priority_map_3): 10381 res = priority_map_3[priority][subpriority]; 10382 break; 10383 case RTE_DIM(priority_map_5): 10384 res = priority_map_5[priority][subpriority]; 10385 break; 10386 } 10387 return res; 10388 } 10389