1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /* Optional expand field. The expansion alg will not go deeper. */ 128 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 129 130 /* The node is not added implicitly as expansion to the flow pattern. 131 * If the node type does not match the flow pattern item type, the 132 * expansion alg will go deeper to its next items. 133 * In the current implementation, the list of next nodes indexes can 134 * have up to one node with this flag set and it has to be the last 135 * node index (before the list terminator). 136 */ 137 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 138 139 /** Object returned by mlx5_flow_expand_rss(). */ 140 struct mlx5_flow_expand_rss { 141 uint32_t entries; 142 /**< Number of entries @p patterns and @p priorities. */ 143 struct { 144 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 145 uint32_t priority; /**< Priority offset for each expansion. */ 146 } entry[]; 147 }; 148 149 static void 150 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 151 152 static const struct mlx5_flow_expand_node * 153 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 154 unsigned int item_idx, 155 const struct mlx5_flow_expand_node graph[], 156 const struct mlx5_flow_expand_node *node); 157 158 static bool 159 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 160 { 161 switch (item->type) { 162 case RTE_FLOW_ITEM_TYPE_ETH: 163 case RTE_FLOW_ITEM_TYPE_VLAN: 164 case RTE_FLOW_ITEM_TYPE_IPV4: 165 case RTE_FLOW_ITEM_TYPE_IPV6: 166 case RTE_FLOW_ITEM_TYPE_UDP: 167 case RTE_FLOW_ITEM_TYPE_TCP: 168 case RTE_FLOW_ITEM_TYPE_ESP: 169 case RTE_FLOW_ITEM_TYPE_ICMP: 170 case RTE_FLOW_ITEM_TYPE_ICMP6: 171 case RTE_FLOW_ITEM_TYPE_VXLAN: 172 case RTE_FLOW_ITEM_TYPE_NVGRE: 173 case RTE_FLOW_ITEM_TYPE_GRE: 174 case RTE_FLOW_ITEM_TYPE_GENEVE: 175 case RTE_FLOW_ITEM_TYPE_MPLS: 176 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 177 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 178 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 179 case RTE_FLOW_ITEM_TYPE_GTP: 180 return true; 181 default: 182 break; 183 } 184 return false; 185 } 186 187 /** 188 * Network Service Header (NSH) and its next protocol values 189 * are described in RFC-8393. 190 */ 191 static enum rte_flow_item_type 192 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 193 { 194 enum rte_flow_item_type type; 195 196 switch (proto_mask & proto_spec) { 197 case 0: 198 type = RTE_FLOW_ITEM_TYPE_VOID; 199 break; 200 case RTE_VXLAN_GPE_TYPE_IPV4: 201 type = RTE_FLOW_ITEM_TYPE_IPV4; 202 break; 203 case RTE_VXLAN_GPE_TYPE_IPV6: 204 type = RTE_VXLAN_GPE_TYPE_IPV6; 205 break; 206 case RTE_VXLAN_GPE_TYPE_ETH: 207 type = RTE_FLOW_ITEM_TYPE_ETH; 208 break; 209 default: 210 type = RTE_FLOW_ITEM_TYPE_END; 211 } 212 return type; 213 } 214 215 static enum rte_flow_item_type 216 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 217 { 218 enum rte_flow_item_type type; 219 220 switch (proto_mask & proto_spec) { 221 case 0: 222 type = RTE_FLOW_ITEM_TYPE_VOID; 223 break; 224 case IPPROTO_UDP: 225 type = RTE_FLOW_ITEM_TYPE_UDP; 226 break; 227 case IPPROTO_TCP: 228 type = RTE_FLOW_ITEM_TYPE_TCP; 229 break; 230 case IPPROTO_IPIP: 231 type = RTE_FLOW_ITEM_TYPE_IPV4; 232 break; 233 case IPPROTO_IPV6: 234 type = RTE_FLOW_ITEM_TYPE_IPV6; 235 break; 236 case IPPROTO_ESP: 237 type = RTE_FLOW_ITEM_TYPE_ESP; 238 break; 239 default: 240 type = RTE_FLOW_ITEM_TYPE_END; 241 } 242 return type; 243 } 244 245 static enum rte_flow_item_type 246 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 247 rte_be16_t type_mask, bool is_tunnel) 248 { 249 enum rte_flow_item_type type; 250 251 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 252 case 0: 253 type = RTE_FLOW_ITEM_TYPE_VOID; 254 break; 255 case RTE_ETHER_TYPE_TEB: 256 type = is_tunnel ? 257 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 258 break; 259 case RTE_ETHER_TYPE_VLAN: 260 type = !is_tunnel ? 261 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 262 break; 263 case RTE_ETHER_TYPE_IPV4: 264 type = RTE_FLOW_ITEM_TYPE_IPV4; 265 break; 266 case RTE_ETHER_TYPE_IPV6: 267 type = RTE_FLOW_ITEM_TYPE_IPV6; 268 break; 269 default: 270 type = RTE_FLOW_ITEM_TYPE_END; 271 } 272 return type; 273 } 274 275 static enum rte_flow_item_type 276 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 277 { 278 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 279 do { \ 280 const void *m = item->mask; \ 281 const void *s = item->spec; \ 282 mask = m ? \ 283 ((const struct rte_flow_item_##type *)m)->fld : \ 284 rte_flow_item_##type##_mask.fld; \ 285 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 286 } while (0) 287 288 enum rte_flow_item_type ret; 289 uint16_t spec, mask; 290 291 if (item == NULL || item->spec == NULL) 292 return RTE_FLOW_ITEM_TYPE_VOID; 293 switch (item->type) { 294 case RTE_FLOW_ITEM_TYPE_ETH: 295 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 296 if (!mask) 297 return RTE_FLOW_ITEM_TYPE_VOID; 298 ret = mlx5_ethertype_to_item_type(spec, mask, false); 299 break; 300 case RTE_FLOW_ITEM_TYPE_VLAN: 301 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 302 if (!mask) 303 return RTE_FLOW_ITEM_TYPE_VOID; 304 ret = mlx5_ethertype_to_item_type(spec, mask, false); 305 break; 306 case RTE_FLOW_ITEM_TYPE_IPV4: 307 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 308 if (!mask) 309 return RTE_FLOW_ITEM_TYPE_VOID; 310 ret = mlx5_inet_proto_to_item_type(spec, mask); 311 break; 312 case RTE_FLOW_ITEM_TYPE_IPV6: 313 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 314 if (!mask) 315 return RTE_FLOW_ITEM_TYPE_VOID; 316 ret = mlx5_inet_proto_to_item_type(spec, mask); 317 break; 318 case RTE_FLOW_ITEM_TYPE_GENEVE: 319 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 320 ret = mlx5_ethertype_to_item_type(spec, mask, true); 321 break; 322 case RTE_FLOW_ITEM_TYPE_GRE: 323 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 324 ret = mlx5_ethertype_to_item_type(spec, mask, true); 325 break; 326 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 327 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 328 ret = mlx5_nsh_proto_to_item_type(spec, mask); 329 break; 330 default: 331 ret = RTE_FLOW_ITEM_TYPE_VOID; 332 break; 333 } 334 return ret; 335 #undef MLX5_XSET_ITEM_MASK_SPEC 336 } 337 338 static const int * 339 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 340 const int *next_node) 341 { 342 const struct mlx5_flow_expand_node *node = NULL; 343 const int *next = next_node; 344 345 while (next && *next) { 346 /* 347 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 348 * flag set, because they were not found in the flow pattern. 349 */ 350 node = &graph[*next]; 351 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 352 break; 353 next = node->next; 354 } 355 return next; 356 } 357 358 #define MLX5_RSS_EXP_ELT_N 16 359 360 /** 361 * Expand RSS flows into several possible flows according to the RSS hash 362 * fields requested and the driver capabilities. 363 * 364 * @param[out] buf 365 * Buffer to store the result expansion. 366 * @param[in] size 367 * Buffer size in bytes. If 0, @p buf can be NULL. 368 * @param[in] pattern 369 * User flow pattern. 370 * @param[in] types 371 * RSS types to expand (see RTE_ETH_RSS_* definitions). 372 * @param[in] graph 373 * Input graph to expand @p pattern according to @p types. 374 * @param[in] graph_root_index 375 * Index of root node in @p graph, typically 0. 376 * 377 * @return 378 * A positive value representing the size of @p buf in bytes regardless of 379 * @p size on success, a negative errno value otherwise and rte_errno is 380 * set, the following errors are defined: 381 * 382 * -E2BIG: graph-depth @p graph is too deep. 383 * -EINVAL: @p size has not enough space for expanded pattern. 384 */ 385 static int 386 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 387 const struct rte_flow_item *pattern, uint64_t types, 388 const struct mlx5_flow_expand_node graph[], 389 int graph_root_index) 390 { 391 const struct rte_flow_item *item; 392 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 393 const int *next_node; 394 const int *stack[MLX5_RSS_EXP_ELT_N]; 395 int stack_pos = 0; 396 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 397 unsigned int i, item_idx, last_expand_item_idx = 0; 398 size_t lsize; 399 size_t user_pattern_size = 0; 400 void *addr = NULL; 401 const struct mlx5_flow_expand_node *next = NULL; 402 struct rte_flow_item missed_item; 403 int missed = 0; 404 int elt = 0; 405 const struct rte_flow_item *last_expand_item = NULL; 406 407 memset(&missed_item, 0, sizeof(missed_item)); 408 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 409 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 410 if (lsize > size) 411 return -EINVAL; 412 buf->entry[0].priority = 0; 413 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 414 buf->entries = 0; 415 addr = buf->entry[0].pattern; 416 for (item = pattern, item_idx = 0; 417 item->type != RTE_FLOW_ITEM_TYPE_END; 418 item++, item_idx++) { 419 if (!mlx5_flow_is_rss_expandable_item(item)) { 420 user_pattern_size += sizeof(*item); 421 continue; 422 } 423 last_expand_item = item; 424 last_expand_item_idx = item_idx; 425 i = 0; 426 while (node->next && node->next[i]) { 427 next = &graph[node->next[i]]; 428 if (next->type == item->type) 429 break; 430 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 431 node = next; 432 i = 0; 433 } else { 434 ++i; 435 } 436 } 437 if (next) 438 node = next; 439 user_pattern_size += sizeof(*item); 440 } 441 user_pattern_size += sizeof(*item); /* Handle END item. */ 442 lsize += user_pattern_size; 443 if (lsize > size) 444 return -EINVAL; 445 /* Copy the user pattern in the first entry of the buffer. */ 446 rte_memcpy(addr, pattern, user_pattern_size); 447 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 448 buf->entries = 1; 449 /* Start expanding. */ 450 memset(flow_items, 0, sizeof(flow_items)); 451 user_pattern_size -= sizeof(*item); 452 /* 453 * Check if the last valid item has spec set, need complete pattern, 454 * and the pattern can be used for expansion. 455 */ 456 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 457 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 458 /* Item type END indicates expansion is not required. */ 459 return lsize; 460 } 461 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 462 next = NULL; 463 missed = 1; 464 i = 0; 465 while (node->next && node->next[i]) { 466 next = &graph[node->next[i]]; 467 if (next->type == missed_item.type) { 468 flow_items[0].type = missed_item.type; 469 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 470 break; 471 } 472 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 473 node = next; 474 i = 0; 475 } else { 476 ++i; 477 } 478 next = NULL; 479 } 480 } 481 if (next && missed) { 482 elt = 2; /* missed item + item end. */ 483 node = next; 484 lsize += elt * sizeof(*item) + user_pattern_size; 485 if (lsize > size) 486 return -EINVAL; 487 if (node->rss_types & types) { 488 buf->entry[buf->entries].priority = 1; 489 buf->entry[buf->entries].pattern = addr; 490 buf->entries++; 491 rte_memcpy(addr, buf->entry[0].pattern, 492 user_pattern_size); 493 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 494 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 495 addr = (void *)(((uintptr_t)addr) + 496 elt * sizeof(*item)); 497 } 498 } else if (last_expand_item != NULL) { 499 node = mlx5_flow_expand_rss_adjust_node(pattern, 500 last_expand_item_idx, graph, node); 501 } 502 memset(flow_items, 0, sizeof(flow_items)); 503 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 504 node->next); 505 stack[stack_pos] = next_node; 506 node = next_node ? &graph[*next_node] : NULL; 507 while (node) { 508 flow_items[stack_pos].type = node->type; 509 if (node->rss_types & types) { 510 size_t n; 511 /* 512 * compute the number of items to copy from the 513 * expansion and copy it. 514 * When the stack_pos is 0, there are 1 element in it, 515 * plus the addition END item. 516 */ 517 elt = stack_pos + 2; 518 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 519 lsize += elt * sizeof(*item) + user_pattern_size; 520 if (lsize > size) 521 return -EINVAL; 522 n = elt * sizeof(*item); 523 buf->entry[buf->entries].priority = 524 stack_pos + 1 + missed; 525 buf->entry[buf->entries].pattern = addr; 526 buf->entries++; 527 rte_memcpy(addr, buf->entry[0].pattern, 528 user_pattern_size); 529 addr = (void *)(((uintptr_t)addr) + 530 user_pattern_size); 531 rte_memcpy(addr, &missed_item, 532 missed * sizeof(*item)); 533 addr = (void *)(((uintptr_t)addr) + 534 missed * sizeof(*item)); 535 rte_memcpy(addr, flow_items, n); 536 addr = (void *)(((uintptr_t)addr) + n); 537 } 538 /* Go deeper. */ 539 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 540 node->next) { 541 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 542 node->next); 543 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 544 rte_errno = E2BIG; 545 return -rte_errno; 546 } 547 stack[stack_pos] = next_node; 548 } else if (*(next_node + 1)) { 549 /* Follow up with the next possibility. */ 550 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 551 ++next_node); 552 } else if (!stack_pos) { 553 /* 554 * Completing the traverse over the different paths. 555 * The next_node is advanced to the terminator. 556 */ 557 ++next_node; 558 } else { 559 /* Move to the next path. */ 560 while (stack_pos) { 561 next_node = stack[--stack_pos]; 562 next_node++; 563 if (*next_node) 564 break; 565 } 566 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 567 next_node); 568 stack[stack_pos] = next_node; 569 } 570 node = next_node && *next_node ? &graph[*next_node] : NULL; 571 }; 572 return lsize; 573 } 574 575 enum mlx5_expansion { 576 MLX5_EXPANSION_ROOT, 577 MLX5_EXPANSION_ROOT_OUTER, 578 MLX5_EXPANSION_OUTER_ETH, 579 MLX5_EXPANSION_OUTER_VLAN, 580 MLX5_EXPANSION_OUTER_IPV4, 581 MLX5_EXPANSION_OUTER_IPV4_UDP, 582 MLX5_EXPANSION_OUTER_IPV4_TCP, 583 MLX5_EXPANSION_OUTER_IPV4_ESP, 584 MLX5_EXPANSION_OUTER_IPV4_ICMP, 585 MLX5_EXPANSION_OUTER_IPV6, 586 MLX5_EXPANSION_OUTER_IPV6_UDP, 587 MLX5_EXPANSION_OUTER_IPV6_TCP, 588 MLX5_EXPANSION_OUTER_IPV6_ESP, 589 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 590 MLX5_EXPANSION_VXLAN, 591 MLX5_EXPANSION_STD_VXLAN, 592 MLX5_EXPANSION_L3_VXLAN, 593 MLX5_EXPANSION_VXLAN_GPE, 594 MLX5_EXPANSION_GRE, 595 MLX5_EXPANSION_NVGRE, 596 MLX5_EXPANSION_GRE_KEY, 597 MLX5_EXPANSION_MPLS, 598 MLX5_EXPANSION_ETH, 599 MLX5_EXPANSION_VLAN, 600 MLX5_EXPANSION_IPV4, 601 MLX5_EXPANSION_IPV4_UDP, 602 MLX5_EXPANSION_IPV4_TCP, 603 MLX5_EXPANSION_IPV4_ESP, 604 MLX5_EXPANSION_IPV4_ICMP, 605 MLX5_EXPANSION_IPV6, 606 MLX5_EXPANSION_IPV6_UDP, 607 MLX5_EXPANSION_IPV6_TCP, 608 MLX5_EXPANSION_IPV6_ESP, 609 MLX5_EXPANSION_IPV6_ICMP6, 610 MLX5_EXPANSION_IPV6_FRAG_EXT, 611 MLX5_EXPANSION_GTP, 612 MLX5_EXPANSION_GENEVE, 613 }; 614 615 /** Supported expansion of items. */ 616 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 617 [MLX5_EXPANSION_ROOT] = { 618 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 619 MLX5_EXPANSION_IPV4, 620 MLX5_EXPANSION_IPV6), 621 .type = RTE_FLOW_ITEM_TYPE_END, 622 }, 623 [MLX5_EXPANSION_ROOT_OUTER] = { 624 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 625 MLX5_EXPANSION_OUTER_IPV4, 626 MLX5_EXPANSION_OUTER_IPV6), 627 .type = RTE_FLOW_ITEM_TYPE_END, 628 }, 629 [MLX5_EXPANSION_OUTER_ETH] = { 630 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 631 .type = RTE_FLOW_ITEM_TYPE_ETH, 632 .rss_types = 0, 633 }, 634 [MLX5_EXPANSION_OUTER_VLAN] = { 635 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 636 MLX5_EXPANSION_OUTER_IPV6), 637 .type = RTE_FLOW_ITEM_TYPE_VLAN, 638 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 639 }, 640 [MLX5_EXPANSION_OUTER_IPV4] = { 641 .next = MLX5_FLOW_EXPAND_RSS_NEXT 642 (MLX5_EXPANSION_OUTER_IPV4_UDP, 643 MLX5_EXPANSION_OUTER_IPV4_TCP, 644 MLX5_EXPANSION_OUTER_IPV4_ESP, 645 MLX5_EXPANSION_OUTER_IPV4_ICMP, 646 MLX5_EXPANSION_GRE, 647 MLX5_EXPANSION_NVGRE, 648 MLX5_EXPANSION_IPV4, 649 MLX5_EXPANSION_IPV6), 650 .type = RTE_FLOW_ITEM_TYPE_IPV4, 651 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 652 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 653 }, 654 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 655 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 656 MLX5_EXPANSION_VXLAN_GPE, 657 MLX5_EXPANSION_MPLS, 658 MLX5_EXPANSION_GENEVE, 659 MLX5_EXPANSION_GTP), 660 .type = RTE_FLOW_ITEM_TYPE_UDP, 661 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 662 }, 663 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 664 .type = RTE_FLOW_ITEM_TYPE_TCP, 665 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 666 }, 667 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 668 .type = RTE_FLOW_ITEM_TYPE_ESP, 669 .rss_types = RTE_ETH_RSS_ESP, 670 }, 671 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 672 .type = RTE_FLOW_ITEM_TYPE_ICMP, 673 }, 674 [MLX5_EXPANSION_OUTER_IPV6] = { 675 .next = MLX5_FLOW_EXPAND_RSS_NEXT 676 (MLX5_EXPANSION_OUTER_IPV6_UDP, 677 MLX5_EXPANSION_OUTER_IPV6_TCP, 678 MLX5_EXPANSION_OUTER_IPV6_ESP, 679 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 680 MLX5_EXPANSION_IPV4, 681 MLX5_EXPANSION_IPV6, 682 MLX5_EXPANSION_GRE, 683 MLX5_EXPANSION_NVGRE), 684 .type = RTE_FLOW_ITEM_TYPE_IPV6, 685 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 686 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 687 }, 688 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 689 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 690 MLX5_EXPANSION_VXLAN_GPE, 691 MLX5_EXPANSION_MPLS, 692 MLX5_EXPANSION_GENEVE, 693 MLX5_EXPANSION_GTP), 694 .type = RTE_FLOW_ITEM_TYPE_UDP, 695 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 696 }, 697 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 698 .type = RTE_FLOW_ITEM_TYPE_TCP, 699 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 700 }, 701 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 702 .type = RTE_FLOW_ITEM_TYPE_ESP, 703 .rss_types = RTE_ETH_RSS_ESP, 704 }, 705 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 706 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 707 }, 708 [MLX5_EXPANSION_VXLAN] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 710 MLX5_EXPANSION_IPV4, 711 MLX5_EXPANSION_IPV6), 712 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 713 }, 714 [MLX5_EXPANSION_STD_VXLAN] = { 715 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 716 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 717 }, 718 [MLX5_EXPANSION_L3_VXLAN] = { 719 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 720 MLX5_EXPANSION_IPV6), 721 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 722 }, 723 [MLX5_EXPANSION_VXLAN_GPE] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 725 MLX5_EXPANSION_IPV4, 726 MLX5_EXPANSION_IPV6), 727 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 728 }, 729 [MLX5_EXPANSION_GRE] = { 730 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 731 MLX5_EXPANSION_IPV4, 732 MLX5_EXPANSION_IPV6, 733 MLX5_EXPANSION_GRE_KEY, 734 MLX5_EXPANSION_MPLS), 735 .type = RTE_FLOW_ITEM_TYPE_GRE, 736 }, 737 [MLX5_EXPANSION_GRE_KEY] = { 738 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 739 MLX5_EXPANSION_IPV6, 740 MLX5_EXPANSION_MPLS), 741 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 742 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 743 }, 744 [MLX5_EXPANSION_NVGRE] = { 745 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 746 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 747 }, 748 [MLX5_EXPANSION_MPLS] = { 749 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6, 751 MLX5_EXPANSION_ETH), 752 .type = RTE_FLOW_ITEM_TYPE_MPLS, 753 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 754 }, 755 [MLX5_EXPANSION_ETH] = { 756 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 757 .type = RTE_FLOW_ITEM_TYPE_ETH, 758 }, 759 [MLX5_EXPANSION_VLAN] = { 760 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 761 MLX5_EXPANSION_IPV6), 762 .type = RTE_FLOW_ITEM_TYPE_VLAN, 763 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 764 }, 765 [MLX5_EXPANSION_IPV4] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 767 MLX5_EXPANSION_IPV4_TCP, 768 MLX5_EXPANSION_IPV4_ESP, 769 MLX5_EXPANSION_IPV4_ICMP), 770 .type = RTE_FLOW_ITEM_TYPE_IPV4, 771 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 772 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 773 }, 774 [MLX5_EXPANSION_IPV4_UDP] = { 775 .type = RTE_FLOW_ITEM_TYPE_UDP, 776 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 777 }, 778 [MLX5_EXPANSION_IPV4_TCP] = { 779 .type = RTE_FLOW_ITEM_TYPE_TCP, 780 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 781 }, 782 [MLX5_EXPANSION_IPV4_ESP] = { 783 .type = RTE_FLOW_ITEM_TYPE_ESP, 784 .rss_types = RTE_ETH_RSS_ESP, 785 }, 786 [MLX5_EXPANSION_IPV4_ICMP] = { 787 .type = RTE_FLOW_ITEM_TYPE_ICMP, 788 }, 789 [MLX5_EXPANSION_IPV6] = { 790 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 791 MLX5_EXPANSION_IPV6_TCP, 792 MLX5_EXPANSION_IPV6_ESP, 793 MLX5_EXPANSION_IPV6_ICMP6, 794 MLX5_EXPANSION_IPV6_FRAG_EXT), 795 .type = RTE_FLOW_ITEM_TYPE_IPV6, 796 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 797 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 798 }, 799 [MLX5_EXPANSION_IPV6_UDP] = { 800 .type = RTE_FLOW_ITEM_TYPE_UDP, 801 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 802 }, 803 [MLX5_EXPANSION_IPV6_TCP] = { 804 .type = RTE_FLOW_ITEM_TYPE_TCP, 805 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 806 }, 807 [MLX5_EXPANSION_IPV6_ESP] = { 808 .type = RTE_FLOW_ITEM_TYPE_ESP, 809 .rss_types = RTE_ETH_RSS_ESP, 810 }, 811 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 812 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 813 }, 814 [MLX5_EXPANSION_IPV6_ICMP6] = { 815 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 816 }, 817 [MLX5_EXPANSION_GTP] = { 818 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 819 MLX5_EXPANSION_IPV6), 820 .type = RTE_FLOW_ITEM_TYPE_GTP, 821 }, 822 [MLX5_EXPANSION_GENEVE] = { 823 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 824 MLX5_EXPANSION_IPV4, 825 MLX5_EXPANSION_IPV6), 826 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 827 }, 828 }; 829 830 static struct rte_flow_action_handle * 831 mlx5_action_handle_create(struct rte_eth_dev *dev, 832 const struct rte_flow_indir_action_conf *conf, 833 const struct rte_flow_action *action, 834 struct rte_flow_error *error); 835 static int mlx5_action_handle_destroy 836 (struct rte_eth_dev *dev, 837 struct rte_flow_action_handle *handle, 838 struct rte_flow_error *error); 839 static int mlx5_action_handle_update 840 (struct rte_eth_dev *dev, 841 struct rte_flow_action_handle *handle, 842 const void *update, 843 struct rte_flow_error *error); 844 static int mlx5_action_handle_query 845 (struct rte_eth_dev *dev, 846 const struct rte_flow_action_handle *handle, 847 void *data, 848 struct rte_flow_error *error); 849 static int 850 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 851 struct rte_flow_tunnel *app_tunnel, 852 struct rte_flow_action **actions, 853 uint32_t *num_of_actions, 854 struct rte_flow_error *error); 855 static int 856 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 857 struct rte_flow_tunnel *app_tunnel, 858 struct rte_flow_item **items, 859 uint32_t *num_of_items, 860 struct rte_flow_error *error); 861 static int 862 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 863 struct rte_flow_item *pmd_items, 864 uint32_t num_items, struct rte_flow_error *err); 865 static int 866 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 867 struct rte_flow_action *pmd_actions, 868 uint32_t num_actions, 869 struct rte_flow_error *err); 870 static int 871 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 872 struct rte_mbuf *m, 873 struct rte_flow_restore_info *info, 874 struct rte_flow_error *err); 875 static struct rte_flow_item_flex_handle * 876 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 877 const struct rte_flow_item_flex_conf *conf, 878 struct rte_flow_error *error); 879 static int 880 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 881 const struct rte_flow_item_flex_handle *handle, 882 struct rte_flow_error *error); 883 static int 884 mlx5_flow_info_get(struct rte_eth_dev *dev, 885 struct rte_flow_port_info *port_info, 886 struct rte_flow_queue_info *queue_info, 887 struct rte_flow_error *error); 888 static int 889 mlx5_flow_port_configure(struct rte_eth_dev *dev, 890 const struct rte_flow_port_attr *port_attr, 891 uint16_t nb_queue, 892 const struct rte_flow_queue_attr *queue_attr[], 893 struct rte_flow_error *err); 894 895 static struct rte_flow_pattern_template * 896 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 897 const struct rte_flow_pattern_template_attr *attr, 898 const struct rte_flow_item items[], 899 struct rte_flow_error *error); 900 901 static int 902 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 903 struct rte_flow_pattern_template *template, 904 struct rte_flow_error *error); 905 static struct rte_flow_actions_template * 906 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 907 const struct rte_flow_actions_template_attr *attr, 908 const struct rte_flow_action actions[], 909 const struct rte_flow_action masks[], 910 struct rte_flow_error *error); 911 static int 912 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 913 struct rte_flow_actions_template *template, 914 struct rte_flow_error *error); 915 916 static struct rte_flow_template_table * 917 mlx5_flow_table_create(struct rte_eth_dev *dev, 918 const struct rte_flow_template_table_attr *attr, 919 struct rte_flow_pattern_template *item_templates[], 920 uint8_t nb_item_templates, 921 struct rte_flow_actions_template *action_templates[], 922 uint8_t nb_action_templates, 923 struct rte_flow_error *error); 924 static int 925 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 926 struct rte_flow_template_table *table, 927 struct rte_flow_error *error); 928 static struct rte_flow * 929 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 930 uint32_t queue, 931 const struct rte_flow_op_attr *attr, 932 struct rte_flow_template_table *table, 933 const struct rte_flow_item items[], 934 uint8_t pattern_template_index, 935 const struct rte_flow_action actions[], 936 uint8_t action_template_index, 937 void *user_data, 938 struct rte_flow_error *error); 939 static int 940 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 941 uint32_t queue, 942 const struct rte_flow_op_attr *attr, 943 struct rte_flow *flow, 944 void *user_data, 945 struct rte_flow_error *error); 946 static int 947 mlx5_flow_pull(struct rte_eth_dev *dev, 948 uint32_t queue, 949 struct rte_flow_op_result res[], 950 uint16_t n_res, 951 struct rte_flow_error *error); 952 static int 953 mlx5_flow_push(struct rte_eth_dev *dev, 954 uint32_t queue, 955 struct rte_flow_error *error); 956 957 static struct rte_flow_action_handle * 958 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 959 const struct rte_flow_op_attr *attr, 960 const struct rte_flow_indir_action_conf *conf, 961 const struct rte_flow_action *action, 962 void *user_data, 963 struct rte_flow_error *error); 964 965 static int 966 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 967 const struct rte_flow_op_attr *attr, 968 struct rte_flow_action_handle *handle, 969 const void *update, 970 void *user_data, 971 struct rte_flow_error *error); 972 973 static int 974 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 975 const struct rte_flow_op_attr *attr, 976 struct rte_flow_action_handle *handle, 977 void *user_data, 978 struct rte_flow_error *error); 979 980 static int 981 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 982 const struct rte_flow_op_attr *attr, 983 const struct rte_flow_action_handle *handle, 984 void *data, 985 void *user_data, 986 struct rte_flow_error *error); 987 988 static const struct rte_flow_ops mlx5_flow_ops = { 989 .validate = mlx5_flow_validate, 990 .create = mlx5_flow_create, 991 .destroy = mlx5_flow_destroy, 992 .flush = mlx5_flow_flush, 993 .isolate = mlx5_flow_isolate, 994 .query = mlx5_flow_query, 995 .dev_dump = mlx5_flow_dev_dump, 996 #ifdef MLX5_HAVE_RTE_FLOW_Q_AGE 997 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 998 #endif 999 .get_aged_flows = mlx5_flow_get_aged_flows, 1000 .action_handle_create = mlx5_action_handle_create, 1001 .action_handle_destroy = mlx5_action_handle_destroy, 1002 .action_handle_update = mlx5_action_handle_update, 1003 .action_handle_query = mlx5_action_handle_query, 1004 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1005 .tunnel_match = mlx5_flow_tunnel_match, 1006 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1007 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1008 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1009 .flex_item_create = mlx5_flow_flex_item_create, 1010 .flex_item_release = mlx5_flow_flex_item_release, 1011 .info_get = mlx5_flow_info_get, 1012 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1013 .configure = mlx5_flow_port_configure, 1014 .pattern_template_create = mlx5_flow_pattern_template_create, 1015 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1016 .actions_template_create = mlx5_flow_actions_template_create, 1017 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1018 .template_table_create = mlx5_flow_table_create, 1019 .template_table_destroy = mlx5_flow_table_destroy, 1020 .async_create = mlx5_flow_async_flow_create, 1021 .async_destroy = mlx5_flow_async_flow_destroy, 1022 .pull = mlx5_flow_pull, 1023 .push = mlx5_flow_push, 1024 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1025 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1026 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1027 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1028 }; 1029 1030 /* Tunnel information. */ 1031 struct mlx5_flow_tunnel_info { 1032 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1033 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1034 }; 1035 1036 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1037 { 1038 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1039 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1040 }, 1041 { 1042 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1043 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1044 }, 1045 { 1046 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1047 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1048 }, 1049 { 1050 .tunnel = MLX5_FLOW_LAYER_GRE, 1051 .ptype = RTE_PTYPE_TUNNEL_GRE, 1052 }, 1053 { 1054 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1055 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1056 }, 1057 { 1058 .tunnel = MLX5_FLOW_LAYER_MPLS, 1059 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1060 }, 1061 { 1062 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1063 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1064 }, 1065 { 1066 .tunnel = MLX5_FLOW_LAYER_IPIP, 1067 .ptype = RTE_PTYPE_TUNNEL_IP, 1068 }, 1069 { 1070 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1071 .ptype = RTE_PTYPE_TUNNEL_IP, 1072 }, 1073 { 1074 .tunnel = MLX5_FLOW_LAYER_GTP, 1075 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1076 }, 1077 }; 1078 1079 1080 1081 /** 1082 * Translate tag ID to register. 1083 * 1084 * @param[in] dev 1085 * Pointer to the Ethernet device structure. 1086 * @param[in] feature 1087 * The feature that request the register. 1088 * @param[in] id 1089 * The request register ID. 1090 * @param[out] error 1091 * Error description in case of any. 1092 * 1093 * @return 1094 * The request register on success, a negative errno 1095 * value otherwise and rte_errno is set. 1096 */ 1097 int 1098 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1099 enum mlx5_feature_name feature, 1100 uint32_t id, 1101 struct rte_flow_error *error) 1102 { 1103 struct mlx5_priv *priv = dev->data->dev_private; 1104 struct mlx5_sh_config *config = &priv->sh->config; 1105 enum modify_reg start_reg; 1106 bool skip_mtr_reg = false; 1107 1108 switch (feature) { 1109 case MLX5_HAIRPIN_RX: 1110 return REG_B; 1111 case MLX5_HAIRPIN_TX: 1112 return REG_A; 1113 case MLX5_METADATA_RX: 1114 switch (config->dv_xmeta_en) { 1115 case MLX5_XMETA_MODE_LEGACY: 1116 return REG_B; 1117 case MLX5_XMETA_MODE_META16: 1118 return REG_C_0; 1119 case MLX5_XMETA_MODE_META32: 1120 return REG_C_1; 1121 case MLX5_XMETA_MODE_META32_HWS: 1122 return REG_C_1; 1123 } 1124 break; 1125 case MLX5_METADATA_TX: 1126 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1127 return REG_C_1; 1128 } else { 1129 return REG_A; 1130 } 1131 case MLX5_METADATA_FDB: 1132 switch (config->dv_xmeta_en) { 1133 case MLX5_XMETA_MODE_LEGACY: 1134 return REG_NON; 1135 case MLX5_XMETA_MODE_META16: 1136 return REG_C_0; 1137 case MLX5_XMETA_MODE_META32: 1138 return REG_C_1; 1139 case MLX5_XMETA_MODE_META32_HWS: 1140 return REG_C_1; 1141 } 1142 break; 1143 case MLX5_FLOW_MARK: 1144 switch (config->dv_xmeta_en) { 1145 case MLX5_XMETA_MODE_LEGACY: 1146 case MLX5_XMETA_MODE_META32_HWS: 1147 return REG_NON; 1148 case MLX5_XMETA_MODE_META16: 1149 return REG_C_1; 1150 case MLX5_XMETA_MODE_META32: 1151 return REG_C_0; 1152 } 1153 break; 1154 case MLX5_MTR_ID: 1155 /* 1156 * If meter color and meter id share one register, flow match 1157 * should use the meter color register for match. 1158 */ 1159 if (priv->mtr_reg_share) 1160 return priv->mtr_color_reg; 1161 else 1162 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1163 REG_C_3; 1164 case MLX5_MTR_COLOR: 1165 case MLX5_ASO_FLOW_HIT: 1166 case MLX5_ASO_CONNTRACK: 1167 case MLX5_SAMPLE_ID: 1168 /* All features use the same REG_C. */ 1169 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1170 return priv->mtr_color_reg; 1171 case MLX5_COPY_MARK: 1172 /* 1173 * Metadata COPY_MARK register using is in meter suffix sub 1174 * flow while with meter. It's safe to share the same register. 1175 */ 1176 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1177 case MLX5_APP_TAG: 1178 /* 1179 * If meter is enable, it will engage the register for color 1180 * match and flow match. If meter color match is not using the 1181 * REG_C_2, need to skip the REG_C_x be used by meter color 1182 * match. 1183 * If meter is disable, free to use all available registers. 1184 */ 1185 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1186 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1187 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1188 if (id > (uint32_t)(REG_C_7 - start_reg)) 1189 return rte_flow_error_set(error, EINVAL, 1190 RTE_FLOW_ERROR_TYPE_ITEM, 1191 NULL, "invalid tag id"); 1192 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1193 return rte_flow_error_set(error, ENOTSUP, 1194 RTE_FLOW_ERROR_TYPE_ITEM, 1195 NULL, "unsupported tag id"); 1196 /* 1197 * This case means meter is using the REG_C_x great than 2. 1198 * Take care not to conflict with meter color REG_C_x. 1199 * If the available index REG_C_y >= REG_C_x, skip the 1200 * color register. 1201 */ 1202 if (skip_mtr_reg && priv->sh->flow_mreg_c 1203 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1204 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1205 return rte_flow_error_set(error, EINVAL, 1206 RTE_FLOW_ERROR_TYPE_ITEM, 1207 NULL, "invalid tag id"); 1208 if (priv->sh->flow_mreg_c 1209 [id + 1 + start_reg - REG_C_0] != REG_NON) 1210 return priv->sh->flow_mreg_c 1211 [id + 1 + start_reg - REG_C_0]; 1212 return rte_flow_error_set(error, ENOTSUP, 1213 RTE_FLOW_ERROR_TYPE_ITEM, 1214 NULL, "unsupported tag id"); 1215 } 1216 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1217 } 1218 MLX5_ASSERT(false); 1219 return rte_flow_error_set(error, EINVAL, 1220 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1221 NULL, "invalid feature name"); 1222 } 1223 1224 /** 1225 * Check extensive flow metadata register support. 1226 * 1227 * @param dev 1228 * Pointer to rte_eth_dev structure. 1229 * 1230 * @return 1231 * True if device supports extensive flow metadata register, otherwise false. 1232 */ 1233 bool 1234 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1235 { 1236 struct mlx5_priv *priv = dev->data->dev_private; 1237 1238 /* 1239 * Having available reg_c can be regarded inclusively as supporting 1240 * extensive flow metadata register, which could mean, 1241 * - metadata register copy action by modify header. 1242 * - 16 modify header actions is supported. 1243 * - reg_c's are preserved across different domain (FDB and NIC) on 1244 * packet loopback by flow lookup miss. 1245 */ 1246 return priv->sh->flow_mreg_c[2] != REG_NON; 1247 } 1248 1249 /** 1250 * Get the lowest priority. 1251 * 1252 * @param[in] dev 1253 * Pointer to the Ethernet device structure. 1254 * @param[in] attributes 1255 * Pointer to device flow rule attributes. 1256 * 1257 * @return 1258 * The value of lowest priority of flow. 1259 */ 1260 uint32_t 1261 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1262 const struct rte_flow_attr *attr) 1263 { 1264 struct mlx5_priv *priv = dev->data->dev_private; 1265 1266 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1267 return priv->sh->flow_max_priority - 2; 1268 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1269 } 1270 1271 /** 1272 * Calculate matcher priority of the flow. 1273 * 1274 * @param[in] dev 1275 * Pointer to the Ethernet device structure. 1276 * @param[in] attr 1277 * Pointer to device flow rule attributes. 1278 * @param[in] subpriority 1279 * The priority based on the items. 1280 * @param[in] external 1281 * Flow is user flow. 1282 * @return 1283 * The matcher priority of the flow. 1284 */ 1285 uint16_t 1286 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1287 const struct rte_flow_attr *attr, 1288 uint32_t subpriority, bool external) 1289 { 1290 uint16_t priority = (uint16_t)attr->priority; 1291 struct mlx5_priv *priv = dev->data->dev_private; 1292 1293 /* NIC root rules */ 1294 if (!attr->group && !attr->transfer) { 1295 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1296 priority = priv->sh->flow_max_priority - 1; 1297 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1298 /* FDB root rules */ 1299 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1300 attr->group == 0 && 1301 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1302 return (priv->sh->flow_max_priority - 1) * 3; 1303 } 1304 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1305 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1306 return priority * 3 + subpriority; 1307 } 1308 1309 /** 1310 * Verify the @p item specifications (spec, last, mask) are compatible with the 1311 * NIC capabilities. 1312 * 1313 * @param[in] item 1314 * Item specification. 1315 * @param[in] mask 1316 * @p item->mask or flow default bit-masks. 1317 * @param[in] nic_mask 1318 * Bit-masks covering supported fields by the NIC to compare with user mask. 1319 * @param[in] size 1320 * Bit-masks size in bytes. 1321 * @param[in] range_accepted 1322 * True if range of values is accepted for specific fields, false otherwise. 1323 * @param[out] error 1324 * Pointer to error structure. 1325 * 1326 * @return 1327 * 0 on success, a negative errno value otherwise and rte_errno is set. 1328 */ 1329 int 1330 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1331 const uint8_t *mask, 1332 const uint8_t *nic_mask, 1333 unsigned int size, 1334 bool range_accepted, 1335 struct rte_flow_error *error) 1336 { 1337 unsigned int i; 1338 1339 MLX5_ASSERT(nic_mask); 1340 for (i = 0; i < size; ++i) 1341 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1342 return rte_flow_error_set(error, ENOTSUP, 1343 RTE_FLOW_ERROR_TYPE_ITEM, 1344 item, 1345 "mask enables non supported" 1346 " bits"); 1347 if (!item->spec && (item->mask || item->last)) 1348 return rte_flow_error_set(error, EINVAL, 1349 RTE_FLOW_ERROR_TYPE_ITEM, item, 1350 "mask/last without a spec is not" 1351 " supported"); 1352 if (item->spec && item->last && !range_accepted) { 1353 uint8_t spec[size]; 1354 uint8_t last[size]; 1355 unsigned int i; 1356 int ret; 1357 1358 for (i = 0; i < size; ++i) { 1359 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1360 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1361 } 1362 ret = memcmp(spec, last, size); 1363 if (ret != 0) 1364 return rte_flow_error_set(error, EINVAL, 1365 RTE_FLOW_ERROR_TYPE_ITEM, 1366 item, 1367 "range is not valid"); 1368 } 1369 return 0; 1370 } 1371 1372 /** 1373 * Adjust the hash fields according to the @p flow information. 1374 * 1375 * @param[in] dev_flow. 1376 * Pointer to the mlx5_flow. 1377 * @param[in] tunnel 1378 * 1 when the hash field is for a tunnel item. 1379 * @param[in] layer_types 1380 * RTE_ETH_RSS_* types. 1381 * @param[in] hash_fields 1382 * Item hash fields. 1383 * 1384 * @return 1385 * The hash fields that should be used. 1386 */ 1387 uint64_t 1388 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1389 int tunnel __rte_unused, uint64_t layer_types, 1390 uint64_t hash_fields) 1391 { 1392 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1393 int rss_request_inner = rss_desc->level >= 2; 1394 1395 /* Check RSS hash level for tunnel. */ 1396 if (tunnel && rss_request_inner) 1397 hash_fields |= IBV_RX_HASH_INNER; 1398 else if (tunnel || rss_request_inner) 1399 return 0; 1400 #endif 1401 /* Check if requested layer matches RSS hash fields. */ 1402 if (!(rss_desc->types & layer_types)) 1403 return 0; 1404 return hash_fields; 1405 } 1406 1407 /** 1408 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1409 * if several tunnel rules are used on this queue, the tunnel ptype will be 1410 * cleared. 1411 * 1412 * @param rxq_ctrl 1413 * Rx queue to update. 1414 */ 1415 static void 1416 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1417 { 1418 unsigned int i; 1419 uint32_t tunnel_ptype = 0; 1420 1421 /* Look up for the ptype to use. */ 1422 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1423 if (!rxq_ctrl->flow_tunnels_n[i]) 1424 continue; 1425 if (!tunnel_ptype) { 1426 tunnel_ptype = tunnels_info[i].ptype; 1427 } else { 1428 tunnel_ptype = 0; 1429 break; 1430 } 1431 } 1432 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1433 } 1434 1435 /** 1436 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1437 * flow. 1438 * 1439 * @param[in] dev 1440 * Pointer to the Ethernet device structure. 1441 * @param[in] dev_handle 1442 * Pointer to device flow handle structure. 1443 */ 1444 void 1445 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1446 struct mlx5_flow_handle *dev_handle) 1447 { 1448 struct mlx5_priv *priv = dev->data->dev_private; 1449 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1450 struct mlx5_ind_table_obj *ind_tbl = NULL; 1451 unsigned int i; 1452 1453 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1454 struct mlx5_hrxq *hrxq; 1455 1456 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1457 dev_handle->rix_hrxq); 1458 if (hrxq) 1459 ind_tbl = hrxq->ind_table; 1460 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1461 struct mlx5_shared_action_rss *shared_rss; 1462 1463 shared_rss = mlx5_ipool_get 1464 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1465 dev_handle->rix_srss); 1466 if (shared_rss) 1467 ind_tbl = shared_rss->ind_tbl; 1468 } 1469 if (!ind_tbl) 1470 return; 1471 for (i = 0; i != ind_tbl->queues_n; ++i) { 1472 int idx = ind_tbl->queues[i]; 1473 struct mlx5_rxq_ctrl *rxq_ctrl; 1474 1475 if (mlx5_is_external_rxq(dev, idx)) 1476 continue; 1477 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1478 MLX5_ASSERT(rxq_ctrl != NULL); 1479 if (rxq_ctrl == NULL) 1480 continue; 1481 /* 1482 * To support metadata register copy on Tx loopback, 1483 * this must be always enabled (metadata may arive 1484 * from other port - not from local flows only. 1485 */ 1486 if (tunnel) { 1487 unsigned int j; 1488 1489 /* Increase the counter matching the flow. */ 1490 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1491 if ((tunnels_info[j].tunnel & 1492 dev_handle->layers) == 1493 tunnels_info[j].tunnel) { 1494 rxq_ctrl->flow_tunnels_n[j]++; 1495 break; 1496 } 1497 } 1498 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1499 } 1500 } 1501 } 1502 1503 static void 1504 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1505 { 1506 struct mlx5_priv *priv = dev->data->dev_private; 1507 struct mlx5_rxq_ctrl *rxq_ctrl; 1508 uint16_t port_id; 1509 1510 if (priv->sh->shared_mark_enabled) 1511 return; 1512 if (priv->master || priv->representor) { 1513 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1514 struct mlx5_priv *opriv = 1515 rte_eth_devices[port_id].data->dev_private; 1516 1517 if (!opriv || 1518 opriv->sh != priv->sh || 1519 opriv->domain_id != priv->domain_id || 1520 opriv->mark_enabled) 1521 continue; 1522 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1523 rxq_ctrl->rxq.mark = 1; 1524 } 1525 opriv->mark_enabled = 1; 1526 } 1527 } else { 1528 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1529 rxq_ctrl->rxq.mark = 1; 1530 } 1531 priv->mark_enabled = 1; 1532 } 1533 priv->sh->shared_mark_enabled = 1; 1534 } 1535 1536 /** 1537 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1538 * 1539 * @param[in] dev 1540 * Pointer to the Ethernet device structure. 1541 * @param[in] flow 1542 * Pointer to flow structure. 1543 */ 1544 static void 1545 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1546 { 1547 struct mlx5_priv *priv = dev->data->dev_private; 1548 uint32_t handle_idx; 1549 struct mlx5_flow_handle *dev_handle; 1550 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1551 1552 MLX5_ASSERT(wks); 1553 if (wks->mark) 1554 flow_rxq_mark_flag_set(dev); 1555 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1556 handle_idx, dev_handle, next) 1557 flow_drv_rxq_flags_set(dev, dev_handle); 1558 } 1559 1560 /** 1561 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1562 * device flow if no other flow uses it with the same kind of request. 1563 * 1564 * @param dev 1565 * Pointer to Ethernet device. 1566 * @param[in] dev_handle 1567 * Pointer to the device flow handle structure. 1568 */ 1569 static void 1570 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1571 struct mlx5_flow_handle *dev_handle) 1572 { 1573 struct mlx5_priv *priv = dev->data->dev_private; 1574 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1575 struct mlx5_ind_table_obj *ind_tbl = NULL; 1576 unsigned int i; 1577 1578 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1579 struct mlx5_hrxq *hrxq; 1580 1581 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1582 dev_handle->rix_hrxq); 1583 if (hrxq) 1584 ind_tbl = hrxq->ind_table; 1585 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1586 struct mlx5_shared_action_rss *shared_rss; 1587 1588 shared_rss = mlx5_ipool_get 1589 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1590 dev_handle->rix_srss); 1591 if (shared_rss) 1592 ind_tbl = shared_rss->ind_tbl; 1593 } 1594 if (!ind_tbl) 1595 return; 1596 MLX5_ASSERT(dev->data->dev_started); 1597 for (i = 0; i != ind_tbl->queues_n; ++i) { 1598 int idx = ind_tbl->queues[i]; 1599 struct mlx5_rxq_ctrl *rxq_ctrl; 1600 1601 if (mlx5_is_external_rxq(dev, idx)) 1602 continue; 1603 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1604 MLX5_ASSERT(rxq_ctrl != NULL); 1605 if (rxq_ctrl == NULL) 1606 continue; 1607 if (tunnel) { 1608 unsigned int j; 1609 1610 /* Decrease the counter matching the flow. */ 1611 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1612 if ((tunnels_info[j].tunnel & 1613 dev_handle->layers) == 1614 tunnels_info[j].tunnel) { 1615 rxq_ctrl->flow_tunnels_n[j]--; 1616 break; 1617 } 1618 } 1619 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1620 } 1621 } 1622 } 1623 1624 /** 1625 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1626 * @p flow if no other flow uses it with the same kind of request. 1627 * 1628 * @param dev 1629 * Pointer to Ethernet device. 1630 * @param[in] flow 1631 * Pointer to the flow. 1632 */ 1633 static void 1634 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1635 { 1636 struct mlx5_priv *priv = dev->data->dev_private; 1637 uint32_t handle_idx; 1638 struct mlx5_flow_handle *dev_handle; 1639 1640 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1641 handle_idx, dev_handle, next) 1642 flow_drv_rxq_flags_trim(dev, dev_handle); 1643 } 1644 1645 /** 1646 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1647 * 1648 * @param dev 1649 * Pointer to Ethernet device. 1650 */ 1651 static void 1652 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1653 { 1654 struct mlx5_priv *priv = dev->data->dev_private; 1655 unsigned int i; 1656 1657 for (i = 0; i != priv->rxqs_n; ++i) { 1658 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1659 unsigned int j; 1660 1661 if (rxq == NULL || rxq->ctrl == NULL) 1662 continue; 1663 rxq->ctrl->rxq.mark = 0; 1664 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1665 rxq->ctrl->flow_tunnels_n[j] = 0; 1666 rxq->ctrl->rxq.tunnel = 0; 1667 } 1668 priv->mark_enabled = 0; 1669 priv->sh->shared_mark_enabled = 0; 1670 } 1671 1672 /** 1673 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1674 * 1675 * @param[in] dev 1676 * Pointer to the Ethernet device structure. 1677 */ 1678 void 1679 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1680 { 1681 struct mlx5_priv *priv = dev->data->dev_private; 1682 unsigned int i; 1683 1684 for (i = 0; i != priv->rxqs_n; ++i) { 1685 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1686 struct mlx5_rxq_data *data; 1687 1688 if (rxq == NULL || rxq->ctrl == NULL) 1689 continue; 1690 data = &rxq->ctrl->rxq; 1691 if (!rte_flow_dynf_metadata_avail()) { 1692 data->dynf_meta = 0; 1693 data->flow_meta_mask = 0; 1694 data->flow_meta_offset = -1; 1695 data->flow_meta_port_mask = 0; 1696 } else { 1697 data->dynf_meta = 1; 1698 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1699 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1700 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1701 } 1702 } 1703 } 1704 1705 /* 1706 * return a pointer to the desired action in the list of actions. 1707 * 1708 * @param[in] actions 1709 * The list of actions to search the action in. 1710 * @param[in] action 1711 * The action to find. 1712 * 1713 * @return 1714 * Pointer to the action in the list, if found. NULL otherwise. 1715 */ 1716 const struct rte_flow_action * 1717 mlx5_flow_find_action(const struct rte_flow_action *actions, 1718 enum rte_flow_action_type action) 1719 { 1720 if (actions == NULL) 1721 return NULL; 1722 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1723 if (actions->type == action) 1724 return actions; 1725 return NULL; 1726 } 1727 1728 /* 1729 * Validate the flag action. 1730 * 1731 * @param[in] action_flags 1732 * Bit-fields that holds the actions detected until now. 1733 * @param[in] attr 1734 * Attributes of flow that includes this action. 1735 * @param[out] error 1736 * Pointer to error structure. 1737 * 1738 * @return 1739 * 0 on success, a negative errno value otherwise and rte_errno is set. 1740 */ 1741 int 1742 mlx5_flow_validate_action_flag(uint64_t action_flags, 1743 const struct rte_flow_attr *attr, 1744 struct rte_flow_error *error) 1745 { 1746 if (action_flags & MLX5_FLOW_ACTION_MARK) 1747 return rte_flow_error_set(error, EINVAL, 1748 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1749 "can't mark and flag in same flow"); 1750 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1751 return rte_flow_error_set(error, EINVAL, 1752 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1753 "can't have 2 flag" 1754 " actions in same flow"); 1755 if (attr->egress) 1756 return rte_flow_error_set(error, ENOTSUP, 1757 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1758 "flag action not supported for " 1759 "egress"); 1760 return 0; 1761 } 1762 1763 /* 1764 * Validate the mark action. 1765 * 1766 * @param[in] action 1767 * Pointer to the queue action. 1768 * @param[in] action_flags 1769 * Bit-fields that holds the actions detected until now. 1770 * @param[in] attr 1771 * Attributes of flow that includes this action. 1772 * @param[out] error 1773 * Pointer to error structure. 1774 * 1775 * @return 1776 * 0 on success, a negative errno value otherwise and rte_errno is set. 1777 */ 1778 int 1779 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1780 uint64_t action_flags, 1781 const struct rte_flow_attr *attr, 1782 struct rte_flow_error *error) 1783 { 1784 const struct rte_flow_action_mark *mark = action->conf; 1785 1786 if (!mark) 1787 return rte_flow_error_set(error, EINVAL, 1788 RTE_FLOW_ERROR_TYPE_ACTION, 1789 action, 1790 "configuration cannot be null"); 1791 if (mark->id >= MLX5_FLOW_MARK_MAX) 1792 return rte_flow_error_set(error, EINVAL, 1793 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1794 &mark->id, 1795 "mark id must in 0 <= id < " 1796 RTE_STR(MLX5_FLOW_MARK_MAX)); 1797 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1798 return rte_flow_error_set(error, EINVAL, 1799 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1800 "can't flag and mark in same flow"); 1801 if (action_flags & MLX5_FLOW_ACTION_MARK) 1802 return rte_flow_error_set(error, EINVAL, 1803 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1804 "can't have 2 mark actions in same" 1805 " flow"); 1806 if (attr->egress) 1807 return rte_flow_error_set(error, ENOTSUP, 1808 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1809 "mark action not supported for " 1810 "egress"); 1811 return 0; 1812 } 1813 1814 /* 1815 * Validate the drop action. 1816 * 1817 * @param[in] action_flags 1818 * Bit-fields that holds the actions detected until now. 1819 * @param[in] attr 1820 * Attributes of flow that includes this action. 1821 * @param[out] error 1822 * Pointer to error structure. 1823 * 1824 * @return 1825 * 0 on success, a negative errno value otherwise and rte_errno is set. 1826 */ 1827 int 1828 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1829 const struct rte_flow_attr *attr, 1830 struct rte_flow_error *error) 1831 { 1832 if (attr->egress) 1833 return rte_flow_error_set(error, ENOTSUP, 1834 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1835 "drop action not supported for " 1836 "egress"); 1837 return 0; 1838 } 1839 1840 /* 1841 * Validate the queue action. 1842 * 1843 * @param[in] action 1844 * Pointer to the queue action. 1845 * @param[in] action_flags 1846 * Bit-fields that holds the actions detected until now. 1847 * @param[in] dev 1848 * Pointer to the Ethernet device structure. 1849 * @param[in] attr 1850 * Attributes of flow that includes this action. 1851 * @param[out] error 1852 * Pointer to error structure. 1853 * 1854 * @return 1855 * 0 on success, a negative errno value otherwise and rte_errno is set. 1856 */ 1857 int 1858 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1859 uint64_t action_flags, 1860 struct rte_eth_dev *dev, 1861 const struct rte_flow_attr *attr, 1862 struct rte_flow_error *error) 1863 { 1864 struct mlx5_priv *priv = dev->data->dev_private; 1865 const struct rte_flow_action_queue *queue = action->conf; 1866 1867 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1868 return rte_flow_error_set(error, EINVAL, 1869 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1870 "can't have 2 fate actions in" 1871 " same flow"); 1872 if (attr->egress) 1873 return rte_flow_error_set(error, ENOTSUP, 1874 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1875 "queue action not supported for egress."); 1876 if (mlx5_is_external_rxq(dev, queue->index)) 1877 return 0; 1878 if (!priv->rxqs_n) 1879 return rte_flow_error_set(error, EINVAL, 1880 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1881 NULL, "No Rx queues configured"); 1882 if (queue->index >= priv->rxqs_n) 1883 return rte_flow_error_set(error, EINVAL, 1884 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1885 &queue->index, 1886 "queue index out of range"); 1887 if (mlx5_rxq_get(dev, queue->index) == NULL) 1888 return rte_flow_error_set(error, EINVAL, 1889 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1890 &queue->index, 1891 "queue is not configured"); 1892 return 0; 1893 } 1894 1895 /** 1896 * Validate queue numbers for device RSS. 1897 * 1898 * @param[in] dev 1899 * Configured device. 1900 * @param[in] queues 1901 * Array of queue numbers. 1902 * @param[in] queues_n 1903 * Size of the @p queues array. 1904 * @param[out] error 1905 * On error, filled with a textual error description. 1906 * @param[out] queue_idx 1907 * On error, filled with an offending queue index in @p queues array. 1908 * 1909 * @return 1910 * 0 on success, a negative errno code on error. 1911 */ 1912 static int 1913 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1914 const uint16_t *queues, uint32_t queues_n, 1915 const char **error, uint32_t *queue_idx) 1916 { 1917 const struct mlx5_priv *priv = dev->data->dev_private; 1918 bool is_hairpin = false; 1919 bool is_ext_rss = false; 1920 uint32_t i; 1921 1922 for (i = 0; i != queues_n; ++i) { 1923 struct mlx5_rxq_ctrl *rxq_ctrl; 1924 1925 if (mlx5_is_external_rxq(dev, queues[0])) { 1926 is_ext_rss = true; 1927 continue; 1928 } 1929 if (is_ext_rss) { 1930 *error = "Combining external and regular RSS queues is not supported"; 1931 *queue_idx = i; 1932 return -ENOTSUP; 1933 } 1934 if (queues[i] >= priv->rxqs_n) { 1935 *error = "queue index out of range"; 1936 *queue_idx = i; 1937 return -EINVAL; 1938 } 1939 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 1940 if (rxq_ctrl == NULL) { 1941 *error = "queue is not configured"; 1942 *queue_idx = i; 1943 return -EINVAL; 1944 } 1945 if (i == 0 && rxq_ctrl->is_hairpin) 1946 is_hairpin = true; 1947 if (is_hairpin != rxq_ctrl->is_hairpin) { 1948 *error = "combining hairpin and regular RSS queues is not supported"; 1949 *queue_idx = i; 1950 return -ENOTSUP; 1951 } 1952 } 1953 return 0; 1954 } 1955 1956 /* 1957 * Validate the rss action. 1958 * 1959 * @param[in] dev 1960 * Pointer to the Ethernet device structure. 1961 * @param[in] action 1962 * Pointer to the queue action. 1963 * @param[out] error 1964 * Pointer to error structure. 1965 * 1966 * @return 1967 * 0 on success, a negative errno value otherwise and rte_errno is set. 1968 */ 1969 int 1970 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1971 const struct rte_flow_action *action, 1972 struct rte_flow_error *error) 1973 { 1974 struct mlx5_priv *priv = dev->data->dev_private; 1975 const struct rte_flow_action_rss *rss = action->conf; 1976 int ret; 1977 const char *message; 1978 uint32_t queue_idx; 1979 1980 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1981 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1982 return rte_flow_error_set(error, ENOTSUP, 1983 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1984 &rss->func, 1985 "RSS hash function not supported"); 1986 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1987 if (rss->level > 2) 1988 #else 1989 if (rss->level > 1) 1990 #endif 1991 return rte_flow_error_set(error, ENOTSUP, 1992 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1993 &rss->level, 1994 "tunnel RSS is not supported"); 1995 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1996 if (rss->key_len == 0 && rss->key != NULL) 1997 return rte_flow_error_set(error, ENOTSUP, 1998 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1999 &rss->key_len, 2000 "RSS hash key length 0"); 2001 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2002 return rte_flow_error_set(error, ENOTSUP, 2003 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2004 &rss->key_len, 2005 "RSS hash key too small"); 2006 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2007 return rte_flow_error_set(error, ENOTSUP, 2008 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2009 &rss->key_len, 2010 "RSS hash key too large"); 2011 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2012 return rte_flow_error_set(error, ENOTSUP, 2013 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2014 &rss->queue_num, 2015 "number of queues too large"); 2016 if (rss->types & MLX5_RSS_HF_MASK) 2017 return rte_flow_error_set(error, ENOTSUP, 2018 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2019 &rss->types, 2020 "some RSS protocols are not" 2021 " supported"); 2022 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2023 !(rss->types & RTE_ETH_RSS_IP)) 2024 return rte_flow_error_set(error, EINVAL, 2025 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2026 "L3 partial RSS requested but L3 RSS" 2027 " type not specified"); 2028 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2029 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2030 return rte_flow_error_set(error, EINVAL, 2031 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2032 "L4 partial RSS requested but L4 RSS" 2033 " type not specified"); 2034 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2035 return rte_flow_error_set(error, EINVAL, 2036 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2037 NULL, "No Rx queues configured"); 2038 if (!rss->queue_num) 2039 return rte_flow_error_set(error, EINVAL, 2040 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2041 NULL, "No queues configured"); 2042 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2043 &message, &queue_idx); 2044 if (ret != 0) { 2045 return rte_flow_error_set(error, -ret, 2046 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2047 &rss->queue[queue_idx], message); 2048 } 2049 return 0; 2050 } 2051 2052 /* 2053 * Validate the rss action. 2054 * 2055 * @param[in] action 2056 * Pointer to the queue action. 2057 * @param[in] action_flags 2058 * Bit-fields that holds the actions detected until now. 2059 * @param[in] dev 2060 * Pointer to the Ethernet device structure. 2061 * @param[in] attr 2062 * Attributes of flow that includes this action. 2063 * @param[in] item_flags 2064 * Items that were detected. 2065 * @param[out] error 2066 * Pointer to error structure. 2067 * 2068 * @return 2069 * 0 on success, a negative errno value otherwise and rte_errno is set. 2070 */ 2071 int 2072 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2073 uint64_t action_flags, 2074 struct rte_eth_dev *dev, 2075 const struct rte_flow_attr *attr, 2076 uint64_t item_flags, 2077 struct rte_flow_error *error) 2078 { 2079 const struct rte_flow_action_rss *rss = action->conf; 2080 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2081 int ret; 2082 2083 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2084 return rte_flow_error_set(error, EINVAL, 2085 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2086 "can't have 2 fate actions" 2087 " in same flow"); 2088 ret = mlx5_validate_action_rss(dev, action, error); 2089 if (ret) 2090 return ret; 2091 if (attr->egress) 2092 return rte_flow_error_set(error, ENOTSUP, 2093 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2094 "rss action not supported for " 2095 "egress"); 2096 if (rss->level > 1 && !tunnel) 2097 return rte_flow_error_set(error, EINVAL, 2098 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2099 "inner RSS is not supported for " 2100 "non-tunnel flows"); 2101 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2102 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2103 return rte_flow_error_set(error, EINVAL, 2104 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2105 "RSS on eCPRI is not supported now"); 2106 } 2107 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2108 !(item_flags & 2109 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2110 rss->level > 1) 2111 return rte_flow_error_set(error, EINVAL, 2112 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2113 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2114 return 0; 2115 } 2116 2117 /* 2118 * Validate the default miss action. 2119 * 2120 * @param[in] action_flags 2121 * Bit-fields that holds the actions detected until now. 2122 * @param[out] error 2123 * Pointer to error structure. 2124 * 2125 * @return 2126 * 0 on success, a negative errno value otherwise and rte_errno is set. 2127 */ 2128 int 2129 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2130 const struct rte_flow_attr *attr, 2131 struct rte_flow_error *error) 2132 { 2133 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2134 return rte_flow_error_set(error, EINVAL, 2135 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2136 "can't have 2 fate actions in" 2137 " same flow"); 2138 if (attr->egress) 2139 return rte_flow_error_set(error, ENOTSUP, 2140 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2141 "default miss action not supported " 2142 "for egress"); 2143 if (attr->group) 2144 return rte_flow_error_set(error, ENOTSUP, 2145 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2146 "only group 0 is supported"); 2147 if (attr->transfer) 2148 return rte_flow_error_set(error, ENOTSUP, 2149 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2150 NULL, "transfer is not supported"); 2151 return 0; 2152 } 2153 2154 /* 2155 * Validate the count action. 2156 * 2157 * @param[in] dev 2158 * Pointer to the Ethernet device structure. 2159 * @param[in] attr 2160 * Attributes of flow that includes this action. 2161 * @param[out] error 2162 * Pointer to error structure. 2163 * 2164 * @return 2165 * 0 on success, a negative errno value otherwise and rte_errno is set. 2166 */ 2167 int 2168 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2169 const struct rte_flow_attr *attr, 2170 struct rte_flow_error *error) 2171 { 2172 if (attr->egress) 2173 return rte_flow_error_set(error, ENOTSUP, 2174 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2175 "count action not supported for " 2176 "egress"); 2177 return 0; 2178 } 2179 2180 /* 2181 * Validate the ASO CT action. 2182 * 2183 * @param[in] dev 2184 * Pointer to the Ethernet device structure. 2185 * @param[in] conntrack 2186 * Pointer to the CT action profile. 2187 * @param[out] error 2188 * Pointer to error structure. 2189 * 2190 * @return 2191 * 0 on success, a negative errno value otherwise and rte_errno is set. 2192 */ 2193 int 2194 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2195 const struct rte_flow_action_conntrack *conntrack, 2196 struct rte_flow_error *error) 2197 { 2198 RTE_SET_USED(dev); 2199 2200 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2201 return rte_flow_error_set(error, EINVAL, 2202 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2203 "Invalid CT state"); 2204 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2205 return rte_flow_error_set(error, EINVAL, 2206 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2207 "Invalid last TCP packet flag"); 2208 return 0; 2209 } 2210 2211 /** 2212 * Verify the @p attributes will be correctly understood by the NIC and store 2213 * them in the @p flow if everything is correct. 2214 * 2215 * @param[in] dev 2216 * Pointer to the Ethernet device structure. 2217 * @param[in] attributes 2218 * Pointer to flow attributes 2219 * @param[out] error 2220 * Pointer to error structure. 2221 * 2222 * @return 2223 * 0 on success, a negative errno value otherwise and rte_errno is set. 2224 */ 2225 int 2226 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2227 const struct rte_flow_attr *attributes, 2228 struct rte_flow_error *error) 2229 { 2230 struct mlx5_priv *priv = dev->data->dev_private; 2231 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2232 2233 if (attributes->group) 2234 return rte_flow_error_set(error, ENOTSUP, 2235 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2236 NULL, "groups is not supported"); 2237 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2238 attributes->priority >= priority_max) 2239 return rte_flow_error_set(error, ENOTSUP, 2240 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2241 NULL, "priority out of range"); 2242 if (attributes->egress) 2243 return rte_flow_error_set(error, ENOTSUP, 2244 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2245 "egress is not supported"); 2246 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2247 return rte_flow_error_set(error, ENOTSUP, 2248 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2249 NULL, "transfer is not supported"); 2250 if (!attributes->ingress) 2251 return rte_flow_error_set(error, EINVAL, 2252 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2253 NULL, 2254 "ingress attribute is mandatory"); 2255 return 0; 2256 } 2257 2258 /** 2259 * Validate ICMP6 item. 2260 * 2261 * @param[in] item 2262 * Item specification. 2263 * @param[in] item_flags 2264 * Bit-fields that holds the items detected until now. 2265 * @param[in] ext_vlan_sup 2266 * Whether extended VLAN features are supported or not. 2267 * @param[out] error 2268 * Pointer to error structure. 2269 * 2270 * @return 2271 * 0 on success, a negative errno value otherwise and rte_errno is set. 2272 */ 2273 int 2274 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2275 uint64_t item_flags, 2276 uint8_t target_protocol, 2277 struct rte_flow_error *error) 2278 { 2279 const struct rte_flow_item_icmp6 *mask = item->mask; 2280 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2281 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2282 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2283 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2284 MLX5_FLOW_LAYER_OUTER_L4; 2285 int ret; 2286 2287 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2288 return rte_flow_error_set(error, EINVAL, 2289 RTE_FLOW_ERROR_TYPE_ITEM, item, 2290 "protocol filtering not compatible" 2291 " with ICMP6 layer"); 2292 if (!(item_flags & l3m)) 2293 return rte_flow_error_set(error, EINVAL, 2294 RTE_FLOW_ERROR_TYPE_ITEM, item, 2295 "IPv6 is mandatory to filter on" 2296 " ICMP6"); 2297 if (item_flags & l4m) 2298 return rte_flow_error_set(error, EINVAL, 2299 RTE_FLOW_ERROR_TYPE_ITEM, item, 2300 "multiple L4 layers not supported"); 2301 if (!mask) 2302 mask = &rte_flow_item_icmp6_mask; 2303 ret = mlx5_flow_item_acceptable 2304 (item, (const uint8_t *)mask, 2305 (const uint8_t *)&rte_flow_item_icmp6_mask, 2306 sizeof(struct rte_flow_item_icmp6), 2307 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2308 if (ret < 0) 2309 return ret; 2310 return 0; 2311 } 2312 2313 /** 2314 * Validate ICMP item. 2315 * 2316 * @param[in] item 2317 * Item specification. 2318 * @param[in] item_flags 2319 * Bit-fields that holds the items detected until now. 2320 * @param[out] error 2321 * Pointer to error structure. 2322 * 2323 * @return 2324 * 0 on success, a negative errno value otherwise and rte_errno is set. 2325 */ 2326 int 2327 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2328 uint64_t item_flags, 2329 uint8_t target_protocol, 2330 struct rte_flow_error *error) 2331 { 2332 const struct rte_flow_item_icmp *mask = item->mask; 2333 const struct rte_flow_item_icmp nic_mask = { 2334 .hdr.icmp_type = 0xff, 2335 .hdr.icmp_code = 0xff, 2336 .hdr.icmp_ident = RTE_BE16(0xffff), 2337 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2338 }; 2339 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2340 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2341 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2342 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2343 MLX5_FLOW_LAYER_OUTER_L4; 2344 int ret; 2345 2346 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2347 return rte_flow_error_set(error, EINVAL, 2348 RTE_FLOW_ERROR_TYPE_ITEM, item, 2349 "protocol filtering not compatible" 2350 " with ICMP layer"); 2351 if (!(item_flags & l3m)) 2352 return rte_flow_error_set(error, EINVAL, 2353 RTE_FLOW_ERROR_TYPE_ITEM, item, 2354 "IPv4 is mandatory to filter" 2355 " on ICMP"); 2356 if (item_flags & l4m) 2357 return rte_flow_error_set(error, EINVAL, 2358 RTE_FLOW_ERROR_TYPE_ITEM, item, 2359 "multiple L4 layers not supported"); 2360 if (!mask) 2361 mask = &nic_mask; 2362 ret = mlx5_flow_item_acceptable 2363 (item, (const uint8_t *)mask, 2364 (const uint8_t *)&nic_mask, 2365 sizeof(struct rte_flow_item_icmp), 2366 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2367 if (ret < 0) 2368 return ret; 2369 return 0; 2370 } 2371 2372 /** 2373 * Validate Ethernet item. 2374 * 2375 * @param[in] item 2376 * Item specification. 2377 * @param[in] item_flags 2378 * Bit-fields that holds the items detected until now. 2379 * @param[out] error 2380 * Pointer to error structure. 2381 * 2382 * @return 2383 * 0 on success, a negative errno value otherwise and rte_errno is set. 2384 */ 2385 int 2386 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2387 uint64_t item_flags, bool ext_vlan_sup, 2388 struct rte_flow_error *error) 2389 { 2390 const struct rte_flow_item_eth *mask = item->mask; 2391 const struct rte_flow_item_eth nic_mask = { 2392 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2393 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2394 .type = RTE_BE16(0xffff), 2395 .has_vlan = ext_vlan_sup ? 1 : 0, 2396 }; 2397 int ret; 2398 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2399 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2400 MLX5_FLOW_LAYER_OUTER_L2; 2401 2402 if (item_flags & ethm) 2403 return rte_flow_error_set(error, ENOTSUP, 2404 RTE_FLOW_ERROR_TYPE_ITEM, item, 2405 "multiple L2 layers not supported"); 2406 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2407 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2408 return rte_flow_error_set(error, EINVAL, 2409 RTE_FLOW_ERROR_TYPE_ITEM, item, 2410 "L2 layer should not follow " 2411 "L3 layers"); 2412 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2413 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2414 return rte_flow_error_set(error, EINVAL, 2415 RTE_FLOW_ERROR_TYPE_ITEM, item, 2416 "L2 layer should not follow VLAN"); 2417 if (item_flags & MLX5_FLOW_LAYER_GTP) 2418 return rte_flow_error_set(error, EINVAL, 2419 RTE_FLOW_ERROR_TYPE_ITEM, item, 2420 "L2 layer should not follow GTP"); 2421 if (!mask) 2422 mask = &rte_flow_item_eth_mask; 2423 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2424 (const uint8_t *)&nic_mask, 2425 sizeof(struct rte_flow_item_eth), 2426 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2427 return ret; 2428 } 2429 2430 /** 2431 * Validate VLAN item. 2432 * 2433 * @param[in] item 2434 * Item specification. 2435 * @param[in] item_flags 2436 * Bit-fields that holds the items detected until now. 2437 * @param[in] dev 2438 * Ethernet device flow is being created on. 2439 * @param[out] error 2440 * Pointer to error structure. 2441 * 2442 * @return 2443 * 0 on success, a negative errno value otherwise and rte_errno is set. 2444 */ 2445 int 2446 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2447 uint64_t item_flags, 2448 struct rte_eth_dev *dev, 2449 struct rte_flow_error *error) 2450 { 2451 const struct rte_flow_item_vlan *spec = item->spec; 2452 const struct rte_flow_item_vlan *mask = item->mask; 2453 const struct rte_flow_item_vlan nic_mask = { 2454 .tci = RTE_BE16(UINT16_MAX), 2455 .inner_type = RTE_BE16(UINT16_MAX), 2456 }; 2457 uint16_t vlan_tag = 0; 2458 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2459 int ret; 2460 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2461 MLX5_FLOW_LAYER_INNER_L4) : 2462 (MLX5_FLOW_LAYER_OUTER_L3 | 2463 MLX5_FLOW_LAYER_OUTER_L4); 2464 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2465 MLX5_FLOW_LAYER_OUTER_VLAN; 2466 2467 if (item_flags & vlanm) 2468 return rte_flow_error_set(error, EINVAL, 2469 RTE_FLOW_ERROR_TYPE_ITEM, item, 2470 "multiple VLAN layers not supported"); 2471 else if ((item_flags & l34m) != 0) 2472 return rte_flow_error_set(error, EINVAL, 2473 RTE_FLOW_ERROR_TYPE_ITEM, item, 2474 "VLAN cannot follow L3/L4 layer"); 2475 if (!mask) 2476 mask = &rte_flow_item_vlan_mask; 2477 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2478 (const uint8_t *)&nic_mask, 2479 sizeof(struct rte_flow_item_vlan), 2480 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2481 if (ret) 2482 return ret; 2483 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2484 struct mlx5_priv *priv = dev->data->dev_private; 2485 2486 if (priv->vmwa_context) { 2487 /* 2488 * Non-NULL context means we have a virtual machine 2489 * and SR-IOV enabled, we have to create VLAN interface 2490 * to make hypervisor to setup E-Switch vport 2491 * context correctly. We avoid creating the multiple 2492 * VLAN interfaces, so we cannot support VLAN tag mask. 2493 */ 2494 return rte_flow_error_set(error, EINVAL, 2495 RTE_FLOW_ERROR_TYPE_ITEM, 2496 item, 2497 "VLAN tag mask is not" 2498 " supported in virtual" 2499 " environment"); 2500 } 2501 } 2502 if (spec) { 2503 vlan_tag = spec->tci; 2504 vlan_tag &= mask->tci; 2505 } 2506 /* 2507 * From verbs perspective an empty VLAN is equivalent 2508 * to a packet without VLAN layer. 2509 */ 2510 if (!vlan_tag) 2511 return rte_flow_error_set(error, EINVAL, 2512 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2513 item->spec, 2514 "VLAN cannot be empty"); 2515 return 0; 2516 } 2517 2518 /** 2519 * Validate IPV4 item. 2520 * 2521 * @param[in] item 2522 * Item specification. 2523 * @param[in] item_flags 2524 * Bit-fields that holds the items detected until now. 2525 * @param[in] last_item 2526 * Previous validated item in the pattern items. 2527 * @param[in] ether_type 2528 * Type in the ethernet layer header (including dot1q). 2529 * @param[in] acc_mask 2530 * Acceptable mask, if NULL default internal default mask 2531 * will be used to check whether item fields are supported. 2532 * @param[in] range_accepted 2533 * True if range of values is accepted for specific fields, false otherwise. 2534 * @param[out] error 2535 * Pointer to error structure. 2536 * 2537 * @return 2538 * 0 on success, a negative errno value otherwise and rte_errno is set. 2539 */ 2540 int 2541 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2542 uint64_t item_flags, 2543 uint64_t last_item, 2544 uint16_t ether_type, 2545 const struct rte_flow_item_ipv4 *acc_mask, 2546 bool range_accepted, 2547 struct rte_flow_error *error) 2548 { 2549 const struct rte_flow_item_ipv4 *mask = item->mask; 2550 const struct rte_flow_item_ipv4 *spec = item->spec; 2551 const struct rte_flow_item_ipv4 nic_mask = { 2552 .hdr = { 2553 .src_addr = RTE_BE32(0xffffffff), 2554 .dst_addr = RTE_BE32(0xffffffff), 2555 .type_of_service = 0xff, 2556 .next_proto_id = 0xff, 2557 }, 2558 }; 2559 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2560 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2561 MLX5_FLOW_LAYER_OUTER_L3; 2562 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2563 MLX5_FLOW_LAYER_OUTER_L4; 2564 int ret; 2565 uint8_t next_proto = 0xFF; 2566 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2567 MLX5_FLOW_LAYER_OUTER_VLAN | 2568 MLX5_FLOW_LAYER_INNER_VLAN); 2569 2570 if ((last_item & l2_vlan) && ether_type && 2571 ether_type != RTE_ETHER_TYPE_IPV4) 2572 return rte_flow_error_set(error, EINVAL, 2573 RTE_FLOW_ERROR_TYPE_ITEM, item, 2574 "IPv4 cannot follow L2/VLAN layer " 2575 "which ether type is not IPv4"); 2576 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2577 if (mask && spec) 2578 next_proto = mask->hdr.next_proto_id & 2579 spec->hdr.next_proto_id; 2580 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2581 return rte_flow_error_set(error, EINVAL, 2582 RTE_FLOW_ERROR_TYPE_ITEM, 2583 item, 2584 "multiple tunnel " 2585 "not supported"); 2586 } 2587 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2588 return rte_flow_error_set(error, EINVAL, 2589 RTE_FLOW_ERROR_TYPE_ITEM, item, 2590 "wrong tunnel type - IPv6 specified " 2591 "but IPv4 item provided"); 2592 if (item_flags & l3m) 2593 return rte_flow_error_set(error, ENOTSUP, 2594 RTE_FLOW_ERROR_TYPE_ITEM, item, 2595 "multiple L3 layers not supported"); 2596 else if (item_flags & l4m) 2597 return rte_flow_error_set(error, EINVAL, 2598 RTE_FLOW_ERROR_TYPE_ITEM, item, 2599 "L3 cannot follow an L4 layer."); 2600 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2601 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2602 return rte_flow_error_set(error, EINVAL, 2603 RTE_FLOW_ERROR_TYPE_ITEM, item, 2604 "L3 cannot follow an NVGRE layer."); 2605 if (!mask) 2606 mask = &rte_flow_item_ipv4_mask; 2607 else if (mask->hdr.next_proto_id != 0 && 2608 mask->hdr.next_proto_id != 0xff) 2609 return rte_flow_error_set(error, EINVAL, 2610 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2611 "partial mask is not supported" 2612 " for protocol"); 2613 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2614 acc_mask ? (const uint8_t *)acc_mask 2615 : (const uint8_t *)&nic_mask, 2616 sizeof(struct rte_flow_item_ipv4), 2617 range_accepted, error); 2618 if (ret < 0) 2619 return ret; 2620 return 0; 2621 } 2622 2623 /** 2624 * Validate IPV6 item. 2625 * 2626 * @param[in] item 2627 * Item specification. 2628 * @param[in] item_flags 2629 * Bit-fields that holds the items detected until now. 2630 * @param[in] last_item 2631 * Previous validated item in the pattern items. 2632 * @param[in] ether_type 2633 * Type in the ethernet layer header (including dot1q). 2634 * @param[in] acc_mask 2635 * Acceptable mask, if NULL default internal default mask 2636 * will be used to check whether item fields are supported. 2637 * @param[out] error 2638 * Pointer to error structure. 2639 * 2640 * @return 2641 * 0 on success, a negative errno value otherwise and rte_errno is set. 2642 */ 2643 int 2644 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2645 uint64_t item_flags, 2646 uint64_t last_item, 2647 uint16_t ether_type, 2648 const struct rte_flow_item_ipv6 *acc_mask, 2649 struct rte_flow_error *error) 2650 { 2651 const struct rte_flow_item_ipv6 *mask = item->mask; 2652 const struct rte_flow_item_ipv6 *spec = item->spec; 2653 const struct rte_flow_item_ipv6 nic_mask = { 2654 .hdr = { 2655 .src_addr = 2656 "\xff\xff\xff\xff\xff\xff\xff\xff" 2657 "\xff\xff\xff\xff\xff\xff\xff\xff", 2658 .dst_addr = 2659 "\xff\xff\xff\xff\xff\xff\xff\xff" 2660 "\xff\xff\xff\xff\xff\xff\xff\xff", 2661 .vtc_flow = RTE_BE32(0xffffffff), 2662 .proto = 0xff, 2663 }, 2664 }; 2665 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2666 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2667 MLX5_FLOW_LAYER_OUTER_L3; 2668 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2669 MLX5_FLOW_LAYER_OUTER_L4; 2670 int ret; 2671 uint8_t next_proto = 0xFF; 2672 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2673 MLX5_FLOW_LAYER_OUTER_VLAN | 2674 MLX5_FLOW_LAYER_INNER_VLAN); 2675 2676 if ((last_item & l2_vlan) && ether_type && 2677 ether_type != RTE_ETHER_TYPE_IPV6) 2678 return rte_flow_error_set(error, EINVAL, 2679 RTE_FLOW_ERROR_TYPE_ITEM, item, 2680 "IPv6 cannot follow L2/VLAN layer " 2681 "which ether type is not IPv6"); 2682 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2683 next_proto = spec->hdr.proto; 2684 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2685 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2686 return rte_flow_error_set(error, EINVAL, 2687 RTE_FLOW_ERROR_TYPE_ITEM, 2688 item, 2689 "multiple tunnel " 2690 "not supported"); 2691 } 2692 if (next_proto == IPPROTO_HOPOPTS || 2693 next_proto == IPPROTO_ROUTING || 2694 next_proto == IPPROTO_FRAGMENT || 2695 next_proto == IPPROTO_ESP || 2696 next_proto == IPPROTO_AH || 2697 next_proto == IPPROTO_DSTOPTS) 2698 return rte_flow_error_set(error, EINVAL, 2699 RTE_FLOW_ERROR_TYPE_ITEM, item, 2700 "IPv6 proto (next header) should " 2701 "not be set as extension header"); 2702 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2703 return rte_flow_error_set(error, EINVAL, 2704 RTE_FLOW_ERROR_TYPE_ITEM, item, 2705 "wrong tunnel type - IPv4 specified " 2706 "but IPv6 item provided"); 2707 if (item_flags & l3m) 2708 return rte_flow_error_set(error, ENOTSUP, 2709 RTE_FLOW_ERROR_TYPE_ITEM, item, 2710 "multiple L3 layers not supported"); 2711 else if (item_flags & l4m) 2712 return rte_flow_error_set(error, EINVAL, 2713 RTE_FLOW_ERROR_TYPE_ITEM, item, 2714 "L3 cannot follow an L4 layer."); 2715 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2716 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2717 return rte_flow_error_set(error, EINVAL, 2718 RTE_FLOW_ERROR_TYPE_ITEM, item, 2719 "L3 cannot follow an NVGRE layer."); 2720 if (!mask) 2721 mask = &rte_flow_item_ipv6_mask; 2722 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2723 acc_mask ? (const uint8_t *)acc_mask 2724 : (const uint8_t *)&nic_mask, 2725 sizeof(struct rte_flow_item_ipv6), 2726 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2727 if (ret < 0) 2728 return ret; 2729 return 0; 2730 } 2731 2732 /** 2733 * Validate UDP item. 2734 * 2735 * @param[in] item 2736 * Item specification. 2737 * @param[in] item_flags 2738 * Bit-fields that holds the items detected until now. 2739 * @param[in] target_protocol 2740 * The next protocol in the previous item. 2741 * @param[in] flow_mask 2742 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2743 * @param[out] error 2744 * Pointer to error structure. 2745 * 2746 * @return 2747 * 0 on success, a negative errno value otherwise and rte_errno is set. 2748 */ 2749 int 2750 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2751 uint64_t item_flags, 2752 uint8_t target_protocol, 2753 struct rte_flow_error *error) 2754 { 2755 const struct rte_flow_item_udp *mask = item->mask; 2756 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2757 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2758 MLX5_FLOW_LAYER_OUTER_L3; 2759 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2760 MLX5_FLOW_LAYER_OUTER_L4; 2761 int ret; 2762 2763 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2764 return rte_flow_error_set(error, EINVAL, 2765 RTE_FLOW_ERROR_TYPE_ITEM, item, 2766 "protocol filtering not compatible" 2767 " with UDP layer"); 2768 if (!(item_flags & l3m)) 2769 return rte_flow_error_set(error, EINVAL, 2770 RTE_FLOW_ERROR_TYPE_ITEM, item, 2771 "L3 is mandatory to filter on L4"); 2772 if (item_flags & l4m) 2773 return rte_flow_error_set(error, EINVAL, 2774 RTE_FLOW_ERROR_TYPE_ITEM, item, 2775 "multiple L4 layers not supported"); 2776 if (!mask) 2777 mask = &rte_flow_item_udp_mask; 2778 ret = mlx5_flow_item_acceptable 2779 (item, (const uint8_t *)mask, 2780 (const uint8_t *)&rte_flow_item_udp_mask, 2781 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2782 error); 2783 if (ret < 0) 2784 return ret; 2785 return 0; 2786 } 2787 2788 /** 2789 * Validate TCP item. 2790 * 2791 * @param[in] item 2792 * Item specification. 2793 * @param[in] item_flags 2794 * Bit-fields that holds the items detected until now. 2795 * @param[in] target_protocol 2796 * The next protocol in the previous item. 2797 * @param[out] error 2798 * Pointer to error structure. 2799 * 2800 * @return 2801 * 0 on success, a negative errno value otherwise and rte_errno is set. 2802 */ 2803 int 2804 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2805 uint64_t item_flags, 2806 uint8_t target_protocol, 2807 const struct rte_flow_item_tcp *flow_mask, 2808 struct rte_flow_error *error) 2809 { 2810 const struct rte_flow_item_tcp *mask = item->mask; 2811 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2812 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2813 MLX5_FLOW_LAYER_OUTER_L3; 2814 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2815 MLX5_FLOW_LAYER_OUTER_L4; 2816 int ret; 2817 2818 MLX5_ASSERT(flow_mask); 2819 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2820 return rte_flow_error_set(error, EINVAL, 2821 RTE_FLOW_ERROR_TYPE_ITEM, item, 2822 "protocol filtering not compatible" 2823 " with TCP layer"); 2824 if (!(item_flags & l3m)) 2825 return rte_flow_error_set(error, EINVAL, 2826 RTE_FLOW_ERROR_TYPE_ITEM, item, 2827 "L3 is mandatory to filter on L4"); 2828 if (item_flags & l4m) 2829 return rte_flow_error_set(error, EINVAL, 2830 RTE_FLOW_ERROR_TYPE_ITEM, item, 2831 "multiple L4 layers not supported"); 2832 if (!mask) 2833 mask = &rte_flow_item_tcp_mask; 2834 ret = mlx5_flow_item_acceptable 2835 (item, (const uint8_t *)mask, 2836 (const uint8_t *)flow_mask, 2837 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2838 error); 2839 if (ret < 0) 2840 return ret; 2841 return 0; 2842 } 2843 2844 /** 2845 * Validate VXLAN item. 2846 * 2847 * @param[in] dev 2848 * Pointer to the Ethernet device structure. 2849 * @param[in] udp_dport 2850 * UDP destination port 2851 * @param[in] item 2852 * Item specification. 2853 * @param[in] item_flags 2854 * Bit-fields that holds the items detected until now. 2855 * @param root 2856 * Whether action is on root table. 2857 * @param[out] error 2858 * Pointer to error structure. 2859 * 2860 * @return 2861 * 0 on success, a negative errno value otherwise and rte_errno is set. 2862 */ 2863 int 2864 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2865 uint16_t udp_dport, 2866 const struct rte_flow_item *item, 2867 uint64_t item_flags, 2868 bool root, 2869 struct rte_flow_error *error) 2870 { 2871 const struct rte_flow_item_vxlan *spec = item->spec; 2872 const struct rte_flow_item_vxlan *mask = item->mask; 2873 int ret; 2874 struct mlx5_priv *priv = dev->data->dev_private; 2875 union vni { 2876 uint32_t vlan_id; 2877 uint8_t vni[4]; 2878 } id = { .vlan_id = 0, }; 2879 const struct rte_flow_item_vxlan nic_mask = { 2880 .vni = "\xff\xff\xff", 2881 .rsvd1 = 0xff, 2882 }; 2883 const struct rte_flow_item_vxlan *valid_mask; 2884 2885 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2886 return rte_flow_error_set(error, ENOTSUP, 2887 RTE_FLOW_ERROR_TYPE_ITEM, item, 2888 "multiple tunnel layers not" 2889 " supported"); 2890 valid_mask = &rte_flow_item_vxlan_mask; 2891 /* 2892 * Verify only UDPv4 is present as defined in 2893 * https://tools.ietf.org/html/rfc7348 2894 */ 2895 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2896 return rte_flow_error_set(error, EINVAL, 2897 RTE_FLOW_ERROR_TYPE_ITEM, item, 2898 "no outer UDP layer found"); 2899 if (!mask) 2900 mask = &rte_flow_item_vxlan_mask; 2901 2902 if (priv->sh->steering_format_version != 2903 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2904 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2905 /* non-root table */ 2906 if (!root && priv->sh->misc5_cap) 2907 valid_mask = &nic_mask; 2908 /* Group zero in NIC domain */ 2909 if (!root && priv->sh->tunnel_header_0_1) 2910 valid_mask = &nic_mask; 2911 } 2912 ret = mlx5_flow_item_acceptable 2913 (item, (const uint8_t *)mask, 2914 (const uint8_t *)valid_mask, 2915 sizeof(struct rte_flow_item_vxlan), 2916 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2917 if (ret < 0) 2918 return ret; 2919 if (spec) { 2920 memcpy(&id.vni[1], spec->vni, 3); 2921 memcpy(&id.vni[1], mask->vni, 3); 2922 } 2923 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2924 return rte_flow_error_set(error, ENOTSUP, 2925 RTE_FLOW_ERROR_TYPE_ITEM, item, 2926 "VXLAN tunnel must be fully defined"); 2927 return 0; 2928 } 2929 2930 /** 2931 * Validate VXLAN_GPE item. 2932 * 2933 * @param[in] item 2934 * Item specification. 2935 * @param[in] item_flags 2936 * Bit-fields that holds the items detected until now. 2937 * @param[in] priv 2938 * Pointer to the private data structure. 2939 * @param[in] target_protocol 2940 * The next protocol in the previous item. 2941 * @param[out] error 2942 * Pointer to error structure. 2943 * 2944 * @return 2945 * 0 on success, a negative errno value otherwise and rte_errno is set. 2946 */ 2947 int 2948 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2949 uint64_t item_flags, 2950 struct rte_eth_dev *dev, 2951 struct rte_flow_error *error) 2952 { 2953 struct mlx5_priv *priv = dev->data->dev_private; 2954 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2955 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2956 int ret; 2957 union vni { 2958 uint32_t vlan_id; 2959 uint8_t vni[4]; 2960 } id = { .vlan_id = 0, }; 2961 2962 if (!priv->sh->config.l3_vxlan_en) 2963 return rte_flow_error_set(error, ENOTSUP, 2964 RTE_FLOW_ERROR_TYPE_ITEM, item, 2965 "L3 VXLAN is not enabled by device" 2966 " parameter and/or not configured in" 2967 " firmware"); 2968 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2969 return rte_flow_error_set(error, ENOTSUP, 2970 RTE_FLOW_ERROR_TYPE_ITEM, item, 2971 "multiple tunnel layers not" 2972 " supported"); 2973 /* 2974 * Verify only UDPv4 is present as defined in 2975 * https://tools.ietf.org/html/rfc7348 2976 */ 2977 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2978 return rte_flow_error_set(error, EINVAL, 2979 RTE_FLOW_ERROR_TYPE_ITEM, item, 2980 "no outer UDP layer found"); 2981 if (!mask) 2982 mask = &rte_flow_item_vxlan_gpe_mask; 2983 ret = mlx5_flow_item_acceptable 2984 (item, (const uint8_t *)mask, 2985 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2986 sizeof(struct rte_flow_item_vxlan_gpe), 2987 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2988 if (ret < 0) 2989 return ret; 2990 if (spec) { 2991 if (spec->protocol) 2992 return rte_flow_error_set(error, ENOTSUP, 2993 RTE_FLOW_ERROR_TYPE_ITEM, 2994 item, 2995 "VxLAN-GPE protocol" 2996 " not supported"); 2997 memcpy(&id.vni[1], spec->vni, 3); 2998 memcpy(&id.vni[1], mask->vni, 3); 2999 } 3000 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3001 return rte_flow_error_set(error, ENOTSUP, 3002 RTE_FLOW_ERROR_TYPE_ITEM, item, 3003 "VXLAN-GPE tunnel must be fully" 3004 " defined"); 3005 return 0; 3006 } 3007 /** 3008 * Validate GRE Key item. 3009 * 3010 * @param[in] item 3011 * Item specification. 3012 * @param[in] item_flags 3013 * Bit flags to mark detected items. 3014 * @param[in] gre_item 3015 * Pointer to gre_item 3016 * @param[out] error 3017 * Pointer to error structure. 3018 * 3019 * @return 3020 * 0 on success, a negative errno value otherwise and rte_errno is set. 3021 */ 3022 int 3023 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3024 uint64_t item_flags, 3025 const struct rte_flow_item *gre_item, 3026 struct rte_flow_error *error) 3027 { 3028 const rte_be32_t *mask = item->mask; 3029 int ret = 0; 3030 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3031 const struct rte_flow_item_gre *gre_spec; 3032 const struct rte_flow_item_gre *gre_mask; 3033 3034 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3035 return rte_flow_error_set(error, ENOTSUP, 3036 RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "Multiple GRE key not support"); 3038 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3039 return rte_flow_error_set(error, ENOTSUP, 3040 RTE_FLOW_ERROR_TYPE_ITEM, item, 3041 "No preceding GRE header"); 3042 if (item_flags & MLX5_FLOW_LAYER_INNER) 3043 return rte_flow_error_set(error, ENOTSUP, 3044 RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "GRE key following a wrong item"); 3046 gre_mask = gre_item->mask; 3047 if (!gre_mask) 3048 gre_mask = &rte_flow_item_gre_mask; 3049 gre_spec = gre_item->spec; 3050 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3051 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3052 return rte_flow_error_set(error, EINVAL, 3053 RTE_FLOW_ERROR_TYPE_ITEM, item, 3054 "Key bit must be on"); 3055 3056 if (!mask) 3057 mask = &gre_key_default_mask; 3058 ret = mlx5_flow_item_acceptable 3059 (item, (const uint8_t *)mask, 3060 (const uint8_t *)&gre_key_default_mask, 3061 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3062 return ret; 3063 } 3064 3065 /** 3066 * Validate GRE optional item. 3067 * 3068 * @param[in] dev 3069 * Pointer to the Ethernet device structure. 3070 * @param[in] item 3071 * Item specification. 3072 * @param[in] item_flags 3073 * Bit flags to mark detected items. 3074 * @param[in] attr 3075 * Flow rule attributes. 3076 * @param[in] gre_item 3077 * Pointer to gre_item 3078 * @param[out] error 3079 * Pointer to error structure. 3080 * 3081 * @return 3082 * 0 on success, a negative errno value otherwise and rte_errno is set. 3083 */ 3084 int 3085 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3086 const struct rte_flow_item *item, 3087 uint64_t item_flags, 3088 const struct rte_flow_attr *attr, 3089 const struct rte_flow_item *gre_item, 3090 struct rte_flow_error *error) 3091 { 3092 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3093 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3094 const struct rte_flow_item_gre_opt *spec = item->spec; 3095 const struct rte_flow_item_gre_opt *mask = item->mask; 3096 struct mlx5_priv *priv = dev->data->dev_private; 3097 int ret = 0; 3098 struct rte_flow_item_gre_opt nic_mask = { 3099 .checksum_rsvd = { 3100 .checksum = RTE_BE16(UINT16_MAX), 3101 .reserved1 = 0x0, 3102 }, 3103 .key = { 3104 .key = RTE_BE32(UINT32_MAX), 3105 }, 3106 .sequence = { 3107 .sequence = RTE_BE32(UINT32_MAX), 3108 }, 3109 }; 3110 3111 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3112 return rte_flow_error_set(error, ENOTSUP, 3113 RTE_FLOW_ERROR_TYPE_ITEM, item, 3114 "No preceding GRE header"); 3115 if (item_flags & MLX5_FLOW_LAYER_INNER) 3116 return rte_flow_error_set(error, ENOTSUP, 3117 RTE_FLOW_ERROR_TYPE_ITEM, item, 3118 "GRE option following a wrong item"); 3119 if (!spec || !mask) 3120 return rte_flow_error_set(error, EINVAL, 3121 RTE_FLOW_ERROR_TYPE_ITEM, item, 3122 "At least one field gre_option(checksum/key/sequence) must be specified"); 3123 if (!gre_mask) 3124 gre_mask = &rte_flow_item_gre_mask; 3125 if (mask->checksum_rsvd.checksum) 3126 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3127 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3128 return rte_flow_error_set(error, EINVAL, 3129 RTE_FLOW_ERROR_TYPE_ITEM, 3130 item, 3131 "Checksum bit must be on"); 3132 if (mask->key.key) 3133 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3134 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3135 return rte_flow_error_set(error, EINVAL, 3136 RTE_FLOW_ERROR_TYPE_ITEM, 3137 item, "Key bit must be on"); 3138 if (mask->sequence.sequence) 3139 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3140 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3141 return rte_flow_error_set(error, EINVAL, 3142 RTE_FLOW_ERROR_TYPE_ITEM, 3143 item, 3144 "Sequence bit must be on"); 3145 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3146 if (priv->sh->steering_format_version == 3147 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3148 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3149 !priv->sh->misc5_cap) || 3150 (!(priv->sh->tunnel_header_0_1 && 3151 priv->sh->tunnel_header_2_3) && 3152 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3153 return rte_flow_error_set(error, EINVAL, 3154 RTE_FLOW_ERROR_TYPE_ITEM, 3155 item, 3156 "Checksum/Sequence not supported"); 3157 } 3158 ret = mlx5_flow_item_acceptable 3159 (item, (const uint8_t *)mask, 3160 (const uint8_t *)&nic_mask, 3161 sizeof(struct rte_flow_item_gre_opt), 3162 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3163 return ret; 3164 } 3165 3166 /** 3167 * Validate GRE item. 3168 * 3169 * @param[in] item 3170 * Item specification. 3171 * @param[in] item_flags 3172 * Bit flags to mark detected items. 3173 * @param[in] target_protocol 3174 * The next protocol in the previous item. 3175 * @param[out] error 3176 * Pointer to error structure. 3177 * 3178 * @return 3179 * 0 on success, a negative errno value otherwise and rte_errno is set. 3180 */ 3181 int 3182 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3183 uint64_t item_flags, 3184 uint8_t target_protocol, 3185 struct rte_flow_error *error) 3186 { 3187 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3188 const struct rte_flow_item_gre *mask = item->mask; 3189 int ret; 3190 const struct rte_flow_item_gre nic_mask = { 3191 .c_rsvd0_ver = RTE_BE16(0xB000), 3192 .protocol = RTE_BE16(UINT16_MAX), 3193 }; 3194 3195 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3196 return rte_flow_error_set(error, EINVAL, 3197 RTE_FLOW_ERROR_TYPE_ITEM, item, 3198 "protocol filtering not compatible" 3199 " with this GRE layer"); 3200 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3201 return rte_flow_error_set(error, ENOTSUP, 3202 RTE_FLOW_ERROR_TYPE_ITEM, item, 3203 "multiple tunnel layers not" 3204 " supported"); 3205 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3206 return rte_flow_error_set(error, ENOTSUP, 3207 RTE_FLOW_ERROR_TYPE_ITEM, item, 3208 "L3 Layer is missing"); 3209 if (!mask) 3210 mask = &rte_flow_item_gre_mask; 3211 ret = mlx5_flow_item_acceptable 3212 (item, (const uint8_t *)mask, 3213 (const uint8_t *)&nic_mask, 3214 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3215 error); 3216 if (ret < 0) 3217 return ret; 3218 #ifndef HAVE_MLX5DV_DR 3219 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3220 if (spec && (spec->protocol & mask->protocol)) 3221 return rte_flow_error_set(error, ENOTSUP, 3222 RTE_FLOW_ERROR_TYPE_ITEM, item, 3223 "without MPLS support the" 3224 " specification cannot be used for" 3225 " filtering"); 3226 #endif 3227 #endif 3228 return 0; 3229 } 3230 3231 /** 3232 * Validate Geneve item. 3233 * 3234 * @param[in] item 3235 * Item specification. 3236 * @param[in] itemFlags 3237 * Bit-fields that holds the items detected until now. 3238 * @param[in] enPriv 3239 * Pointer to the private data structure. 3240 * @param[out] error 3241 * Pointer to error structure. 3242 * 3243 * @return 3244 * 0 on success, a negative errno value otherwise and rte_errno is set. 3245 */ 3246 3247 int 3248 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3249 uint64_t item_flags, 3250 struct rte_eth_dev *dev, 3251 struct rte_flow_error *error) 3252 { 3253 struct mlx5_priv *priv = dev->data->dev_private; 3254 const struct rte_flow_item_geneve *spec = item->spec; 3255 const struct rte_flow_item_geneve *mask = item->mask; 3256 int ret; 3257 uint16_t gbhdr; 3258 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3259 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3260 const struct rte_flow_item_geneve nic_mask = { 3261 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3262 .vni = "\xff\xff\xff", 3263 .protocol = RTE_BE16(UINT16_MAX), 3264 }; 3265 3266 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3267 return rte_flow_error_set(error, ENOTSUP, 3268 RTE_FLOW_ERROR_TYPE_ITEM, item, 3269 "L3 Geneve is not enabled by device" 3270 " parameter and/or not configured in" 3271 " firmware"); 3272 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3273 return rte_flow_error_set(error, ENOTSUP, 3274 RTE_FLOW_ERROR_TYPE_ITEM, item, 3275 "multiple tunnel layers not" 3276 " supported"); 3277 /* 3278 * Verify only UDPv4 is present as defined in 3279 * https://tools.ietf.org/html/rfc7348 3280 */ 3281 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3282 return rte_flow_error_set(error, EINVAL, 3283 RTE_FLOW_ERROR_TYPE_ITEM, item, 3284 "no outer UDP layer found"); 3285 if (!mask) 3286 mask = &rte_flow_item_geneve_mask; 3287 ret = mlx5_flow_item_acceptable 3288 (item, (const uint8_t *)mask, 3289 (const uint8_t *)&nic_mask, 3290 sizeof(struct rte_flow_item_geneve), 3291 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3292 if (ret) 3293 return ret; 3294 if (spec) { 3295 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3296 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3297 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3298 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3299 return rte_flow_error_set(error, ENOTSUP, 3300 RTE_FLOW_ERROR_TYPE_ITEM, 3301 item, 3302 "Geneve protocol unsupported" 3303 " fields are being used"); 3304 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3305 return rte_flow_error_set 3306 (error, ENOTSUP, 3307 RTE_FLOW_ERROR_TYPE_ITEM, 3308 item, 3309 "Unsupported Geneve options length"); 3310 } 3311 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3312 return rte_flow_error_set 3313 (error, ENOTSUP, 3314 RTE_FLOW_ERROR_TYPE_ITEM, item, 3315 "Geneve tunnel must be fully defined"); 3316 return 0; 3317 } 3318 3319 /** 3320 * Validate Geneve TLV option item. 3321 * 3322 * @param[in] item 3323 * Item specification. 3324 * @param[in] last_item 3325 * Previous validated item in the pattern items. 3326 * @param[in] geneve_item 3327 * Previous GENEVE item specification. 3328 * @param[in] dev 3329 * Pointer to the rte_eth_dev structure. 3330 * @param[out] error 3331 * Pointer to error structure. 3332 * 3333 * @return 3334 * 0 on success, a negative errno value otherwise and rte_errno is set. 3335 */ 3336 int 3337 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3338 uint64_t last_item, 3339 const struct rte_flow_item *geneve_item, 3340 struct rte_eth_dev *dev, 3341 struct rte_flow_error *error) 3342 { 3343 struct mlx5_priv *priv = dev->data->dev_private; 3344 struct mlx5_dev_ctx_shared *sh = priv->sh; 3345 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3346 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3347 uint8_t data_max_supported = 3348 hca_attr->max_geneve_tlv_option_data_len * 4; 3349 const struct rte_flow_item_geneve *geneve_spec; 3350 const struct rte_flow_item_geneve *geneve_mask; 3351 const struct rte_flow_item_geneve_opt *spec = item->spec; 3352 const struct rte_flow_item_geneve_opt *mask = item->mask; 3353 unsigned int i; 3354 unsigned int data_len; 3355 uint8_t tlv_option_len; 3356 uint16_t optlen_m, optlen_v; 3357 const struct rte_flow_item_geneve_opt full_mask = { 3358 .option_class = RTE_BE16(0xffff), 3359 .option_type = 0xff, 3360 .option_len = 0x1f, 3361 }; 3362 3363 if (!mask) 3364 mask = &rte_flow_item_geneve_opt_mask; 3365 if (!spec) 3366 return rte_flow_error_set 3367 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3368 "Geneve TLV opt class/type/length must be specified"); 3369 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3370 return rte_flow_error_set 3371 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3372 "Geneve TLV opt length exceeds the limit (31)"); 3373 /* Check if class type and length masks are full. */ 3374 if (full_mask.option_class != mask->option_class || 3375 full_mask.option_type != mask->option_type || 3376 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3377 return rte_flow_error_set 3378 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3379 "Geneve TLV opt class/type/length masks must be full"); 3380 /* Check if length is supported */ 3381 if ((uint32_t)spec->option_len > 3382 hca_attr->max_geneve_tlv_option_data_len) 3383 return rte_flow_error_set 3384 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3385 "Geneve TLV opt length not supported"); 3386 if (hca_attr->max_geneve_tlv_options > 1) 3387 DRV_LOG(DEBUG, 3388 "max_geneve_tlv_options supports more than 1 option"); 3389 /* Check GENEVE item preceding. */ 3390 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3391 return rte_flow_error_set 3392 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3393 "Geneve opt item must be preceded with Geneve item"); 3394 geneve_spec = geneve_item->spec; 3395 geneve_mask = geneve_item->mask ? geneve_item->mask : 3396 &rte_flow_item_geneve_mask; 3397 /* Check if GENEVE TLV option size doesn't exceed option length */ 3398 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3399 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3400 tlv_option_len = spec->option_len & mask->option_len; 3401 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3402 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3403 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3404 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3405 if ((optlen_v & optlen_m) <= tlv_option_len) 3406 return rte_flow_error_set 3407 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3408 "GENEVE TLV option length exceeds optlen"); 3409 } 3410 /* Check if length is 0 or data is 0. */ 3411 if (spec->data == NULL || spec->option_len == 0) 3412 return rte_flow_error_set 3413 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3414 "Geneve TLV opt with zero data/length not supported"); 3415 /* Check not all data & mask are 0. */ 3416 data_len = spec->option_len * 4; 3417 if (mask->data == NULL) { 3418 for (i = 0; i < data_len; i++) 3419 if (spec->data[i]) 3420 break; 3421 if (i == data_len) 3422 return rte_flow_error_set(error, ENOTSUP, 3423 RTE_FLOW_ERROR_TYPE_ITEM, item, 3424 "Can't match on Geneve option data 0"); 3425 } else { 3426 for (i = 0; i < data_len; i++) 3427 if (spec->data[i] & mask->data[i]) 3428 break; 3429 if (i == data_len) 3430 return rte_flow_error_set(error, ENOTSUP, 3431 RTE_FLOW_ERROR_TYPE_ITEM, item, 3432 "Can't match on Geneve option data and mask 0"); 3433 /* Check data mask supported. */ 3434 for (i = data_max_supported; i < data_len ; i++) 3435 if (mask->data[i]) 3436 return rte_flow_error_set(error, ENOTSUP, 3437 RTE_FLOW_ERROR_TYPE_ITEM, item, 3438 "Data mask is of unsupported size"); 3439 } 3440 /* Check GENEVE option is supported in NIC. */ 3441 if (!hca_attr->geneve_tlv_opt) 3442 return rte_flow_error_set 3443 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3444 "Geneve TLV opt not supported"); 3445 /* Check if we already have geneve option with different type/class. */ 3446 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3447 geneve_opt_resource = sh->geneve_tlv_option_resource; 3448 if (geneve_opt_resource != NULL) 3449 if (geneve_opt_resource->option_class != spec->option_class || 3450 geneve_opt_resource->option_type != spec->option_type || 3451 geneve_opt_resource->length != spec->option_len) { 3452 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3453 return rte_flow_error_set(error, ENOTSUP, 3454 RTE_FLOW_ERROR_TYPE_ITEM, item, 3455 "Only one Geneve TLV option supported"); 3456 } 3457 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3458 return 0; 3459 } 3460 3461 /** 3462 * Validate MPLS item. 3463 * 3464 * @param[in] dev 3465 * Pointer to the rte_eth_dev structure. 3466 * @param[in] item 3467 * Item specification. 3468 * @param[in] item_flags 3469 * Bit-fields that holds the items detected until now. 3470 * @param[in] prev_layer 3471 * The protocol layer indicated in previous item. 3472 * @param[out] error 3473 * Pointer to error structure. 3474 * 3475 * @return 3476 * 0 on success, a negative errno value otherwise and rte_errno is set. 3477 */ 3478 int 3479 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3480 const struct rte_flow_item *item __rte_unused, 3481 uint64_t item_flags __rte_unused, 3482 uint64_t prev_layer __rte_unused, 3483 struct rte_flow_error *error) 3484 { 3485 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3486 const struct rte_flow_item_mpls *mask = item->mask; 3487 struct mlx5_priv *priv = dev->data->dev_private; 3488 int ret; 3489 3490 if (!priv->sh->dev_cap.mpls_en) 3491 return rte_flow_error_set(error, ENOTSUP, 3492 RTE_FLOW_ERROR_TYPE_ITEM, item, 3493 "MPLS not supported or" 3494 " disabled in firmware" 3495 " configuration."); 3496 /* MPLS over UDP, GRE is allowed */ 3497 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3498 MLX5_FLOW_LAYER_GRE | 3499 MLX5_FLOW_LAYER_GRE_KEY))) 3500 return rte_flow_error_set(error, EINVAL, 3501 RTE_FLOW_ERROR_TYPE_ITEM, item, 3502 "protocol filtering not compatible" 3503 " with MPLS layer"); 3504 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3505 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3506 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3507 return rte_flow_error_set(error, ENOTSUP, 3508 RTE_FLOW_ERROR_TYPE_ITEM, item, 3509 "multiple tunnel layers not" 3510 " supported"); 3511 if (!mask) 3512 mask = &rte_flow_item_mpls_mask; 3513 ret = mlx5_flow_item_acceptable 3514 (item, (const uint8_t *)mask, 3515 (const uint8_t *)&rte_flow_item_mpls_mask, 3516 sizeof(struct rte_flow_item_mpls), 3517 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3518 if (ret < 0) 3519 return ret; 3520 return 0; 3521 #else 3522 return rte_flow_error_set(error, ENOTSUP, 3523 RTE_FLOW_ERROR_TYPE_ITEM, item, 3524 "MPLS is not supported by Verbs, please" 3525 " update."); 3526 #endif 3527 } 3528 3529 /** 3530 * Validate NVGRE item. 3531 * 3532 * @param[in] item 3533 * Item specification. 3534 * @param[in] item_flags 3535 * Bit flags to mark detected items. 3536 * @param[in] target_protocol 3537 * The next protocol in the previous item. 3538 * @param[out] error 3539 * Pointer to error structure. 3540 * 3541 * @return 3542 * 0 on success, a negative errno value otherwise and rte_errno is set. 3543 */ 3544 int 3545 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3546 uint64_t item_flags, 3547 uint8_t target_protocol, 3548 struct rte_flow_error *error) 3549 { 3550 const struct rte_flow_item_nvgre *mask = item->mask; 3551 int ret; 3552 3553 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3554 return rte_flow_error_set(error, EINVAL, 3555 RTE_FLOW_ERROR_TYPE_ITEM, item, 3556 "protocol filtering not compatible" 3557 " with this GRE layer"); 3558 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3559 return rte_flow_error_set(error, ENOTSUP, 3560 RTE_FLOW_ERROR_TYPE_ITEM, item, 3561 "multiple tunnel layers not" 3562 " supported"); 3563 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3564 return rte_flow_error_set(error, ENOTSUP, 3565 RTE_FLOW_ERROR_TYPE_ITEM, item, 3566 "L3 Layer is missing"); 3567 if (!mask) 3568 mask = &rte_flow_item_nvgre_mask; 3569 ret = mlx5_flow_item_acceptable 3570 (item, (const uint8_t *)mask, 3571 (const uint8_t *)&rte_flow_item_nvgre_mask, 3572 sizeof(struct rte_flow_item_nvgre), 3573 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3574 if (ret < 0) 3575 return ret; 3576 return 0; 3577 } 3578 3579 /** 3580 * Validate eCPRI item. 3581 * 3582 * @param[in] item 3583 * Item specification. 3584 * @param[in] item_flags 3585 * Bit-fields that holds the items detected until now. 3586 * @param[in] last_item 3587 * Previous validated item in the pattern items. 3588 * @param[in] ether_type 3589 * Type in the ethernet layer header (including dot1q). 3590 * @param[in] acc_mask 3591 * Acceptable mask, if NULL default internal default mask 3592 * will be used to check whether item fields are supported. 3593 * @param[out] error 3594 * Pointer to error structure. 3595 * 3596 * @return 3597 * 0 on success, a negative errno value otherwise and rte_errno is set. 3598 */ 3599 int 3600 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3601 uint64_t item_flags, 3602 uint64_t last_item, 3603 uint16_t ether_type, 3604 const struct rte_flow_item_ecpri *acc_mask, 3605 struct rte_flow_error *error) 3606 { 3607 const struct rte_flow_item_ecpri *mask = item->mask; 3608 const struct rte_flow_item_ecpri nic_mask = { 3609 .hdr = { 3610 .common = { 3611 .u32 = 3612 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3613 .type = 0xFF, 3614 }).u32), 3615 }, 3616 .dummy[0] = 0xFFFFFFFF, 3617 }, 3618 }; 3619 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3620 MLX5_FLOW_LAYER_OUTER_VLAN); 3621 struct rte_flow_item_ecpri mask_lo; 3622 3623 if (!(last_item & outer_l2_vlan) && 3624 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3625 return rte_flow_error_set(error, EINVAL, 3626 RTE_FLOW_ERROR_TYPE_ITEM, item, 3627 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3628 if ((last_item & outer_l2_vlan) && ether_type && 3629 ether_type != RTE_ETHER_TYPE_ECPRI) 3630 return rte_flow_error_set(error, EINVAL, 3631 RTE_FLOW_ERROR_TYPE_ITEM, item, 3632 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3633 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3634 return rte_flow_error_set(error, EINVAL, 3635 RTE_FLOW_ERROR_TYPE_ITEM, item, 3636 "eCPRI with tunnel is not supported right now"); 3637 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3638 return rte_flow_error_set(error, ENOTSUP, 3639 RTE_FLOW_ERROR_TYPE_ITEM, item, 3640 "multiple L3 layers not supported"); 3641 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3642 return rte_flow_error_set(error, EINVAL, 3643 RTE_FLOW_ERROR_TYPE_ITEM, item, 3644 "eCPRI cannot coexist with a TCP layer"); 3645 /* In specification, eCPRI could be over UDP layer. */ 3646 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3647 return rte_flow_error_set(error, EINVAL, 3648 RTE_FLOW_ERROR_TYPE_ITEM, item, 3649 "eCPRI over UDP layer is not yet supported right now"); 3650 /* Mask for type field in common header could be zero. */ 3651 if (!mask) 3652 mask = &rte_flow_item_ecpri_mask; 3653 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3654 /* Input mask is in big-endian format. */ 3655 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3656 return rte_flow_error_set(error, EINVAL, 3657 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3658 "partial mask is not supported for protocol"); 3659 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3660 return rte_flow_error_set(error, EINVAL, 3661 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3662 "message header mask must be after a type mask"); 3663 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3664 acc_mask ? (const uint8_t *)acc_mask 3665 : (const uint8_t *)&nic_mask, 3666 sizeof(struct rte_flow_item_ecpri), 3667 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3668 } 3669 3670 static int 3671 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3672 const struct rte_flow_attr *attr __rte_unused, 3673 const struct rte_flow_item items[] __rte_unused, 3674 const struct rte_flow_action actions[] __rte_unused, 3675 bool external __rte_unused, 3676 int hairpin __rte_unused, 3677 struct rte_flow_error *error) 3678 { 3679 return rte_flow_error_set(error, ENOTSUP, 3680 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3681 } 3682 3683 static struct mlx5_flow * 3684 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3685 const struct rte_flow_attr *attr __rte_unused, 3686 const struct rte_flow_item items[] __rte_unused, 3687 const struct rte_flow_action actions[] __rte_unused, 3688 struct rte_flow_error *error) 3689 { 3690 rte_flow_error_set(error, ENOTSUP, 3691 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3692 return NULL; 3693 } 3694 3695 static int 3696 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3697 struct mlx5_flow *dev_flow __rte_unused, 3698 const struct rte_flow_attr *attr __rte_unused, 3699 const struct rte_flow_item items[] __rte_unused, 3700 const struct rte_flow_action actions[] __rte_unused, 3701 struct rte_flow_error *error) 3702 { 3703 return rte_flow_error_set(error, ENOTSUP, 3704 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3705 } 3706 3707 static int 3708 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3709 struct rte_flow *flow __rte_unused, 3710 struct rte_flow_error *error) 3711 { 3712 return rte_flow_error_set(error, ENOTSUP, 3713 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3714 } 3715 3716 static void 3717 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3718 struct rte_flow *flow __rte_unused) 3719 { 3720 } 3721 3722 static void 3723 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3724 struct rte_flow *flow __rte_unused) 3725 { 3726 } 3727 3728 static int 3729 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3730 struct rte_flow *flow __rte_unused, 3731 const struct rte_flow_action *actions __rte_unused, 3732 void *data __rte_unused, 3733 struct rte_flow_error *error) 3734 { 3735 return rte_flow_error_set(error, ENOTSUP, 3736 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3737 } 3738 3739 static int 3740 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3741 uint32_t domains __rte_unused, 3742 uint32_t flags __rte_unused) 3743 { 3744 return 0; 3745 } 3746 3747 /* Void driver to protect from null pointer reference. */ 3748 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3749 .validate = flow_null_validate, 3750 .prepare = flow_null_prepare, 3751 .translate = flow_null_translate, 3752 .apply = flow_null_apply, 3753 .remove = flow_null_remove, 3754 .destroy = flow_null_destroy, 3755 .query = flow_null_query, 3756 .sync_domain = flow_null_sync_domain, 3757 }; 3758 3759 /** 3760 * Select flow driver type according to flow attributes and device 3761 * configuration. 3762 * 3763 * @param[in] dev 3764 * Pointer to the dev structure. 3765 * @param[in] attr 3766 * Pointer to the flow attributes. 3767 * 3768 * @return 3769 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3770 */ 3771 static enum mlx5_flow_drv_type 3772 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3773 { 3774 struct mlx5_priv *priv = dev->data->dev_private; 3775 /* The OS can determine first a specific flow type (DV, VERBS) */ 3776 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3777 3778 if (type != MLX5_FLOW_TYPE_MAX) 3779 return type; 3780 /* 3781 * Currently when dv_flow_en == 2, only HW steering engine is 3782 * supported. New engines can also be chosen here if ready. 3783 */ 3784 if (priv->sh->config.dv_flow_en == 2) 3785 return MLX5_FLOW_TYPE_HW; 3786 if (!attr) 3787 return MLX5_FLOW_TYPE_MIN; 3788 /* If no OS specific type - continue with DV/VERBS selection */ 3789 if (attr->transfer && priv->sh->config.dv_esw_en) 3790 type = MLX5_FLOW_TYPE_DV; 3791 if (!attr->transfer) 3792 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3793 MLX5_FLOW_TYPE_VERBS; 3794 return type; 3795 } 3796 3797 #define flow_get_drv_ops(type) flow_drv_ops[type] 3798 3799 /** 3800 * Flow driver validation API. This abstracts calling driver specific functions. 3801 * The type of flow driver is determined according to flow attributes. 3802 * 3803 * @param[in] dev 3804 * Pointer to the dev structure. 3805 * @param[in] attr 3806 * Pointer to the flow attributes. 3807 * @param[in] items 3808 * Pointer to the list of items. 3809 * @param[in] actions 3810 * Pointer to the list of actions. 3811 * @param[in] external 3812 * This flow rule is created by request external to PMD. 3813 * @param[in] hairpin 3814 * Number of hairpin TX actions, 0 means classic flow. 3815 * @param[out] error 3816 * Pointer to the error structure. 3817 * 3818 * @return 3819 * 0 on success, a negative errno value otherwise and rte_errno is set. 3820 */ 3821 static inline int 3822 flow_drv_validate(struct rte_eth_dev *dev, 3823 const struct rte_flow_attr *attr, 3824 const struct rte_flow_item items[], 3825 const struct rte_flow_action actions[], 3826 bool external, int hairpin, struct rte_flow_error *error) 3827 { 3828 const struct mlx5_flow_driver_ops *fops; 3829 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3830 3831 fops = flow_get_drv_ops(type); 3832 return fops->validate(dev, attr, items, actions, external, 3833 hairpin, error); 3834 } 3835 3836 /** 3837 * Flow driver preparation API. This abstracts calling driver specific 3838 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3839 * calculates the size of memory required for device flow, allocates the memory, 3840 * initializes the device flow and returns the pointer. 3841 * 3842 * @note 3843 * This function initializes device flow structure such as dv or verbs in 3844 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3845 * rest. For example, adding returning device flow to flow->dev_flow list and 3846 * setting backward reference to the flow should be done out of this function. 3847 * layers field is not filled either. 3848 * 3849 * @param[in] dev 3850 * Pointer to the dev structure. 3851 * @param[in] attr 3852 * Pointer to the flow attributes. 3853 * @param[in] items 3854 * Pointer to the list of items. 3855 * @param[in] actions 3856 * Pointer to the list of actions. 3857 * @param[in] flow_idx 3858 * This memory pool index to the flow. 3859 * @param[out] error 3860 * Pointer to the error structure. 3861 * 3862 * @return 3863 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3864 */ 3865 static inline struct mlx5_flow * 3866 flow_drv_prepare(struct rte_eth_dev *dev, 3867 const struct rte_flow *flow, 3868 const struct rte_flow_attr *attr, 3869 const struct rte_flow_item items[], 3870 const struct rte_flow_action actions[], 3871 uint32_t flow_idx, 3872 struct rte_flow_error *error) 3873 { 3874 const struct mlx5_flow_driver_ops *fops; 3875 enum mlx5_flow_drv_type type = flow->drv_type; 3876 struct mlx5_flow *mlx5_flow = NULL; 3877 3878 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3879 fops = flow_get_drv_ops(type); 3880 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3881 if (mlx5_flow) 3882 mlx5_flow->flow_idx = flow_idx; 3883 return mlx5_flow; 3884 } 3885 3886 /** 3887 * Flow driver translation API. This abstracts calling driver specific 3888 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3889 * translates a generic flow into a driver flow. flow_drv_prepare() must 3890 * precede. 3891 * 3892 * @note 3893 * dev_flow->layers could be filled as a result of parsing during translation 3894 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3895 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3896 * flow->actions could be overwritten even though all the expanded dev_flows 3897 * have the same actions. 3898 * 3899 * @param[in] dev 3900 * Pointer to the rte dev structure. 3901 * @param[in, out] dev_flow 3902 * Pointer to the mlx5 flow. 3903 * @param[in] attr 3904 * Pointer to the flow attributes. 3905 * @param[in] items 3906 * Pointer to the list of items. 3907 * @param[in] actions 3908 * Pointer to the list of actions. 3909 * @param[out] error 3910 * Pointer to the error structure. 3911 * 3912 * @return 3913 * 0 on success, a negative errno value otherwise and rte_errno is set. 3914 */ 3915 static inline int 3916 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3917 const struct rte_flow_attr *attr, 3918 const struct rte_flow_item items[], 3919 const struct rte_flow_action actions[], 3920 struct rte_flow_error *error) 3921 { 3922 const struct mlx5_flow_driver_ops *fops; 3923 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3924 3925 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3926 fops = flow_get_drv_ops(type); 3927 return fops->translate(dev, dev_flow, attr, items, actions, error); 3928 } 3929 3930 /** 3931 * Flow driver apply API. This abstracts calling driver specific functions. 3932 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3933 * translated driver flows on to device. flow_drv_translate() must precede. 3934 * 3935 * @param[in] dev 3936 * Pointer to Ethernet device structure. 3937 * @param[in, out] flow 3938 * Pointer to flow structure. 3939 * @param[out] error 3940 * Pointer to error structure. 3941 * 3942 * @return 3943 * 0 on success, a negative errno value otherwise and rte_errno is set. 3944 */ 3945 static inline int 3946 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3947 struct rte_flow_error *error) 3948 { 3949 const struct mlx5_flow_driver_ops *fops; 3950 enum mlx5_flow_drv_type type = flow->drv_type; 3951 3952 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3953 fops = flow_get_drv_ops(type); 3954 return fops->apply(dev, flow, error); 3955 } 3956 3957 /** 3958 * Flow driver destroy API. This abstracts calling driver specific functions. 3959 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3960 * on device and releases resources of the flow. 3961 * 3962 * @param[in] dev 3963 * Pointer to Ethernet device. 3964 * @param[in, out] flow 3965 * Pointer to flow structure. 3966 */ 3967 static inline void 3968 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3969 { 3970 const struct mlx5_flow_driver_ops *fops; 3971 enum mlx5_flow_drv_type type = flow->drv_type; 3972 3973 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3974 fops = flow_get_drv_ops(type); 3975 fops->destroy(dev, flow); 3976 } 3977 3978 /** 3979 * Flow driver find RSS policy tbl API. This abstracts calling driver 3980 * specific functions. Parent flow (rte_flow) should have driver 3981 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3982 * 3983 * @param[in] dev 3984 * Pointer to Ethernet device. 3985 * @param[in, out] flow 3986 * Pointer to flow structure. 3987 * @param[in] policy 3988 * Pointer to meter policy table. 3989 * @param[in] rss_desc 3990 * Pointer to rss_desc 3991 */ 3992 static struct mlx5_flow_meter_sub_policy * 3993 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3994 struct rte_flow *flow, 3995 struct mlx5_flow_meter_policy *policy, 3996 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3997 { 3998 const struct mlx5_flow_driver_ops *fops; 3999 enum mlx5_flow_drv_type type = flow->drv_type; 4000 4001 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4002 fops = flow_get_drv_ops(type); 4003 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4004 } 4005 4006 /** 4007 * Flow driver color tag rule API. This abstracts calling driver 4008 * specific functions. Parent flow (rte_flow) should have driver 4009 * type (drv_type). It will create the color tag rules in hierarchy meter. 4010 * 4011 * @param[in] dev 4012 * Pointer to Ethernet device. 4013 * @param[in, out] flow 4014 * Pointer to flow structure. 4015 * @param[in] fm 4016 * Pointer to flow meter structure. 4017 * @param[in] src_port 4018 * The src port this extra rule should use. 4019 * @param[in] item 4020 * The src port id match item. 4021 * @param[out] error 4022 * Pointer to error structure. 4023 */ 4024 static int 4025 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4026 struct rte_flow *flow, 4027 struct mlx5_flow_meter_info *fm, 4028 int32_t src_port, 4029 const struct rte_flow_item *item, 4030 struct rte_flow_error *error) 4031 { 4032 const struct mlx5_flow_driver_ops *fops; 4033 enum mlx5_flow_drv_type type = flow->drv_type; 4034 4035 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4036 fops = flow_get_drv_ops(type); 4037 return fops->meter_hierarchy_rule_create(dev, fm, 4038 src_port, item, error); 4039 } 4040 4041 /** 4042 * Get RSS action from the action list. 4043 * 4044 * @param[in] dev 4045 * Pointer to Ethernet device. 4046 * @param[in] actions 4047 * Pointer to the list of actions. 4048 * @param[in] flow 4049 * Parent flow structure pointer. 4050 * 4051 * @return 4052 * Pointer to the RSS action if exist, else return NULL. 4053 */ 4054 static const struct rte_flow_action_rss* 4055 flow_get_rss_action(struct rte_eth_dev *dev, 4056 const struct rte_flow_action actions[]) 4057 { 4058 struct mlx5_priv *priv = dev->data->dev_private; 4059 const struct rte_flow_action_rss *rss = NULL; 4060 struct mlx5_meter_policy_action_container *acg; 4061 struct mlx5_meter_policy_action_container *acy; 4062 4063 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4064 switch (actions->type) { 4065 case RTE_FLOW_ACTION_TYPE_RSS: 4066 rss = actions->conf; 4067 break; 4068 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4069 { 4070 const struct rte_flow_action_sample *sample = 4071 actions->conf; 4072 const struct rte_flow_action *act = sample->actions; 4073 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4074 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4075 rss = act->conf; 4076 break; 4077 } 4078 case RTE_FLOW_ACTION_TYPE_METER: 4079 { 4080 uint32_t mtr_idx; 4081 struct mlx5_flow_meter_info *fm; 4082 struct mlx5_flow_meter_policy *policy; 4083 const struct rte_flow_action_meter *mtr = actions->conf; 4084 4085 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4086 if (fm && !fm->def_policy) { 4087 policy = mlx5_flow_meter_policy_find(dev, 4088 fm->policy_id, NULL); 4089 MLX5_ASSERT(policy); 4090 if (policy->is_hierarchy) { 4091 policy = 4092 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4093 policy); 4094 if (!policy) 4095 return NULL; 4096 } 4097 if (policy->is_rss) { 4098 acg = 4099 &policy->act_cnt[RTE_COLOR_GREEN]; 4100 acy = 4101 &policy->act_cnt[RTE_COLOR_YELLOW]; 4102 if (acg->fate_action == 4103 MLX5_FLOW_FATE_SHARED_RSS) 4104 rss = acg->rss->conf; 4105 else if (acy->fate_action == 4106 MLX5_FLOW_FATE_SHARED_RSS) 4107 rss = acy->rss->conf; 4108 } 4109 } 4110 break; 4111 } 4112 default: 4113 break; 4114 } 4115 } 4116 return rss; 4117 } 4118 4119 /** 4120 * Get ASO age action by index. 4121 * 4122 * @param[in] dev 4123 * Pointer to the Ethernet device structure. 4124 * @param[in] age_idx 4125 * Index to the ASO age action. 4126 * 4127 * @return 4128 * The specified ASO age action. 4129 */ 4130 struct mlx5_aso_age_action* 4131 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4132 { 4133 uint16_t pool_idx = age_idx & UINT16_MAX; 4134 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4135 struct mlx5_priv *priv = dev->data->dev_private; 4136 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4137 struct mlx5_aso_age_pool *pool; 4138 4139 rte_rwlock_read_lock(&mng->resize_rwl); 4140 pool = mng->pools[pool_idx]; 4141 rte_rwlock_read_unlock(&mng->resize_rwl); 4142 return &pool->actions[offset - 1]; 4143 } 4144 4145 /* maps indirect action to translated direct in some actions array */ 4146 struct mlx5_translated_action_handle { 4147 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4148 int index; /**< Index in related array of rte_flow_action. */ 4149 }; 4150 4151 /** 4152 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4153 * direct action if translation possible. 4154 * This functionality used to run same execution path for both direct and 4155 * indirect actions on flow create. All necessary preparations for indirect 4156 * action handling should be performed on *handle* actions list returned 4157 * from this call. 4158 * 4159 * @param[in] dev 4160 * Pointer to Ethernet device. 4161 * @param[in] actions 4162 * List of actions to translate. 4163 * @param[out] handle 4164 * List to store translated indirect action object handles. 4165 * @param[in, out] indir_n 4166 * Size of *handle* array. On return should be updated with number of 4167 * indirect actions retrieved from the *actions* list. 4168 * @param[out] translated_actions 4169 * List of actions where all indirect actions were translated to direct 4170 * if possible. NULL if no translation took place. 4171 * @param[out] error 4172 * Pointer to the error structure. 4173 * 4174 * @return 4175 * 0 on success, a negative errno value otherwise and rte_errno is set. 4176 */ 4177 static int 4178 flow_action_handles_translate(struct rte_eth_dev *dev, 4179 const struct rte_flow_action actions[], 4180 struct mlx5_translated_action_handle *handle, 4181 int *indir_n, 4182 struct rte_flow_action **translated_actions, 4183 struct rte_flow_error *error) 4184 { 4185 struct mlx5_priv *priv = dev->data->dev_private; 4186 struct rte_flow_action *translated = NULL; 4187 size_t actions_size; 4188 int n; 4189 int copied_n = 0; 4190 struct mlx5_translated_action_handle *handle_end = NULL; 4191 4192 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4193 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4194 continue; 4195 if (copied_n == *indir_n) { 4196 return rte_flow_error_set 4197 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4198 NULL, "too many shared actions"); 4199 } 4200 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4201 sizeof(actions[n].conf)); 4202 handle[copied_n].index = n; 4203 copied_n++; 4204 } 4205 n++; 4206 *indir_n = copied_n; 4207 if (!copied_n) 4208 return 0; 4209 actions_size = sizeof(struct rte_flow_action) * n; 4210 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4211 if (!translated) { 4212 rte_errno = ENOMEM; 4213 return -ENOMEM; 4214 } 4215 memcpy(translated, actions, actions_size); 4216 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4217 struct mlx5_shared_action_rss *shared_rss; 4218 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4219 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4220 uint32_t idx = act_idx & 4221 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4222 4223 switch (type) { 4224 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4225 shared_rss = mlx5_ipool_get 4226 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4227 translated[handle->index].type = 4228 RTE_FLOW_ACTION_TYPE_RSS; 4229 translated[handle->index].conf = 4230 &shared_rss->origin; 4231 break; 4232 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4233 translated[handle->index].type = 4234 (enum rte_flow_action_type) 4235 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4236 translated[handle->index].conf = (void *)(uintptr_t)idx; 4237 break; 4238 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4239 translated[handle->index].type = 4240 (enum rte_flow_action_type) 4241 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4242 translated[handle->index].conf = (void *)(uintptr_t)idx; 4243 break; 4244 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4245 if (priv->sh->flow_hit_aso_en) { 4246 translated[handle->index].type = 4247 (enum rte_flow_action_type) 4248 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4249 translated[handle->index].conf = 4250 (void *)(uintptr_t)idx; 4251 break; 4252 } 4253 /* Fall-through */ 4254 case MLX5_INDIRECT_ACTION_TYPE_CT: 4255 if (priv->sh->ct_aso_en) { 4256 translated[handle->index].type = 4257 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4258 translated[handle->index].conf = 4259 (void *)(uintptr_t)idx; 4260 break; 4261 } 4262 /* Fall-through */ 4263 default: 4264 mlx5_free(translated); 4265 return rte_flow_error_set 4266 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4267 NULL, "invalid indirect action type"); 4268 } 4269 } 4270 *translated_actions = translated; 4271 return 0; 4272 } 4273 4274 /** 4275 * Get Shared RSS action from the action list. 4276 * 4277 * @param[in] dev 4278 * Pointer to Ethernet device. 4279 * @param[in] shared 4280 * Pointer to the list of actions. 4281 * @param[in] shared_n 4282 * Actions list length. 4283 * 4284 * @return 4285 * The MLX5 RSS action ID if exists, otherwise return 0. 4286 */ 4287 static uint32_t 4288 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4289 struct mlx5_translated_action_handle *handle, 4290 int shared_n) 4291 { 4292 struct mlx5_translated_action_handle *handle_end; 4293 struct mlx5_priv *priv = dev->data->dev_private; 4294 struct mlx5_shared_action_rss *shared_rss; 4295 4296 4297 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4298 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4299 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4300 uint32_t idx = act_idx & 4301 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4302 switch (type) { 4303 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4304 shared_rss = mlx5_ipool_get 4305 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4306 idx); 4307 __atomic_add_fetch(&shared_rss->refcnt, 1, 4308 __ATOMIC_RELAXED); 4309 return idx; 4310 default: 4311 break; 4312 } 4313 } 4314 return 0; 4315 } 4316 4317 static unsigned int 4318 find_graph_root(uint32_t rss_level) 4319 { 4320 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4321 MLX5_EXPANSION_ROOT_OUTER; 4322 } 4323 4324 /** 4325 * Get layer flags from the prefix flow. 4326 * 4327 * Some flows may be split to several subflows, the prefix subflow gets the 4328 * match items and the suffix sub flow gets the actions. 4329 * Some actions need the user defined match item flags to get the detail for 4330 * the action. 4331 * This function helps the suffix flow to get the item layer flags from prefix 4332 * subflow. 4333 * 4334 * @param[in] dev_flow 4335 * Pointer the created prefix subflow. 4336 * 4337 * @return 4338 * The layers get from prefix subflow. 4339 */ 4340 static inline uint64_t 4341 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4342 { 4343 uint64_t layers = 0; 4344 4345 /* 4346 * Layers bits could be localization, but usually the compiler will 4347 * help to do the optimization work for source code. 4348 * If no decap actions, use the layers directly. 4349 */ 4350 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4351 return dev_flow->handle->layers; 4352 /* Convert L3 layers with decap action. */ 4353 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4354 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4355 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4356 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4357 /* Convert L4 layers with decap action. */ 4358 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4359 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4360 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4361 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4362 return layers; 4363 } 4364 4365 /** 4366 * Get metadata split action information. 4367 * 4368 * @param[in] actions 4369 * Pointer to the list of actions. 4370 * @param[out] qrss 4371 * Pointer to the return pointer. 4372 * @param[out] qrss_type 4373 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4374 * if no QUEUE/RSS is found. 4375 * @param[out] encap_idx 4376 * Pointer to the index of the encap action if exists, otherwise the last 4377 * action index. 4378 * 4379 * @return 4380 * Total number of actions. 4381 */ 4382 static int 4383 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4384 const struct rte_flow_action **qrss, 4385 int *encap_idx) 4386 { 4387 const struct rte_flow_action_raw_encap *raw_encap; 4388 int actions_n = 0; 4389 int raw_decap_idx = -1; 4390 4391 *encap_idx = -1; 4392 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4393 switch (actions->type) { 4394 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4395 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4396 *encap_idx = actions_n; 4397 break; 4398 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4399 raw_decap_idx = actions_n; 4400 break; 4401 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4402 raw_encap = actions->conf; 4403 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4404 *encap_idx = raw_decap_idx != -1 ? 4405 raw_decap_idx : actions_n; 4406 break; 4407 case RTE_FLOW_ACTION_TYPE_QUEUE: 4408 case RTE_FLOW_ACTION_TYPE_RSS: 4409 *qrss = actions; 4410 break; 4411 default: 4412 break; 4413 } 4414 actions_n++; 4415 } 4416 if (*encap_idx == -1) 4417 *encap_idx = actions_n; 4418 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4419 return actions_n + 1; 4420 } 4421 4422 /** 4423 * Check if the action will change packet. 4424 * 4425 * @param dev 4426 * Pointer to Ethernet device. 4427 * @param[in] type 4428 * action type. 4429 * 4430 * @return 4431 * true if action will change packet, false otherwise. 4432 */ 4433 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4434 enum rte_flow_action_type type) 4435 { 4436 struct mlx5_priv *priv = dev->data->dev_private; 4437 4438 switch (type) { 4439 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4440 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4441 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4442 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4443 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4444 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4445 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4446 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4447 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4448 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4449 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4450 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4451 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4452 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4453 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4454 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4455 case RTE_FLOW_ACTION_TYPE_SET_META: 4456 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4457 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4458 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4459 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4460 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4461 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4462 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4463 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4464 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4465 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4466 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4467 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4468 return true; 4469 case RTE_FLOW_ACTION_TYPE_FLAG: 4470 case RTE_FLOW_ACTION_TYPE_MARK: 4471 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4472 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4473 return true; 4474 else 4475 return false; 4476 default: 4477 return false; 4478 } 4479 } 4480 4481 /** 4482 * Check meter action from the action list. 4483 * 4484 * @param dev 4485 * Pointer to Ethernet device. 4486 * @param[in] actions 4487 * Pointer to the list of actions. 4488 * @param[out] has_mtr 4489 * Pointer to the meter exist flag. 4490 * @param[out] has_modify 4491 * Pointer to the flag showing there's packet change action. 4492 * @param[out] meter_id 4493 * Pointer to the meter id. 4494 * 4495 * @return 4496 * Total number of actions. 4497 */ 4498 static int 4499 flow_check_meter_action(struct rte_eth_dev *dev, 4500 const struct rte_flow_action actions[], 4501 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4502 { 4503 const struct rte_flow_action_meter *mtr = NULL; 4504 int actions_n = 0; 4505 4506 MLX5_ASSERT(has_mtr); 4507 *has_mtr = false; 4508 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4509 switch (actions->type) { 4510 case RTE_FLOW_ACTION_TYPE_METER: 4511 mtr = actions->conf; 4512 *meter_id = mtr->mtr_id; 4513 *has_mtr = true; 4514 break; 4515 default: 4516 break; 4517 } 4518 if (!*has_mtr) 4519 *has_modify |= flow_check_modify_action_type(dev, 4520 actions->type); 4521 actions_n++; 4522 } 4523 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4524 return actions_n + 1; 4525 } 4526 4527 /** 4528 * Check if the flow should be split due to hairpin. 4529 * The reason for the split is that in current HW we can't 4530 * support encap and push-vlan on Rx, so if a flow contains 4531 * these actions we move it to Tx. 4532 * 4533 * @param dev 4534 * Pointer to Ethernet device. 4535 * @param[in] attr 4536 * Flow rule attributes. 4537 * @param[in] actions 4538 * Associated actions (list terminated by the END action). 4539 * 4540 * @return 4541 * > 0 the number of actions and the flow should be split, 4542 * 0 when no split required. 4543 */ 4544 static int 4545 flow_check_hairpin_split(struct rte_eth_dev *dev, 4546 const struct rte_flow_attr *attr, 4547 const struct rte_flow_action actions[]) 4548 { 4549 int queue_action = 0; 4550 int action_n = 0; 4551 int split = 0; 4552 const struct rte_flow_action_queue *queue; 4553 const struct rte_flow_action_rss *rss; 4554 const struct rte_flow_action_raw_encap *raw_encap; 4555 const struct rte_eth_hairpin_conf *conf; 4556 4557 if (!attr->ingress) 4558 return 0; 4559 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4560 switch (actions->type) { 4561 case RTE_FLOW_ACTION_TYPE_QUEUE: 4562 queue = actions->conf; 4563 if (queue == NULL) 4564 return 0; 4565 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4566 if (conf == NULL || conf->tx_explicit != 0) 4567 return 0; 4568 queue_action = 1; 4569 action_n++; 4570 break; 4571 case RTE_FLOW_ACTION_TYPE_RSS: 4572 rss = actions->conf; 4573 if (rss == NULL || rss->queue_num == 0) 4574 return 0; 4575 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4576 if (conf == NULL || conf->tx_explicit != 0) 4577 return 0; 4578 queue_action = 1; 4579 action_n++; 4580 break; 4581 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4582 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4583 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4584 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4585 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4586 split++; 4587 action_n++; 4588 break; 4589 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4590 raw_encap = actions->conf; 4591 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4592 split++; 4593 action_n++; 4594 break; 4595 default: 4596 action_n++; 4597 break; 4598 } 4599 } 4600 if (split && queue_action) 4601 return action_n; 4602 return 0; 4603 } 4604 4605 /* Declare flow create/destroy prototype in advance. */ 4606 static uint32_t 4607 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4608 const struct rte_flow_attr *attr, 4609 const struct rte_flow_item items[], 4610 const struct rte_flow_action actions[], 4611 bool external, struct rte_flow_error *error); 4612 4613 static void 4614 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4615 uint32_t flow_idx); 4616 4617 int 4618 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4619 struct mlx5_list_entry *entry, void *cb_ctx) 4620 { 4621 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4622 struct mlx5_flow_mreg_copy_resource *mcp_res = 4623 container_of(entry, typeof(*mcp_res), hlist_ent); 4624 4625 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4626 } 4627 4628 struct mlx5_list_entry * 4629 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4630 { 4631 struct rte_eth_dev *dev = tool_ctx; 4632 struct mlx5_priv *priv = dev->data->dev_private; 4633 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4634 struct mlx5_flow_mreg_copy_resource *mcp_res; 4635 struct rte_flow_error *error = ctx->error; 4636 uint32_t idx = 0; 4637 int ret; 4638 uint32_t mark_id = *(uint32_t *)(ctx->data); 4639 struct rte_flow_attr attr = { 4640 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4641 .ingress = 1, 4642 }; 4643 struct mlx5_rte_flow_item_tag tag_spec = { 4644 .data = mark_id, 4645 }; 4646 struct rte_flow_item items[] = { 4647 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4648 }; 4649 struct rte_flow_action_mark ftag = { 4650 .id = mark_id, 4651 }; 4652 struct mlx5_flow_action_copy_mreg cp_mreg = { 4653 .dst = REG_B, 4654 .src = REG_NON, 4655 }; 4656 struct rte_flow_action_jump jump = { 4657 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4658 }; 4659 struct rte_flow_action actions[] = { 4660 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4661 }; 4662 4663 /* Fill the register fields in the flow. */ 4664 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4665 if (ret < 0) 4666 return NULL; 4667 tag_spec.id = ret; 4668 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4669 if (ret < 0) 4670 return NULL; 4671 cp_mreg.src = ret; 4672 /* Provide the full width of FLAG specific value. */ 4673 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4674 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4675 /* Build a new flow. */ 4676 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4677 items[0] = (struct rte_flow_item){ 4678 .type = (enum rte_flow_item_type) 4679 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4680 .spec = &tag_spec, 4681 }; 4682 items[1] = (struct rte_flow_item){ 4683 .type = RTE_FLOW_ITEM_TYPE_END, 4684 }; 4685 actions[0] = (struct rte_flow_action){ 4686 .type = (enum rte_flow_action_type) 4687 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4688 .conf = &ftag, 4689 }; 4690 actions[1] = (struct rte_flow_action){ 4691 .type = (enum rte_flow_action_type) 4692 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4693 .conf = &cp_mreg, 4694 }; 4695 actions[2] = (struct rte_flow_action){ 4696 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4697 .conf = &jump, 4698 }; 4699 actions[3] = (struct rte_flow_action){ 4700 .type = RTE_FLOW_ACTION_TYPE_END, 4701 }; 4702 } else { 4703 /* Default rule, wildcard match. */ 4704 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4705 items[0] = (struct rte_flow_item){ 4706 .type = RTE_FLOW_ITEM_TYPE_END, 4707 }; 4708 actions[0] = (struct rte_flow_action){ 4709 .type = (enum rte_flow_action_type) 4710 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4711 .conf = &cp_mreg, 4712 }; 4713 actions[1] = (struct rte_flow_action){ 4714 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4715 .conf = &jump, 4716 }; 4717 actions[2] = (struct rte_flow_action){ 4718 .type = RTE_FLOW_ACTION_TYPE_END, 4719 }; 4720 } 4721 /* Build a new entry. */ 4722 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4723 if (!mcp_res) { 4724 rte_errno = ENOMEM; 4725 return NULL; 4726 } 4727 mcp_res->idx = idx; 4728 mcp_res->mark_id = mark_id; 4729 /* 4730 * The copy Flows are not included in any list. There 4731 * ones are referenced from other Flows and can not 4732 * be applied, removed, deleted in arbitrary order 4733 * by list traversing. 4734 */ 4735 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4736 &attr, items, actions, false, error); 4737 if (!mcp_res->rix_flow) { 4738 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4739 return NULL; 4740 } 4741 return &mcp_res->hlist_ent; 4742 } 4743 4744 struct mlx5_list_entry * 4745 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4746 void *cb_ctx __rte_unused) 4747 { 4748 struct rte_eth_dev *dev = tool_ctx; 4749 struct mlx5_priv *priv = dev->data->dev_private; 4750 struct mlx5_flow_mreg_copy_resource *mcp_res; 4751 uint32_t idx = 0; 4752 4753 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4754 if (!mcp_res) { 4755 rte_errno = ENOMEM; 4756 return NULL; 4757 } 4758 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4759 mcp_res->idx = idx; 4760 return &mcp_res->hlist_ent; 4761 } 4762 4763 void 4764 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4765 { 4766 struct mlx5_flow_mreg_copy_resource *mcp_res = 4767 container_of(entry, typeof(*mcp_res), hlist_ent); 4768 struct rte_eth_dev *dev = tool_ctx; 4769 struct mlx5_priv *priv = dev->data->dev_private; 4770 4771 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4772 } 4773 4774 /** 4775 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4776 * 4777 * As mark_id is unique, if there's already a registered flow for the mark_id, 4778 * return by increasing the reference counter of the resource. Otherwise, create 4779 * the resource (mcp_res) and flow. 4780 * 4781 * Flow looks like, 4782 * - If ingress port is ANY and reg_c[1] is mark_id, 4783 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4784 * 4785 * For default flow (zero mark_id), flow is like, 4786 * - If ingress port is ANY, 4787 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4788 * 4789 * @param dev 4790 * Pointer to Ethernet device. 4791 * @param mark_id 4792 * ID of MARK action, zero means default flow for META. 4793 * @param[out] error 4794 * Perform verbose error reporting if not NULL. 4795 * 4796 * @return 4797 * Associated resource on success, NULL otherwise and rte_errno is set. 4798 */ 4799 static struct mlx5_flow_mreg_copy_resource * 4800 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4801 struct rte_flow_error *error) 4802 { 4803 struct mlx5_priv *priv = dev->data->dev_private; 4804 struct mlx5_list_entry *entry; 4805 struct mlx5_flow_cb_ctx ctx = { 4806 .dev = dev, 4807 .error = error, 4808 .data = &mark_id, 4809 }; 4810 4811 /* Check if already registered. */ 4812 MLX5_ASSERT(priv->mreg_cp_tbl); 4813 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4814 if (!entry) 4815 return NULL; 4816 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4817 hlist_ent); 4818 } 4819 4820 void 4821 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4822 { 4823 struct mlx5_flow_mreg_copy_resource *mcp_res = 4824 container_of(entry, typeof(*mcp_res), hlist_ent); 4825 struct rte_eth_dev *dev = tool_ctx; 4826 struct mlx5_priv *priv = dev->data->dev_private; 4827 4828 MLX5_ASSERT(mcp_res->rix_flow); 4829 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4830 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4831 } 4832 4833 /** 4834 * Release flow in RX_CP_TBL. 4835 * 4836 * @param dev 4837 * Pointer to Ethernet device. 4838 * @flow 4839 * Parent flow for wich copying is provided. 4840 */ 4841 static void 4842 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4843 struct rte_flow *flow) 4844 { 4845 struct mlx5_flow_mreg_copy_resource *mcp_res; 4846 struct mlx5_priv *priv = dev->data->dev_private; 4847 4848 if (!flow->rix_mreg_copy) 4849 return; 4850 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4851 flow->rix_mreg_copy); 4852 if (!mcp_res || !priv->mreg_cp_tbl) 4853 return; 4854 MLX5_ASSERT(mcp_res->rix_flow); 4855 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4856 flow->rix_mreg_copy = 0; 4857 } 4858 4859 /** 4860 * Remove the default copy action from RX_CP_TBL. 4861 * 4862 * This functions is called in the mlx5_dev_start(). No thread safe 4863 * is guaranteed. 4864 * 4865 * @param dev 4866 * Pointer to Ethernet device. 4867 */ 4868 static void 4869 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4870 { 4871 struct mlx5_list_entry *entry; 4872 struct mlx5_priv *priv = dev->data->dev_private; 4873 struct mlx5_flow_cb_ctx ctx; 4874 uint32_t mark_id; 4875 4876 /* Check if default flow is registered. */ 4877 if (!priv->mreg_cp_tbl) 4878 return; 4879 mark_id = MLX5_DEFAULT_COPY_ID; 4880 ctx.data = &mark_id; 4881 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4882 if (!entry) 4883 return; 4884 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4885 } 4886 4887 /** 4888 * Add the default copy action in in RX_CP_TBL. 4889 * 4890 * This functions is called in the mlx5_dev_start(). No thread safe 4891 * is guaranteed. 4892 * 4893 * @param dev 4894 * Pointer to Ethernet device. 4895 * @param[out] error 4896 * Perform verbose error reporting if not NULL. 4897 * 4898 * @return 4899 * 0 for success, negative value otherwise and rte_errno is set. 4900 */ 4901 static int 4902 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4903 struct rte_flow_error *error) 4904 { 4905 struct mlx5_priv *priv = dev->data->dev_private; 4906 struct mlx5_flow_mreg_copy_resource *mcp_res; 4907 struct mlx5_flow_cb_ctx ctx; 4908 uint32_t mark_id; 4909 4910 /* Check whether extensive metadata feature is engaged. */ 4911 if (!priv->sh->config.dv_flow_en || 4912 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4913 !mlx5_flow_ext_mreg_supported(dev) || 4914 !priv->sh->dv_regc0_mask) 4915 return 0; 4916 /* 4917 * Add default mreg copy flow may be called multiple time, but 4918 * only be called once in stop. Avoid register it twice. 4919 */ 4920 mark_id = MLX5_DEFAULT_COPY_ID; 4921 ctx.data = &mark_id; 4922 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4923 return 0; 4924 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4925 if (!mcp_res) 4926 return -rte_errno; 4927 return 0; 4928 } 4929 4930 /** 4931 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4932 * 4933 * All the flow having Q/RSS action should be split by 4934 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4935 * performs the following, 4936 * - CQE->flow_tag := reg_c[1] (MARK) 4937 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4938 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4939 * but there should be a flow per each MARK ID set by MARK action. 4940 * 4941 * For the aforementioned reason, if there's a MARK action in flow's action 4942 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4943 * the MARK ID to CQE's flow_tag like, 4944 * - If reg_c[1] is mark_id, 4945 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4946 * 4947 * For SET_META action which stores value in reg_c[0], as the destination is 4948 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4949 * MARK ID means the default flow. The default flow looks like, 4950 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4951 * 4952 * @param dev 4953 * Pointer to Ethernet device. 4954 * @param flow 4955 * Pointer to flow structure. 4956 * @param[in] actions 4957 * Pointer to the list of actions. 4958 * @param[out] error 4959 * Perform verbose error reporting if not NULL. 4960 * 4961 * @return 4962 * 0 on success, negative value otherwise and rte_errno is set. 4963 */ 4964 static int 4965 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4966 struct rte_flow *flow, 4967 const struct rte_flow_action *actions, 4968 struct rte_flow_error *error) 4969 { 4970 struct mlx5_priv *priv = dev->data->dev_private; 4971 struct mlx5_sh_config *config = &priv->sh->config; 4972 struct mlx5_flow_mreg_copy_resource *mcp_res; 4973 const struct rte_flow_action_mark *mark; 4974 4975 /* Check whether extensive metadata feature is engaged. */ 4976 if (!config->dv_flow_en || 4977 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4978 !mlx5_flow_ext_mreg_supported(dev) || 4979 !priv->sh->dv_regc0_mask) 4980 return 0; 4981 /* Find MARK action. */ 4982 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4983 switch (actions->type) { 4984 case RTE_FLOW_ACTION_TYPE_FLAG: 4985 mcp_res = flow_mreg_add_copy_action 4986 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4987 if (!mcp_res) 4988 return -rte_errno; 4989 flow->rix_mreg_copy = mcp_res->idx; 4990 return 0; 4991 case RTE_FLOW_ACTION_TYPE_MARK: 4992 mark = (const struct rte_flow_action_mark *) 4993 actions->conf; 4994 mcp_res = 4995 flow_mreg_add_copy_action(dev, mark->id, error); 4996 if (!mcp_res) 4997 return -rte_errno; 4998 flow->rix_mreg_copy = mcp_res->idx; 4999 return 0; 5000 default: 5001 break; 5002 } 5003 } 5004 return 0; 5005 } 5006 5007 #define MLX5_MAX_SPLIT_ACTIONS 24 5008 #define MLX5_MAX_SPLIT_ITEMS 24 5009 5010 /** 5011 * Split the hairpin flow. 5012 * Since HW can't support encap and push-vlan on Rx, we move these 5013 * actions to Tx. 5014 * If the count action is after the encap then we also 5015 * move the count action. in this case the count will also measure 5016 * the outer bytes. 5017 * 5018 * @param dev 5019 * Pointer to Ethernet device. 5020 * @param[in] actions 5021 * Associated actions (list terminated by the END action). 5022 * @param[out] actions_rx 5023 * Rx flow actions. 5024 * @param[out] actions_tx 5025 * Tx flow actions.. 5026 * @param[out] pattern_tx 5027 * The pattern items for the Tx flow. 5028 * @param[out] flow_id 5029 * The flow ID connected to this flow. 5030 * 5031 * @return 5032 * 0 on success. 5033 */ 5034 static int 5035 flow_hairpin_split(struct rte_eth_dev *dev, 5036 const struct rte_flow_action actions[], 5037 struct rte_flow_action actions_rx[], 5038 struct rte_flow_action actions_tx[], 5039 struct rte_flow_item pattern_tx[], 5040 uint32_t flow_id) 5041 { 5042 const struct rte_flow_action_raw_encap *raw_encap; 5043 const struct rte_flow_action_raw_decap *raw_decap; 5044 struct mlx5_rte_flow_action_set_tag *set_tag; 5045 struct rte_flow_action *tag_action; 5046 struct mlx5_rte_flow_item_tag *tag_item; 5047 struct rte_flow_item *item; 5048 char *addr; 5049 int encap = 0; 5050 5051 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5052 switch (actions->type) { 5053 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5054 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5055 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5056 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5057 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5058 rte_memcpy(actions_tx, actions, 5059 sizeof(struct rte_flow_action)); 5060 actions_tx++; 5061 break; 5062 case RTE_FLOW_ACTION_TYPE_COUNT: 5063 if (encap) { 5064 rte_memcpy(actions_tx, actions, 5065 sizeof(struct rte_flow_action)); 5066 actions_tx++; 5067 } else { 5068 rte_memcpy(actions_rx, actions, 5069 sizeof(struct rte_flow_action)); 5070 actions_rx++; 5071 } 5072 break; 5073 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5074 raw_encap = actions->conf; 5075 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5076 memcpy(actions_tx, actions, 5077 sizeof(struct rte_flow_action)); 5078 actions_tx++; 5079 encap = 1; 5080 } else { 5081 rte_memcpy(actions_rx, actions, 5082 sizeof(struct rte_flow_action)); 5083 actions_rx++; 5084 } 5085 break; 5086 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5087 raw_decap = actions->conf; 5088 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5089 memcpy(actions_tx, actions, 5090 sizeof(struct rte_flow_action)); 5091 actions_tx++; 5092 } else { 5093 rte_memcpy(actions_rx, actions, 5094 sizeof(struct rte_flow_action)); 5095 actions_rx++; 5096 } 5097 break; 5098 default: 5099 rte_memcpy(actions_rx, actions, 5100 sizeof(struct rte_flow_action)); 5101 actions_rx++; 5102 break; 5103 } 5104 } 5105 /* Add set meta action and end action for the Rx flow. */ 5106 tag_action = actions_rx; 5107 tag_action->type = (enum rte_flow_action_type) 5108 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5109 actions_rx++; 5110 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5111 actions_rx++; 5112 set_tag = (void *)actions_rx; 5113 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5114 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5115 .data = flow_id, 5116 }; 5117 MLX5_ASSERT(set_tag->id > REG_NON); 5118 tag_action->conf = set_tag; 5119 /* Create Tx item list. */ 5120 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5121 addr = (void *)&pattern_tx[2]; 5122 item = pattern_tx; 5123 item->type = (enum rte_flow_item_type) 5124 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5125 tag_item = (void *)addr; 5126 tag_item->data = flow_id; 5127 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5128 MLX5_ASSERT(set_tag->id > REG_NON); 5129 item->spec = tag_item; 5130 addr += sizeof(struct mlx5_rte_flow_item_tag); 5131 tag_item = (void *)addr; 5132 tag_item->data = UINT32_MAX; 5133 tag_item->id = UINT16_MAX; 5134 item->mask = tag_item; 5135 item->last = NULL; 5136 item++; 5137 item->type = RTE_FLOW_ITEM_TYPE_END; 5138 return 0; 5139 } 5140 5141 /** 5142 * The last stage of splitting chain, just creates the subflow 5143 * without any modification. 5144 * 5145 * @param[in] dev 5146 * Pointer to Ethernet device. 5147 * @param[in] flow 5148 * Parent flow structure pointer. 5149 * @param[in, out] sub_flow 5150 * Pointer to return the created subflow, may be NULL. 5151 * @param[in] attr 5152 * Flow rule attributes. 5153 * @param[in] items 5154 * Pattern specification (list terminated by the END pattern item). 5155 * @param[in] actions 5156 * Associated actions (list terminated by the END action). 5157 * @param[in] flow_split_info 5158 * Pointer to flow split info structure. 5159 * @param[out] error 5160 * Perform verbose error reporting if not NULL. 5161 * @return 5162 * 0 on success, negative value otherwise 5163 */ 5164 static int 5165 flow_create_split_inner(struct rte_eth_dev *dev, 5166 struct rte_flow *flow, 5167 struct mlx5_flow **sub_flow, 5168 const struct rte_flow_attr *attr, 5169 const struct rte_flow_item items[], 5170 const struct rte_flow_action actions[], 5171 struct mlx5_flow_split_info *flow_split_info, 5172 struct rte_flow_error *error) 5173 { 5174 struct mlx5_flow *dev_flow; 5175 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5176 5177 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5178 flow_split_info->flow_idx, error); 5179 if (!dev_flow) 5180 return -rte_errno; 5181 dev_flow->flow = flow; 5182 dev_flow->external = flow_split_info->external; 5183 dev_flow->skip_scale = flow_split_info->skip_scale; 5184 /* Subflow object was created, we must include one in the list. */ 5185 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5186 dev_flow->handle, next); 5187 /* 5188 * If dev_flow is as one of the suffix flow, some actions in suffix 5189 * flow may need some user defined item layer flags, and pass the 5190 * Metadata rxq mark flag to suffix flow as well. 5191 */ 5192 if (flow_split_info->prefix_layers) 5193 dev_flow->handle->layers = flow_split_info->prefix_layers; 5194 if (flow_split_info->prefix_mark) { 5195 MLX5_ASSERT(wks); 5196 wks->mark = 1; 5197 } 5198 if (sub_flow) 5199 *sub_flow = dev_flow; 5200 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5201 dev_flow->dv.table_id = flow_split_info->table_id; 5202 #endif 5203 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5204 } 5205 5206 /** 5207 * Get the sub policy of a meter. 5208 * 5209 * @param[in] dev 5210 * Pointer to Ethernet device. 5211 * @param[in] flow 5212 * Parent flow structure pointer. 5213 * @param wks 5214 * Pointer to thread flow work space. 5215 * @param[in] attr 5216 * Flow rule attributes. 5217 * @param[in] items 5218 * Pattern specification (list terminated by the END pattern item). 5219 * @param[out] error 5220 * Perform verbose error reporting if not NULL. 5221 * 5222 * @return 5223 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5224 */ 5225 static struct mlx5_flow_meter_sub_policy * 5226 get_meter_sub_policy(struct rte_eth_dev *dev, 5227 struct rte_flow *flow, 5228 struct mlx5_flow_workspace *wks, 5229 const struct rte_flow_attr *attr, 5230 const struct rte_flow_item items[], 5231 struct rte_flow_error *error) 5232 { 5233 struct mlx5_flow_meter_policy *policy; 5234 struct mlx5_flow_meter_policy *final_policy; 5235 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5236 5237 policy = wks->policy; 5238 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5239 if (final_policy->is_rss || final_policy->is_queue) { 5240 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5241 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5242 uint32_t i; 5243 5244 /* 5245 * This is a tmp dev_flow, 5246 * no need to register any matcher for it in translate. 5247 */ 5248 wks->skip_matcher_reg = 1; 5249 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5250 struct mlx5_flow dev_flow = {0}; 5251 struct mlx5_flow_handle dev_handle = { {0} }; 5252 uint8_t fate = final_policy->act_cnt[i].fate_action; 5253 5254 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5255 const struct rte_flow_action_rss *rss_act = 5256 final_policy->act_cnt[i].rss->conf; 5257 struct rte_flow_action rss_actions[2] = { 5258 [0] = { 5259 .type = RTE_FLOW_ACTION_TYPE_RSS, 5260 .conf = rss_act, 5261 }, 5262 [1] = { 5263 .type = RTE_FLOW_ACTION_TYPE_END, 5264 .conf = NULL, 5265 } 5266 }; 5267 5268 dev_flow.handle = &dev_handle; 5269 dev_flow.ingress = attr->ingress; 5270 dev_flow.flow = flow; 5271 dev_flow.external = 0; 5272 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5273 dev_flow.dv.transfer = attr->transfer; 5274 #endif 5275 /** 5276 * Translate RSS action to get rss hash fields. 5277 */ 5278 if (flow_drv_translate(dev, &dev_flow, attr, 5279 items, rss_actions, error)) 5280 goto exit; 5281 rss_desc_v[i] = wks->rss_desc; 5282 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5283 rss_desc_v[i].hash_fields = 5284 dev_flow.hash_fields; 5285 rss_desc_v[i].queue_num = 5286 rss_desc_v[i].hash_fields ? 5287 rss_desc_v[i].queue_num : 1; 5288 rss_desc_v[i].tunnel = 5289 !!(dev_flow.handle->layers & 5290 MLX5_FLOW_LAYER_TUNNEL); 5291 /* Use the RSS queues in the containers. */ 5292 rss_desc_v[i].queue = 5293 (uint16_t *)(uintptr_t)rss_act->queue; 5294 rss_desc[i] = &rss_desc_v[i]; 5295 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5296 /* This is queue action. */ 5297 rss_desc_v[i] = wks->rss_desc; 5298 rss_desc_v[i].key_len = 0; 5299 rss_desc_v[i].hash_fields = 0; 5300 rss_desc_v[i].queue = 5301 &final_policy->act_cnt[i].queue; 5302 rss_desc_v[i].queue_num = 1; 5303 rss_desc[i] = &rss_desc_v[i]; 5304 } else { 5305 rss_desc[i] = NULL; 5306 } 5307 } 5308 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5309 flow, policy, rss_desc); 5310 } else { 5311 enum mlx5_meter_domain mtr_domain = 5312 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5313 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5314 MLX5_MTR_DOMAIN_INGRESS); 5315 sub_policy = policy->sub_policys[mtr_domain][0]; 5316 } 5317 if (!sub_policy) 5318 rte_flow_error_set(error, EINVAL, 5319 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5320 "Failed to get meter sub-policy."); 5321 exit: 5322 return sub_policy; 5323 } 5324 5325 /** 5326 * Split the meter flow. 5327 * 5328 * As meter flow will split to three sub flow, other than meter 5329 * action, the other actions make sense to only meter accepts 5330 * the packet. If it need to be dropped, no other additional 5331 * actions should be take. 5332 * 5333 * One kind of special action which decapsulates the L3 tunnel 5334 * header will be in the prefix sub flow, as not to take the 5335 * L3 tunnel header into account. 5336 * 5337 * @param[in] dev 5338 * Pointer to Ethernet device. 5339 * @param[in] flow 5340 * Parent flow structure pointer. 5341 * @param wks 5342 * Pointer to thread flow work space. 5343 * @param[in] attr 5344 * Flow rule attributes. 5345 * @param[in] items 5346 * Pattern specification (list terminated by the END pattern item). 5347 * @param[out] sfx_items 5348 * Suffix flow match items (list terminated by the END pattern item). 5349 * @param[in] actions 5350 * Associated actions (list terminated by the END action). 5351 * @param[out] actions_sfx 5352 * Suffix flow actions. 5353 * @param[out] actions_pre 5354 * Prefix flow actions. 5355 * @param[out] mtr_flow_id 5356 * Pointer to meter flow id. 5357 * @param[out] error 5358 * Perform verbose error reporting if not NULL. 5359 * 5360 * @return 5361 * 0 on success, a negative errno value otherwise and rte_errno is set. 5362 */ 5363 static int 5364 flow_meter_split_prep(struct rte_eth_dev *dev, 5365 struct rte_flow *flow, 5366 struct mlx5_flow_workspace *wks, 5367 const struct rte_flow_attr *attr, 5368 const struct rte_flow_item items[], 5369 struct rte_flow_item sfx_items[], 5370 const struct rte_flow_action actions[], 5371 struct rte_flow_action actions_sfx[], 5372 struct rte_flow_action actions_pre[], 5373 uint32_t *mtr_flow_id, 5374 struct rte_flow_error *error) 5375 { 5376 struct mlx5_priv *priv = dev->data->dev_private; 5377 struct mlx5_flow_meter_info *fm = wks->fm; 5378 struct rte_flow_action *tag_action = NULL; 5379 struct rte_flow_item *tag_item; 5380 struct mlx5_rte_flow_action_set_tag *set_tag; 5381 const struct rte_flow_action_raw_encap *raw_encap; 5382 const struct rte_flow_action_raw_decap *raw_decap; 5383 struct mlx5_rte_flow_item_tag *tag_item_spec; 5384 struct mlx5_rte_flow_item_tag *tag_item_mask; 5385 uint32_t tag_id = 0; 5386 struct rte_flow_item *vlan_item_dst = NULL; 5387 const struct rte_flow_item *vlan_item_src = NULL; 5388 const struct rte_flow_item *orig_items = items; 5389 struct rte_flow_action *hw_mtr_action; 5390 struct rte_flow_action *action_pre_head = NULL; 5391 uint16_t flow_src_port = priv->representor_id; 5392 bool mtr_first; 5393 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5394 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5395 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5396 uint32_t flow_id = 0; 5397 uint32_t flow_id_reversed = 0; 5398 uint8_t flow_id_bits = 0; 5399 bool after_meter = false; 5400 int shift; 5401 5402 /* Prepare the suffix subflow items. */ 5403 tag_item = sfx_items++; 5404 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5405 int item_type = items->type; 5406 5407 switch (item_type) { 5408 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5409 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5410 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5411 return -rte_errno; 5412 if (!fm->def_policy && wks->policy->is_hierarchy && 5413 flow_src_port != priv->representor_id) { 5414 if (flow_drv_mtr_hierarchy_rule_create(dev, 5415 flow, fm, 5416 flow_src_port, 5417 items, 5418 error)) 5419 return -rte_errno; 5420 } 5421 memcpy(sfx_items, items, sizeof(*sfx_items)); 5422 sfx_items++; 5423 break; 5424 case RTE_FLOW_ITEM_TYPE_VLAN: 5425 /* Determine if copy vlan item below. */ 5426 vlan_item_src = items; 5427 vlan_item_dst = sfx_items++; 5428 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5429 break; 5430 default: 5431 break; 5432 } 5433 } 5434 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5435 sfx_items++; 5436 mtr_first = priv->sh->meter_aso_en && 5437 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5438 /* For ASO meter, meter must be before tag in TX direction. */ 5439 if (mtr_first) { 5440 action_pre_head = actions_pre++; 5441 /* Leave space for tag action. */ 5442 tag_action = actions_pre++; 5443 } 5444 /* Prepare the actions for prefix and suffix flow. */ 5445 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5446 struct rte_flow_action *action_cur = NULL; 5447 5448 switch (actions->type) { 5449 case RTE_FLOW_ACTION_TYPE_METER: 5450 if (mtr_first) { 5451 action_cur = action_pre_head; 5452 } else { 5453 /* Leave space for tag action. */ 5454 tag_action = actions_pre++; 5455 action_cur = actions_pre++; 5456 } 5457 after_meter = true; 5458 break; 5459 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5460 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5461 action_cur = actions_pre++; 5462 break; 5463 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5464 raw_encap = actions->conf; 5465 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5466 action_cur = actions_pre++; 5467 break; 5468 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5469 raw_decap = actions->conf; 5470 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5471 action_cur = actions_pre++; 5472 break; 5473 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5474 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5475 if (vlan_item_dst && vlan_item_src) { 5476 memcpy(vlan_item_dst, vlan_item_src, 5477 sizeof(*vlan_item_dst)); 5478 /* 5479 * Convert to internal match item, it is used 5480 * for vlan push and set vid. 5481 */ 5482 vlan_item_dst->type = (enum rte_flow_item_type) 5483 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5484 } 5485 break; 5486 case RTE_FLOW_ACTION_TYPE_COUNT: 5487 if (fm->def_policy) 5488 action_cur = after_meter ? 5489 actions_sfx++ : actions_pre++; 5490 break; 5491 default: 5492 break; 5493 } 5494 if (!action_cur) 5495 action_cur = (fm->def_policy) ? 5496 actions_sfx++ : actions_pre++; 5497 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5498 } 5499 /* Add end action to the actions. */ 5500 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5501 if (priv->sh->meter_aso_en) { 5502 /** 5503 * For ASO meter, need to add an extra jump action explicitly, 5504 * to jump from meter to policer table. 5505 */ 5506 struct mlx5_flow_meter_sub_policy *sub_policy; 5507 struct mlx5_flow_tbl_data_entry *tbl_data; 5508 5509 if (!fm->def_policy) { 5510 sub_policy = get_meter_sub_policy(dev, flow, wks, 5511 attr, orig_items, 5512 error); 5513 if (!sub_policy) 5514 return -rte_errno; 5515 } else { 5516 enum mlx5_meter_domain mtr_domain = 5517 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5518 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5519 MLX5_MTR_DOMAIN_INGRESS); 5520 5521 sub_policy = 5522 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5523 } 5524 tbl_data = container_of(sub_policy->tbl_rsc, 5525 struct mlx5_flow_tbl_data_entry, tbl); 5526 hw_mtr_action = actions_pre++; 5527 hw_mtr_action->type = (enum rte_flow_action_type) 5528 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5529 hw_mtr_action->conf = tbl_data->jump.action; 5530 } 5531 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5532 actions_pre++; 5533 if (!tag_action) 5534 return rte_flow_error_set(error, ENOMEM, 5535 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5536 NULL, "No tag action space."); 5537 if (!mtr_flow_id) { 5538 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5539 goto exit; 5540 } 5541 /* Only default-policy Meter creates mtr flow id. */ 5542 if (fm->def_policy) { 5543 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5544 if (!tag_id) 5545 return rte_flow_error_set(error, ENOMEM, 5546 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5547 "Failed to allocate meter flow id."); 5548 flow_id = tag_id - 1; 5549 flow_id_bits = (!flow_id) ? 1 : 5550 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5551 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5552 mtr_reg_bits) { 5553 mlx5_ipool_free(fm->flow_ipool, tag_id); 5554 return rte_flow_error_set(error, EINVAL, 5555 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5556 "Meter flow id exceeds max limit."); 5557 } 5558 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5559 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5560 } 5561 /* Build tag actions and items for meter_id/meter flow_id. */ 5562 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5563 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5564 tag_item_mask = tag_item_spec + 1; 5565 /* Both flow_id and meter_id share the same register. */ 5566 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5567 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5568 0, error), 5569 .offset = mtr_id_offset, 5570 .length = mtr_reg_bits, 5571 .data = flow->meter, 5572 }; 5573 /* 5574 * The color Reg bits used by flow_id are growing from 5575 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5576 */ 5577 for (shift = 0; shift < flow_id_bits; shift++) 5578 flow_id_reversed = (flow_id_reversed << 1) | 5579 ((flow_id >> shift) & 0x1); 5580 set_tag->data |= 5581 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5582 tag_item_spec->id = set_tag->id; 5583 tag_item_spec->data = set_tag->data << mtr_id_offset; 5584 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5585 tag_action->type = (enum rte_flow_action_type) 5586 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5587 tag_action->conf = set_tag; 5588 tag_item->type = (enum rte_flow_item_type) 5589 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5590 tag_item->spec = tag_item_spec; 5591 tag_item->last = NULL; 5592 tag_item->mask = tag_item_mask; 5593 exit: 5594 if (mtr_flow_id) 5595 *mtr_flow_id = tag_id; 5596 return 0; 5597 } 5598 5599 /** 5600 * Split action list having QUEUE/RSS for metadata register copy. 5601 * 5602 * Once Q/RSS action is detected in user's action list, the flow action 5603 * should be split in order to copy metadata registers, which will happen in 5604 * RX_CP_TBL like, 5605 * - CQE->flow_tag := reg_c[1] (MARK) 5606 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5607 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5608 * This is because the last action of each flow must be a terminal action 5609 * (QUEUE, RSS or DROP). 5610 * 5611 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5612 * stored and kept in the mlx5_flow structure per each sub_flow. 5613 * 5614 * The Q/RSS action is replaced with, 5615 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5616 * And the following JUMP action is added at the end, 5617 * - JUMP, to RX_CP_TBL. 5618 * 5619 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5620 * flow_create_split_metadata() routine. The flow will look like, 5621 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5622 * 5623 * @param dev 5624 * Pointer to Ethernet device. 5625 * @param[out] split_actions 5626 * Pointer to store split actions to jump to CP_TBL. 5627 * @param[in] actions 5628 * Pointer to the list of original flow actions. 5629 * @param[in] qrss 5630 * Pointer to the Q/RSS action. 5631 * @param[in] actions_n 5632 * Number of original actions. 5633 * @param[in] mtr_sfx 5634 * Check if it is in meter suffix table. 5635 * @param[out] error 5636 * Perform verbose error reporting if not NULL. 5637 * 5638 * @return 5639 * non-zero unique flow_id on success, otherwise 0 and 5640 * error/rte_error are set. 5641 */ 5642 static uint32_t 5643 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5644 struct rte_flow_action *split_actions, 5645 const struct rte_flow_action *actions, 5646 const struct rte_flow_action *qrss, 5647 int actions_n, int mtr_sfx, 5648 struct rte_flow_error *error) 5649 { 5650 struct mlx5_priv *priv = dev->data->dev_private; 5651 struct mlx5_rte_flow_action_set_tag *set_tag; 5652 struct rte_flow_action_jump *jump; 5653 const int qrss_idx = qrss - actions; 5654 uint32_t flow_id = 0; 5655 int ret = 0; 5656 5657 /* 5658 * Given actions will be split 5659 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5660 * - Add jump to mreg CP_TBL. 5661 * As a result, there will be one more action. 5662 */ 5663 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5664 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5665 ++actions_n; 5666 set_tag = (void *)(split_actions + actions_n); 5667 /* 5668 * If we are not the meter suffix flow, add the tag action. 5669 * Since meter suffix flow already has the tag added. 5670 */ 5671 if (!mtr_sfx) { 5672 /* 5673 * Allocate the new subflow ID. This one is unique within 5674 * device and not shared with representors. Otherwise, 5675 * we would have to resolve multi-thread access synch 5676 * issue. Each flow on the shared device is appended 5677 * with source vport identifier, so the resulting 5678 * flows will be unique in the shared (by master and 5679 * representors) domain even if they have coinciding 5680 * IDs. 5681 */ 5682 mlx5_ipool_malloc(priv->sh->ipool 5683 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5684 if (!flow_id) 5685 return rte_flow_error_set(error, ENOMEM, 5686 RTE_FLOW_ERROR_TYPE_ACTION, 5687 NULL, "can't allocate id " 5688 "for split Q/RSS subflow"); 5689 /* Internal SET_TAG action to set flow ID. */ 5690 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5691 .data = flow_id, 5692 }; 5693 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5694 if (ret < 0) 5695 return ret; 5696 set_tag->id = ret; 5697 /* Construct new actions array. */ 5698 /* Replace QUEUE/RSS action. */ 5699 split_actions[qrss_idx] = (struct rte_flow_action){ 5700 .type = (enum rte_flow_action_type) 5701 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5702 .conf = set_tag, 5703 }; 5704 } else { 5705 /* 5706 * If we are the suffix flow of meter, tag already exist. 5707 * Set the QUEUE/RSS action to void. 5708 */ 5709 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5710 } 5711 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5712 jump = (void *)(set_tag + 1); 5713 *jump = (struct rte_flow_action_jump){ 5714 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5715 }; 5716 split_actions[actions_n - 2] = (struct rte_flow_action){ 5717 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5718 .conf = jump, 5719 }; 5720 split_actions[actions_n - 1] = (struct rte_flow_action){ 5721 .type = RTE_FLOW_ACTION_TYPE_END, 5722 }; 5723 return flow_id; 5724 } 5725 5726 /** 5727 * Extend the given action list for Tx metadata copy. 5728 * 5729 * Copy the given action list to the ext_actions and add flow metadata register 5730 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5731 * 5732 * @param[out] ext_actions 5733 * Pointer to the extended action list. 5734 * @param[in] actions 5735 * Pointer to the list of actions. 5736 * @param[in] actions_n 5737 * Number of actions in the list. 5738 * @param[out] error 5739 * Perform verbose error reporting if not NULL. 5740 * @param[in] encap_idx 5741 * The encap action index. 5742 * 5743 * @return 5744 * 0 on success, negative value otherwise 5745 */ 5746 static int 5747 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5748 struct rte_flow_action *ext_actions, 5749 const struct rte_flow_action *actions, 5750 int actions_n, struct rte_flow_error *error, 5751 int encap_idx) 5752 { 5753 struct mlx5_flow_action_copy_mreg *cp_mreg = 5754 (struct mlx5_flow_action_copy_mreg *) 5755 (ext_actions + actions_n + 1); 5756 int ret; 5757 5758 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5759 if (ret < 0) 5760 return ret; 5761 cp_mreg->dst = ret; 5762 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5763 if (ret < 0) 5764 return ret; 5765 cp_mreg->src = ret; 5766 if (encap_idx != 0) 5767 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5768 if (encap_idx == actions_n - 1) { 5769 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5770 .type = (enum rte_flow_action_type) 5771 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5772 .conf = cp_mreg, 5773 }; 5774 ext_actions[actions_n] = (struct rte_flow_action){ 5775 .type = RTE_FLOW_ACTION_TYPE_END, 5776 }; 5777 } else { 5778 ext_actions[encap_idx] = (struct rte_flow_action){ 5779 .type = (enum rte_flow_action_type) 5780 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5781 .conf = cp_mreg, 5782 }; 5783 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5784 sizeof(*ext_actions) * (actions_n - encap_idx)); 5785 } 5786 return 0; 5787 } 5788 5789 /** 5790 * Check the match action from the action list. 5791 * 5792 * @param[in] actions 5793 * Pointer to the list of actions. 5794 * @param[in] attr 5795 * Flow rule attributes. 5796 * @param[in] action 5797 * The action to be check if exist. 5798 * @param[out] match_action_pos 5799 * Pointer to the position of the matched action if exists, otherwise is -1. 5800 * @param[out] qrss_action_pos 5801 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5802 * @param[out] modify_after_mirror 5803 * Pointer to the flag of modify action after FDB mirroring. 5804 * 5805 * @return 5806 * > 0 the total number of actions. 5807 * 0 if not found match action in action list. 5808 */ 5809 static int 5810 flow_check_match_action(const struct rte_flow_action actions[], 5811 const struct rte_flow_attr *attr, 5812 enum rte_flow_action_type action, 5813 int *match_action_pos, int *qrss_action_pos, 5814 int *modify_after_mirror) 5815 { 5816 const struct rte_flow_action_sample *sample; 5817 const struct rte_flow_action_raw_decap *decap; 5818 int actions_n = 0; 5819 uint32_t ratio = 0; 5820 int sub_type = 0; 5821 int flag = 0; 5822 int fdb_mirror = 0; 5823 5824 *match_action_pos = -1; 5825 *qrss_action_pos = -1; 5826 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5827 if (actions->type == action) { 5828 flag = 1; 5829 *match_action_pos = actions_n; 5830 } 5831 switch (actions->type) { 5832 case RTE_FLOW_ACTION_TYPE_QUEUE: 5833 case RTE_FLOW_ACTION_TYPE_RSS: 5834 *qrss_action_pos = actions_n; 5835 break; 5836 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5837 sample = actions->conf; 5838 ratio = sample->ratio; 5839 sub_type = ((const struct rte_flow_action *) 5840 (sample->actions))->type; 5841 if (ratio == 1 && attr->transfer) 5842 fdb_mirror = 1; 5843 break; 5844 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5845 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5846 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5847 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5848 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5849 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5850 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5851 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5852 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5853 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5854 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5855 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5856 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5857 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5858 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5859 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5860 case RTE_FLOW_ACTION_TYPE_FLAG: 5861 case RTE_FLOW_ACTION_TYPE_MARK: 5862 case RTE_FLOW_ACTION_TYPE_SET_META: 5863 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5864 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5865 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5866 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5867 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5868 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5869 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5870 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5871 case RTE_FLOW_ACTION_TYPE_METER: 5872 if (fdb_mirror) 5873 *modify_after_mirror = 1; 5874 break; 5875 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5876 decap = actions->conf; 5877 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5878 ; 5879 actions_n++; 5880 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5881 const struct rte_flow_action_raw_encap *encap = 5882 actions->conf; 5883 if (decap->size <= 5884 MLX5_ENCAPSULATION_DECISION_SIZE && 5885 encap->size > 5886 MLX5_ENCAPSULATION_DECISION_SIZE) 5887 /* L3 encap. */ 5888 break; 5889 } 5890 if (fdb_mirror) 5891 *modify_after_mirror = 1; 5892 break; 5893 default: 5894 break; 5895 } 5896 actions_n++; 5897 } 5898 if (flag && fdb_mirror && !*modify_after_mirror) { 5899 /* FDB mirroring uses the destination array to implement 5900 * instead of FLOW_SAMPLER object. 5901 */ 5902 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5903 flag = 0; 5904 } 5905 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5906 return flag ? actions_n + 1 : 0; 5907 } 5908 5909 #define SAMPLE_SUFFIX_ITEM 3 5910 5911 /** 5912 * Split the sample flow. 5913 * 5914 * As sample flow will split to two sub flow, sample flow with 5915 * sample action, the other actions will move to new suffix flow. 5916 * 5917 * Also add unique tag id with tag action in the sample flow, 5918 * the same tag id will be as match in the suffix flow. 5919 * 5920 * @param dev 5921 * Pointer to Ethernet device. 5922 * @param[in] add_tag 5923 * Add extra tag action flag. 5924 * @param[out] sfx_items 5925 * Suffix flow match items (list terminated by the END pattern item). 5926 * @param[in] actions 5927 * Associated actions (list terminated by the END action). 5928 * @param[out] actions_sfx 5929 * Suffix flow actions. 5930 * @param[out] actions_pre 5931 * Prefix flow actions. 5932 * @param[in] actions_n 5933 * The total number of actions. 5934 * @param[in] sample_action_pos 5935 * The sample action position. 5936 * @param[in] qrss_action_pos 5937 * The Queue/RSS action position. 5938 * @param[in] jump_table 5939 * Add extra jump action flag. 5940 * @param[out] error 5941 * Perform verbose error reporting if not NULL. 5942 * 5943 * @return 5944 * 0 on success, or unique flow_id, a negative errno value 5945 * otherwise and rte_errno is set. 5946 */ 5947 static int 5948 flow_sample_split_prep(struct rte_eth_dev *dev, 5949 int add_tag, 5950 const struct rte_flow_item items[], 5951 struct rte_flow_item sfx_items[], 5952 const struct rte_flow_action actions[], 5953 struct rte_flow_action actions_sfx[], 5954 struct rte_flow_action actions_pre[], 5955 int actions_n, 5956 int sample_action_pos, 5957 int qrss_action_pos, 5958 int jump_table, 5959 struct rte_flow_error *error) 5960 { 5961 struct mlx5_priv *priv = dev->data->dev_private; 5962 struct mlx5_rte_flow_action_set_tag *set_tag; 5963 struct mlx5_rte_flow_item_tag *tag_spec; 5964 struct mlx5_rte_flow_item_tag *tag_mask; 5965 struct rte_flow_action_jump *jump_action; 5966 uint32_t tag_id = 0; 5967 int append_index = 0; 5968 int set_tag_idx = -1; 5969 int index; 5970 int ret; 5971 5972 if (sample_action_pos < 0) 5973 return rte_flow_error_set(error, EINVAL, 5974 RTE_FLOW_ERROR_TYPE_ACTION, 5975 NULL, "invalid position of sample " 5976 "action in list"); 5977 /* Prepare the actions for prefix and suffix flow. */ 5978 if (add_tag) { 5979 /* Update the new added tag action index preceding 5980 * the PUSH_VLAN or ENCAP action. 5981 */ 5982 const struct rte_flow_action_raw_encap *raw_encap; 5983 const struct rte_flow_action *action = actions; 5984 int encap_idx; 5985 int action_idx = 0; 5986 int raw_decap_idx = -1; 5987 int push_vlan_idx = -1; 5988 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 5989 switch (action->type) { 5990 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5991 raw_decap_idx = action_idx; 5992 break; 5993 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5994 raw_encap = action->conf; 5995 if (raw_encap->size > 5996 MLX5_ENCAPSULATION_DECISION_SIZE) { 5997 encap_idx = raw_decap_idx != -1 ? 5998 raw_decap_idx : action_idx; 5999 if (encap_idx < sample_action_pos && 6000 push_vlan_idx == -1) 6001 set_tag_idx = encap_idx; 6002 } 6003 break; 6004 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6005 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6006 encap_idx = action_idx; 6007 if (encap_idx < sample_action_pos && 6008 push_vlan_idx == -1) 6009 set_tag_idx = encap_idx; 6010 break; 6011 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6012 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6013 push_vlan_idx = action_idx; 6014 if (push_vlan_idx < sample_action_pos) 6015 set_tag_idx = action_idx; 6016 break; 6017 default: 6018 break; 6019 } 6020 action_idx++; 6021 } 6022 } 6023 /* Prepare the actions for prefix and suffix flow. */ 6024 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6025 index = qrss_action_pos; 6026 /* Put the preceding the Queue/RSS action into prefix flow. */ 6027 if (index != 0) 6028 memcpy(actions_pre, actions, 6029 sizeof(struct rte_flow_action) * index); 6030 /* Put others preceding the sample action into prefix flow. */ 6031 if (sample_action_pos > index + 1) 6032 memcpy(actions_pre + index, actions + index + 1, 6033 sizeof(struct rte_flow_action) * 6034 (sample_action_pos - index - 1)); 6035 index = sample_action_pos - 1; 6036 /* Put Queue/RSS action into Suffix flow. */ 6037 memcpy(actions_sfx, actions + qrss_action_pos, 6038 sizeof(struct rte_flow_action)); 6039 actions_sfx++; 6040 } else if (add_tag && set_tag_idx >= 0) { 6041 if (set_tag_idx > 0) 6042 memcpy(actions_pre, actions, 6043 sizeof(struct rte_flow_action) * set_tag_idx); 6044 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6045 sizeof(struct rte_flow_action) * 6046 (sample_action_pos - set_tag_idx)); 6047 index = sample_action_pos; 6048 } else { 6049 index = sample_action_pos; 6050 if (index != 0) 6051 memcpy(actions_pre, actions, 6052 sizeof(struct rte_flow_action) * index); 6053 } 6054 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6055 * For CX6DX and above, metadata registers Cx preserve their value, 6056 * add an extra tag action for NIC-RX and E-Switch Domain. 6057 */ 6058 if (add_tag) { 6059 /* Prepare the prefix tag action. */ 6060 append_index++; 6061 set_tag = (void *)(actions_pre + actions_n + append_index); 6062 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6063 /* Trust VF/SF on CX5 not supported meter so that the reserved 6064 * metadata regC is REG_NON, back to use application tag 6065 * index 0. 6066 */ 6067 if (unlikely(ret == REG_NON)) 6068 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6069 if (ret < 0) 6070 return ret; 6071 mlx5_ipool_malloc(priv->sh->ipool 6072 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6073 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6074 .id = ret, 6075 .data = tag_id, 6076 }; 6077 /* Prepare the suffix subflow items. */ 6078 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6079 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6080 memcpy(sfx_items, items, sizeof(*sfx_items)); 6081 sfx_items++; 6082 } 6083 } 6084 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6085 tag_spec->data = tag_id; 6086 tag_spec->id = set_tag->id; 6087 tag_mask = tag_spec + 1; 6088 tag_mask->data = UINT32_MAX; 6089 sfx_items[0] = (struct rte_flow_item){ 6090 .type = (enum rte_flow_item_type) 6091 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6092 .spec = tag_spec, 6093 .last = NULL, 6094 .mask = tag_mask, 6095 }; 6096 sfx_items[1] = (struct rte_flow_item){ 6097 .type = (enum rte_flow_item_type) 6098 RTE_FLOW_ITEM_TYPE_END, 6099 }; 6100 /* Prepare the tag action in prefix subflow. */ 6101 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6102 actions_pre[set_tag_idx] = 6103 (struct rte_flow_action){ 6104 .type = (enum rte_flow_action_type) 6105 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6106 .conf = set_tag, 6107 }; 6108 /* Update next sample position due to add one tag action */ 6109 index += 1; 6110 } 6111 /* Copy the sample action into prefix flow. */ 6112 memcpy(actions_pre + index, actions + sample_action_pos, 6113 sizeof(struct rte_flow_action)); 6114 index += 1; 6115 /* For the modify action after the sample action in E-Switch mirroring, 6116 * Add the extra jump action in prefix subflow and jump into the next 6117 * table, then do the modify action in the new table. 6118 */ 6119 if (jump_table) { 6120 /* Prepare the prefix jump action. */ 6121 append_index++; 6122 jump_action = (void *)(actions_pre + actions_n + append_index); 6123 jump_action->group = jump_table; 6124 actions_pre[index++] = 6125 (struct rte_flow_action){ 6126 .type = (enum rte_flow_action_type) 6127 RTE_FLOW_ACTION_TYPE_JUMP, 6128 .conf = jump_action, 6129 }; 6130 } 6131 actions_pre[index] = (struct rte_flow_action){ 6132 .type = (enum rte_flow_action_type) 6133 RTE_FLOW_ACTION_TYPE_END, 6134 }; 6135 /* Put the actions after sample into Suffix flow. */ 6136 memcpy(actions_sfx, actions + sample_action_pos + 1, 6137 sizeof(struct rte_flow_action) * 6138 (actions_n - sample_action_pos - 1)); 6139 return tag_id; 6140 } 6141 6142 /** 6143 * The splitting for metadata feature. 6144 * 6145 * - Q/RSS action on NIC Rx should be split in order to pass by 6146 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6147 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6148 * 6149 * - All the actions on NIC Tx should have a mreg copy action to 6150 * copy reg_a from WQE to reg_c[0]. 6151 * 6152 * @param dev 6153 * Pointer to Ethernet device. 6154 * @param[in] flow 6155 * Parent flow structure pointer. 6156 * @param[in] attr 6157 * Flow rule attributes. 6158 * @param[in] items 6159 * Pattern specification (list terminated by the END pattern item). 6160 * @param[in] actions 6161 * Associated actions (list terminated by the END action). 6162 * @param[in] flow_split_info 6163 * Pointer to flow split info structure. 6164 * @param[out] error 6165 * Perform verbose error reporting if not NULL. 6166 * @return 6167 * 0 on success, negative value otherwise 6168 */ 6169 static int 6170 flow_create_split_metadata(struct rte_eth_dev *dev, 6171 struct rte_flow *flow, 6172 const struct rte_flow_attr *attr, 6173 const struct rte_flow_item items[], 6174 const struct rte_flow_action actions[], 6175 struct mlx5_flow_split_info *flow_split_info, 6176 struct rte_flow_error *error) 6177 { 6178 struct mlx5_priv *priv = dev->data->dev_private; 6179 struct mlx5_sh_config *config = &priv->sh->config; 6180 const struct rte_flow_action *qrss = NULL; 6181 struct rte_flow_action *ext_actions = NULL; 6182 struct mlx5_flow *dev_flow = NULL; 6183 uint32_t qrss_id = 0; 6184 int mtr_sfx = 0; 6185 size_t act_size; 6186 int actions_n; 6187 int encap_idx; 6188 int ret; 6189 6190 /* Check whether extensive metadata feature is engaged. */ 6191 if (!config->dv_flow_en || 6192 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6193 !mlx5_flow_ext_mreg_supported(dev)) 6194 return flow_create_split_inner(dev, flow, NULL, attr, items, 6195 actions, flow_split_info, error); 6196 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6197 &encap_idx); 6198 if (qrss) { 6199 /* Exclude hairpin flows from splitting. */ 6200 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6201 const struct rte_flow_action_queue *queue; 6202 6203 queue = qrss->conf; 6204 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6205 qrss = NULL; 6206 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6207 const struct rte_flow_action_rss *rss; 6208 6209 rss = qrss->conf; 6210 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6211 qrss = NULL; 6212 } 6213 } 6214 if (qrss) { 6215 /* Check if it is in meter suffix table. */ 6216 mtr_sfx = attr->group == 6217 ((attr->transfer && priv->fdb_def_rule) ? 6218 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6219 MLX5_FLOW_TABLE_LEVEL_METER); 6220 /* 6221 * Q/RSS action on NIC Rx should be split in order to pass by 6222 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6223 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6224 */ 6225 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6226 sizeof(struct rte_flow_action_set_tag) + 6227 sizeof(struct rte_flow_action_jump); 6228 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6229 SOCKET_ID_ANY); 6230 if (!ext_actions) 6231 return rte_flow_error_set(error, ENOMEM, 6232 RTE_FLOW_ERROR_TYPE_ACTION, 6233 NULL, "no memory to split " 6234 "metadata flow"); 6235 /* 6236 * Create the new actions list with removed Q/RSS action 6237 * and appended set tag and jump to register copy table 6238 * (RX_CP_TBL). We should preallocate unique tag ID here 6239 * in advance, because it is needed for set tag action. 6240 */ 6241 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6242 qrss, actions_n, 6243 mtr_sfx, error); 6244 if (!mtr_sfx && !qrss_id) { 6245 ret = -rte_errno; 6246 goto exit; 6247 } 6248 } else if (attr->egress && !attr->transfer) { 6249 /* 6250 * All the actions on NIC Tx should have a metadata register 6251 * copy action to copy reg_a from WQE to reg_c[meta] 6252 */ 6253 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6254 sizeof(struct mlx5_flow_action_copy_mreg); 6255 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6256 SOCKET_ID_ANY); 6257 if (!ext_actions) 6258 return rte_flow_error_set(error, ENOMEM, 6259 RTE_FLOW_ERROR_TYPE_ACTION, 6260 NULL, "no memory to split " 6261 "metadata flow"); 6262 /* Create the action list appended with copy register. */ 6263 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6264 actions_n, error, encap_idx); 6265 if (ret < 0) 6266 goto exit; 6267 } 6268 /* Add the unmodified original or prefix subflow. */ 6269 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6270 items, ext_actions ? ext_actions : 6271 actions, flow_split_info, error); 6272 if (ret < 0) 6273 goto exit; 6274 MLX5_ASSERT(dev_flow); 6275 if (qrss) { 6276 const struct rte_flow_attr q_attr = { 6277 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6278 .ingress = 1, 6279 }; 6280 /* Internal PMD action to set register. */ 6281 struct mlx5_rte_flow_item_tag q_tag_spec = { 6282 .data = qrss_id, 6283 .id = REG_NON, 6284 }; 6285 struct rte_flow_item q_items[] = { 6286 { 6287 .type = (enum rte_flow_item_type) 6288 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6289 .spec = &q_tag_spec, 6290 .last = NULL, 6291 .mask = NULL, 6292 }, 6293 { 6294 .type = RTE_FLOW_ITEM_TYPE_END, 6295 }, 6296 }; 6297 struct rte_flow_action q_actions[] = { 6298 { 6299 .type = qrss->type, 6300 .conf = qrss->conf, 6301 }, 6302 { 6303 .type = RTE_FLOW_ACTION_TYPE_END, 6304 }, 6305 }; 6306 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6307 6308 /* 6309 * Configure the tag item only if there is no meter subflow. 6310 * Since tag is already marked in the meter suffix subflow 6311 * we can just use the meter suffix items as is. 6312 */ 6313 if (qrss_id) { 6314 /* Not meter subflow. */ 6315 MLX5_ASSERT(!mtr_sfx); 6316 /* 6317 * Put unique id in prefix flow due to it is destroyed 6318 * after suffix flow and id will be freed after there 6319 * is no actual flows with this id and identifier 6320 * reallocation becomes possible (for example, for 6321 * other flows in other threads). 6322 */ 6323 dev_flow->handle->split_flow_id = qrss_id; 6324 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6325 error); 6326 if (ret < 0) 6327 goto exit; 6328 q_tag_spec.id = ret; 6329 } 6330 dev_flow = NULL; 6331 /* Add suffix subflow to execute Q/RSS. */ 6332 flow_split_info->prefix_layers = layers; 6333 flow_split_info->prefix_mark = 0; 6334 flow_split_info->table_id = 0; 6335 ret = flow_create_split_inner(dev, flow, &dev_flow, 6336 &q_attr, mtr_sfx ? items : 6337 q_items, q_actions, 6338 flow_split_info, error); 6339 if (ret < 0) 6340 goto exit; 6341 /* qrss ID should be freed if failed. */ 6342 qrss_id = 0; 6343 MLX5_ASSERT(dev_flow); 6344 } 6345 6346 exit: 6347 /* 6348 * We do not destroy the partially created sub_flows in case of error. 6349 * These ones are included into parent flow list and will be destroyed 6350 * by flow_drv_destroy. 6351 */ 6352 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6353 qrss_id); 6354 mlx5_free(ext_actions); 6355 return ret; 6356 } 6357 6358 /** 6359 * Create meter internal drop flow with the original pattern. 6360 * 6361 * @param dev 6362 * Pointer to Ethernet device. 6363 * @param[in] flow 6364 * Parent flow structure pointer. 6365 * @param[in] attr 6366 * Flow rule attributes. 6367 * @param[in] items 6368 * Pattern specification (list terminated by the END pattern item). 6369 * @param[in] flow_split_info 6370 * Pointer to flow split info structure. 6371 * @param[in] fm 6372 * Pointer to flow meter structure. 6373 * @param[out] error 6374 * Perform verbose error reporting if not NULL. 6375 * @return 6376 * 0 on success, negative value otherwise 6377 */ 6378 static uint32_t 6379 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6380 struct rte_flow *flow, 6381 const struct rte_flow_attr *attr, 6382 const struct rte_flow_item items[], 6383 struct mlx5_flow_split_info *flow_split_info, 6384 struct mlx5_flow_meter_info *fm, 6385 struct rte_flow_error *error) 6386 { 6387 struct mlx5_flow *dev_flow = NULL; 6388 struct rte_flow_attr drop_attr = *attr; 6389 struct rte_flow_action drop_actions[3]; 6390 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6391 6392 MLX5_ASSERT(fm->drop_cnt); 6393 drop_actions[0].type = 6394 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6395 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6396 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6397 drop_actions[1].conf = NULL; 6398 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6399 drop_actions[2].conf = NULL; 6400 drop_split_info.external = false; 6401 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6402 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6403 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6404 return flow_create_split_inner(dev, flow, &dev_flow, 6405 &drop_attr, items, drop_actions, 6406 &drop_split_info, error); 6407 } 6408 6409 /** 6410 * The splitting for meter feature. 6411 * 6412 * - The meter flow will be split to two flows as prefix and 6413 * suffix flow. The packets make sense only it pass the prefix 6414 * meter action. 6415 * 6416 * - Reg_C_5 is used for the packet to match betweend prefix and 6417 * suffix flow. 6418 * 6419 * @param dev 6420 * Pointer to Ethernet device. 6421 * @param[in] flow 6422 * Parent flow structure pointer. 6423 * @param[in] attr 6424 * Flow rule attributes. 6425 * @param[in] items 6426 * Pattern specification (list terminated by the END pattern item). 6427 * @param[in] actions 6428 * Associated actions (list terminated by the END action). 6429 * @param[in] flow_split_info 6430 * Pointer to flow split info structure. 6431 * @param[out] error 6432 * Perform verbose error reporting if not NULL. 6433 * @return 6434 * 0 on success, negative value otherwise 6435 */ 6436 static int 6437 flow_create_split_meter(struct rte_eth_dev *dev, 6438 struct rte_flow *flow, 6439 const struct rte_flow_attr *attr, 6440 const struct rte_flow_item items[], 6441 const struct rte_flow_action actions[], 6442 struct mlx5_flow_split_info *flow_split_info, 6443 struct rte_flow_error *error) 6444 { 6445 struct mlx5_priv *priv = dev->data->dev_private; 6446 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6447 struct rte_flow_action *sfx_actions = NULL; 6448 struct rte_flow_action *pre_actions = NULL; 6449 struct rte_flow_item *sfx_items = NULL; 6450 struct mlx5_flow *dev_flow = NULL; 6451 struct rte_flow_attr sfx_attr = *attr; 6452 struct mlx5_flow_meter_info *fm = NULL; 6453 uint8_t skip_scale_restore; 6454 bool has_mtr = false; 6455 bool has_modify = false; 6456 bool set_mtr_reg = true; 6457 bool is_mtr_hierarchy = false; 6458 uint32_t meter_id = 0; 6459 uint32_t mtr_idx = 0; 6460 uint32_t mtr_flow_id = 0; 6461 size_t act_size; 6462 size_t item_size; 6463 int actions_n = 0; 6464 int ret = 0; 6465 6466 if (priv->mtr_en) 6467 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6468 &has_modify, &meter_id); 6469 if (has_mtr) { 6470 if (flow->meter) { 6471 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6472 if (!fm) 6473 return rte_flow_error_set(error, EINVAL, 6474 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6475 NULL, "Meter not found."); 6476 } else { 6477 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6478 if (!fm) 6479 return rte_flow_error_set(error, EINVAL, 6480 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6481 NULL, "Meter not found."); 6482 ret = mlx5_flow_meter_attach(priv, fm, 6483 &sfx_attr, error); 6484 if (ret) 6485 return -rte_errno; 6486 flow->meter = mtr_idx; 6487 } 6488 MLX5_ASSERT(wks); 6489 wks->fm = fm; 6490 if (!fm->def_policy) { 6491 wks->policy = mlx5_flow_meter_policy_find(dev, 6492 fm->policy_id, 6493 NULL); 6494 MLX5_ASSERT(wks->policy); 6495 if (wks->policy->mark) 6496 wks->mark = 1; 6497 if (wks->policy->is_hierarchy) { 6498 wks->final_policy = 6499 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6500 wks->policy); 6501 if (!wks->final_policy) 6502 return rte_flow_error_set(error, 6503 EINVAL, 6504 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6505 "Failed to find terminal policy of hierarchy."); 6506 is_mtr_hierarchy = true; 6507 } 6508 } 6509 /* 6510 * If it isn't default-policy Meter, and 6511 * 1. Not meter hierarchy and there's no action in flow to change 6512 * packet (modify/encap/decap etc.), OR 6513 * 2. No drop count needed for this meter. 6514 * Then no need to use regC to save meter id anymore. 6515 */ 6516 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6517 set_mtr_reg = false; 6518 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6519 #define METER_PREFIX_ACTION 7 6520 act_size = (sizeof(struct rte_flow_action) * 6521 (actions_n + METER_PREFIX_ACTION)) + 6522 sizeof(struct mlx5_rte_flow_action_set_tag); 6523 /* Suffix items: tag, vlan, port id, end. */ 6524 #define METER_SUFFIX_ITEM 4 6525 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6526 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6527 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6528 0, SOCKET_ID_ANY); 6529 if (!sfx_actions) 6530 return rte_flow_error_set(error, ENOMEM, 6531 RTE_FLOW_ERROR_TYPE_ACTION, 6532 NULL, "no memory to split " 6533 "meter flow"); 6534 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6535 act_size); 6536 /* There's no suffix flow for meter of non-default policy. */ 6537 if (!fm->def_policy) 6538 pre_actions = sfx_actions + 1; 6539 else 6540 pre_actions = sfx_actions + actions_n; 6541 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6542 items, sfx_items, actions, 6543 sfx_actions, pre_actions, 6544 (set_mtr_reg ? &mtr_flow_id : NULL), 6545 error); 6546 if (ret) { 6547 ret = -rte_errno; 6548 goto exit; 6549 } 6550 /* Add the prefix subflow. */ 6551 skip_scale_restore = flow_split_info->skip_scale; 6552 flow_split_info->skip_scale |= 6553 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6554 ret = flow_create_split_inner(dev, flow, &dev_flow, 6555 attr, items, pre_actions, 6556 flow_split_info, error); 6557 flow_split_info->skip_scale = skip_scale_restore; 6558 if (ret) { 6559 if (mtr_flow_id) 6560 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6561 ret = -rte_errno; 6562 goto exit; 6563 } 6564 if (mtr_flow_id) { 6565 dev_flow->handle->split_flow_id = mtr_flow_id; 6566 dev_flow->handle->is_meter_flow_id = 1; 6567 } 6568 if (!fm->def_policy) { 6569 if (!set_mtr_reg && fm->drop_cnt) 6570 ret = 6571 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6572 &sfx_attr, items, 6573 flow_split_info, 6574 fm, error); 6575 goto exit; 6576 } 6577 /* Setting the sfx group atrr. */ 6578 sfx_attr.group = sfx_attr.transfer ? 6579 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6580 MLX5_FLOW_TABLE_LEVEL_METER; 6581 flow_split_info->prefix_layers = 6582 flow_get_prefix_layer_flags(dev_flow); 6583 flow_split_info->prefix_mark |= wks->mark; 6584 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6585 } 6586 /* Add the prefix subflow. */ 6587 ret = flow_create_split_metadata(dev, flow, 6588 &sfx_attr, sfx_items ? 6589 sfx_items : items, 6590 sfx_actions ? sfx_actions : actions, 6591 flow_split_info, error); 6592 exit: 6593 if (sfx_actions) 6594 mlx5_free(sfx_actions); 6595 return ret; 6596 } 6597 6598 /** 6599 * The splitting for sample feature. 6600 * 6601 * Once Sample action is detected in the action list, the flow actions should 6602 * be split into prefix sub flow and suffix sub flow. 6603 * 6604 * The original items remain in the prefix sub flow, all actions preceding the 6605 * sample action and the sample action itself will be copied to the prefix 6606 * sub flow, the actions following the sample action will be copied to the 6607 * suffix sub flow, Queue action always be located in the suffix sub flow. 6608 * 6609 * In order to make the packet from prefix sub flow matches with suffix sub 6610 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6611 * flow uses tag item with the unique flow id. 6612 * 6613 * @param dev 6614 * Pointer to Ethernet device. 6615 * @param[in] flow 6616 * Parent flow structure pointer. 6617 * @param[in] attr 6618 * Flow rule attributes. 6619 * @param[in] items 6620 * Pattern specification (list terminated by the END pattern item). 6621 * @param[in] actions 6622 * Associated actions (list terminated by the END action). 6623 * @param[in] flow_split_info 6624 * Pointer to flow split info structure. 6625 * @param[out] error 6626 * Perform verbose error reporting if not NULL. 6627 * @return 6628 * 0 on success, negative value otherwise 6629 */ 6630 static int 6631 flow_create_split_sample(struct rte_eth_dev *dev, 6632 struct rte_flow *flow, 6633 const struct rte_flow_attr *attr, 6634 const struct rte_flow_item items[], 6635 const struct rte_flow_action actions[], 6636 struct mlx5_flow_split_info *flow_split_info, 6637 struct rte_flow_error *error) 6638 { 6639 struct mlx5_priv *priv = dev->data->dev_private; 6640 struct rte_flow_action *sfx_actions = NULL; 6641 struct rte_flow_action *pre_actions = NULL; 6642 struct rte_flow_item *sfx_items = NULL; 6643 struct mlx5_flow *dev_flow = NULL; 6644 struct rte_flow_attr sfx_attr = *attr; 6645 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6646 struct mlx5_flow_dv_sample_resource *sample_res; 6647 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6648 struct mlx5_flow_tbl_resource *sfx_tbl; 6649 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6650 #endif 6651 size_t act_size; 6652 size_t item_size; 6653 uint32_t fdb_tx = 0; 6654 int32_t tag_id = 0; 6655 int actions_n = 0; 6656 int sample_action_pos; 6657 int qrss_action_pos; 6658 int add_tag = 0; 6659 int modify_after_mirror = 0; 6660 uint16_t jump_table = 0; 6661 const uint32_t next_ft_step = 1; 6662 int ret = 0; 6663 6664 if (priv->sampler_en) 6665 actions_n = flow_check_match_action(actions, attr, 6666 RTE_FLOW_ACTION_TYPE_SAMPLE, 6667 &sample_action_pos, &qrss_action_pos, 6668 &modify_after_mirror); 6669 if (actions_n) { 6670 /* The prefix actions must includes sample, tag, end. */ 6671 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6672 + sizeof(struct mlx5_rte_flow_action_set_tag); 6673 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6674 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6675 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6676 item_size), 0, SOCKET_ID_ANY); 6677 if (!sfx_actions) 6678 return rte_flow_error_set(error, ENOMEM, 6679 RTE_FLOW_ERROR_TYPE_ACTION, 6680 NULL, "no memory to split " 6681 "sample flow"); 6682 /* The representor_id is UINT16_MAX for uplink. */ 6683 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6684 /* 6685 * When reg_c_preserve is set, metadata registers Cx preserve 6686 * their value even through packet duplication. 6687 */ 6688 add_tag = (!fdb_tx || 6689 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6690 if (add_tag) 6691 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6692 + act_size); 6693 if (modify_after_mirror) 6694 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6695 next_ft_step; 6696 pre_actions = sfx_actions + actions_n; 6697 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6698 actions, sfx_actions, 6699 pre_actions, actions_n, 6700 sample_action_pos, 6701 qrss_action_pos, jump_table, 6702 error); 6703 if (tag_id < 0 || (add_tag && !tag_id)) { 6704 ret = -rte_errno; 6705 goto exit; 6706 } 6707 if (modify_after_mirror) 6708 flow_split_info->skip_scale = 6709 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6710 /* Add the prefix subflow. */ 6711 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6712 items, pre_actions, 6713 flow_split_info, error); 6714 if (ret) { 6715 ret = -rte_errno; 6716 goto exit; 6717 } 6718 dev_flow->handle->split_flow_id = tag_id; 6719 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6720 if (!modify_after_mirror) { 6721 /* Set the sfx group attr. */ 6722 sample_res = (struct mlx5_flow_dv_sample_resource *) 6723 dev_flow->dv.sample_res; 6724 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6725 sample_res->normal_path_tbl; 6726 sfx_tbl_data = container_of(sfx_tbl, 6727 struct mlx5_flow_tbl_data_entry, 6728 tbl); 6729 sfx_attr.group = sfx_attr.transfer ? 6730 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6731 } else { 6732 MLX5_ASSERT(attr->transfer); 6733 sfx_attr.group = jump_table; 6734 } 6735 flow_split_info->prefix_layers = 6736 flow_get_prefix_layer_flags(dev_flow); 6737 MLX5_ASSERT(wks); 6738 flow_split_info->prefix_mark |= wks->mark; 6739 /* Suffix group level already be scaled with factor, set 6740 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6741 * again in translation. 6742 */ 6743 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6744 #endif 6745 } 6746 /* Add the suffix subflow. */ 6747 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6748 sfx_items ? sfx_items : items, 6749 sfx_actions ? sfx_actions : actions, 6750 flow_split_info, error); 6751 exit: 6752 if (sfx_actions) 6753 mlx5_free(sfx_actions); 6754 return ret; 6755 } 6756 6757 /** 6758 * Split the flow to subflow set. The splitters might be linked 6759 * in the chain, like this: 6760 * flow_create_split_outer() calls: 6761 * flow_create_split_meter() calls: 6762 * flow_create_split_metadata(meter_subflow_0) calls: 6763 * flow_create_split_inner(metadata_subflow_0) 6764 * flow_create_split_inner(metadata_subflow_1) 6765 * flow_create_split_inner(metadata_subflow_2) 6766 * flow_create_split_metadata(meter_subflow_1) calls: 6767 * flow_create_split_inner(metadata_subflow_0) 6768 * flow_create_split_inner(metadata_subflow_1) 6769 * flow_create_split_inner(metadata_subflow_2) 6770 * 6771 * This provide flexible way to add new levels of flow splitting. 6772 * The all of successfully created subflows are included to the 6773 * parent flow dev_flow list. 6774 * 6775 * @param dev 6776 * Pointer to Ethernet device. 6777 * @param[in] flow 6778 * Parent flow structure pointer. 6779 * @param[in] attr 6780 * Flow rule attributes. 6781 * @param[in] items 6782 * Pattern specification (list terminated by the END pattern item). 6783 * @param[in] actions 6784 * Associated actions (list terminated by the END action). 6785 * @param[in] flow_split_info 6786 * Pointer to flow split info structure. 6787 * @param[out] error 6788 * Perform verbose error reporting if not NULL. 6789 * @return 6790 * 0 on success, negative value otherwise 6791 */ 6792 static int 6793 flow_create_split_outer(struct rte_eth_dev *dev, 6794 struct rte_flow *flow, 6795 const struct rte_flow_attr *attr, 6796 const struct rte_flow_item items[], 6797 const struct rte_flow_action actions[], 6798 struct mlx5_flow_split_info *flow_split_info, 6799 struct rte_flow_error *error) 6800 { 6801 int ret; 6802 6803 ret = flow_create_split_sample(dev, flow, attr, items, 6804 actions, flow_split_info, error); 6805 MLX5_ASSERT(ret <= 0); 6806 return ret; 6807 } 6808 6809 static inline struct mlx5_flow_tunnel * 6810 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6811 { 6812 struct mlx5_flow_tunnel *tunnel; 6813 6814 #pragma GCC diagnostic push 6815 #pragma GCC diagnostic ignored "-Wcast-qual" 6816 tunnel = (typeof(tunnel))flow->tunnel; 6817 #pragma GCC diagnostic pop 6818 6819 return tunnel; 6820 } 6821 6822 /** 6823 * Adjust flow RSS workspace if needed. 6824 * 6825 * @param wks 6826 * Pointer to thread flow work space. 6827 * @param rss_desc 6828 * Pointer to RSS descriptor. 6829 * @param[in] nrssq_num 6830 * New RSS queue number. 6831 * 6832 * @return 6833 * 0 on success, -1 otherwise and rte_errno is set. 6834 */ 6835 static int 6836 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6837 struct mlx5_flow_rss_desc *rss_desc, 6838 uint32_t nrssq_num) 6839 { 6840 if (likely(nrssq_num <= wks->rssq_num)) 6841 return 0; 6842 rss_desc->queue = realloc(rss_desc->queue, 6843 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6844 if (!rss_desc->queue) { 6845 rte_errno = ENOMEM; 6846 return -1; 6847 } 6848 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6849 return 0; 6850 } 6851 6852 /** 6853 * Create a flow and add it to @p list. 6854 * 6855 * @param dev 6856 * Pointer to Ethernet device. 6857 * @param list 6858 * Pointer to a TAILQ flow list. If this parameter NULL, 6859 * no list insertion occurred, flow is just created, 6860 * this is caller's responsibility to track the 6861 * created flow. 6862 * @param[in] attr 6863 * Flow rule attributes. 6864 * @param[in] items 6865 * Pattern specification (list terminated by the END pattern item). 6866 * @param[in] actions 6867 * Associated actions (list terminated by the END action). 6868 * @param[in] external 6869 * This flow rule is created by request external to PMD. 6870 * @param[out] error 6871 * Perform verbose error reporting if not NULL. 6872 * 6873 * @return 6874 * A flow index on success, 0 otherwise and rte_errno is set. 6875 */ 6876 static uint32_t 6877 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6878 const struct rte_flow_attr *attr, 6879 const struct rte_flow_item items[], 6880 const struct rte_flow_action original_actions[], 6881 bool external, struct rte_flow_error *error) 6882 { 6883 struct mlx5_priv *priv = dev->data->dev_private; 6884 struct rte_flow *flow = NULL; 6885 struct mlx5_flow *dev_flow; 6886 const struct rte_flow_action_rss *rss = NULL; 6887 struct mlx5_translated_action_handle 6888 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6889 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6890 union { 6891 struct mlx5_flow_expand_rss buf; 6892 uint8_t buffer[8192]; 6893 } expand_buffer; 6894 union { 6895 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6896 uint8_t buffer[2048]; 6897 } actions_rx; 6898 union { 6899 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6900 uint8_t buffer[2048]; 6901 } actions_hairpin_tx; 6902 union { 6903 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6904 uint8_t buffer[2048]; 6905 } items_tx; 6906 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6907 struct mlx5_flow_rss_desc *rss_desc; 6908 const struct rte_flow_action *p_actions_rx; 6909 uint32_t i; 6910 uint32_t idx = 0; 6911 int hairpin_flow; 6912 struct rte_flow_attr attr_tx = { .priority = 0 }; 6913 const struct rte_flow_action *actions; 6914 struct rte_flow_action *translated_actions = NULL; 6915 struct mlx5_flow_tunnel *tunnel; 6916 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6917 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6918 struct mlx5_flow_split_info flow_split_info = { 6919 .external = !!external, 6920 .skip_scale = 0, 6921 .flow_idx = 0, 6922 .prefix_mark = 0, 6923 .prefix_layers = 0, 6924 .table_id = 0 6925 }; 6926 int ret; 6927 6928 MLX5_ASSERT(wks); 6929 rss_desc = &wks->rss_desc; 6930 ret = flow_action_handles_translate(dev, original_actions, 6931 indir_actions, 6932 &indir_actions_n, 6933 &translated_actions, error); 6934 if (ret < 0) { 6935 MLX5_ASSERT(translated_actions == NULL); 6936 return 0; 6937 } 6938 actions = translated_actions ? translated_actions : original_actions; 6939 p_actions_rx = actions; 6940 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6941 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6942 external, hairpin_flow, error); 6943 if (ret < 0) 6944 goto error_before_hairpin_split; 6945 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6946 if (!flow) { 6947 rte_errno = ENOMEM; 6948 goto error_before_hairpin_split; 6949 } 6950 if (hairpin_flow > 0) { 6951 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6952 rte_errno = EINVAL; 6953 goto error_before_hairpin_split; 6954 } 6955 flow_hairpin_split(dev, actions, actions_rx.actions, 6956 actions_hairpin_tx.actions, items_tx.items, 6957 idx); 6958 p_actions_rx = actions_rx.actions; 6959 } 6960 flow_split_info.flow_idx = idx; 6961 flow->drv_type = flow_get_drv_type(dev, attr); 6962 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6963 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6964 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6965 /* RSS Action only works on NIC RX domain */ 6966 if (attr->ingress && !attr->transfer) 6967 rss = flow_get_rss_action(dev, p_actions_rx); 6968 if (rss) { 6969 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6970 return 0; 6971 /* 6972 * The following information is required by 6973 * mlx5_flow_hashfields_adjust() in advance. 6974 */ 6975 rss_desc->level = rss->level; 6976 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6977 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6978 } 6979 flow->dev_handles = 0; 6980 if (rss && rss->types) { 6981 unsigned int graph_root; 6982 6983 graph_root = find_graph_root(rss->level); 6984 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6985 items, rss->types, 6986 mlx5_support_expansion, graph_root); 6987 MLX5_ASSERT(ret > 0 && 6988 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6989 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6990 for (i = 0; i < buf->entries; ++i) 6991 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6992 } 6993 } else { 6994 buf->entries = 1; 6995 buf->entry[0].pattern = (void *)(uintptr_t)items; 6996 } 6997 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6998 indir_actions_n); 6999 for (i = 0; i < buf->entries; ++i) { 7000 /* Initialize flow split data. */ 7001 flow_split_info.prefix_layers = 0; 7002 flow_split_info.prefix_mark = 0; 7003 flow_split_info.skip_scale = 0; 7004 /* 7005 * The splitter may create multiple dev_flows, 7006 * depending on configuration. In the simplest 7007 * case it just creates unmodified original flow. 7008 */ 7009 ret = flow_create_split_outer(dev, flow, attr, 7010 buf->entry[i].pattern, 7011 p_actions_rx, &flow_split_info, 7012 error); 7013 if (ret < 0) 7014 goto error; 7015 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7016 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7017 p_actions_rx, 7018 idx, 7019 wks->flows[0].tunnel, 7020 &default_miss_ctx, 7021 error); 7022 if (ret < 0) { 7023 mlx5_free(default_miss_ctx.queue); 7024 goto error; 7025 } 7026 } 7027 } 7028 /* Create the tx flow. */ 7029 if (hairpin_flow) { 7030 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7031 attr_tx.ingress = 0; 7032 attr_tx.egress = 1; 7033 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7034 actions_hairpin_tx.actions, 7035 idx, error); 7036 if (!dev_flow) 7037 goto error; 7038 dev_flow->flow = flow; 7039 dev_flow->external = 0; 7040 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7041 dev_flow->handle, next); 7042 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7043 items_tx.items, 7044 actions_hairpin_tx.actions, error); 7045 if (ret < 0) 7046 goto error; 7047 } 7048 /* 7049 * Update the metadata register copy table. If extensive 7050 * metadata feature is enabled and registers are supported 7051 * we might create the extra rte_flow for each unique 7052 * MARK/FLAG action ID. 7053 * 7054 * The table is updated for ingress Flows only, because 7055 * the egress Flows belong to the different device and 7056 * copy table should be updated in peer NIC Rx domain. 7057 */ 7058 if (attr->ingress && 7059 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7060 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7061 if (ret) 7062 goto error; 7063 } 7064 /* 7065 * If the flow is external (from application) OR device is started, 7066 * OR mreg discover, then apply immediately. 7067 */ 7068 if (external || dev->data->dev_started || 7069 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7070 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7071 ret = flow_drv_apply(dev, flow, error); 7072 if (ret < 0) 7073 goto error; 7074 } 7075 flow->type = type; 7076 flow_rxq_flags_set(dev, flow); 7077 rte_free(translated_actions); 7078 tunnel = flow_tunnel_from_rule(wks->flows); 7079 if (tunnel) { 7080 flow->tunnel = 1; 7081 flow->tunnel_id = tunnel->tunnel_id; 7082 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7083 mlx5_free(default_miss_ctx.queue); 7084 } 7085 mlx5_flow_pop_thread_workspace(); 7086 return idx; 7087 error: 7088 MLX5_ASSERT(flow); 7089 ret = rte_errno; /* Save rte_errno before cleanup. */ 7090 flow_mreg_del_copy_action(dev, flow); 7091 flow_drv_destroy(dev, flow); 7092 if (rss_desc->shared_rss) 7093 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7094 mlx5_ipool_get 7095 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7096 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7097 mlx5_ipool_free(priv->flows[type], idx); 7098 rte_errno = ret; /* Restore rte_errno. */ 7099 ret = rte_errno; 7100 rte_errno = ret; 7101 error_before_hairpin_split: 7102 mlx5_flow_pop_thread_workspace(); 7103 rte_free(translated_actions); 7104 return 0; 7105 } 7106 7107 /** 7108 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7109 * incoming packets to table 1. 7110 * 7111 * Other flow rules, requested for group n, will be created in 7112 * e-switch table n+1. 7113 * Jump action to e-switch group n will be created to group n+1. 7114 * 7115 * Used when working in switchdev mode, to utilise advantages of table 1 7116 * and above. 7117 * 7118 * @param dev 7119 * Pointer to Ethernet device. 7120 * 7121 * @return 7122 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7123 */ 7124 struct rte_flow * 7125 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7126 { 7127 const struct rte_flow_attr attr = { 7128 .group = 0, 7129 .priority = 0, 7130 .ingress = 1, 7131 .egress = 0, 7132 .transfer = 1, 7133 }; 7134 const struct rte_flow_item pattern = { 7135 .type = RTE_FLOW_ITEM_TYPE_END, 7136 }; 7137 struct rte_flow_action_jump jump = { 7138 .group = 1, 7139 }; 7140 const struct rte_flow_action actions[] = { 7141 { 7142 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7143 .conf = &jump, 7144 }, 7145 { 7146 .type = RTE_FLOW_ACTION_TYPE_END, 7147 }, 7148 }; 7149 struct rte_flow_error error; 7150 7151 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7152 &attr, &pattern, 7153 actions, false, &error); 7154 } 7155 7156 /** 7157 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7158 * and sq number, directs all packets to peer vport. 7159 * 7160 * @param dev 7161 * Pointer to Ethernet device. 7162 * @param sq_num 7163 * SQ number. 7164 * 7165 * @return 7166 * Flow ID on success, 0 otherwise and rte_errno is set. 7167 */ 7168 uint32_t 7169 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7170 { 7171 struct rte_flow_attr attr = { 7172 .group = 0, 7173 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7174 .ingress = 1, 7175 .egress = 0, 7176 .transfer = 1, 7177 }; 7178 struct rte_flow_item_port_id port_spec = { 7179 .id = MLX5_PORT_ESW_MGR, 7180 }; 7181 struct mlx5_rte_flow_item_sq sq_spec = { 7182 .queue = sq_num, 7183 }; 7184 struct rte_flow_item pattern[] = { 7185 { 7186 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7187 .spec = &port_spec, 7188 }, 7189 { 7190 .type = (enum rte_flow_item_type) 7191 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7192 .spec = &sq_spec, 7193 }, 7194 { 7195 .type = RTE_FLOW_ITEM_TYPE_END, 7196 }, 7197 }; 7198 struct rte_flow_action_jump jump = { 7199 .group = 1, 7200 }; 7201 struct rte_flow_action_port_id port = { 7202 .id = dev->data->port_id, 7203 }; 7204 struct rte_flow_action actions[] = { 7205 { 7206 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7207 .conf = &jump, 7208 }, 7209 { 7210 .type = RTE_FLOW_ACTION_TYPE_END, 7211 }, 7212 }; 7213 struct rte_flow_error error; 7214 7215 /* 7216 * Creates group 0, highest priority jump flow. 7217 * Matches txq to bypass kernel packets. 7218 */ 7219 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7220 false, &error) == 0) 7221 return 0; 7222 /* Create group 1, lowest priority redirect flow for txq. */ 7223 attr.group = 1; 7224 actions[0].conf = &port; 7225 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7226 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7227 actions, false, &error); 7228 } 7229 7230 /** 7231 * Validate a flow supported by the NIC. 7232 * 7233 * @see rte_flow_validate() 7234 * @see rte_flow_ops 7235 */ 7236 int 7237 mlx5_flow_validate(struct rte_eth_dev *dev, 7238 const struct rte_flow_attr *attr, 7239 const struct rte_flow_item items[], 7240 const struct rte_flow_action original_actions[], 7241 struct rte_flow_error *error) 7242 { 7243 int hairpin_flow; 7244 struct mlx5_translated_action_handle 7245 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7246 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7247 const struct rte_flow_action *actions; 7248 struct rte_flow_action *translated_actions = NULL; 7249 int ret = flow_action_handles_translate(dev, original_actions, 7250 indir_actions, 7251 &indir_actions_n, 7252 &translated_actions, error); 7253 7254 if (ret) 7255 return ret; 7256 actions = translated_actions ? translated_actions : original_actions; 7257 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7258 ret = flow_drv_validate(dev, attr, items, actions, 7259 true, hairpin_flow, error); 7260 rte_free(translated_actions); 7261 return ret; 7262 } 7263 7264 /** 7265 * Create a flow. 7266 * 7267 * @see rte_flow_create() 7268 * @see rte_flow_ops 7269 */ 7270 struct rte_flow * 7271 mlx5_flow_create(struct rte_eth_dev *dev, 7272 const struct rte_flow_attr *attr, 7273 const struct rte_flow_item items[], 7274 const struct rte_flow_action actions[], 7275 struct rte_flow_error *error) 7276 { 7277 struct mlx5_priv *priv = dev->data->dev_private; 7278 7279 if (priv->sh->config.dv_flow_en == 2) { 7280 rte_flow_error_set(error, ENOTSUP, 7281 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7282 NULL, 7283 "Flow non-Q creation not supported"); 7284 return NULL; 7285 } 7286 /* 7287 * If the device is not started yet, it is not allowed to created a 7288 * flow from application. PMD default flows and traffic control flows 7289 * are not affected. 7290 */ 7291 if (unlikely(!dev->data->dev_started)) { 7292 DRV_LOG(DEBUG, "port %u is not started when " 7293 "inserting a flow", dev->data->port_id); 7294 rte_flow_error_set(error, ENODEV, 7295 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7296 NULL, 7297 "port not started"); 7298 return NULL; 7299 } 7300 7301 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7302 attr, items, actions, 7303 true, error); 7304 } 7305 7306 /** 7307 * Destroy a flow in a list. 7308 * 7309 * @param dev 7310 * Pointer to Ethernet device. 7311 * @param[in] flow_idx 7312 * Index of flow to destroy. 7313 */ 7314 static void 7315 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7316 uint32_t flow_idx) 7317 { 7318 struct mlx5_priv *priv = dev->data->dev_private; 7319 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7320 7321 if (!flow) 7322 return; 7323 MLX5_ASSERT(flow->type == type); 7324 /* 7325 * Update RX queue flags only if port is started, otherwise it is 7326 * already clean. 7327 */ 7328 if (dev->data->dev_started) 7329 flow_rxq_flags_trim(dev, flow); 7330 flow_drv_destroy(dev, flow); 7331 if (flow->tunnel) { 7332 struct mlx5_flow_tunnel *tunnel; 7333 7334 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7335 RTE_VERIFY(tunnel); 7336 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7337 mlx5_flow_tunnel_free(dev, tunnel); 7338 } 7339 flow_mreg_del_copy_action(dev, flow); 7340 mlx5_ipool_free(priv->flows[type], flow_idx); 7341 } 7342 7343 /** 7344 * Destroy all flows. 7345 * 7346 * @param dev 7347 * Pointer to Ethernet device. 7348 * @param type 7349 * Flow type to be flushed. 7350 * @param active 7351 * If flushing is called actively. 7352 */ 7353 void 7354 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7355 bool active) 7356 { 7357 struct mlx5_priv *priv = dev->data->dev_private; 7358 uint32_t num_flushed = 0, fidx = 1; 7359 struct rte_flow *flow; 7360 7361 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7362 if (priv->sh->config.dv_flow_en == 2 && 7363 type == MLX5_FLOW_TYPE_GEN) { 7364 flow_hw_q_flow_flush(dev, NULL); 7365 return; 7366 } 7367 #endif 7368 7369 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7370 flow_list_destroy(dev, type, fidx); 7371 num_flushed++; 7372 } 7373 if (active) { 7374 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7375 dev->data->port_id, num_flushed); 7376 } 7377 } 7378 7379 /** 7380 * Stop all default actions for flows. 7381 * 7382 * @param dev 7383 * Pointer to Ethernet device. 7384 */ 7385 void 7386 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7387 { 7388 flow_mreg_del_default_copy_action(dev); 7389 flow_rxq_flags_clear(dev); 7390 } 7391 7392 /** 7393 * Start all default actions for flows. 7394 * 7395 * @param dev 7396 * Pointer to Ethernet device. 7397 * @return 7398 * 0 on success, a negative errno value otherwise and rte_errno is set. 7399 */ 7400 int 7401 mlx5_flow_start_default(struct rte_eth_dev *dev) 7402 { 7403 struct rte_flow_error error; 7404 7405 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7406 return flow_mreg_add_default_copy_action(dev, &error); 7407 } 7408 7409 /** 7410 * Release key of thread specific flow workspace data. 7411 */ 7412 void 7413 flow_release_workspace(void *data) 7414 { 7415 struct mlx5_flow_workspace *wks = data; 7416 struct mlx5_flow_workspace *next; 7417 7418 while (wks) { 7419 next = wks->next; 7420 free(wks->rss_desc.queue); 7421 free(wks); 7422 wks = next; 7423 } 7424 } 7425 7426 /** 7427 * Get thread specific current flow workspace. 7428 * 7429 * @return pointer to thread specific flow workspace data, NULL on error. 7430 */ 7431 struct mlx5_flow_workspace* 7432 mlx5_flow_get_thread_workspace(void) 7433 { 7434 struct mlx5_flow_workspace *data; 7435 7436 data = mlx5_flow_os_get_specific_workspace(); 7437 MLX5_ASSERT(data && data->inuse); 7438 if (!data || !data->inuse) 7439 DRV_LOG(ERR, "flow workspace not initialized."); 7440 return data; 7441 } 7442 7443 /** 7444 * Allocate and init new flow workspace. 7445 * 7446 * @return pointer to flow workspace data, NULL on error. 7447 */ 7448 static struct mlx5_flow_workspace* 7449 flow_alloc_thread_workspace(void) 7450 { 7451 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7452 7453 if (!data) { 7454 DRV_LOG(ERR, "Failed to allocate flow workspace " 7455 "memory."); 7456 return NULL; 7457 } 7458 data->rss_desc.queue = calloc(1, 7459 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7460 if (!data->rss_desc.queue) 7461 goto err; 7462 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7463 return data; 7464 err: 7465 free(data->rss_desc.queue); 7466 free(data); 7467 return NULL; 7468 } 7469 7470 /** 7471 * Get new thread specific flow workspace. 7472 * 7473 * If current workspace inuse, create new one and set as current. 7474 * 7475 * @return pointer to thread specific flow workspace data, NULL on error. 7476 */ 7477 struct mlx5_flow_workspace* 7478 mlx5_flow_push_thread_workspace(void) 7479 { 7480 struct mlx5_flow_workspace *curr; 7481 struct mlx5_flow_workspace *data; 7482 7483 curr = mlx5_flow_os_get_specific_workspace(); 7484 if (!curr) { 7485 data = flow_alloc_thread_workspace(); 7486 if (!data) 7487 return NULL; 7488 } else if (!curr->inuse) { 7489 data = curr; 7490 } else if (curr->next) { 7491 data = curr->next; 7492 } else { 7493 data = flow_alloc_thread_workspace(); 7494 if (!data) 7495 return NULL; 7496 curr->next = data; 7497 data->prev = curr; 7498 } 7499 data->inuse = 1; 7500 data->flow_idx = 0; 7501 /* Set as current workspace */ 7502 if (mlx5_flow_os_set_specific_workspace(data)) 7503 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7504 return data; 7505 } 7506 7507 /** 7508 * Close current thread specific flow workspace. 7509 * 7510 * If previous workspace available, set it as current. 7511 * 7512 * @return pointer to thread specific flow workspace data, NULL on error. 7513 */ 7514 void 7515 mlx5_flow_pop_thread_workspace(void) 7516 { 7517 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7518 7519 if (!data) 7520 return; 7521 if (!data->inuse) { 7522 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7523 return; 7524 } 7525 data->inuse = 0; 7526 if (!data->prev) 7527 return; 7528 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7529 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7530 } 7531 7532 /** 7533 * Verify the flow list is empty 7534 * 7535 * @param dev 7536 * Pointer to Ethernet device. 7537 * 7538 * @return the number of flows not released. 7539 */ 7540 int 7541 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7542 { 7543 struct mlx5_priv *priv = dev->data->dev_private; 7544 struct rte_flow *flow; 7545 uint32_t idx = 0; 7546 int ret = 0, i; 7547 7548 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7549 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7550 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7551 dev->data->port_id, (void *)flow); 7552 ret++; 7553 } 7554 } 7555 return ret; 7556 } 7557 7558 /** 7559 * Enable default hairpin egress flow. 7560 * 7561 * @param dev 7562 * Pointer to Ethernet device. 7563 * @param sq_num 7564 * The SQ hw number. 7565 * 7566 * @return 7567 * 0 on success, a negative errno value otherwise and rte_errno is set. 7568 */ 7569 int 7570 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7571 uint32_t sq_num) 7572 { 7573 const struct rte_flow_attr attr = { 7574 .egress = 1, 7575 .priority = 0, 7576 }; 7577 struct mlx5_rte_flow_item_sq queue_spec = { 7578 .queue = sq_num, 7579 }; 7580 struct mlx5_rte_flow_item_sq queue_mask = { 7581 .queue = UINT32_MAX, 7582 }; 7583 struct rte_flow_item items[] = { 7584 { 7585 .type = (enum rte_flow_item_type) 7586 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7587 .spec = &queue_spec, 7588 .last = NULL, 7589 .mask = &queue_mask, 7590 }, 7591 { 7592 .type = RTE_FLOW_ITEM_TYPE_END, 7593 }, 7594 }; 7595 struct rte_flow_action_jump jump = { 7596 .group = MLX5_HAIRPIN_TX_TABLE, 7597 }; 7598 struct rte_flow_action actions[2]; 7599 uint32_t flow_idx; 7600 struct rte_flow_error error; 7601 7602 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7603 actions[0].conf = &jump; 7604 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7605 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7606 &attr, items, actions, false, &error); 7607 if (!flow_idx) { 7608 DRV_LOG(DEBUG, 7609 "Failed to create ctrl flow: rte_errno(%d)," 7610 " type(%d), message(%s)", 7611 rte_errno, error.type, 7612 error.message ? error.message : " (no stated reason)"); 7613 return -rte_errno; 7614 } 7615 return 0; 7616 } 7617 7618 /** 7619 * Enable a control flow configured from the control plane. 7620 * 7621 * @param dev 7622 * Pointer to Ethernet device. 7623 * @param eth_spec 7624 * An Ethernet flow spec to apply. 7625 * @param eth_mask 7626 * An Ethernet flow mask to apply. 7627 * @param vlan_spec 7628 * A VLAN flow spec to apply. 7629 * @param vlan_mask 7630 * A VLAN flow mask to apply. 7631 * 7632 * @return 7633 * 0 on success, a negative errno value otherwise and rte_errno is set. 7634 */ 7635 int 7636 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7637 struct rte_flow_item_eth *eth_spec, 7638 struct rte_flow_item_eth *eth_mask, 7639 struct rte_flow_item_vlan *vlan_spec, 7640 struct rte_flow_item_vlan *vlan_mask) 7641 { 7642 struct mlx5_priv *priv = dev->data->dev_private; 7643 const struct rte_flow_attr attr = { 7644 .ingress = 1, 7645 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7646 }; 7647 struct rte_flow_item items[] = { 7648 { 7649 .type = RTE_FLOW_ITEM_TYPE_ETH, 7650 .spec = eth_spec, 7651 .last = NULL, 7652 .mask = eth_mask, 7653 }, 7654 { 7655 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7656 RTE_FLOW_ITEM_TYPE_END, 7657 .spec = vlan_spec, 7658 .last = NULL, 7659 .mask = vlan_mask, 7660 }, 7661 { 7662 .type = RTE_FLOW_ITEM_TYPE_END, 7663 }, 7664 }; 7665 uint16_t queue[priv->reta_idx_n]; 7666 struct rte_flow_action_rss action_rss = { 7667 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7668 .level = 0, 7669 .types = priv->rss_conf.rss_hf, 7670 .key_len = priv->rss_conf.rss_key_len, 7671 .queue_num = priv->reta_idx_n, 7672 .key = priv->rss_conf.rss_key, 7673 .queue = queue, 7674 }; 7675 struct rte_flow_action actions[] = { 7676 { 7677 .type = RTE_FLOW_ACTION_TYPE_RSS, 7678 .conf = &action_rss, 7679 }, 7680 { 7681 .type = RTE_FLOW_ACTION_TYPE_END, 7682 }, 7683 }; 7684 uint32_t flow_idx; 7685 struct rte_flow_error error; 7686 unsigned int i; 7687 7688 if (!priv->reta_idx_n || !priv->rxqs_n) { 7689 return 0; 7690 } 7691 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7692 action_rss.types = 0; 7693 for (i = 0; i != priv->reta_idx_n; ++i) 7694 queue[i] = (*priv->reta_idx)[i]; 7695 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7696 &attr, items, actions, false, &error); 7697 if (!flow_idx) 7698 return -rte_errno; 7699 return 0; 7700 } 7701 7702 /** 7703 * Enable a flow control configured from the control plane. 7704 * 7705 * @param dev 7706 * Pointer to Ethernet device. 7707 * @param eth_spec 7708 * An Ethernet flow spec to apply. 7709 * @param eth_mask 7710 * An Ethernet flow mask to apply. 7711 * 7712 * @return 7713 * 0 on success, a negative errno value otherwise and rte_errno is set. 7714 */ 7715 int 7716 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7717 struct rte_flow_item_eth *eth_spec, 7718 struct rte_flow_item_eth *eth_mask) 7719 { 7720 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7721 } 7722 7723 /** 7724 * Create default miss flow rule matching lacp traffic 7725 * 7726 * @param dev 7727 * Pointer to Ethernet device. 7728 * @param eth_spec 7729 * An Ethernet flow spec to apply. 7730 * 7731 * @return 7732 * 0 on success, a negative errno value otherwise and rte_errno is set. 7733 */ 7734 int 7735 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7736 { 7737 /* 7738 * The LACP matching is done by only using ether type since using 7739 * a multicast dst mac causes kernel to give low priority to this flow. 7740 */ 7741 static const struct rte_flow_item_eth lacp_spec = { 7742 .type = RTE_BE16(0x8809), 7743 }; 7744 static const struct rte_flow_item_eth lacp_mask = { 7745 .type = 0xffff, 7746 }; 7747 const struct rte_flow_attr attr = { 7748 .ingress = 1, 7749 }; 7750 struct rte_flow_item items[] = { 7751 { 7752 .type = RTE_FLOW_ITEM_TYPE_ETH, 7753 .spec = &lacp_spec, 7754 .mask = &lacp_mask, 7755 }, 7756 { 7757 .type = RTE_FLOW_ITEM_TYPE_END, 7758 }, 7759 }; 7760 struct rte_flow_action actions[] = { 7761 { 7762 .type = (enum rte_flow_action_type) 7763 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7764 }, 7765 { 7766 .type = RTE_FLOW_ACTION_TYPE_END, 7767 }, 7768 }; 7769 struct rte_flow_error error; 7770 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7771 &attr, items, actions, 7772 false, &error); 7773 7774 if (!flow_idx) 7775 return -rte_errno; 7776 return 0; 7777 } 7778 7779 /** 7780 * Destroy a flow. 7781 * 7782 * @see rte_flow_destroy() 7783 * @see rte_flow_ops 7784 */ 7785 int 7786 mlx5_flow_destroy(struct rte_eth_dev *dev, 7787 struct rte_flow *flow, 7788 struct rte_flow_error *error __rte_unused) 7789 { 7790 struct mlx5_priv *priv = dev->data->dev_private; 7791 7792 if (priv->sh->config.dv_flow_en == 2) 7793 return rte_flow_error_set(error, ENOTSUP, 7794 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7795 NULL, 7796 "Flow non-Q destruction not supported"); 7797 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7798 (uintptr_t)(void *)flow); 7799 return 0; 7800 } 7801 7802 /** 7803 * Destroy all flows. 7804 * 7805 * @see rte_flow_flush() 7806 * @see rte_flow_ops 7807 */ 7808 int 7809 mlx5_flow_flush(struct rte_eth_dev *dev, 7810 struct rte_flow_error *error __rte_unused) 7811 { 7812 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7813 return 0; 7814 } 7815 7816 /** 7817 * Isolated mode. 7818 * 7819 * @see rte_flow_isolate() 7820 * @see rte_flow_ops 7821 */ 7822 int 7823 mlx5_flow_isolate(struct rte_eth_dev *dev, 7824 int enable, 7825 struct rte_flow_error *error) 7826 { 7827 struct mlx5_priv *priv = dev->data->dev_private; 7828 7829 if (dev->data->dev_started) { 7830 rte_flow_error_set(error, EBUSY, 7831 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7832 NULL, 7833 "port must be stopped first"); 7834 return -rte_errno; 7835 } 7836 priv->isolated = !!enable; 7837 if (enable) 7838 dev->dev_ops = &mlx5_dev_ops_isolate; 7839 else 7840 dev->dev_ops = &mlx5_dev_ops; 7841 7842 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7843 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7844 7845 return 0; 7846 } 7847 7848 /** 7849 * Query a flow. 7850 * 7851 * @see rte_flow_query() 7852 * @see rte_flow_ops 7853 */ 7854 static int 7855 flow_drv_query(struct rte_eth_dev *dev, 7856 struct rte_flow *eflow, 7857 const struct rte_flow_action *actions, 7858 void *data, 7859 struct rte_flow_error *error) 7860 { 7861 struct mlx5_priv *priv = dev->data->dev_private; 7862 const struct mlx5_flow_driver_ops *fops; 7863 struct rte_flow *flow = NULL; 7864 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 7865 7866 if (priv->sh->config.dv_flow_en == 2) { 7867 #ifdef HAVE_MLX5_HWS_SUPPORT 7868 flow = eflow; 7869 ftype = MLX5_FLOW_TYPE_HW; 7870 #endif 7871 } else { 7872 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7873 (uintptr_t)(void *)eflow); 7874 } 7875 if (!flow) { 7876 return rte_flow_error_set(error, ENOENT, 7877 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7878 NULL, 7879 "invalid flow handle"); 7880 } 7881 if (ftype == MLX5_FLOW_TYPE_MIN) 7882 ftype = flow->drv_type; 7883 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7884 fops = flow_get_drv_ops(ftype); 7885 7886 return fops->query(dev, flow, actions, data, error); 7887 } 7888 7889 /** 7890 * Query a flow. 7891 * 7892 * @see rte_flow_query() 7893 * @see rte_flow_ops 7894 */ 7895 int 7896 mlx5_flow_query(struct rte_eth_dev *dev, 7897 struct rte_flow *flow, 7898 const struct rte_flow_action *actions, 7899 void *data, 7900 struct rte_flow_error *error) 7901 { 7902 int ret; 7903 7904 ret = flow_drv_query(dev, flow, actions, data, 7905 error); 7906 if (ret < 0) 7907 return ret; 7908 return 0; 7909 } 7910 7911 /** 7912 * Get rte_flow callbacks. 7913 * 7914 * @param dev 7915 * Pointer to Ethernet device structure. 7916 * @param ops 7917 * Pointer to operation-specific structure. 7918 * 7919 * @return 0 7920 */ 7921 int 7922 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7923 const struct rte_flow_ops **ops) 7924 { 7925 *ops = &mlx5_flow_ops; 7926 return 0; 7927 } 7928 7929 /** 7930 * Validate meter policy actions. 7931 * Dispatcher for action type specific validation. 7932 * 7933 * @param[in] dev 7934 * Pointer to the Ethernet device structure. 7935 * @param[in] action 7936 * The meter policy action object to validate. 7937 * @param[in] attr 7938 * Attributes of flow to determine steering domain. 7939 * @param[out] is_rss 7940 * Is RSS or not. 7941 * @param[out] domain_bitmap 7942 * Domain bitmap. 7943 * @param[out] is_def_policy 7944 * Is default policy or not. 7945 * @param[out] error 7946 * Perform verbose error reporting if not NULL. Initialized in case of 7947 * error only. 7948 * 7949 * @return 7950 * 0 on success, otherwise negative errno value. 7951 */ 7952 int 7953 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7954 const struct rte_flow_action *actions[RTE_COLORS], 7955 struct rte_flow_attr *attr, 7956 bool *is_rss, 7957 uint8_t *domain_bitmap, 7958 uint8_t *policy_mode, 7959 struct rte_mtr_error *error) 7960 { 7961 const struct mlx5_flow_driver_ops *fops; 7962 7963 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7964 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7965 domain_bitmap, policy_mode, error); 7966 } 7967 7968 /** 7969 * Destroy the meter table set. 7970 * 7971 * @param[in] dev 7972 * Pointer to Ethernet device. 7973 * @param[in] mtr_policy 7974 * Meter policy struct. 7975 */ 7976 void 7977 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7978 struct mlx5_flow_meter_policy *mtr_policy) 7979 { 7980 const struct mlx5_flow_driver_ops *fops; 7981 7982 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7983 fops->destroy_mtr_acts(dev, mtr_policy); 7984 } 7985 7986 /** 7987 * Create policy action, lock free, 7988 * (mutex should be acquired by caller). 7989 * Dispatcher for action type specific call. 7990 * 7991 * @param[in] dev 7992 * Pointer to the Ethernet device structure. 7993 * @param[in] mtr_policy 7994 * Meter policy struct. 7995 * @param[in] action 7996 * Action specification used to create meter actions. 7997 * @param[in] attr 7998 * Flow rule attributes. 7999 * @param[out] error 8000 * Perform verbose error reporting if not NULL. Initialized in case of 8001 * error only. 8002 * 8003 * @return 8004 * 0 on success, otherwise negative errno value. 8005 */ 8006 int 8007 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8008 struct mlx5_flow_meter_policy *mtr_policy, 8009 const struct rte_flow_action *actions[RTE_COLORS], 8010 struct rte_flow_attr *attr, 8011 struct rte_mtr_error *error) 8012 { 8013 const struct mlx5_flow_driver_ops *fops; 8014 8015 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8016 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8017 } 8018 8019 /** 8020 * Create policy rules, lock free, 8021 * (mutex should be acquired by caller). 8022 * Dispatcher for action type specific call. 8023 * 8024 * @param[in] dev 8025 * Pointer to the Ethernet device structure. 8026 * @param[in] mtr_policy 8027 * Meter policy struct. 8028 * 8029 * @return 8030 * 0 on success, -1 otherwise. 8031 */ 8032 int 8033 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8034 struct mlx5_flow_meter_policy *mtr_policy) 8035 { 8036 const struct mlx5_flow_driver_ops *fops; 8037 8038 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8039 return fops->create_policy_rules(dev, mtr_policy); 8040 } 8041 8042 /** 8043 * Destroy policy rules, lock free, 8044 * (mutex should be acquired by caller). 8045 * Dispatcher for action type specific call. 8046 * 8047 * @param[in] dev 8048 * Pointer to the Ethernet device structure. 8049 * @param[in] mtr_policy 8050 * Meter policy struct. 8051 */ 8052 void 8053 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8054 struct mlx5_flow_meter_policy *mtr_policy) 8055 { 8056 const struct mlx5_flow_driver_ops *fops; 8057 8058 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8059 fops->destroy_policy_rules(dev, mtr_policy); 8060 } 8061 8062 /** 8063 * Destroy the default policy table set. 8064 * 8065 * @param[in] dev 8066 * Pointer to Ethernet device. 8067 */ 8068 void 8069 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8070 { 8071 const struct mlx5_flow_driver_ops *fops; 8072 8073 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8074 fops->destroy_def_policy(dev); 8075 } 8076 8077 /** 8078 * Destroy the default policy table set. 8079 * 8080 * @param[in] dev 8081 * Pointer to Ethernet device. 8082 * 8083 * @return 8084 * 0 on success, -1 otherwise. 8085 */ 8086 int 8087 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8088 { 8089 const struct mlx5_flow_driver_ops *fops; 8090 8091 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8092 return fops->create_def_policy(dev); 8093 } 8094 8095 /** 8096 * Create the needed meter and suffix tables. 8097 * 8098 * @param[in] dev 8099 * Pointer to Ethernet device. 8100 * 8101 * @return 8102 * 0 on success, -1 otherwise. 8103 */ 8104 int 8105 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8106 struct mlx5_flow_meter_info *fm, 8107 uint32_t mtr_idx, 8108 uint8_t domain_bitmap) 8109 { 8110 const struct mlx5_flow_driver_ops *fops; 8111 8112 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8113 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8114 } 8115 8116 /** 8117 * Destroy the meter table set. 8118 * 8119 * @param[in] dev 8120 * Pointer to Ethernet device. 8121 * @param[in] tbl 8122 * Pointer to the meter table set. 8123 */ 8124 void 8125 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8126 struct mlx5_flow_meter_info *fm) 8127 { 8128 const struct mlx5_flow_driver_ops *fops; 8129 8130 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8131 fops->destroy_mtr_tbls(dev, fm); 8132 } 8133 8134 /** 8135 * Destroy the global meter drop table. 8136 * 8137 * @param[in] dev 8138 * Pointer to Ethernet device. 8139 */ 8140 void 8141 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8142 { 8143 const struct mlx5_flow_driver_ops *fops; 8144 8145 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8146 fops->destroy_mtr_drop_tbls(dev); 8147 } 8148 8149 /** 8150 * Destroy the sub policy table with RX queue. 8151 * 8152 * @param[in] dev 8153 * Pointer to Ethernet device. 8154 * @param[in] mtr_policy 8155 * Pointer to meter policy table. 8156 */ 8157 void 8158 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8159 struct mlx5_flow_meter_policy *mtr_policy) 8160 { 8161 const struct mlx5_flow_driver_ops *fops; 8162 8163 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8164 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8165 } 8166 8167 /** 8168 * Allocate the needed aso flow meter id. 8169 * 8170 * @param[in] dev 8171 * Pointer to Ethernet device. 8172 * 8173 * @return 8174 * Index to aso flow meter on success, NULL otherwise. 8175 */ 8176 uint32_t 8177 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8178 { 8179 const struct mlx5_flow_driver_ops *fops; 8180 8181 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8182 return fops->create_meter(dev); 8183 } 8184 8185 /** 8186 * Free the aso flow meter id. 8187 * 8188 * @param[in] dev 8189 * Pointer to Ethernet device. 8190 * @param[in] mtr_idx 8191 * Index to aso flow meter to be free. 8192 * 8193 * @return 8194 * 0 on success. 8195 */ 8196 void 8197 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8198 { 8199 const struct mlx5_flow_driver_ops *fops; 8200 8201 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8202 fops->free_meter(dev, mtr_idx); 8203 } 8204 8205 /** 8206 * Allocate a counter. 8207 * 8208 * @param[in] dev 8209 * Pointer to Ethernet device structure. 8210 * 8211 * @return 8212 * Index to allocated counter on success, 0 otherwise. 8213 */ 8214 uint32_t 8215 mlx5_counter_alloc(struct rte_eth_dev *dev) 8216 { 8217 const struct mlx5_flow_driver_ops *fops; 8218 struct rte_flow_attr attr = { .transfer = 0 }; 8219 8220 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8221 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8222 return fops->counter_alloc(dev); 8223 } 8224 DRV_LOG(ERR, 8225 "port %u counter allocate is not supported.", 8226 dev->data->port_id); 8227 return 0; 8228 } 8229 8230 /** 8231 * Free a counter. 8232 * 8233 * @param[in] dev 8234 * Pointer to Ethernet device structure. 8235 * @param[in] cnt 8236 * Index to counter to be free. 8237 */ 8238 void 8239 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8240 { 8241 const struct mlx5_flow_driver_ops *fops; 8242 struct rte_flow_attr attr = { .transfer = 0 }; 8243 8244 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8245 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8246 fops->counter_free(dev, cnt); 8247 return; 8248 } 8249 DRV_LOG(ERR, 8250 "port %u counter free is not supported.", 8251 dev->data->port_id); 8252 } 8253 8254 /** 8255 * Query counter statistics. 8256 * 8257 * @param[in] dev 8258 * Pointer to Ethernet device structure. 8259 * @param[in] cnt 8260 * Index to counter to query. 8261 * @param[in] clear 8262 * Set to clear counter statistics. 8263 * @param[out] pkts 8264 * The counter hits packets number to save. 8265 * @param[out] bytes 8266 * The counter hits bytes number to save. 8267 * 8268 * @return 8269 * 0 on success, a negative errno value otherwise. 8270 */ 8271 int 8272 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8273 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8274 { 8275 const struct mlx5_flow_driver_ops *fops; 8276 struct rte_flow_attr attr = { .transfer = 0 }; 8277 8278 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8279 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8280 return fops->counter_query(dev, cnt, clear, pkts, 8281 bytes, action); 8282 } 8283 DRV_LOG(ERR, 8284 "port %u counter query is not supported.", 8285 dev->data->port_id); 8286 return -ENOTSUP; 8287 } 8288 8289 /** 8290 * Get information about HWS pre-configurable resources. 8291 * 8292 * @param[in] dev 8293 * Pointer to the rte_eth_dev structure. 8294 * @param[out] port_info 8295 * Pointer to port information. 8296 * @param[out] queue_info 8297 * Pointer to queue information. 8298 * @param[out] error 8299 * Pointer to error structure. 8300 * 8301 * @return 8302 * 0 on success, a negative errno value otherwise and rte_errno is set. 8303 */ 8304 static int 8305 mlx5_flow_info_get(struct rte_eth_dev *dev, 8306 struct rte_flow_port_info *port_info, 8307 struct rte_flow_queue_info *queue_info, 8308 struct rte_flow_error *error) 8309 { 8310 const struct mlx5_flow_driver_ops *fops; 8311 struct rte_flow_attr attr = {0}; 8312 8313 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8314 return rte_flow_error_set(error, ENOTSUP, 8315 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8316 NULL, 8317 "info get with incorrect steering mode"); 8318 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8319 return fops->info_get(dev, port_info, queue_info, error); 8320 } 8321 8322 /** 8323 * Configure port HWS resources. 8324 * 8325 * @param[in] dev 8326 * Pointer to the rte_eth_dev structure. 8327 * @param[in] port_attr 8328 * Port configuration attributes. 8329 * @param[in] nb_queue 8330 * Number of queue. 8331 * @param[in] queue_attr 8332 * Array that holds attributes for each flow queue. 8333 * @param[out] error 8334 * Pointer to error structure. 8335 * 8336 * @return 8337 * 0 on success, a negative errno value otherwise and rte_errno is set. 8338 */ 8339 static int 8340 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8341 const struct rte_flow_port_attr *port_attr, 8342 uint16_t nb_queue, 8343 const struct rte_flow_queue_attr *queue_attr[], 8344 struct rte_flow_error *error) 8345 { 8346 const struct mlx5_flow_driver_ops *fops; 8347 struct rte_flow_attr attr = {0}; 8348 8349 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8350 return rte_flow_error_set(error, ENOTSUP, 8351 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8352 NULL, 8353 "port configure with incorrect steering mode"); 8354 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8355 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8356 } 8357 8358 /** 8359 * Validate item template. 8360 * 8361 * @param[in] dev 8362 * Pointer to the rte_eth_dev structure. 8363 * @param[in] attr 8364 * Pointer to the item template attributes. 8365 * @param[in] items 8366 * The template item pattern. 8367 * @param[out] error 8368 * Pointer to error structure. 8369 * 8370 * @return 8371 * 0 on success, a negative errno value otherwise and rte_errno is set. 8372 */ 8373 int 8374 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8375 const struct rte_flow_pattern_template_attr *attr, 8376 const struct rte_flow_item items[], 8377 struct rte_flow_error *error) 8378 { 8379 const struct mlx5_flow_driver_ops *fops; 8380 struct rte_flow_attr fattr = {0}; 8381 8382 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8383 rte_flow_error_set(error, ENOTSUP, 8384 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8385 "pattern validate with incorrect steering mode"); 8386 return -ENOTSUP; 8387 } 8388 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8389 return fops->pattern_validate(dev, attr, items, error); 8390 } 8391 8392 /** 8393 * Create flow item template. 8394 * 8395 * @param[in] dev 8396 * Pointer to the rte_eth_dev structure. 8397 * @param[in] attr 8398 * Pointer to the item template attributes. 8399 * @param[in] items 8400 * The template item pattern. 8401 * @param[out] error 8402 * Pointer to error structure. 8403 * 8404 * @return 8405 * 0 on success, a negative errno value otherwise and rte_errno is set. 8406 */ 8407 static struct rte_flow_pattern_template * 8408 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8409 const struct rte_flow_pattern_template_attr *attr, 8410 const struct rte_flow_item items[], 8411 struct rte_flow_error *error) 8412 { 8413 const struct mlx5_flow_driver_ops *fops; 8414 struct rte_flow_attr fattr = {0}; 8415 8416 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8417 rte_flow_error_set(error, ENOTSUP, 8418 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8419 NULL, 8420 "pattern create with incorrect steering mode"); 8421 return NULL; 8422 } 8423 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8424 return fops->pattern_template_create(dev, attr, items, error); 8425 } 8426 8427 /** 8428 * Destroy flow item template. 8429 * 8430 * @param[in] dev 8431 * Pointer to the rte_eth_dev structure. 8432 * @param[in] template 8433 * Pointer to the item template to be destroyed. 8434 * @param[out] error 8435 * Pointer to error structure. 8436 * 8437 * @return 8438 * 0 on success, a negative errno value otherwise and rte_errno is set. 8439 */ 8440 static int 8441 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8442 struct rte_flow_pattern_template *template, 8443 struct rte_flow_error *error) 8444 { 8445 const struct mlx5_flow_driver_ops *fops; 8446 struct rte_flow_attr attr = {0}; 8447 8448 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8449 return rte_flow_error_set(error, ENOTSUP, 8450 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8451 NULL, 8452 "pattern destroy with incorrect steering mode"); 8453 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8454 return fops->pattern_template_destroy(dev, template, error); 8455 } 8456 8457 /** 8458 * Validate flow actions template. 8459 * 8460 * @param[in] dev 8461 * Pointer to the rte_eth_dev structure. 8462 * @param[in] attr 8463 * Pointer to the action template attributes. 8464 * @param[in] actions 8465 * Associated actions (list terminated by the END action). 8466 * @param[in] masks 8467 * List of actions that marks which of the action's member is constant. 8468 * @param[out] error 8469 * Pointer to error structure. 8470 * 8471 * @return 8472 * 0 on success, a negative errno value otherwise and rte_errno is set. 8473 */ 8474 int 8475 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8476 const struct rte_flow_actions_template_attr *attr, 8477 const struct rte_flow_action actions[], 8478 const struct rte_flow_action masks[], 8479 struct rte_flow_error *error) 8480 { 8481 const struct mlx5_flow_driver_ops *fops; 8482 struct rte_flow_attr fattr = {0}; 8483 8484 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8485 rte_flow_error_set(error, ENOTSUP, 8486 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8487 "actions validate with incorrect steering mode"); 8488 return -ENOTSUP; 8489 } 8490 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8491 return fops->actions_validate(dev, attr, actions, masks, error); 8492 } 8493 8494 /** 8495 * Create flow item template. 8496 * 8497 * @param[in] dev 8498 * Pointer to the rte_eth_dev structure. 8499 * @param[in] attr 8500 * Pointer to the action template attributes. 8501 * @param[in] actions 8502 * Associated actions (list terminated by the END action). 8503 * @param[in] masks 8504 * List of actions that marks which of the action's member is constant. 8505 * @param[out] error 8506 * Pointer to error structure. 8507 * 8508 * @return 8509 * 0 on success, a negative errno value otherwise and rte_errno is set. 8510 */ 8511 static struct rte_flow_actions_template * 8512 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8513 const struct rte_flow_actions_template_attr *attr, 8514 const struct rte_flow_action actions[], 8515 const struct rte_flow_action masks[], 8516 struct rte_flow_error *error) 8517 { 8518 const struct mlx5_flow_driver_ops *fops; 8519 struct rte_flow_attr fattr = {0}; 8520 8521 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8522 rte_flow_error_set(error, ENOTSUP, 8523 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8524 NULL, 8525 "action create with incorrect steering mode"); 8526 return NULL; 8527 } 8528 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8529 return fops->actions_template_create(dev, attr, actions, masks, error); 8530 } 8531 8532 /** 8533 * Destroy flow action template. 8534 * 8535 * @param[in] dev 8536 * Pointer to the rte_eth_dev structure. 8537 * @param[in] template 8538 * Pointer to the action template to be destroyed. 8539 * @param[out] error 8540 * Pointer to error structure. 8541 * 8542 * @return 8543 * 0 on success, a negative errno value otherwise and rte_errno is set. 8544 */ 8545 static int 8546 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8547 struct rte_flow_actions_template *template, 8548 struct rte_flow_error *error) 8549 { 8550 const struct mlx5_flow_driver_ops *fops; 8551 struct rte_flow_attr attr = {0}; 8552 8553 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8554 return rte_flow_error_set(error, ENOTSUP, 8555 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8556 NULL, 8557 "action destroy with incorrect steering mode"); 8558 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8559 return fops->actions_template_destroy(dev, template, error); 8560 } 8561 8562 /** 8563 * Create flow table. 8564 * 8565 * @param[in] dev 8566 * Pointer to the rte_eth_dev structure. 8567 * @param[in] attr 8568 * Pointer to the table attributes. 8569 * @param[in] item_templates 8570 * Item template array to be binded to the table. 8571 * @param[in] nb_item_templates 8572 * Number of item template. 8573 * @param[in] action_templates 8574 * Action template array to be binded to the table. 8575 * @param[in] nb_action_templates 8576 * Number of action template. 8577 * @param[out] error 8578 * Pointer to error structure. 8579 * 8580 * @return 8581 * Table on success, NULL otherwise and rte_errno is set. 8582 */ 8583 static struct rte_flow_template_table * 8584 mlx5_flow_table_create(struct rte_eth_dev *dev, 8585 const struct rte_flow_template_table_attr *attr, 8586 struct rte_flow_pattern_template *item_templates[], 8587 uint8_t nb_item_templates, 8588 struct rte_flow_actions_template *action_templates[], 8589 uint8_t nb_action_templates, 8590 struct rte_flow_error *error) 8591 { 8592 const struct mlx5_flow_driver_ops *fops; 8593 struct rte_flow_attr fattr = {0}; 8594 8595 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8596 rte_flow_error_set(error, ENOTSUP, 8597 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8598 NULL, 8599 "table create with incorrect steering mode"); 8600 return NULL; 8601 } 8602 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8603 return fops->template_table_create(dev, 8604 attr, 8605 item_templates, 8606 nb_item_templates, 8607 action_templates, 8608 nb_action_templates, 8609 error); 8610 } 8611 8612 /** 8613 * PMD destroy flow table. 8614 * 8615 * @param[in] dev 8616 * Pointer to the rte_eth_dev structure. 8617 * @param[in] table 8618 * Pointer to the table to be destroyed. 8619 * @param[out] error 8620 * Pointer to error structure. 8621 * 8622 * @return 8623 * 0 on success, a negative errno value otherwise and rte_errno is set. 8624 */ 8625 static int 8626 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8627 struct rte_flow_template_table *table, 8628 struct rte_flow_error *error) 8629 { 8630 const struct mlx5_flow_driver_ops *fops; 8631 struct rte_flow_attr attr = {0}; 8632 8633 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8634 return rte_flow_error_set(error, ENOTSUP, 8635 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8636 NULL, 8637 "table destroy with incorrect steering mode"); 8638 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8639 return fops->template_table_destroy(dev, table, error); 8640 } 8641 8642 /** 8643 * Enqueue flow creation. 8644 * 8645 * @param[in] dev 8646 * Pointer to the rte_eth_dev structure. 8647 * @param[in] queue_id 8648 * The queue to create the flow. 8649 * @param[in] attr 8650 * Pointer to the flow operation attributes. 8651 * @param[in] items 8652 * Items with flow spec value. 8653 * @param[in] pattern_template_index 8654 * The item pattern flow follows from the table. 8655 * @param[in] actions 8656 * Action with flow spec value. 8657 * @param[in] action_template_index 8658 * The action pattern flow follows from the table. 8659 * @param[in] user_data 8660 * Pointer to the user_data. 8661 * @param[out] error 8662 * Pointer to error structure. 8663 * 8664 * @return 8665 * Flow pointer on success, NULL otherwise and rte_errno is set. 8666 */ 8667 static struct rte_flow * 8668 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8669 uint32_t queue_id, 8670 const struct rte_flow_op_attr *attr, 8671 struct rte_flow_template_table *table, 8672 const struct rte_flow_item items[], 8673 uint8_t pattern_template_index, 8674 const struct rte_flow_action actions[], 8675 uint8_t action_template_index, 8676 void *user_data, 8677 struct rte_flow_error *error) 8678 { 8679 const struct mlx5_flow_driver_ops *fops; 8680 struct rte_flow_attr fattr = {0}; 8681 8682 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8683 rte_flow_error_set(error, ENOTSUP, 8684 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8685 NULL, 8686 "flow_q create with incorrect steering mode"); 8687 return NULL; 8688 } 8689 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8690 return fops->async_flow_create(dev, queue_id, attr, table, 8691 items, pattern_template_index, 8692 actions, action_template_index, 8693 user_data, error); 8694 } 8695 8696 /** 8697 * Enqueue flow destruction. 8698 * 8699 * @param[in] dev 8700 * Pointer to the rte_eth_dev structure. 8701 * @param[in] queue 8702 * The queue to destroy the flow. 8703 * @param[in] attr 8704 * Pointer to the flow operation attributes. 8705 * @param[in] flow 8706 * Pointer to the flow to be destroyed. 8707 * @param[in] user_data 8708 * Pointer to the user_data. 8709 * @param[out] error 8710 * Pointer to error structure. 8711 * 8712 * @return 8713 * 0 on success, negative value otherwise and rte_errno is set. 8714 */ 8715 static int 8716 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8717 uint32_t queue, 8718 const struct rte_flow_op_attr *attr, 8719 struct rte_flow *flow, 8720 void *user_data, 8721 struct rte_flow_error *error) 8722 { 8723 const struct mlx5_flow_driver_ops *fops; 8724 struct rte_flow_attr fattr = {0}; 8725 8726 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8727 return rte_flow_error_set(error, ENOTSUP, 8728 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8729 NULL, 8730 "flow_q destroy with incorrect steering mode"); 8731 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8732 return fops->async_flow_destroy(dev, queue, attr, flow, 8733 user_data, error); 8734 } 8735 8736 /** 8737 * Pull the enqueued flows. 8738 * 8739 * @param[in] dev 8740 * Pointer to the rte_eth_dev structure. 8741 * @param[in] queue 8742 * The queue to pull the result. 8743 * @param[in/out] res 8744 * Array to save the results. 8745 * @param[in] n_res 8746 * Available result with the array. 8747 * @param[out] error 8748 * Pointer to error structure. 8749 * 8750 * @return 8751 * Result number on success, negative value otherwise and rte_errno is set. 8752 */ 8753 static int 8754 mlx5_flow_pull(struct rte_eth_dev *dev, 8755 uint32_t queue, 8756 struct rte_flow_op_result res[], 8757 uint16_t n_res, 8758 struct rte_flow_error *error) 8759 { 8760 const struct mlx5_flow_driver_ops *fops; 8761 struct rte_flow_attr attr = {0}; 8762 8763 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8764 return rte_flow_error_set(error, ENOTSUP, 8765 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8766 NULL, 8767 "flow_q pull with incorrect steering mode"); 8768 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8769 return fops->pull(dev, queue, res, n_res, error); 8770 } 8771 8772 /** 8773 * Push the enqueued flows. 8774 * 8775 * @param[in] dev 8776 * Pointer to the rte_eth_dev structure. 8777 * @param[in] queue 8778 * The queue to push the flows. 8779 * @param[out] error 8780 * Pointer to error structure. 8781 * 8782 * @return 8783 * 0 on success, negative value otherwise and rte_errno is set. 8784 */ 8785 static int 8786 mlx5_flow_push(struct rte_eth_dev *dev, 8787 uint32_t queue, 8788 struct rte_flow_error *error) 8789 { 8790 const struct mlx5_flow_driver_ops *fops; 8791 struct rte_flow_attr attr = {0}; 8792 8793 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8794 return rte_flow_error_set(error, ENOTSUP, 8795 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8796 NULL, 8797 "flow_q push with incorrect steering mode"); 8798 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8799 return fops->push(dev, queue, error); 8800 } 8801 8802 /** 8803 * Create shared action. 8804 * 8805 * @param[in] dev 8806 * Pointer to the rte_eth_dev structure. 8807 * @param[in] queue 8808 * Which queue to be used.. 8809 * @param[in] attr 8810 * Operation attribute. 8811 * @param[in] conf 8812 * Indirect action configuration. 8813 * @param[in] action 8814 * rte_flow action detail. 8815 * @param[in] user_data 8816 * Pointer to the user_data. 8817 * @param[out] error 8818 * Pointer to error structure. 8819 * 8820 * @return 8821 * Action handle on success, NULL otherwise and rte_errno is set. 8822 */ 8823 static struct rte_flow_action_handle * 8824 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8825 const struct rte_flow_op_attr *attr, 8826 const struct rte_flow_indir_action_conf *conf, 8827 const struct rte_flow_action *action, 8828 void *user_data, 8829 struct rte_flow_error *error) 8830 { 8831 const struct mlx5_flow_driver_ops *fops = 8832 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8833 8834 return fops->async_action_create(dev, queue, attr, conf, action, 8835 user_data, error); 8836 } 8837 8838 /** 8839 * Update shared action. 8840 * 8841 * @param[in] dev 8842 * Pointer to the rte_eth_dev structure. 8843 * @param[in] queue 8844 * Which queue to be used.. 8845 * @param[in] attr 8846 * Operation attribute. 8847 * @param[in] handle 8848 * Action handle to be updated. 8849 * @param[in] update 8850 * Update value. 8851 * @param[in] user_data 8852 * Pointer to the user_data. 8853 * @param[out] error 8854 * Pointer to error structure. 8855 * 8856 * @return 8857 * 0 on success, negative value otherwise and rte_errno is set. 8858 */ 8859 static int 8860 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8861 const struct rte_flow_op_attr *attr, 8862 struct rte_flow_action_handle *handle, 8863 const void *update, 8864 void *user_data, 8865 struct rte_flow_error *error) 8866 { 8867 const struct mlx5_flow_driver_ops *fops = 8868 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8869 8870 return fops->async_action_update(dev, queue, attr, handle, 8871 update, user_data, error); 8872 } 8873 8874 /** 8875 * Query shared action. 8876 * 8877 * @param[in] dev 8878 * Pointer to the rte_eth_dev structure. 8879 * @param[in] queue 8880 * Which queue to be used.. 8881 * @param[in] attr 8882 * Operation attribute. 8883 * @param[in] handle 8884 * Action handle to be updated. 8885 * @param[in] data 8886 * Pointer query result data. 8887 * @param[in] user_data 8888 * Pointer to the user_data. 8889 * @param[out] error 8890 * Pointer to error structure. 8891 * 8892 * @return 8893 * 0 on success, negative value otherwise and rte_errno is set. 8894 */ 8895 static int 8896 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 8897 const struct rte_flow_op_attr *attr, 8898 const struct rte_flow_action_handle *handle, 8899 void *data, 8900 void *user_data, 8901 struct rte_flow_error *error) 8902 { 8903 const struct mlx5_flow_driver_ops *fops = 8904 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8905 8906 return fops->async_action_query(dev, queue, attr, handle, 8907 data, user_data, error); 8908 } 8909 8910 /** 8911 * Destroy shared action. 8912 * 8913 * @param[in] dev 8914 * Pointer to the rte_eth_dev structure. 8915 * @param[in] queue 8916 * Which queue to be used.. 8917 * @param[in] attr 8918 * Operation attribute. 8919 * @param[in] handle 8920 * Action handle to be destroyed. 8921 * @param[in] user_data 8922 * Pointer to the user_data. 8923 * @param[out] error 8924 * Pointer to error structure. 8925 * 8926 * @return 8927 * 0 on success, negative value otherwise and rte_errno is set. 8928 */ 8929 static int 8930 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 8931 const struct rte_flow_op_attr *attr, 8932 struct rte_flow_action_handle *handle, 8933 void *user_data, 8934 struct rte_flow_error *error) 8935 { 8936 const struct mlx5_flow_driver_ops *fops = 8937 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8938 8939 return fops->async_action_destroy(dev, queue, attr, handle, 8940 user_data, error); 8941 } 8942 8943 /** 8944 * Allocate a new memory for the counter values wrapped by all the needed 8945 * management. 8946 * 8947 * @param[in] sh 8948 * Pointer to mlx5_dev_ctx_shared object. 8949 * 8950 * @return 8951 * 0 on success, a negative errno value otherwise. 8952 */ 8953 static int 8954 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8955 { 8956 struct mlx5_counter_stats_mem_mng *mem_mng; 8957 volatile struct flow_counter_stats *raw_data; 8958 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8959 int size = (sizeof(struct flow_counter_stats) * 8960 MLX5_COUNTERS_PER_POOL + 8961 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8962 sizeof(struct mlx5_counter_stats_mem_mng); 8963 size_t pgsize = rte_mem_page_size(); 8964 uint8_t *mem; 8965 int ret; 8966 int i; 8967 8968 if (pgsize == (size_t)-1) { 8969 DRV_LOG(ERR, "Failed to get mem page size"); 8970 rte_errno = ENOMEM; 8971 return -ENOMEM; 8972 } 8973 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8974 if (!mem) { 8975 rte_errno = ENOMEM; 8976 return -ENOMEM; 8977 } 8978 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8979 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8980 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8981 sh->cdev->pdn, mem, size, 8982 &mem_mng->wm); 8983 if (ret) { 8984 rte_errno = errno; 8985 mlx5_free(mem); 8986 return -rte_errno; 8987 } 8988 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8989 raw_data = (volatile struct flow_counter_stats *)mem; 8990 for (i = 0; i < raws_n; ++i) { 8991 mem_mng->raws[i].mem_mng = mem_mng; 8992 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8993 } 8994 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8995 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 8996 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8997 next); 8998 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 8999 sh->sws_cmng.mem_mng = mem_mng; 9000 return 0; 9001 } 9002 9003 /** 9004 * Set the statistic memory to the new counter pool. 9005 * 9006 * @param[in] sh 9007 * Pointer to mlx5_dev_ctx_shared object. 9008 * @param[in] pool 9009 * Pointer to the pool to set the statistic memory. 9010 * 9011 * @return 9012 * 0 on success, a negative errno value otherwise. 9013 */ 9014 static int 9015 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9016 struct mlx5_flow_counter_pool *pool) 9017 { 9018 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9019 /* Resize statistic memory once used out. */ 9020 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 9021 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9022 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9023 return -1; 9024 } 9025 rte_spinlock_lock(&pool->sl); 9026 pool->raw = cmng->mem_mng->raws + pool->index % 9027 MLX5_CNT_CONTAINER_RESIZE; 9028 rte_spinlock_unlock(&pool->sl); 9029 pool->raw_hw = NULL; 9030 return 0; 9031 } 9032 9033 #define MLX5_POOL_QUERY_FREQ_US 1000000 9034 9035 /** 9036 * Set the periodic procedure for triggering asynchronous batch queries for all 9037 * the counter pools. 9038 * 9039 * @param[in] sh 9040 * Pointer to mlx5_dev_ctx_shared object. 9041 */ 9042 void 9043 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9044 { 9045 uint32_t pools_n, us; 9046 9047 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9048 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9049 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9050 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9051 sh->sws_cmng.query_thread_on = 0; 9052 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9053 } else { 9054 sh->sws_cmng.query_thread_on = 1; 9055 } 9056 } 9057 9058 /** 9059 * The periodic procedure for triggering asynchronous batch queries for all the 9060 * counter pools. This function is probably called by the host thread. 9061 * 9062 * @param[in] arg 9063 * The parameter for the alarm process. 9064 */ 9065 void 9066 mlx5_flow_query_alarm(void *arg) 9067 { 9068 struct mlx5_dev_ctx_shared *sh = arg; 9069 int ret; 9070 uint16_t pool_index = sh->sws_cmng.pool_index; 9071 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9072 struct mlx5_flow_counter_pool *pool; 9073 uint16_t n_valid; 9074 9075 if (sh->sws_cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 9076 goto set_alarm; 9077 rte_spinlock_lock(&cmng->pool_update_sl); 9078 pool = cmng->pools[pool_index]; 9079 n_valid = cmng->n_valid; 9080 rte_spinlock_unlock(&cmng->pool_update_sl); 9081 /* Set the statistic memory to the new created pool. */ 9082 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9083 goto set_alarm; 9084 if (pool->raw_hw) 9085 /* There is a pool query in progress. */ 9086 goto set_alarm; 9087 pool->raw_hw = 9088 LIST_FIRST(&sh->sws_cmng.free_stat_raws); 9089 if (!pool->raw_hw) 9090 /* No free counter statistics raw memory. */ 9091 goto set_alarm; 9092 /* 9093 * Identify the counters released between query trigger and query 9094 * handle more efficiently. The counter released in this gap period 9095 * should wait for a new round of query as the new arrived packets 9096 * will not be taken into account. 9097 */ 9098 pool->query_gen++; 9099 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9100 MLX5_COUNTERS_PER_POOL, 9101 NULL, NULL, 9102 pool->raw_hw->mem_mng->wm.lkey, 9103 (void *)(uintptr_t) 9104 pool->raw_hw->data, 9105 sh->devx_comp, 9106 (uint64_t)(uintptr_t)pool); 9107 if (ret) { 9108 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9109 " %d", pool->min_dcs->id); 9110 pool->raw_hw = NULL; 9111 goto set_alarm; 9112 } 9113 LIST_REMOVE(pool->raw_hw, next); 9114 sh->sws_cmng.pending_queries++; 9115 pool_index++; 9116 if (pool_index >= n_valid) 9117 pool_index = 0; 9118 set_alarm: 9119 sh->sws_cmng.pool_index = pool_index; 9120 mlx5_set_query_alarm(sh); 9121 } 9122 9123 /** 9124 * Check and callback event for new aged flow in the counter pool 9125 * 9126 * @param[in] sh 9127 * Pointer to mlx5_dev_ctx_shared object. 9128 * @param[in] pool 9129 * Pointer to Current counter pool. 9130 */ 9131 static void 9132 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9133 struct mlx5_flow_counter_pool *pool) 9134 { 9135 struct mlx5_priv *priv; 9136 struct mlx5_flow_counter *cnt; 9137 struct mlx5_age_info *age_info; 9138 struct mlx5_age_param *age_param; 9139 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9140 struct mlx5_counter_stats_raw *prev = pool->raw; 9141 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9142 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9143 uint16_t expected = AGE_CANDIDATE; 9144 uint32_t i; 9145 9146 pool->time_of_last_age_check = curr_time; 9147 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9148 cnt = MLX5_POOL_GET_CNT(pool, i); 9149 age_param = MLX5_CNT_TO_AGE(cnt); 9150 if (__atomic_load_n(&age_param->state, 9151 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9152 continue; 9153 if (cur->data[i].hits != prev->data[i].hits) { 9154 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9155 __ATOMIC_RELAXED); 9156 continue; 9157 } 9158 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9159 time_delta, 9160 __ATOMIC_RELAXED) <= age_param->timeout) 9161 continue; 9162 /** 9163 * Hold the lock first, or if between the 9164 * state AGE_TMOUT and tailq operation the 9165 * release happened, the release procedure 9166 * may delete a non-existent tailq node. 9167 */ 9168 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9169 age_info = GET_PORT_AGE_INFO(priv); 9170 rte_spinlock_lock(&age_info->aged_sl); 9171 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9172 AGE_TMOUT, false, 9173 __ATOMIC_RELAXED, 9174 __ATOMIC_RELAXED)) { 9175 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9176 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9177 } 9178 rte_spinlock_unlock(&age_info->aged_sl); 9179 } 9180 mlx5_age_event_prepare(sh); 9181 } 9182 9183 /** 9184 * Handler for the HW respond about ready values from an asynchronous batch 9185 * query. This function is probably called by the host thread. 9186 * 9187 * @param[in] sh 9188 * The pointer to the shared device context. 9189 * @param[in] async_id 9190 * The Devx async ID. 9191 * @param[in] status 9192 * The status of the completion. 9193 */ 9194 void 9195 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9196 uint64_t async_id, int status) 9197 { 9198 struct mlx5_flow_counter_pool *pool = 9199 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9200 struct mlx5_counter_stats_raw *raw_to_free; 9201 uint8_t query_gen = pool->query_gen ^ 1; 9202 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9203 enum mlx5_counter_type cnt_type = 9204 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9205 MLX5_COUNTER_TYPE_ORIGIN; 9206 9207 if (unlikely(status)) { 9208 raw_to_free = pool->raw_hw; 9209 } else { 9210 raw_to_free = pool->raw; 9211 if (pool->is_aged) 9212 mlx5_flow_aging_check(sh, pool); 9213 rte_spinlock_lock(&pool->sl); 9214 pool->raw = pool->raw_hw; 9215 rte_spinlock_unlock(&pool->sl); 9216 /* Be sure the new raw counters data is updated in memory. */ 9217 rte_io_wmb(); 9218 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9219 rte_spinlock_lock(&cmng->csl[cnt_type]); 9220 TAILQ_CONCAT(&cmng->counters[cnt_type], 9221 &pool->counters[query_gen], next); 9222 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9223 } 9224 } 9225 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9226 pool->raw_hw = NULL; 9227 sh->sws_cmng.pending_queries--; 9228 } 9229 9230 static int 9231 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9232 const struct flow_grp_info *grp_info, 9233 struct rte_flow_error *error) 9234 { 9235 if (grp_info->transfer && grp_info->external && 9236 grp_info->fdb_def_rule) { 9237 if (group == UINT32_MAX) 9238 return rte_flow_error_set 9239 (error, EINVAL, 9240 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9241 NULL, 9242 "group index not supported"); 9243 *table = group + 1; 9244 } else { 9245 *table = group; 9246 } 9247 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9248 return 0; 9249 } 9250 9251 /** 9252 * Translate the rte_flow group index to HW table value. 9253 * 9254 * If tunnel offload is disabled, all group ids converted to flow table 9255 * id using the standard method. 9256 * If tunnel offload is enabled, group id can be converted using the 9257 * standard or tunnel conversion method. Group conversion method 9258 * selection depends on flags in `grp_info` parameter: 9259 * - Internal (grp_info.external == 0) groups conversion uses the 9260 * standard method. 9261 * - Group ids in JUMP action converted with the tunnel conversion. 9262 * - Group id in rule attribute conversion depends on a rule type and 9263 * group id value: 9264 * ** non zero group attributes converted with the tunnel method 9265 * ** zero group attribute in non-tunnel rule is converted using the 9266 * standard method - there's only one root table 9267 * ** zero group attribute in steer tunnel rule is converted with the 9268 * standard method - single root table 9269 * ** zero group attribute in match tunnel rule is a special OvS 9270 * case: that value is used for portability reasons. That group 9271 * id is converted with the tunnel conversion method. 9272 * 9273 * @param[in] dev 9274 * Port device 9275 * @param[in] tunnel 9276 * PMD tunnel offload object 9277 * @param[in] group 9278 * rte_flow group index value. 9279 * @param[out] table 9280 * HW table value. 9281 * @param[in] grp_info 9282 * flags used for conversion 9283 * @param[out] error 9284 * Pointer to error structure. 9285 * 9286 * @return 9287 * 0 on success, a negative errno value otherwise and rte_errno is set. 9288 */ 9289 int 9290 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9291 const struct mlx5_flow_tunnel *tunnel, 9292 uint32_t group, uint32_t *table, 9293 const struct flow_grp_info *grp_info, 9294 struct rte_flow_error *error) 9295 { 9296 int ret; 9297 bool standard_translation; 9298 9299 if (!grp_info->skip_scale && grp_info->external && 9300 group < MLX5_MAX_TABLES_EXTERNAL) 9301 group *= MLX5_FLOW_TABLE_FACTOR; 9302 if (is_tunnel_offload_active(dev)) { 9303 standard_translation = !grp_info->external || 9304 grp_info->std_tbl_fix; 9305 } else { 9306 standard_translation = true; 9307 } 9308 DRV_LOG(DEBUG, 9309 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9310 dev->data->port_id, group, grp_info->transfer, 9311 grp_info->external, grp_info->fdb_def_rule, 9312 standard_translation ? "STANDARD" : "TUNNEL"); 9313 if (standard_translation) 9314 ret = flow_group_to_table(dev->data->port_id, group, table, 9315 grp_info, error); 9316 else 9317 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9318 table, error); 9319 9320 return ret; 9321 } 9322 9323 /** 9324 * Discover availability of metadata reg_c's. 9325 * 9326 * Iteratively use test flows to check availability. 9327 * 9328 * @param[in] dev 9329 * Pointer to the Ethernet device structure. 9330 * 9331 * @return 9332 * 0 on success, a negative errno value otherwise and rte_errno is set. 9333 */ 9334 int 9335 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9336 { 9337 struct mlx5_priv *priv = dev->data->dev_private; 9338 enum modify_reg idx; 9339 int n = 0; 9340 9341 /* reg_c[0] and reg_c[1] are reserved. */ 9342 priv->sh->flow_mreg_c[n++] = REG_C_0; 9343 priv->sh->flow_mreg_c[n++] = REG_C_1; 9344 /* Discover availability of other reg_c's. */ 9345 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9346 struct rte_flow_attr attr = { 9347 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9348 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9349 .ingress = 1, 9350 }; 9351 struct rte_flow_item items[] = { 9352 [0] = { 9353 .type = RTE_FLOW_ITEM_TYPE_END, 9354 }, 9355 }; 9356 struct rte_flow_action actions[] = { 9357 [0] = { 9358 .type = (enum rte_flow_action_type) 9359 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9360 .conf = &(struct mlx5_flow_action_copy_mreg){ 9361 .src = REG_C_1, 9362 .dst = idx, 9363 }, 9364 }, 9365 [1] = { 9366 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9367 .conf = &(struct rte_flow_action_jump){ 9368 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9369 }, 9370 }, 9371 [2] = { 9372 .type = RTE_FLOW_ACTION_TYPE_END, 9373 }, 9374 }; 9375 uint32_t flow_idx; 9376 struct rte_flow *flow; 9377 struct rte_flow_error error; 9378 9379 if (!priv->sh->config.dv_flow_en) 9380 break; 9381 /* Create internal flow, validation skips copy action. */ 9382 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9383 items, actions, false, &error); 9384 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9385 flow_idx); 9386 if (!flow) 9387 continue; 9388 priv->sh->flow_mreg_c[n++] = idx; 9389 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9390 } 9391 for (; n < MLX5_MREG_C_NUM; ++n) 9392 priv->sh->flow_mreg_c[n] = REG_NON; 9393 priv->sh->metadata_regc_check_flag = 1; 9394 return 0; 9395 } 9396 9397 int 9398 save_dump_file(const uint8_t *data, uint32_t size, 9399 uint32_t type, uint64_t id, void *arg, FILE *file) 9400 { 9401 char line[BUF_SIZE]; 9402 uint32_t out = 0; 9403 uint32_t k; 9404 uint32_t actions_num; 9405 struct rte_flow_query_count *count; 9406 9407 memset(line, 0, BUF_SIZE); 9408 switch (type) { 9409 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9410 actions_num = *(uint32_t *)(arg); 9411 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9412 type, id, actions_num); 9413 break; 9414 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9415 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9416 type, id); 9417 break; 9418 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9419 count = (struct rte_flow_query_count *)arg; 9420 fprintf(file, 9421 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9422 type, id, count->hits, count->bytes); 9423 return 0; 9424 default: 9425 return -1; 9426 } 9427 9428 for (k = 0; k < size; k++) { 9429 /* Make sure we do not overrun the line buffer length. */ 9430 if (out >= BUF_SIZE - 4) { 9431 line[out] = '\0'; 9432 break; 9433 } 9434 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9435 (data[k]) & 0xff); 9436 } 9437 fprintf(file, "%s\n", line); 9438 return 0; 9439 } 9440 9441 int 9442 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9443 struct rte_flow_query_count *count, struct rte_flow_error *error) 9444 { 9445 struct rte_flow_action action[2]; 9446 enum mlx5_flow_drv_type ftype; 9447 const struct mlx5_flow_driver_ops *fops; 9448 9449 if (!flow) { 9450 return rte_flow_error_set(error, ENOENT, 9451 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9452 NULL, 9453 "invalid flow handle"); 9454 } 9455 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9456 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9457 if (flow->counter) { 9458 memset(count, 0, sizeof(struct rte_flow_query_count)); 9459 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9460 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9461 ftype < MLX5_FLOW_TYPE_MAX); 9462 fops = flow_get_drv_ops(ftype); 9463 return fops->query(dev, flow, action, count, error); 9464 } 9465 return -1; 9466 } 9467 9468 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9469 /** 9470 * Dump flow ipool data to file 9471 * 9472 * @param[in] dev 9473 * The pointer to Ethernet device. 9474 * @param[in] file 9475 * A pointer to a file for output. 9476 * @param[out] error 9477 * Perform verbose error reporting if not NULL. PMDs initialize this 9478 * structure in case of error only. 9479 * @return 9480 * 0 on success, a negative value otherwise. 9481 */ 9482 int 9483 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9484 struct rte_flow *flow, FILE *file, 9485 struct rte_flow_error *error) 9486 { 9487 struct mlx5_priv *priv = dev->data->dev_private; 9488 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9489 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9490 uint32_t handle_idx; 9491 struct mlx5_flow_handle *dh; 9492 struct rte_flow_query_count count; 9493 uint32_t actions_num; 9494 const uint8_t *data; 9495 size_t size; 9496 uint64_t id; 9497 uint32_t type; 9498 void *action = NULL; 9499 9500 if (!flow) { 9501 return rte_flow_error_set(error, ENOENT, 9502 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9503 NULL, 9504 "invalid flow handle"); 9505 } 9506 handle_idx = flow->dev_handles; 9507 /* query counter */ 9508 if (flow->counter && 9509 (!mlx5_counter_query(dev, flow->counter, false, 9510 &count.hits, &count.bytes, &action)) && action) { 9511 id = (uint64_t)(uintptr_t)action; 9512 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9513 save_dump_file(NULL, 0, type, 9514 id, (void *)&count, file); 9515 } 9516 9517 while (handle_idx) { 9518 dh = mlx5_ipool_get(priv->sh->ipool 9519 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9520 if (!dh) 9521 continue; 9522 handle_idx = dh->next.next; 9523 9524 /* Get modify_hdr and encap_decap buf from ipools. */ 9525 encap_decap = NULL; 9526 modify_hdr = dh->dvh.modify_hdr; 9527 9528 if (dh->dvh.rix_encap_decap) { 9529 encap_decap = mlx5_ipool_get(priv->sh->ipool 9530 [MLX5_IPOOL_DECAP_ENCAP], 9531 dh->dvh.rix_encap_decap); 9532 } 9533 if (modify_hdr) { 9534 data = (const uint8_t *)modify_hdr->actions; 9535 size = (size_t)(modify_hdr->actions_num) * 8; 9536 id = (uint64_t)(uintptr_t)modify_hdr->action; 9537 actions_num = modify_hdr->actions_num; 9538 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9539 save_dump_file(data, size, type, id, 9540 (void *)(&actions_num), file); 9541 } 9542 if (encap_decap) { 9543 data = encap_decap->buf; 9544 size = encap_decap->size; 9545 id = (uint64_t)(uintptr_t)encap_decap->action; 9546 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9547 save_dump_file(data, size, type, 9548 id, NULL, file); 9549 } 9550 } 9551 return 0; 9552 } 9553 9554 /** 9555 * Dump all flow's encap_decap/modify_hdr/counter data to file 9556 * 9557 * @param[in] dev 9558 * The pointer to Ethernet device. 9559 * @param[in] file 9560 * A pointer to a file for output. 9561 * @param[out] error 9562 * Perform verbose error reporting if not NULL. PMDs initialize this 9563 * structure in case of error only. 9564 * @return 9565 * 0 on success, a negative value otherwise. 9566 */ 9567 static int 9568 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9569 FILE *file, struct rte_flow_error *error __rte_unused) 9570 { 9571 struct mlx5_priv *priv = dev->data->dev_private; 9572 struct mlx5_dev_ctx_shared *sh = priv->sh; 9573 struct mlx5_hlist *h; 9574 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9575 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9576 struct rte_flow_query_count count; 9577 uint32_t actions_num; 9578 const uint8_t *data; 9579 size_t size; 9580 uint64_t id; 9581 uint32_t type; 9582 uint32_t i; 9583 uint32_t j; 9584 struct mlx5_list_inconst *l_inconst; 9585 struct mlx5_list_entry *e; 9586 int lcore_index; 9587 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9588 uint32_t max; 9589 void *action; 9590 9591 /* encap_decap hlist is lcore_share, get global core cache. */ 9592 i = MLX5_LIST_GLOBAL; 9593 h = sh->encaps_decaps; 9594 if (h) { 9595 for (j = 0; j <= h->mask; j++) { 9596 l_inconst = &h->buckets[j].l; 9597 if (!l_inconst || !l_inconst->cache[i]) 9598 continue; 9599 9600 e = LIST_FIRST(&l_inconst->cache[i]->h); 9601 while (e) { 9602 encap_decap = 9603 (struct mlx5_flow_dv_encap_decap_resource *)e; 9604 data = encap_decap->buf; 9605 size = encap_decap->size; 9606 id = (uint64_t)(uintptr_t)encap_decap->action; 9607 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9608 save_dump_file(data, size, type, 9609 id, NULL, file); 9610 e = LIST_NEXT(e, next); 9611 } 9612 } 9613 } 9614 9615 /* get modify_hdr */ 9616 h = sh->modify_cmds; 9617 if (h) { 9618 lcore_index = rte_lcore_index(rte_lcore_id()); 9619 if (unlikely(lcore_index == -1)) { 9620 lcore_index = MLX5_LIST_NLCORE; 9621 rte_spinlock_lock(&h->l_const.lcore_lock); 9622 } 9623 i = lcore_index; 9624 9625 for (j = 0; j <= h->mask; j++) { 9626 l_inconst = &h->buckets[j].l; 9627 if (!l_inconst || !l_inconst->cache[i]) 9628 continue; 9629 9630 e = LIST_FIRST(&l_inconst->cache[i]->h); 9631 while (e) { 9632 modify_hdr = 9633 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9634 data = (const uint8_t *)modify_hdr->actions; 9635 size = (size_t)(modify_hdr->actions_num) * 8; 9636 actions_num = modify_hdr->actions_num; 9637 id = (uint64_t)(uintptr_t)modify_hdr->action; 9638 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9639 save_dump_file(data, size, type, id, 9640 (void *)(&actions_num), file); 9641 e = LIST_NEXT(e, next); 9642 } 9643 } 9644 9645 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9646 rte_spinlock_unlock(&h->l_const.lcore_lock); 9647 } 9648 9649 /* get counter */ 9650 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9651 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9652 for (j = 1; j <= max; j++) { 9653 action = NULL; 9654 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9655 &count.bytes, &action)) && action) { 9656 id = (uint64_t)(uintptr_t)action; 9657 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9658 save_dump_file(NULL, 0, type, 9659 id, (void *)&count, file); 9660 } 9661 } 9662 return 0; 9663 } 9664 #endif 9665 9666 /** 9667 * Dump flow raw hw data to file 9668 * 9669 * @param[in] dev 9670 * The pointer to Ethernet device. 9671 * @param[in] file 9672 * A pointer to a file for output. 9673 * @param[out] error 9674 * Perform verbose error reporting if not NULL. PMDs initialize this 9675 * structure in case of error only. 9676 * @return 9677 * 0 on success, a negative value otherwise. 9678 */ 9679 int 9680 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9681 FILE *file, 9682 struct rte_flow_error *error __rte_unused) 9683 { 9684 struct mlx5_priv *priv = dev->data->dev_private; 9685 struct mlx5_dev_ctx_shared *sh = priv->sh; 9686 uint32_t handle_idx; 9687 int ret; 9688 struct mlx5_flow_handle *dh; 9689 struct rte_flow *flow; 9690 9691 if (!sh->config.dv_flow_en) { 9692 if (fputs("device dv flow disabled\n", file) <= 0) 9693 return -errno; 9694 return -ENOTSUP; 9695 } 9696 9697 /* dump all */ 9698 if (!flow_idx) { 9699 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9700 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9701 return -EINVAL; 9702 #endif 9703 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9704 sh->rx_domain, 9705 sh->tx_domain, file); 9706 } 9707 /* dump one */ 9708 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9709 (uintptr_t)(void *)flow_idx); 9710 if (!flow) 9711 return -EINVAL; 9712 9713 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9714 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9715 #endif 9716 handle_idx = flow->dev_handles; 9717 while (handle_idx) { 9718 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9719 handle_idx); 9720 if (!dh) 9721 return -ENOENT; 9722 if (dh->drv_flow) { 9723 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9724 file); 9725 if (ret) 9726 return -ENOENT; 9727 } 9728 handle_idx = dh->next.next; 9729 } 9730 return 0; 9731 } 9732 9733 /** 9734 * Get aged-out flows. 9735 * 9736 * @param[in] dev 9737 * Pointer to the Ethernet device structure. 9738 * @param[in] context 9739 * The address of an array of pointers to the aged-out flows contexts. 9740 * @param[in] nb_countexts 9741 * The length of context array pointers. 9742 * @param[out] error 9743 * Perform verbose error reporting if not NULL. Initialized in case of 9744 * error only. 9745 * 9746 * @return 9747 * how many contexts get in success, otherwise negative errno value. 9748 * if nb_contexts is 0, return the amount of all aged contexts. 9749 * if nb_contexts is not 0 , return the amount of aged flows reported 9750 * in the context array. 9751 */ 9752 int 9753 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9754 uint32_t nb_contexts, struct rte_flow_error *error) 9755 { 9756 const struct mlx5_flow_driver_ops *fops; 9757 struct rte_flow_attr attr = { .transfer = 0 }; 9758 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, &attr); 9759 9760 if (type == MLX5_FLOW_TYPE_DV || type == MLX5_FLOW_TYPE_HW) { 9761 fops = flow_get_drv_ops(type); 9762 return fops->get_aged_flows(dev, contexts, nb_contexts, error); 9763 } 9764 DRV_LOG(ERR, "port %u get aged flows is not supported.", 9765 dev->data->port_id); 9766 return -ENOTSUP; 9767 } 9768 9769 /** 9770 * Get aged-out flows per HWS queue. 9771 * 9772 * @param[in] dev 9773 * Pointer to the Ethernet device structure. 9774 * @param[in] queue_id 9775 * Flow queue to query. 9776 * @param[in] context 9777 * The address of an array of pointers to the aged-out flows contexts. 9778 * @param[in] nb_countexts 9779 * The length of context array pointers. 9780 * @param[out] error 9781 * Perform verbose error reporting if not NULL. Initialized in case of 9782 * error only. 9783 * 9784 * @return 9785 * how many contexts get in success, otherwise negative errno value. 9786 * if nb_contexts is 0, return the amount of all aged contexts. 9787 * if nb_contexts is not 0 , return the amount of aged flows reported 9788 * in the context array. 9789 */ 9790 int 9791 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 9792 void **contexts, uint32_t nb_contexts, 9793 struct rte_flow_error *error) 9794 { 9795 const struct mlx5_flow_driver_ops *fops; 9796 struct rte_flow_attr attr = { 0 }; 9797 9798 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 9799 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9800 return fops->get_q_aged_flows(dev, queue_id, contexts, 9801 nb_contexts, error); 9802 } 9803 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 9804 dev->data->port_id, queue_id); 9805 return rte_flow_error_set(error, ENOTSUP, 9806 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9807 "get Q aged flows with incorrect steering mode"); 9808 } 9809 9810 /* Wrapper for driver action_validate op callback */ 9811 static int 9812 flow_drv_action_validate(struct rte_eth_dev *dev, 9813 const struct rte_flow_indir_action_conf *conf, 9814 const struct rte_flow_action *action, 9815 const struct mlx5_flow_driver_ops *fops, 9816 struct rte_flow_error *error) 9817 { 9818 static const char err_msg[] = "indirect action validation unsupported"; 9819 9820 if (!fops->action_validate) { 9821 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9822 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9823 NULL, err_msg); 9824 return -rte_errno; 9825 } 9826 return fops->action_validate(dev, conf, action, error); 9827 } 9828 9829 /** 9830 * Destroys the shared action by handle. 9831 * 9832 * @param dev 9833 * Pointer to Ethernet device structure. 9834 * @param[in] handle 9835 * Handle for the indirect action object to be destroyed. 9836 * @param[out] error 9837 * Perform verbose error reporting if not NULL. PMDs initialize this 9838 * structure in case of error only. 9839 * 9840 * @return 9841 * 0 on success, a negative errno value otherwise and rte_errno is set. 9842 * 9843 * @note: wrapper for driver action_create op callback. 9844 */ 9845 static int 9846 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9847 struct rte_flow_action_handle *handle, 9848 struct rte_flow_error *error) 9849 { 9850 static const char err_msg[] = "indirect action destruction unsupported"; 9851 struct rte_flow_attr attr = { .transfer = 0 }; 9852 const struct mlx5_flow_driver_ops *fops = 9853 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9854 9855 if (!fops->action_destroy) { 9856 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9857 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9858 NULL, err_msg); 9859 return -rte_errno; 9860 } 9861 return fops->action_destroy(dev, handle, error); 9862 } 9863 9864 /* Wrapper for driver action_destroy op callback */ 9865 static int 9866 flow_drv_action_update(struct rte_eth_dev *dev, 9867 struct rte_flow_action_handle *handle, 9868 const void *update, 9869 const struct mlx5_flow_driver_ops *fops, 9870 struct rte_flow_error *error) 9871 { 9872 static const char err_msg[] = "indirect action update unsupported"; 9873 9874 if (!fops->action_update) { 9875 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9876 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9877 NULL, err_msg); 9878 return -rte_errno; 9879 } 9880 return fops->action_update(dev, handle, update, error); 9881 } 9882 9883 /* Wrapper for driver action_destroy op callback */ 9884 static int 9885 flow_drv_action_query(struct rte_eth_dev *dev, 9886 const struct rte_flow_action_handle *handle, 9887 void *data, 9888 const struct mlx5_flow_driver_ops *fops, 9889 struct rte_flow_error *error) 9890 { 9891 static const char err_msg[] = "indirect action query unsupported"; 9892 9893 if (!fops->action_query) { 9894 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9895 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9896 NULL, err_msg); 9897 return -rte_errno; 9898 } 9899 return fops->action_query(dev, handle, data, error); 9900 } 9901 9902 /** 9903 * Create indirect action for reuse in multiple flow rules. 9904 * 9905 * @param dev 9906 * Pointer to Ethernet device structure. 9907 * @param conf 9908 * Pointer to indirect action object configuration. 9909 * @param[in] action 9910 * Action configuration for indirect action object creation. 9911 * @param[out] error 9912 * Perform verbose error reporting if not NULL. PMDs initialize this 9913 * structure in case of error only. 9914 * @return 9915 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9916 */ 9917 static struct rte_flow_action_handle * 9918 mlx5_action_handle_create(struct rte_eth_dev *dev, 9919 const struct rte_flow_indir_action_conf *conf, 9920 const struct rte_flow_action *action, 9921 struct rte_flow_error *error) 9922 { 9923 static const char err_msg[] = "indirect action creation unsupported"; 9924 struct rte_flow_attr attr = { .transfer = 0 }; 9925 const struct mlx5_flow_driver_ops *fops = 9926 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9927 9928 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9929 return NULL; 9930 if (!fops->action_create) { 9931 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9932 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9933 NULL, err_msg); 9934 return NULL; 9935 } 9936 return fops->action_create(dev, conf, action, error); 9937 } 9938 9939 /** 9940 * Updates inplace the indirect action configuration pointed by *handle* 9941 * with the configuration provided as *update* argument. 9942 * The update of the indirect action configuration effects all flow rules 9943 * reusing the action via handle. 9944 * 9945 * @param dev 9946 * Pointer to Ethernet device structure. 9947 * @param[in] handle 9948 * Handle for the indirect action to be updated. 9949 * @param[in] update 9950 * Action specification used to modify the action pointed by handle. 9951 * *update* could be of same type with the action pointed by the *handle* 9952 * handle argument, or some other structures like a wrapper, depending on 9953 * the indirect action type. 9954 * @param[out] error 9955 * Perform verbose error reporting if not NULL. PMDs initialize this 9956 * structure in case of error only. 9957 * 9958 * @return 9959 * 0 on success, a negative errno value otherwise and rte_errno is set. 9960 */ 9961 static int 9962 mlx5_action_handle_update(struct rte_eth_dev *dev, 9963 struct rte_flow_action_handle *handle, 9964 const void *update, 9965 struct rte_flow_error *error) 9966 { 9967 struct rte_flow_attr attr = { .transfer = 0 }; 9968 const struct mlx5_flow_driver_ops *fops = 9969 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9970 int ret; 9971 9972 ret = flow_drv_action_validate(dev, NULL, 9973 (const struct rte_flow_action *)update, fops, error); 9974 if (ret) 9975 return ret; 9976 return flow_drv_action_update(dev, handle, update, fops, 9977 error); 9978 } 9979 9980 /** 9981 * Query the indirect action by handle. 9982 * 9983 * This function allows retrieving action-specific data such as counters. 9984 * Data is gathered by special action which may be present/referenced in 9985 * more than one flow rule definition. 9986 * 9987 * see @RTE_FLOW_ACTION_TYPE_COUNT 9988 * 9989 * @param dev 9990 * Pointer to Ethernet device structure. 9991 * @param[in] handle 9992 * Handle for the indirect action to query. 9993 * @param[in, out] data 9994 * Pointer to storage for the associated query data type. 9995 * @param[out] error 9996 * Perform verbose error reporting if not NULL. PMDs initialize this 9997 * structure in case of error only. 9998 * 9999 * @return 10000 * 0 on success, a negative errno value otherwise and rte_errno is set. 10001 */ 10002 static int 10003 mlx5_action_handle_query(struct rte_eth_dev *dev, 10004 const struct rte_flow_action_handle *handle, 10005 void *data, 10006 struct rte_flow_error *error) 10007 { 10008 struct rte_flow_attr attr = { .transfer = 0 }; 10009 const struct mlx5_flow_driver_ops *fops = 10010 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10011 10012 return flow_drv_action_query(dev, handle, data, fops, error); 10013 } 10014 10015 /** 10016 * Destroy all indirect actions (shared RSS). 10017 * 10018 * @param dev 10019 * Pointer to Ethernet device. 10020 * 10021 * @return 10022 * 0 on success, a negative errno value otherwise and rte_errno is set. 10023 */ 10024 int 10025 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10026 { 10027 struct rte_flow_error error; 10028 struct mlx5_priv *priv = dev->data->dev_private; 10029 struct mlx5_shared_action_rss *shared_rss; 10030 int ret = 0; 10031 uint32_t idx; 10032 10033 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10034 priv->rss_shared_actions, idx, shared_rss, next) { 10035 ret |= mlx5_action_handle_destroy(dev, 10036 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10037 } 10038 return ret; 10039 } 10040 10041 /** 10042 * Validate existing indirect actions against current device configuration 10043 * and attach them to device resources. 10044 * 10045 * @param dev 10046 * Pointer to Ethernet device. 10047 * 10048 * @return 10049 * 0 on success, a negative errno value otherwise and rte_errno is set. 10050 */ 10051 int 10052 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10053 { 10054 struct mlx5_priv *priv = dev->data->dev_private; 10055 int ret = 0; 10056 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10057 10058 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10059 const char *message; 10060 uint32_t queue_idx; 10061 10062 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10063 ind_tbl->queues_n, 10064 &message, &queue_idx); 10065 if (ret != 0) { 10066 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10067 dev->data->port_id, ind_tbl->queues[queue_idx], 10068 message); 10069 break; 10070 } 10071 } 10072 if (ret != 0) 10073 return ret; 10074 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10075 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10076 if (ret != 0) { 10077 DRV_LOG(ERR, "Port %u could not attach " 10078 "indirection table obj %p", 10079 dev->data->port_id, (void *)ind_tbl); 10080 goto error; 10081 } 10082 } 10083 10084 return 0; 10085 error: 10086 ind_tbl_last = ind_tbl; 10087 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10088 if (ind_tbl == ind_tbl_last) 10089 break; 10090 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10091 DRV_LOG(CRIT, "Port %u could not detach " 10092 "indirection table obj %p on rollback", 10093 dev->data->port_id, (void *)ind_tbl); 10094 } 10095 return ret; 10096 } 10097 10098 /** 10099 * Detach indirect actions of the device from its resources. 10100 * 10101 * @param dev 10102 * Pointer to Ethernet device. 10103 * 10104 * @return 10105 * 0 on success, a negative errno value otherwise and rte_errno is set. 10106 */ 10107 int 10108 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10109 { 10110 struct mlx5_priv *priv = dev->data->dev_private; 10111 int ret = 0; 10112 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10113 10114 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10115 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10116 if (ret != 0) { 10117 DRV_LOG(ERR, "Port %u could not detach " 10118 "indirection table obj %p", 10119 dev->data->port_id, (void *)ind_tbl); 10120 goto error; 10121 } 10122 } 10123 return 0; 10124 error: 10125 ind_tbl_last = ind_tbl; 10126 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10127 if (ind_tbl == ind_tbl_last) 10128 break; 10129 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10130 DRV_LOG(CRIT, "Port %u could not attach " 10131 "indirection table obj %p on rollback", 10132 dev->data->port_id, (void *)ind_tbl); 10133 } 10134 return ret; 10135 } 10136 10137 #ifndef HAVE_MLX5DV_DR 10138 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10139 #else 10140 #define MLX5_DOMAIN_SYNC_FLOW \ 10141 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10142 #endif 10143 10144 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10145 { 10146 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10147 const struct mlx5_flow_driver_ops *fops; 10148 int ret; 10149 struct rte_flow_attr attr = { .transfer = 0 }; 10150 10151 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10152 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10153 if (ret > 0) 10154 ret = -ret; 10155 return ret; 10156 } 10157 10158 const struct mlx5_flow_tunnel * 10159 mlx5_get_tof(const struct rte_flow_item *item, 10160 const struct rte_flow_action *action, 10161 enum mlx5_tof_rule_type *rule_type) 10162 { 10163 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10164 if (item->type == (typeof(item->type)) 10165 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10166 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10167 return flow_items_to_tunnel(item); 10168 } 10169 } 10170 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10171 if (action->type == (typeof(action->type)) 10172 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10173 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10174 return flow_actions_to_tunnel(action); 10175 } 10176 } 10177 return NULL; 10178 } 10179 10180 /** 10181 * tunnel offload functionality is defined for DV environment only 10182 */ 10183 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10184 __extension__ 10185 union tunnel_offload_mark { 10186 uint32_t val; 10187 struct { 10188 uint32_t app_reserve:8; 10189 uint32_t table_id:15; 10190 uint32_t transfer:1; 10191 uint32_t _unused_:8; 10192 }; 10193 }; 10194 10195 static bool 10196 mlx5_access_tunnel_offload_db 10197 (struct rte_eth_dev *dev, 10198 bool (*match)(struct rte_eth_dev *, 10199 struct mlx5_flow_tunnel *, const void *), 10200 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10201 void (*miss)(struct rte_eth_dev *, void *), 10202 void *ctx, bool lock_op); 10203 10204 static int 10205 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10206 struct rte_flow *flow, 10207 const struct rte_flow_attr *attr, 10208 const struct rte_flow_action *app_actions, 10209 uint32_t flow_idx, 10210 const struct mlx5_flow_tunnel *tunnel, 10211 struct tunnel_default_miss_ctx *ctx, 10212 struct rte_flow_error *error) 10213 { 10214 struct mlx5_priv *priv = dev->data->dev_private; 10215 struct mlx5_flow *dev_flow; 10216 struct rte_flow_attr miss_attr = *attr; 10217 const struct rte_flow_item miss_items[2] = { 10218 { 10219 .type = RTE_FLOW_ITEM_TYPE_ETH, 10220 .spec = NULL, 10221 .last = NULL, 10222 .mask = NULL 10223 }, 10224 { 10225 .type = RTE_FLOW_ITEM_TYPE_END, 10226 .spec = NULL, 10227 .last = NULL, 10228 .mask = NULL 10229 } 10230 }; 10231 union tunnel_offload_mark mark_id; 10232 struct rte_flow_action_mark miss_mark; 10233 struct rte_flow_action miss_actions[3] = { 10234 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10235 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10236 }; 10237 const struct rte_flow_action_jump *jump_data; 10238 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10239 struct flow_grp_info grp_info = { 10240 .external = 1, 10241 .transfer = attr->transfer, 10242 .fdb_def_rule = !!priv->fdb_def_rule, 10243 .std_tbl_fix = 0, 10244 }; 10245 int ret; 10246 10247 if (!attr->transfer) { 10248 uint32_t q_size; 10249 10250 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10251 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10252 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10253 0, SOCKET_ID_ANY); 10254 if (!ctx->queue) 10255 return rte_flow_error_set 10256 (error, ENOMEM, 10257 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10258 NULL, "invalid default miss RSS"); 10259 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10260 ctx->action_rss.level = 0, 10261 ctx->action_rss.types = priv->rss_conf.rss_hf, 10262 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10263 ctx->action_rss.queue_num = priv->reta_idx_n, 10264 ctx->action_rss.key = priv->rss_conf.rss_key, 10265 ctx->action_rss.queue = ctx->queue; 10266 if (!priv->reta_idx_n || !priv->rxqs_n) 10267 return rte_flow_error_set 10268 (error, EINVAL, 10269 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10270 NULL, "invalid port configuration"); 10271 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10272 ctx->action_rss.types = 0; 10273 for (i = 0; i != priv->reta_idx_n; ++i) 10274 ctx->queue[i] = (*priv->reta_idx)[i]; 10275 } else { 10276 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10277 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10278 } 10279 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10280 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10281 jump_data = app_actions->conf; 10282 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10283 miss_attr.group = jump_data->group; 10284 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10285 &flow_table, &grp_info, error); 10286 if (ret) 10287 return rte_flow_error_set(error, EINVAL, 10288 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10289 NULL, "invalid tunnel id"); 10290 mark_id.app_reserve = 0; 10291 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10292 mark_id.transfer = !!attr->transfer; 10293 mark_id._unused_ = 0; 10294 miss_mark.id = mark_id.val; 10295 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10296 miss_items, miss_actions, flow_idx, error); 10297 if (!dev_flow) 10298 return -rte_errno; 10299 dev_flow->flow = flow; 10300 dev_flow->external = true; 10301 dev_flow->tunnel = tunnel; 10302 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10303 /* Subflow object was created, we must include one in the list. */ 10304 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10305 dev_flow->handle, next); 10306 DRV_LOG(DEBUG, 10307 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10308 dev->data->port_id, tunnel->app_tunnel.type, 10309 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10310 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10311 miss_actions, error); 10312 if (!ret) 10313 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10314 error); 10315 10316 return ret; 10317 } 10318 10319 static const struct mlx5_flow_tbl_data_entry * 10320 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10321 { 10322 struct mlx5_priv *priv = dev->data->dev_private; 10323 struct mlx5_dev_ctx_shared *sh = priv->sh; 10324 struct mlx5_list_entry *he; 10325 union tunnel_offload_mark mbits = { .val = mark }; 10326 union mlx5_flow_tbl_key table_key = { 10327 { 10328 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10329 .id = 0, 10330 .reserved = 0, 10331 .dummy = 0, 10332 .is_fdb = !!mbits.transfer, 10333 .is_egress = 0, 10334 } 10335 }; 10336 struct mlx5_flow_cb_ctx ctx = { 10337 .data = &table_key.v64, 10338 }; 10339 10340 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10341 return he ? 10342 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10343 } 10344 10345 static void 10346 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10347 struct mlx5_list_entry *entry) 10348 { 10349 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10350 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10351 10352 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10353 tunnel_flow_tbl_to_id(tte->flow_table)); 10354 mlx5_free(tte); 10355 } 10356 10357 static int 10358 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10359 struct mlx5_list_entry *entry, void *cb_ctx) 10360 { 10361 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10362 union tunnel_tbl_key tbl = { 10363 .val = *(uint64_t *)(ctx->data), 10364 }; 10365 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10366 10367 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10368 } 10369 10370 static struct mlx5_list_entry * 10371 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10372 { 10373 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10374 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10375 struct tunnel_tbl_entry *tte; 10376 union tunnel_tbl_key tbl = { 10377 .val = *(uint64_t *)(ctx->data), 10378 }; 10379 10380 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10381 sizeof(*tte), 0, 10382 SOCKET_ID_ANY); 10383 if (!tte) 10384 goto err; 10385 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10386 &tte->flow_table); 10387 if (tte->flow_table >= MLX5_MAX_TABLES) { 10388 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10389 tte->flow_table); 10390 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10391 tte->flow_table); 10392 goto err; 10393 } else if (!tte->flow_table) { 10394 goto err; 10395 } 10396 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10397 tte->tunnel_id = tbl.tunnel_id; 10398 tte->group = tbl.group; 10399 return &tte->hash; 10400 err: 10401 if (tte) 10402 mlx5_free(tte); 10403 return NULL; 10404 } 10405 10406 static struct mlx5_list_entry * 10407 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10408 struct mlx5_list_entry *oentry, 10409 void *cb_ctx __rte_unused) 10410 { 10411 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10412 0, SOCKET_ID_ANY); 10413 10414 if (!tte) 10415 return NULL; 10416 memcpy(tte, oentry, sizeof(*tte)); 10417 return &tte->hash; 10418 } 10419 10420 static void 10421 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10422 struct mlx5_list_entry *entry) 10423 { 10424 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10425 10426 mlx5_free(tte); 10427 } 10428 10429 static uint32_t 10430 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10431 const struct mlx5_flow_tunnel *tunnel, 10432 uint32_t group, uint32_t *table, 10433 struct rte_flow_error *error) 10434 { 10435 struct mlx5_list_entry *he; 10436 struct tunnel_tbl_entry *tte; 10437 union tunnel_tbl_key key = { 10438 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10439 .group = group 10440 }; 10441 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10442 struct mlx5_hlist *group_hash; 10443 struct mlx5_flow_cb_ctx ctx = { 10444 .data = &key.val, 10445 }; 10446 10447 group_hash = tunnel ? tunnel->groups : thub->groups; 10448 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10449 if (!he) 10450 return rte_flow_error_set(error, EINVAL, 10451 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10452 NULL, 10453 "tunnel group index not supported"); 10454 tte = container_of(he, typeof(*tte), hash); 10455 *table = tte->flow_table; 10456 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10457 dev->data->port_id, key.tunnel_id, group, *table); 10458 return 0; 10459 } 10460 10461 static void 10462 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10463 struct mlx5_flow_tunnel *tunnel) 10464 { 10465 struct mlx5_priv *priv = dev->data->dev_private; 10466 struct mlx5_indexed_pool *ipool; 10467 10468 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10469 dev->data->port_id, tunnel->tunnel_id); 10470 LIST_REMOVE(tunnel, chain); 10471 mlx5_hlist_destroy(tunnel->groups); 10472 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10473 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10474 } 10475 10476 static bool 10477 mlx5_access_tunnel_offload_db 10478 (struct rte_eth_dev *dev, 10479 bool (*match)(struct rte_eth_dev *, 10480 struct mlx5_flow_tunnel *, const void *), 10481 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10482 void (*miss)(struct rte_eth_dev *, void *), 10483 void *ctx, bool lock_op) 10484 { 10485 bool verdict = false; 10486 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10487 struct mlx5_flow_tunnel *tunnel; 10488 10489 rte_spinlock_lock(&thub->sl); 10490 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10491 verdict = match(dev, tunnel, (const void *)ctx); 10492 if (verdict) 10493 break; 10494 } 10495 if (!lock_op) 10496 rte_spinlock_unlock(&thub->sl); 10497 if (verdict && hit) 10498 hit(dev, tunnel, ctx); 10499 if (!verdict && miss) 10500 miss(dev, ctx); 10501 if (lock_op) 10502 rte_spinlock_unlock(&thub->sl); 10503 10504 return verdict; 10505 } 10506 10507 struct tunnel_db_find_tunnel_id_ctx { 10508 uint32_t tunnel_id; 10509 struct mlx5_flow_tunnel *tunnel; 10510 }; 10511 10512 static bool 10513 find_tunnel_id_match(struct rte_eth_dev *dev, 10514 struct mlx5_flow_tunnel *tunnel, const void *x) 10515 { 10516 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10517 10518 RTE_SET_USED(dev); 10519 return tunnel->tunnel_id == ctx->tunnel_id; 10520 } 10521 10522 static void 10523 find_tunnel_id_hit(struct rte_eth_dev *dev, 10524 struct mlx5_flow_tunnel *tunnel, void *x) 10525 { 10526 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10527 RTE_SET_USED(dev); 10528 ctx->tunnel = tunnel; 10529 } 10530 10531 static struct mlx5_flow_tunnel * 10532 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10533 { 10534 struct tunnel_db_find_tunnel_id_ctx ctx = { 10535 .tunnel_id = id, 10536 }; 10537 10538 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10539 find_tunnel_id_hit, NULL, &ctx, true); 10540 10541 return ctx.tunnel; 10542 } 10543 10544 static struct mlx5_flow_tunnel * 10545 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10546 const struct rte_flow_tunnel *app_tunnel) 10547 { 10548 struct mlx5_priv *priv = dev->data->dev_private; 10549 struct mlx5_indexed_pool *ipool; 10550 struct mlx5_flow_tunnel *tunnel; 10551 uint32_t id; 10552 10553 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10554 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10555 if (!tunnel) 10556 return NULL; 10557 if (id >= MLX5_MAX_TUNNELS) { 10558 mlx5_ipool_free(ipool, id); 10559 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10560 return NULL; 10561 } 10562 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10563 priv->sh, 10564 mlx5_flow_tunnel_grp2tbl_create_cb, 10565 mlx5_flow_tunnel_grp2tbl_match_cb, 10566 mlx5_flow_tunnel_grp2tbl_remove_cb, 10567 mlx5_flow_tunnel_grp2tbl_clone_cb, 10568 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10569 if (!tunnel->groups) { 10570 mlx5_ipool_free(ipool, id); 10571 return NULL; 10572 } 10573 /* initiate new PMD tunnel */ 10574 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10575 tunnel->tunnel_id = id; 10576 tunnel->action.type = (typeof(tunnel->action.type)) 10577 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10578 tunnel->action.conf = tunnel; 10579 tunnel->item.type = (typeof(tunnel->item.type)) 10580 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10581 tunnel->item.spec = tunnel; 10582 tunnel->item.last = NULL; 10583 tunnel->item.mask = NULL; 10584 10585 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10586 dev->data->port_id, tunnel->tunnel_id); 10587 10588 return tunnel; 10589 } 10590 10591 struct tunnel_db_get_tunnel_ctx { 10592 const struct rte_flow_tunnel *app_tunnel; 10593 struct mlx5_flow_tunnel *tunnel; 10594 }; 10595 10596 static bool get_tunnel_match(struct rte_eth_dev *dev, 10597 struct mlx5_flow_tunnel *tunnel, const void *x) 10598 { 10599 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10600 10601 RTE_SET_USED(dev); 10602 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10603 sizeof(*ctx->app_tunnel)); 10604 } 10605 10606 static void get_tunnel_hit(struct rte_eth_dev *dev, 10607 struct mlx5_flow_tunnel *tunnel, void *x) 10608 { 10609 /* called under tunnel spinlock protection */ 10610 struct tunnel_db_get_tunnel_ctx *ctx = x; 10611 10612 RTE_SET_USED(dev); 10613 tunnel->refctn++; 10614 ctx->tunnel = tunnel; 10615 } 10616 10617 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10618 { 10619 /* called under tunnel spinlock protection */ 10620 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10621 struct tunnel_db_get_tunnel_ctx *ctx = x; 10622 10623 rte_spinlock_unlock(&thub->sl); 10624 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10625 rte_spinlock_lock(&thub->sl); 10626 if (ctx->tunnel) { 10627 ctx->tunnel->refctn = 1; 10628 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10629 } 10630 } 10631 10632 10633 static int 10634 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10635 const struct rte_flow_tunnel *app_tunnel, 10636 struct mlx5_flow_tunnel **tunnel) 10637 { 10638 struct tunnel_db_get_tunnel_ctx ctx = { 10639 .app_tunnel = app_tunnel, 10640 }; 10641 10642 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10643 get_tunnel_miss, &ctx, true); 10644 *tunnel = ctx.tunnel; 10645 return ctx.tunnel ? 0 : -ENOMEM; 10646 } 10647 10648 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10649 { 10650 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10651 10652 if (!thub) 10653 return; 10654 if (!LIST_EMPTY(&thub->tunnels)) 10655 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10656 mlx5_hlist_destroy(thub->groups); 10657 mlx5_free(thub); 10658 } 10659 10660 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10661 { 10662 int err; 10663 struct mlx5_flow_tunnel_hub *thub; 10664 10665 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10666 0, SOCKET_ID_ANY); 10667 if (!thub) 10668 return -ENOMEM; 10669 LIST_INIT(&thub->tunnels); 10670 rte_spinlock_init(&thub->sl); 10671 thub->groups = mlx5_hlist_create("flow groups", 64, 10672 false, true, sh, 10673 mlx5_flow_tunnel_grp2tbl_create_cb, 10674 mlx5_flow_tunnel_grp2tbl_match_cb, 10675 mlx5_flow_tunnel_grp2tbl_remove_cb, 10676 mlx5_flow_tunnel_grp2tbl_clone_cb, 10677 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10678 if (!thub->groups) { 10679 err = -rte_errno; 10680 goto err; 10681 } 10682 sh->tunnel_hub = thub; 10683 10684 return 0; 10685 10686 err: 10687 if (thub->groups) 10688 mlx5_hlist_destroy(thub->groups); 10689 if (thub) 10690 mlx5_free(thub); 10691 return err; 10692 } 10693 10694 static inline int 10695 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10696 struct rte_flow_tunnel *tunnel, 10697 struct rte_flow_error *error) 10698 { 10699 struct mlx5_priv *priv = dev->data->dev_private; 10700 10701 if (!priv->sh->config.dv_flow_en) 10702 return rte_flow_error_set(error, ENOTSUP, 10703 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10704 "flow DV interface is off"); 10705 if (!is_tunnel_offload_active(dev)) 10706 return rte_flow_error_set(error, ENOTSUP, 10707 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10708 "tunnel offload was not activated"); 10709 if (!tunnel) 10710 return rte_flow_error_set(error, EINVAL, 10711 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10712 "no application tunnel"); 10713 switch (tunnel->type) { 10714 default: 10715 return rte_flow_error_set(error, EINVAL, 10716 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10717 "unsupported tunnel type"); 10718 case RTE_FLOW_ITEM_TYPE_VXLAN: 10719 case RTE_FLOW_ITEM_TYPE_GRE: 10720 case RTE_FLOW_ITEM_TYPE_NVGRE: 10721 case RTE_FLOW_ITEM_TYPE_GENEVE: 10722 break; 10723 } 10724 return 0; 10725 } 10726 10727 static int 10728 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10729 struct rte_flow_tunnel *app_tunnel, 10730 struct rte_flow_action **actions, 10731 uint32_t *num_of_actions, 10732 struct rte_flow_error *error) 10733 { 10734 struct mlx5_flow_tunnel *tunnel; 10735 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10736 10737 if (ret) 10738 return ret; 10739 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10740 if (ret < 0) { 10741 return rte_flow_error_set(error, ret, 10742 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10743 "failed to initialize pmd tunnel"); 10744 } 10745 *actions = &tunnel->action; 10746 *num_of_actions = 1; 10747 return 0; 10748 } 10749 10750 static int 10751 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10752 struct rte_flow_tunnel *app_tunnel, 10753 struct rte_flow_item **items, 10754 uint32_t *num_of_items, 10755 struct rte_flow_error *error) 10756 { 10757 struct mlx5_flow_tunnel *tunnel; 10758 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10759 10760 if (ret) 10761 return ret; 10762 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10763 if (ret < 0) { 10764 return rte_flow_error_set(error, ret, 10765 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10766 "failed to initialize pmd tunnel"); 10767 } 10768 *items = &tunnel->item; 10769 *num_of_items = 1; 10770 return 0; 10771 } 10772 10773 struct tunnel_db_element_release_ctx { 10774 struct rte_flow_item *items; 10775 struct rte_flow_action *actions; 10776 uint32_t num_elements; 10777 struct rte_flow_error *error; 10778 int ret; 10779 }; 10780 10781 static bool 10782 tunnel_element_release_match(struct rte_eth_dev *dev, 10783 struct mlx5_flow_tunnel *tunnel, const void *x) 10784 { 10785 const struct tunnel_db_element_release_ctx *ctx = x; 10786 10787 RTE_SET_USED(dev); 10788 if (ctx->num_elements != 1) 10789 return false; 10790 else if (ctx->items) 10791 return ctx->items == &tunnel->item; 10792 else if (ctx->actions) 10793 return ctx->actions == &tunnel->action; 10794 10795 return false; 10796 } 10797 10798 static void 10799 tunnel_element_release_hit(struct rte_eth_dev *dev, 10800 struct mlx5_flow_tunnel *tunnel, void *x) 10801 { 10802 struct tunnel_db_element_release_ctx *ctx = x; 10803 ctx->ret = 0; 10804 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10805 mlx5_flow_tunnel_free(dev, tunnel); 10806 } 10807 10808 static void 10809 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10810 { 10811 struct tunnel_db_element_release_ctx *ctx = x; 10812 RTE_SET_USED(dev); 10813 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10814 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10815 "invalid argument"); 10816 } 10817 10818 static int 10819 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10820 struct rte_flow_item *pmd_items, 10821 uint32_t num_items, struct rte_flow_error *err) 10822 { 10823 struct tunnel_db_element_release_ctx ctx = { 10824 .items = pmd_items, 10825 .actions = NULL, 10826 .num_elements = num_items, 10827 .error = err, 10828 }; 10829 10830 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10831 tunnel_element_release_hit, 10832 tunnel_element_release_miss, &ctx, false); 10833 10834 return ctx.ret; 10835 } 10836 10837 static int 10838 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10839 struct rte_flow_action *pmd_actions, 10840 uint32_t num_actions, struct rte_flow_error *err) 10841 { 10842 struct tunnel_db_element_release_ctx ctx = { 10843 .items = NULL, 10844 .actions = pmd_actions, 10845 .num_elements = num_actions, 10846 .error = err, 10847 }; 10848 10849 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10850 tunnel_element_release_hit, 10851 tunnel_element_release_miss, &ctx, false); 10852 10853 return ctx.ret; 10854 } 10855 10856 static int 10857 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10858 struct rte_mbuf *m, 10859 struct rte_flow_restore_info *info, 10860 struct rte_flow_error *err) 10861 { 10862 uint64_t ol_flags = m->ol_flags; 10863 const struct mlx5_flow_tbl_data_entry *tble; 10864 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10865 10866 if (!is_tunnel_offload_active(dev)) { 10867 info->flags = 0; 10868 return 0; 10869 } 10870 10871 if ((ol_flags & mask) != mask) 10872 goto err; 10873 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10874 if (!tble) { 10875 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10876 dev->data->port_id, m->hash.fdir.hi); 10877 goto err; 10878 } 10879 MLX5_ASSERT(tble->tunnel); 10880 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10881 info->group_id = tble->group_id; 10882 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10883 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10884 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10885 10886 return 0; 10887 10888 err: 10889 return rte_flow_error_set(err, EINVAL, 10890 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10891 "failed to get restore info"); 10892 } 10893 10894 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10895 static int 10896 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10897 __rte_unused struct rte_flow_tunnel *app_tunnel, 10898 __rte_unused struct rte_flow_action **actions, 10899 __rte_unused uint32_t *num_of_actions, 10900 __rte_unused struct rte_flow_error *error) 10901 { 10902 return -ENOTSUP; 10903 } 10904 10905 static int 10906 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10907 __rte_unused struct rte_flow_tunnel *app_tunnel, 10908 __rte_unused struct rte_flow_item **items, 10909 __rte_unused uint32_t *num_of_items, 10910 __rte_unused struct rte_flow_error *error) 10911 { 10912 return -ENOTSUP; 10913 } 10914 10915 static int 10916 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10917 __rte_unused struct rte_flow_item *pmd_items, 10918 __rte_unused uint32_t num_items, 10919 __rte_unused struct rte_flow_error *err) 10920 { 10921 return -ENOTSUP; 10922 } 10923 10924 static int 10925 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10926 __rte_unused struct rte_flow_action *pmd_action, 10927 __rte_unused uint32_t num_actions, 10928 __rte_unused struct rte_flow_error *err) 10929 { 10930 return -ENOTSUP; 10931 } 10932 10933 static int 10934 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10935 __rte_unused struct rte_mbuf *m, 10936 __rte_unused struct rte_flow_restore_info *i, 10937 __rte_unused struct rte_flow_error *err) 10938 { 10939 return -ENOTSUP; 10940 } 10941 10942 static int 10943 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10944 __rte_unused struct rte_flow *flow, 10945 __rte_unused const struct rte_flow_attr *attr, 10946 __rte_unused const struct rte_flow_action *actions, 10947 __rte_unused uint32_t flow_idx, 10948 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10949 __rte_unused struct tunnel_default_miss_ctx *ctx, 10950 __rte_unused struct rte_flow_error *error) 10951 { 10952 return -ENOTSUP; 10953 } 10954 10955 static struct mlx5_flow_tunnel * 10956 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10957 __rte_unused uint32_t id) 10958 { 10959 return NULL; 10960 } 10961 10962 static void 10963 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10964 __rte_unused struct mlx5_flow_tunnel *tunnel) 10965 { 10966 } 10967 10968 static uint32_t 10969 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10970 __rte_unused const struct mlx5_flow_tunnel *t, 10971 __rte_unused uint32_t group, 10972 __rte_unused uint32_t *table, 10973 struct rte_flow_error *error) 10974 { 10975 return rte_flow_error_set(error, ENOTSUP, 10976 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10977 "tunnel offload requires DV support"); 10978 } 10979 10980 void 10981 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10982 __rte_unused uint16_t port_id) 10983 { 10984 } 10985 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10986 10987 /* Flex flow item API */ 10988 static struct rte_flow_item_flex_handle * 10989 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10990 const struct rte_flow_item_flex_conf *conf, 10991 struct rte_flow_error *error) 10992 { 10993 static const char err_msg[] = "flex item creation unsupported"; 10994 struct mlx5_priv *priv = dev->data->dev_private; 10995 struct rte_flow_attr attr = { .transfer = 0 }; 10996 const struct mlx5_flow_driver_ops *fops = 10997 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10998 10999 if (!priv->pci_dev) { 11000 rte_flow_error_set(error, ENOTSUP, 11001 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11002 "create flex item on PF only"); 11003 return NULL; 11004 } 11005 switch (priv->pci_dev->id.device_id) { 11006 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11007 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11008 break; 11009 default: 11010 rte_flow_error_set(error, ENOTSUP, 11011 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11012 "flex item available on BlueField ports only"); 11013 return NULL; 11014 } 11015 if (!fops->item_create) { 11016 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11017 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11018 NULL, err_msg); 11019 return NULL; 11020 } 11021 return fops->item_create(dev, conf, error); 11022 } 11023 11024 static int 11025 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11026 const struct rte_flow_item_flex_handle *handle, 11027 struct rte_flow_error *error) 11028 { 11029 static const char err_msg[] = "flex item release unsupported"; 11030 struct rte_flow_attr attr = { .transfer = 0 }; 11031 const struct mlx5_flow_driver_ops *fops = 11032 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11033 11034 if (!fops->item_release) { 11035 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11036 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11037 NULL, err_msg); 11038 return -rte_errno; 11039 } 11040 return fops->item_release(dev, handle, error); 11041 } 11042 11043 static void 11044 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11045 { 11046 int ret; 11047 struct rte_flow_error error; 11048 11049 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11050 char *item_name; 11051 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11052 sizeof(item_name), 11053 (void *)(uintptr_t)item->type, &error); 11054 if (ret > 0) 11055 printf("%s ", item_name); 11056 else 11057 printf("%d\n", (int)item->type); 11058 } 11059 printf("END\n"); 11060 } 11061 11062 static int 11063 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11064 { 11065 const struct rte_flow_item_udp *spec = udp_item->spec; 11066 const struct rte_flow_item_udp *mask = udp_item->mask; 11067 uint16_t udp_dport = 0; 11068 11069 if (spec != NULL) { 11070 if (!mask) 11071 mask = &rte_flow_item_udp_mask; 11072 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11073 mask->hdr.dst_port); 11074 } 11075 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11076 } 11077 11078 static const struct mlx5_flow_expand_node * 11079 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11080 unsigned int item_idx, 11081 const struct mlx5_flow_expand_node graph[], 11082 const struct mlx5_flow_expand_node *node) 11083 { 11084 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11085 11086 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11087 node != NULL && 11088 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11089 /* 11090 * The expansion node is VXLAN and it is also the last 11091 * expandable item in the pattern, so need to continue 11092 * expansion of the inner tunnel. 11093 */ 11094 MLX5_ASSERT(item_idx > 0); 11095 prev_item = pattern + item_idx - 1; 11096 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11097 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11098 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11099 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11100 } 11101 return node; 11102 } 11103 11104 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11105 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11106 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11107 }; 11108 11109 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11110 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11111 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11112 { 9, 10, 11 }, { 12, 13, 14 }, 11113 }; 11114 11115 /** 11116 * Discover the number of available flow priorities. 11117 * 11118 * @param dev 11119 * Ethernet device. 11120 * 11121 * @return 11122 * On success, number of available flow priorities. 11123 * On failure, a negative errno-style code and rte_errno is set. 11124 */ 11125 int 11126 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11127 { 11128 static const uint16_t vprio[] = {8, 16}; 11129 const struct mlx5_priv *priv = dev->data->dev_private; 11130 const struct mlx5_flow_driver_ops *fops; 11131 enum mlx5_flow_drv_type type; 11132 int ret; 11133 11134 type = mlx5_flow_os_get_type(); 11135 if (type == MLX5_FLOW_TYPE_MAX) { 11136 type = MLX5_FLOW_TYPE_VERBS; 11137 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11138 type = MLX5_FLOW_TYPE_DV; 11139 } 11140 fops = flow_get_drv_ops(type); 11141 if (fops->discover_priorities == NULL) { 11142 DRV_LOG(ERR, "Priority discovery not supported"); 11143 rte_errno = ENOTSUP; 11144 return -rte_errno; 11145 } 11146 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11147 if (ret < 0) 11148 return ret; 11149 switch (ret) { 11150 case 8: 11151 ret = RTE_DIM(priority_map_3); 11152 break; 11153 case 16: 11154 ret = RTE_DIM(priority_map_5); 11155 break; 11156 default: 11157 rte_errno = ENOTSUP; 11158 DRV_LOG(ERR, 11159 "port %u maximum priority: %d expected 8/16", 11160 dev->data->port_id, ret); 11161 return -rte_errno; 11162 } 11163 DRV_LOG(INFO, "port %u supported flow priorities:" 11164 " 0-%d for ingress or egress root table," 11165 " 0-%d for non-root table or transfer root table.", 11166 dev->data->port_id, ret - 2, 11167 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11168 return ret; 11169 } 11170 11171 /** 11172 * Adjust flow priority based on the highest layer and the request priority. 11173 * 11174 * @param[in] dev 11175 * Pointer to the Ethernet device structure. 11176 * @param[in] priority 11177 * The rule base priority. 11178 * @param[in] subpriority 11179 * The priority based on the items. 11180 * 11181 * @return 11182 * The new priority. 11183 */ 11184 uint32_t 11185 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11186 uint32_t subpriority) 11187 { 11188 uint32_t res = 0; 11189 struct mlx5_priv *priv = dev->data->dev_private; 11190 11191 switch (priv->sh->flow_max_priority) { 11192 case RTE_DIM(priority_map_3): 11193 res = priority_map_3[priority][subpriority]; 11194 break; 11195 case RTE_DIM(priority_map_5): 11196 res = priority_map_5[priority][subpriority]; 11197 break; 11198 } 11199 return res; 11200 } 11201 11202 /** 11203 * Get the priority for sending traffic to kernel table. 11204 * 11205 * @param[in] dev 11206 * Pointer to the Ethernet device structure. 11207 * 11208 * @return 11209 * On success: the value of priority for sending traffic to kernel table 11210 * On failure: -1 11211 */ 11212 uint32_t 11213 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11214 { 11215 struct mlx5_priv *priv = dev->data->dev_private; 11216 uint32_t res; 11217 11218 switch (priv->sh->flow_max_priority) { 11219 case RTE_DIM(priority_map_5): 11220 res = 15; 11221 break; 11222 case RTE_DIM(priority_map_3): 11223 res = 7; 11224 break; 11225 default: 11226 DRV_LOG(ERR, 11227 "port %u maximum priority: %d expected 8/16", 11228 dev->data->port_id, priv->sh->flow_max_priority); 11229 res = (uint32_t)-1; 11230 } 11231 return res; 11232 } 11233 11234 /** 11235 * Get the E-Switch Manager vport id. 11236 * 11237 * @param[in] dev 11238 * Pointer to the Ethernet device structure. 11239 * 11240 * @return 11241 * The vport id. 11242 */ 11243 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11244 { 11245 struct mlx5_priv *priv = dev->data->dev_private; 11246 struct mlx5_common_device *cdev = priv->sh->cdev; 11247 11248 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11249 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11250 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11251 11252 if (priv->pci_dev == NULL) 11253 return 0; 11254 switch (priv->pci_dev->id.device_id) { 11255 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 11256 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11257 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11258 /* 11259 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11260 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11261 * ECPF vport ID is selected and not the host port (0) in any BF case. 11262 */ 11263 return (int16_t)MLX5_ECPF_VPORT_ID; 11264 default: 11265 return MLX5_PF_VPORT_ID; 11266 } 11267 } 11268 11269 /** 11270 * Parse item to get the vport id. 11271 * 11272 * @param[in] dev 11273 * Pointer to the Ethernet device structure. 11274 * @param[in] item 11275 * The src port id match item. 11276 * @param[out] vport_id 11277 * Pointer to put the vport id. 11278 * @param[out] all_ports 11279 * Indicate if the item matches all ports. 11280 * @param[out] error 11281 * Pointer to error structure. 11282 * 11283 * @return 11284 * 0 on success, a negative errno value otherwise and rte_errno is set. 11285 */ 11286 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11287 const struct rte_flow_item *item, 11288 uint16_t *vport_id, 11289 bool *all_ports, 11290 struct rte_flow_error *error) 11291 { 11292 struct mlx5_priv *port_priv; 11293 const struct rte_flow_item_port_id *pid_v; 11294 uint32_t esw_mgr_port; 11295 11296 if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID && 11297 item->type != RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) 11298 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11299 NULL, "Incorrect item type."); 11300 pid_v = item->spec; 11301 if (!pid_v) { 11302 if (all_ports) 11303 *all_ports = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT); 11304 return 0; 11305 } 11306 if (all_ports) 11307 *all_ports = false; 11308 esw_mgr_port = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) ? 11309 MLX5_REPRESENTED_PORT_ESW_MGR : MLX5_PORT_ESW_MGR; 11310 if (pid_v->id == esw_mgr_port) { 11311 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11312 } else { 11313 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 11314 if (!port_priv) 11315 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11316 NULL, "Failed to get port info."); 11317 *vport_id = port_priv->representor_id; 11318 } 11319 11320 return 0; 11321 } 11322 11323 int 11324 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11325 uint16_t *proxy_port_id, 11326 struct rte_flow_error *error) 11327 { 11328 const struct mlx5_priv *priv = dev->data->dev_private; 11329 uint16_t port_id; 11330 11331 if (!priv->sh->config.dv_esw_en) 11332 return rte_flow_error_set(error, EINVAL, 11333 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11334 NULL, 11335 "unable to provide a proxy port" 11336 " without E-Switch configured"); 11337 if (!priv->master && !priv->representor) 11338 return rte_flow_error_set(error, EINVAL, 11339 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11340 NULL, 11341 "unable to provide a proxy port" 11342 " for port which is not a master" 11343 " or a representor port"); 11344 if (priv->master) { 11345 *proxy_port_id = dev->data->port_id; 11346 return 0; 11347 } 11348 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11349 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11350 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11351 11352 if (port_priv->master && 11353 port_priv->domain_id == priv->domain_id) { 11354 *proxy_port_id = port_id; 11355 return 0; 11356 } 11357 } 11358 return rte_flow_error_set(error, ENODEV, 11359 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11360 NULL, "unable to find a proxy port"); 11361 } 11362