1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 53 struct tunnel_default_miss_ctx { 54 uint16_t *queue; 55 __extension__ 56 union { 57 struct rte_flow_action_rss action_rss; 58 struct rte_flow_action_queue miss_queue; 59 struct rte_flow_action_jump miss_jump; 60 uint8_t raw[0]; 61 }; 62 }; 63 64 static int 65 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 66 struct rte_flow *flow, 67 const struct rte_flow_attr *attr, 68 const struct rte_flow_action *app_actions, 69 uint32_t flow_idx, 70 const struct mlx5_flow_tunnel *tunnel, 71 struct tunnel_default_miss_ctx *ctx, 72 struct rte_flow_error *error); 73 static struct mlx5_flow_tunnel * 74 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 75 static void 76 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 77 static uint32_t 78 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 79 const struct mlx5_flow_tunnel *tunnel, 80 uint32_t group, uint32_t *table, 81 struct rte_flow_error *error); 82 83 /** Device flow drivers. */ 84 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 85 86 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 87 88 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 89 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 90 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 91 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 92 #endif 93 #ifdef HAVE_MLX5_HWS_SUPPORT 94 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 95 #endif 96 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 97 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 98 }; 99 100 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 101 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 102 (const int []){ \ 103 __VA_ARGS__, 0, \ 104 } 105 106 /** Node object of input graph for mlx5_flow_expand_rss(). */ 107 struct mlx5_flow_expand_node { 108 const int *const next; 109 /**< 110 * List of next node indexes. Index 0 is interpreted as a terminator. 111 */ 112 const enum rte_flow_item_type type; 113 /**< Pattern item type of current node. */ 114 uint64_t rss_types; 115 /**< 116 * RSS types bit-field associated with this node 117 * (see RTE_ETH_RSS_* definitions). 118 */ 119 uint64_t node_flags; 120 /**< 121 * Bit-fields that define how the node is used in the expansion. 122 * (see MLX5_EXPANSION_NODE_* definitions). 123 */ 124 }; 125 126 /* Optional expand field. The expansion alg will not go deeper. */ 127 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 128 129 /* The node is not added implicitly as expansion to the flow pattern. 130 * If the node type does not match the flow pattern item type, the 131 * expansion alg will go deeper to its next items. 132 * In the current implementation, the list of next nodes indexes can 133 * have up to one node with this flag set and it has to be the last 134 * node index (before the list terminator). 135 */ 136 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 137 138 /** Object returned by mlx5_flow_expand_rss(). */ 139 struct mlx5_flow_expand_rss { 140 uint32_t entries; 141 /**< Number of entries @p patterns and @p priorities. */ 142 struct { 143 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 144 uint32_t priority; /**< Priority offset for each expansion. */ 145 } entry[]; 146 }; 147 148 static void 149 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 150 151 static const struct mlx5_flow_expand_node * 152 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 153 unsigned int item_idx, 154 const struct mlx5_flow_expand_node graph[], 155 const struct mlx5_flow_expand_node *node); 156 157 static bool 158 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 159 { 160 switch (item->type) { 161 case RTE_FLOW_ITEM_TYPE_ETH: 162 case RTE_FLOW_ITEM_TYPE_VLAN: 163 case RTE_FLOW_ITEM_TYPE_IPV4: 164 case RTE_FLOW_ITEM_TYPE_IPV6: 165 case RTE_FLOW_ITEM_TYPE_UDP: 166 case RTE_FLOW_ITEM_TYPE_TCP: 167 case RTE_FLOW_ITEM_TYPE_ESP: 168 case RTE_FLOW_ITEM_TYPE_ICMP: 169 case RTE_FLOW_ITEM_TYPE_ICMP6: 170 case RTE_FLOW_ITEM_TYPE_VXLAN: 171 case RTE_FLOW_ITEM_TYPE_NVGRE: 172 case RTE_FLOW_ITEM_TYPE_GRE: 173 case RTE_FLOW_ITEM_TYPE_GENEVE: 174 case RTE_FLOW_ITEM_TYPE_MPLS: 175 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 176 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 177 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 178 case RTE_FLOW_ITEM_TYPE_GTP: 179 return true; 180 default: 181 break; 182 } 183 return false; 184 } 185 186 /** 187 * Network Service Header (NSH) and its next protocol values 188 * are described in RFC-8393. 189 */ 190 static enum rte_flow_item_type 191 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 192 { 193 enum rte_flow_item_type type; 194 195 switch (proto_mask & proto_spec) { 196 case 0: 197 type = RTE_FLOW_ITEM_TYPE_VOID; 198 break; 199 case RTE_VXLAN_GPE_TYPE_IPV4: 200 type = RTE_FLOW_ITEM_TYPE_IPV4; 201 break; 202 case RTE_VXLAN_GPE_TYPE_IPV6: 203 type = RTE_VXLAN_GPE_TYPE_IPV6; 204 break; 205 case RTE_VXLAN_GPE_TYPE_ETH: 206 type = RTE_FLOW_ITEM_TYPE_ETH; 207 break; 208 default: 209 type = RTE_FLOW_ITEM_TYPE_END; 210 } 211 return type; 212 } 213 214 static enum rte_flow_item_type 215 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 216 { 217 enum rte_flow_item_type type; 218 219 switch (proto_mask & proto_spec) { 220 case 0: 221 type = RTE_FLOW_ITEM_TYPE_VOID; 222 break; 223 case IPPROTO_UDP: 224 type = RTE_FLOW_ITEM_TYPE_UDP; 225 break; 226 case IPPROTO_TCP: 227 type = RTE_FLOW_ITEM_TYPE_TCP; 228 break; 229 case IPPROTO_IPIP: 230 type = RTE_FLOW_ITEM_TYPE_IPV4; 231 break; 232 case IPPROTO_IPV6: 233 type = RTE_FLOW_ITEM_TYPE_IPV6; 234 break; 235 case IPPROTO_ESP: 236 type = RTE_FLOW_ITEM_TYPE_ESP; 237 break; 238 default: 239 type = RTE_FLOW_ITEM_TYPE_END; 240 } 241 return type; 242 } 243 244 static enum rte_flow_item_type 245 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 246 rte_be16_t type_mask, bool is_tunnel) 247 { 248 enum rte_flow_item_type type; 249 250 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 251 case 0: 252 type = RTE_FLOW_ITEM_TYPE_VOID; 253 break; 254 case RTE_ETHER_TYPE_TEB: 255 type = is_tunnel ? 256 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 257 break; 258 case RTE_ETHER_TYPE_VLAN: 259 type = !is_tunnel ? 260 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 261 break; 262 case RTE_ETHER_TYPE_IPV4: 263 type = RTE_FLOW_ITEM_TYPE_IPV4; 264 break; 265 case RTE_ETHER_TYPE_IPV6: 266 type = RTE_FLOW_ITEM_TYPE_IPV6; 267 break; 268 default: 269 type = RTE_FLOW_ITEM_TYPE_END; 270 } 271 return type; 272 } 273 274 static enum rte_flow_item_type 275 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 276 { 277 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 278 do { \ 279 const void *m = item->mask; \ 280 const void *s = item->spec; \ 281 mask = m ? \ 282 ((const struct rte_flow_item_##type *)m)->fld : \ 283 rte_flow_item_##type##_mask.fld; \ 284 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 285 } while (0) 286 287 enum rte_flow_item_type ret; 288 uint16_t spec, mask; 289 290 if (item == NULL || item->spec == NULL) 291 return RTE_FLOW_ITEM_TYPE_VOID; 292 switch (item->type) { 293 case RTE_FLOW_ITEM_TYPE_ETH: 294 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 295 if (!mask) 296 return RTE_FLOW_ITEM_TYPE_VOID; 297 ret = mlx5_ethertype_to_item_type(spec, mask, false); 298 break; 299 case RTE_FLOW_ITEM_TYPE_VLAN: 300 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 301 if (!mask) 302 return RTE_FLOW_ITEM_TYPE_VOID; 303 ret = mlx5_ethertype_to_item_type(spec, mask, false); 304 break; 305 case RTE_FLOW_ITEM_TYPE_IPV4: 306 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 307 if (!mask) 308 return RTE_FLOW_ITEM_TYPE_VOID; 309 ret = mlx5_inet_proto_to_item_type(spec, mask); 310 break; 311 case RTE_FLOW_ITEM_TYPE_IPV6: 312 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 313 if (!mask) 314 return RTE_FLOW_ITEM_TYPE_VOID; 315 ret = mlx5_inet_proto_to_item_type(spec, mask); 316 break; 317 case RTE_FLOW_ITEM_TYPE_GENEVE: 318 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 319 ret = mlx5_ethertype_to_item_type(spec, mask, true); 320 break; 321 case RTE_FLOW_ITEM_TYPE_GRE: 322 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 323 ret = mlx5_ethertype_to_item_type(spec, mask, true); 324 break; 325 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 326 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 327 ret = mlx5_nsh_proto_to_item_type(spec, mask); 328 break; 329 default: 330 ret = RTE_FLOW_ITEM_TYPE_VOID; 331 break; 332 } 333 return ret; 334 #undef MLX5_XSET_ITEM_MASK_SPEC 335 } 336 337 static const int * 338 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 339 const int *next_node) 340 { 341 const struct mlx5_flow_expand_node *node = NULL; 342 const int *next = next_node; 343 344 while (next && *next) { 345 /* 346 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 347 * flag set, because they were not found in the flow pattern. 348 */ 349 node = &graph[*next]; 350 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 351 break; 352 next = node->next; 353 } 354 return next; 355 } 356 357 #define MLX5_RSS_EXP_ELT_N 16 358 359 /** 360 * Expand RSS flows into several possible flows according to the RSS hash 361 * fields requested and the driver capabilities. 362 * 363 * @param[out] buf 364 * Buffer to store the result expansion. 365 * @param[in] size 366 * Buffer size in bytes. If 0, @p buf can be NULL. 367 * @param[in] pattern 368 * User flow pattern. 369 * @param[in] types 370 * RSS types to expand (see RTE_ETH_RSS_* definitions). 371 * @param[in] graph 372 * Input graph to expand @p pattern according to @p types. 373 * @param[in] graph_root_index 374 * Index of root node in @p graph, typically 0. 375 * 376 * @return 377 * A positive value representing the size of @p buf in bytes regardless of 378 * @p size on success, a negative errno value otherwise and rte_errno is 379 * set, the following errors are defined: 380 * 381 * -E2BIG: graph-depth @p graph is too deep. 382 * -EINVAL: @p size has not enough space for expanded pattern. 383 */ 384 static int 385 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 386 const struct rte_flow_item *pattern, uint64_t types, 387 const struct mlx5_flow_expand_node graph[], 388 int graph_root_index) 389 { 390 const struct rte_flow_item *item; 391 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 392 const int *next_node; 393 const int *stack[MLX5_RSS_EXP_ELT_N]; 394 int stack_pos = 0; 395 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 396 unsigned int i, item_idx, last_expand_item_idx = 0; 397 size_t lsize; 398 size_t user_pattern_size = 0; 399 void *addr = NULL; 400 const struct mlx5_flow_expand_node *next = NULL; 401 struct rte_flow_item missed_item; 402 int missed = 0; 403 int elt = 0; 404 const struct rte_flow_item *last_expand_item = NULL; 405 406 memset(&missed_item, 0, sizeof(missed_item)); 407 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 408 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 409 if (lsize > size) 410 return -EINVAL; 411 buf->entry[0].priority = 0; 412 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 413 buf->entries = 0; 414 addr = buf->entry[0].pattern; 415 for (item = pattern, item_idx = 0; 416 item->type != RTE_FLOW_ITEM_TYPE_END; 417 item++, item_idx++) { 418 if (!mlx5_flow_is_rss_expandable_item(item)) { 419 user_pattern_size += sizeof(*item); 420 continue; 421 } 422 last_expand_item = item; 423 last_expand_item_idx = item_idx; 424 i = 0; 425 while (node->next && node->next[i]) { 426 next = &graph[node->next[i]]; 427 if (next->type == item->type) 428 break; 429 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 430 node = next; 431 i = 0; 432 } else { 433 ++i; 434 } 435 } 436 if (next) 437 node = next; 438 user_pattern_size += sizeof(*item); 439 } 440 user_pattern_size += sizeof(*item); /* Handle END item. */ 441 lsize += user_pattern_size; 442 if (lsize > size) 443 return -EINVAL; 444 /* Copy the user pattern in the first entry of the buffer. */ 445 rte_memcpy(addr, pattern, user_pattern_size); 446 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 447 buf->entries = 1; 448 /* Start expanding. */ 449 memset(flow_items, 0, sizeof(flow_items)); 450 user_pattern_size -= sizeof(*item); 451 /* 452 * Check if the last valid item has spec set, need complete pattern, 453 * and the pattern can be used for expansion. 454 */ 455 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 456 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 457 /* Item type END indicates expansion is not required. */ 458 return lsize; 459 } 460 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 461 next = NULL; 462 missed = 1; 463 i = 0; 464 while (node->next && node->next[i]) { 465 next = &graph[node->next[i]]; 466 if (next->type == missed_item.type) { 467 flow_items[0].type = missed_item.type; 468 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 469 break; 470 } 471 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 472 node = next; 473 i = 0; 474 } else { 475 ++i; 476 } 477 next = NULL; 478 } 479 } 480 if (next && missed) { 481 elt = 2; /* missed item + item end. */ 482 node = next; 483 lsize += elt * sizeof(*item) + user_pattern_size; 484 if (lsize > size) 485 return -EINVAL; 486 if (node->rss_types & types) { 487 buf->entry[buf->entries].priority = 1; 488 buf->entry[buf->entries].pattern = addr; 489 buf->entries++; 490 rte_memcpy(addr, buf->entry[0].pattern, 491 user_pattern_size); 492 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 493 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 494 addr = (void *)(((uintptr_t)addr) + 495 elt * sizeof(*item)); 496 } 497 } else if (last_expand_item != NULL) { 498 node = mlx5_flow_expand_rss_adjust_node(pattern, 499 last_expand_item_idx, graph, node); 500 } 501 memset(flow_items, 0, sizeof(flow_items)); 502 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 503 node->next); 504 stack[stack_pos] = next_node; 505 node = next_node ? &graph[*next_node] : NULL; 506 while (node) { 507 flow_items[stack_pos].type = node->type; 508 if (node->rss_types & types) { 509 size_t n; 510 /* 511 * compute the number of items to copy from the 512 * expansion and copy it. 513 * When the stack_pos is 0, there are 1 element in it, 514 * plus the addition END item. 515 */ 516 elt = stack_pos + 2; 517 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 518 lsize += elt * sizeof(*item) + user_pattern_size; 519 if (lsize > size) 520 return -EINVAL; 521 n = elt * sizeof(*item); 522 buf->entry[buf->entries].priority = 523 stack_pos + 1 + missed; 524 buf->entry[buf->entries].pattern = addr; 525 buf->entries++; 526 rte_memcpy(addr, buf->entry[0].pattern, 527 user_pattern_size); 528 addr = (void *)(((uintptr_t)addr) + 529 user_pattern_size); 530 rte_memcpy(addr, &missed_item, 531 missed * sizeof(*item)); 532 addr = (void *)(((uintptr_t)addr) + 533 missed * sizeof(*item)); 534 rte_memcpy(addr, flow_items, n); 535 addr = (void *)(((uintptr_t)addr) + n); 536 } 537 /* Go deeper. */ 538 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 539 node->next) { 540 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 541 node->next); 542 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 543 rte_errno = E2BIG; 544 return -rte_errno; 545 } 546 stack[stack_pos] = next_node; 547 } else if (*(next_node + 1)) { 548 /* Follow up with the next possibility. */ 549 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 550 ++next_node); 551 } else if (!stack_pos) { 552 /* 553 * Completing the traverse over the different paths. 554 * The next_node is advanced to the terminator. 555 */ 556 ++next_node; 557 } else { 558 /* Move to the next path. */ 559 while (stack_pos) { 560 next_node = stack[--stack_pos]; 561 next_node++; 562 if (*next_node) 563 break; 564 } 565 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 566 next_node); 567 stack[stack_pos] = next_node; 568 } 569 node = next_node && *next_node ? &graph[*next_node] : NULL; 570 }; 571 return lsize; 572 } 573 574 enum mlx5_expansion { 575 MLX5_EXPANSION_ROOT, 576 MLX5_EXPANSION_ROOT_OUTER, 577 MLX5_EXPANSION_OUTER_ETH, 578 MLX5_EXPANSION_OUTER_VLAN, 579 MLX5_EXPANSION_OUTER_IPV4, 580 MLX5_EXPANSION_OUTER_IPV4_UDP, 581 MLX5_EXPANSION_OUTER_IPV4_TCP, 582 MLX5_EXPANSION_OUTER_IPV4_ESP, 583 MLX5_EXPANSION_OUTER_IPV4_ICMP, 584 MLX5_EXPANSION_OUTER_IPV6, 585 MLX5_EXPANSION_OUTER_IPV6_UDP, 586 MLX5_EXPANSION_OUTER_IPV6_TCP, 587 MLX5_EXPANSION_OUTER_IPV6_ESP, 588 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 589 MLX5_EXPANSION_VXLAN, 590 MLX5_EXPANSION_STD_VXLAN, 591 MLX5_EXPANSION_L3_VXLAN, 592 MLX5_EXPANSION_VXLAN_GPE, 593 MLX5_EXPANSION_GRE, 594 MLX5_EXPANSION_NVGRE, 595 MLX5_EXPANSION_GRE_KEY, 596 MLX5_EXPANSION_MPLS, 597 MLX5_EXPANSION_ETH, 598 MLX5_EXPANSION_VLAN, 599 MLX5_EXPANSION_IPV4, 600 MLX5_EXPANSION_IPV4_UDP, 601 MLX5_EXPANSION_IPV4_TCP, 602 MLX5_EXPANSION_IPV4_ESP, 603 MLX5_EXPANSION_IPV4_ICMP, 604 MLX5_EXPANSION_IPV6, 605 MLX5_EXPANSION_IPV6_UDP, 606 MLX5_EXPANSION_IPV6_TCP, 607 MLX5_EXPANSION_IPV6_ESP, 608 MLX5_EXPANSION_IPV6_ICMP6, 609 MLX5_EXPANSION_IPV6_FRAG_EXT, 610 MLX5_EXPANSION_GTP, 611 MLX5_EXPANSION_GENEVE, 612 }; 613 614 /** Supported expansion of items. */ 615 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 616 [MLX5_EXPANSION_ROOT] = { 617 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 618 MLX5_EXPANSION_IPV4, 619 MLX5_EXPANSION_IPV6), 620 .type = RTE_FLOW_ITEM_TYPE_END, 621 }, 622 [MLX5_EXPANSION_ROOT_OUTER] = { 623 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 624 MLX5_EXPANSION_OUTER_IPV4, 625 MLX5_EXPANSION_OUTER_IPV6), 626 .type = RTE_FLOW_ITEM_TYPE_END, 627 }, 628 [MLX5_EXPANSION_OUTER_ETH] = { 629 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 630 .type = RTE_FLOW_ITEM_TYPE_ETH, 631 .rss_types = 0, 632 }, 633 [MLX5_EXPANSION_OUTER_VLAN] = { 634 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 635 MLX5_EXPANSION_OUTER_IPV6), 636 .type = RTE_FLOW_ITEM_TYPE_VLAN, 637 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 638 }, 639 [MLX5_EXPANSION_OUTER_IPV4] = { 640 .next = MLX5_FLOW_EXPAND_RSS_NEXT 641 (MLX5_EXPANSION_OUTER_IPV4_UDP, 642 MLX5_EXPANSION_OUTER_IPV4_TCP, 643 MLX5_EXPANSION_OUTER_IPV4_ESP, 644 MLX5_EXPANSION_OUTER_IPV4_ICMP, 645 MLX5_EXPANSION_GRE, 646 MLX5_EXPANSION_NVGRE, 647 MLX5_EXPANSION_IPV4, 648 MLX5_EXPANSION_IPV6), 649 .type = RTE_FLOW_ITEM_TYPE_IPV4, 650 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 651 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 652 }, 653 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 654 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 655 MLX5_EXPANSION_VXLAN_GPE, 656 MLX5_EXPANSION_MPLS, 657 MLX5_EXPANSION_GENEVE, 658 MLX5_EXPANSION_GTP), 659 .type = RTE_FLOW_ITEM_TYPE_UDP, 660 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 661 }, 662 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 663 .type = RTE_FLOW_ITEM_TYPE_TCP, 664 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 665 }, 666 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 667 .type = RTE_FLOW_ITEM_TYPE_ESP, 668 .rss_types = RTE_ETH_RSS_ESP, 669 }, 670 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 671 .type = RTE_FLOW_ITEM_TYPE_ICMP, 672 }, 673 [MLX5_EXPANSION_OUTER_IPV6] = { 674 .next = MLX5_FLOW_EXPAND_RSS_NEXT 675 (MLX5_EXPANSION_OUTER_IPV6_UDP, 676 MLX5_EXPANSION_OUTER_IPV6_TCP, 677 MLX5_EXPANSION_OUTER_IPV6_ESP, 678 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 679 MLX5_EXPANSION_IPV4, 680 MLX5_EXPANSION_IPV6, 681 MLX5_EXPANSION_GRE, 682 MLX5_EXPANSION_NVGRE), 683 .type = RTE_FLOW_ITEM_TYPE_IPV6, 684 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 685 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 686 }, 687 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 688 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 689 MLX5_EXPANSION_VXLAN_GPE, 690 MLX5_EXPANSION_MPLS, 691 MLX5_EXPANSION_GENEVE, 692 MLX5_EXPANSION_GTP), 693 .type = RTE_FLOW_ITEM_TYPE_UDP, 694 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 695 }, 696 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 697 .type = RTE_FLOW_ITEM_TYPE_TCP, 698 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 699 }, 700 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 701 .type = RTE_FLOW_ITEM_TYPE_ESP, 702 .rss_types = RTE_ETH_RSS_ESP, 703 }, 704 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 705 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 706 }, 707 [MLX5_EXPANSION_VXLAN] = { 708 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 709 MLX5_EXPANSION_IPV4, 710 MLX5_EXPANSION_IPV6), 711 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 712 }, 713 [MLX5_EXPANSION_STD_VXLAN] = { 714 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 715 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 716 }, 717 [MLX5_EXPANSION_L3_VXLAN] = { 718 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 719 MLX5_EXPANSION_IPV6), 720 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 721 }, 722 [MLX5_EXPANSION_VXLAN_GPE] = { 723 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 724 MLX5_EXPANSION_IPV4, 725 MLX5_EXPANSION_IPV6), 726 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 727 }, 728 [MLX5_EXPANSION_GRE] = { 729 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 730 MLX5_EXPANSION_IPV4, 731 MLX5_EXPANSION_IPV6, 732 MLX5_EXPANSION_GRE_KEY, 733 MLX5_EXPANSION_MPLS), 734 .type = RTE_FLOW_ITEM_TYPE_GRE, 735 }, 736 [MLX5_EXPANSION_GRE_KEY] = { 737 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 738 MLX5_EXPANSION_IPV6, 739 MLX5_EXPANSION_MPLS), 740 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 741 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 742 }, 743 [MLX5_EXPANSION_NVGRE] = { 744 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 745 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 746 }, 747 [MLX5_EXPANSION_MPLS] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 749 MLX5_EXPANSION_IPV6, 750 MLX5_EXPANSION_ETH), 751 .type = RTE_FLOW_ITEM_TYPE_MPLS, 752 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 753 }, 754 [MLX5_EXPANSION_ETH] = { 755 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 756 .type = RTE_FLOW_ITEM_TYPE_ETH, 757 }, 758 [MLX5_EXPANSION_VLAN] = { 759 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 760 MLX5_EXPANSION_IPV6), 761 .type = RTE_FLOW_ITEM_TYPE_VLAN, 762 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 763 }, 764 [MLX5_EXPANSION_IPV4] = { 765 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 766 MLX5_EXPANSION_IPV4_TCP, 767 MLX5_EXPANSION_IPV4_ESP, 768 MLX5_EXPANSION_IPV4_ICMP), 769 .type = RTE_FLOW_ITEM_TYPE_IPV4, 770 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 771 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 772 }, 773 [MLX5_EXPANSION_IPV4_UDP] = { 774 .type = RTE_FLOW_ITEM_TYPE_UDP, 775 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 776 }, 777 [MLX5_EXPANSION_IPV4_TCP] = { 778 .type = RTE_FLOW_ITEM_TYPE_TCP, 779 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 780 }, 781 [MLX5_EXPANSION_IPV4_ESP] = { 782 .type = RTE_FLOW_ITEM_TYPE_ESP, 783 .rss_types = RTE_ETH_RSS_ESP, 784 }, 785 [MLX5_EXPANSION_IPV4_ICMP] = { 786 .type = RTE_FLOW_ITEM_TYPE_ICMP, 787 }, 788 [MLX5_EXPANSION_IPV6] = { 789 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 790 MLX5_EXPANSION_IPV6_TCP, 791 MLX5_EXPANSION_IPV6_ESP, 792 MLX5_EXPANSION_IPV6_ICMP6, 793 MLX5_EXPANSION_IPV6_FRAG_EXT), 794 .type = RTE_FLOW_ITEM_TYPE_IPV6, 795 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 796 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 797 }, 798 [MLX5_EXPANSION_IPV6_UDP] = { 799 .type = RTE_FLOW_ITEM_TYPE_UDP, 800 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 801 }, 802 [MLX5_EXPANSION_IPV6_TCP] = { 803 .type = RTE_FLOW_ITEM_TYPE_TCP, 804 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 805 }, 806 [MLX5_EXPANSION_IPV6_ESP] = { 807 .type = RTE_FLOW_ITEM_TYPE_ESP, 808 .rss_types = RTE_ETH_RSS_ESP, 809 }, 810 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 811 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 812 }, 813 [MLX5_EXPANSION_IPV6_ICMP6] = { 814 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 815 }, 816 [MLX5_EXPANSION_GTP] = { 817 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 818 MLX5_EXPANSION_IPV6), 819 .type = RTE_FLOW_ITEM_TYPE_GTP, 820 }, 821 [MLX5_EXPANSION_GENEVE] = { 822 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 823 MLX5_EXPANSION_IPV4, 824 MLX5_EXPANSION_IPV6), 825 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 826 }, 827 }; 828 829 static struct rte_flow_action_handle * 830 mlx5_action_handle_create(struct rte_eth_dev *dev, 831 const struct rte_flow_indir_action_conf *conf, 832 const struct rte_flow_action *action, 833 struct rte_flow_error *error); 834 static int mlx5_action_handle_destroy 835 (struct rte_eth_dev *dev, 836 struct rte_flow_action_handle *handle, 837 struct rte_flow_error *error); 838 static int mlx5_action_handle_update 839 (struct rte_eth_dev *dev, 840 struct rte_flow_action_handle *handle, 841 const void *update, 842 struct rte_flow_error *error); 843 static int mlx5_action_handle_query 844 (struct rte_eth_dev *dev, 845 const struct rte_flow_action_handle *handle, 846 void *data, 847 struct rte_flow_error *error); 848 static int 849 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 850 struct rte_flow_tunnel *app_tunnel, 851 struct rte_flow_action **actions, 852 uint32_t *num_of_actions, 853 struct rte_flow_error *error); 854 static int 855 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 856 struct rte_flow_tunnel *app_tunnel, 857 struct rte_flow_item **items, 858 uint32_t *num_of_items, 859 struct rte_flow_error *error); 860 static int 861 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 862 struct rte_flow_item *pmd_items, 863 uint32_t num_items, struct rte_flow_error *err); 864 static int 865 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 866 struct rte_flow_action *pmd_actions, 867 uint32_t num_actions, 868 struct rte_flow_error *err); 869 static int 870 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 871 struct rte_mbuf *m, 872 struct rte_flow_restore_info *info, 873 struct rte_flow_error *err); 874 static struct rte_flow_item_flex_handle * 875 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 876 const struct rte_flow_item_flex_conf *conf, 877 struct rte_flow_error *error); 878 static int 879 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 880 const struct rte_flow_item_flex_handle *handle, 881 struct rte_flow_error *error); 882 static int 883 mlx5_flow_info_get(struct rte_eth_dev *dev, 884 struct rte_flow_port_info *port_info, 885 struct rte_flow_queue_info *queue_info, 886 struct rte_flow_error *error); 887 static int 888 mlx5_flow_port_configure(struct rte_eth_dev *dev, 889 const struct rte_flow_port_attr *port_attr, 890 uint16_t nb_queue, 891 const struct rte_flow_queue_attr *queue_attr[], 892 struct rte_flow_error *err); 893 894 static struct rte_flow_pattern_template * 895 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 896 const struct rte_flow_pattern_template_attr *attr, 897 const struct rte_flow_item items[], 898 struct rte_flow_error *error); 899 900 static int 901 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 902 struct rte_flow_pattern_template *template, 903 struct rte_flow_error *error); 904 static struct rte_flow_actions_template * 905 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 906 const struct rte_flow_actions_template_attr *attr, 907 const struct rte_flow_action actions[], 908 const struct rte_flow_action masks[], 909 struct rte_flow_error *error); 910 static int 911 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 912 struct rte_flow_actions_template *template, 913 struct rte_flow_error *error); 914 915 static struct rte_flow_template_table * 916 mlx5_flow_table_create(struct rte_eth_dev *dev, 917 const struct rte_flow_template_table_attr *attr, 918 struct rte_flow_pattern_template *item_templates[], 919 uint8_t nb_item_templates, 920 struct rte_flow_actions_template *action_templates[], 921 uint8_t nb_action_templates, 922 struct rte_flow_error *error); 923 static int 924 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 925 struct rte_flow_template_table *table, 926 struct rte_flow_error *error); 927 static struct rte_flow * 928 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 929 uint32_t queue, 930 const struct rte_flow_op_attr *attr, 931 struct rte_flow_template_table *table, 932 const struct rte_flow_item items[], 933 uint8_t pattern_template_index, 934 const struct rte_flow_action actions[], 935 uint8_t action_template_index, 936 void *user_data, 937 struct rte_flow_error *error); 938 static int 939 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 940 uint32_t queue, 941 const struct rte_flow_op_attr *attr, 942 struct rte_flow *flow, 943 void *user_data, 944 struct rte_flow_error *error); 945 static int 946 mlx5_flow_pull(struct rte_eth_dev *dev, 947 uint32_t queue, 948 struct rte_flow_op_result res[], 949 uint16_t n_res, 950 struct rte_flow_error *error); 951 static int 952 mlx5_flow_push(struct rte_eth_dev *dev, 953 uint32_t queue, 954 struct rte_flow_error *error); 955 956 static struct rte_flow_action_handle * 957 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 958 const struct rte_flow_op_attr *attr, 959 const struct rte_flow_indir_action_conf *conf, 960 const struct rte_flow_action *action, 961 void *user_data, 962 struct rte_flow_error *error); 963 964 static int 965 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 966 const struct rte_flow_op_attr *attr, 967 struct rte_flow_action_handle *handle, 968 const void *update, 969 void *user_data, 970 struct rte_flow_error *error); 971 972 static int 973 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 974 const struct rte_flow_op_attr *attr, 975 struct rte_flow_action_handle *handle, 976 void *user_data, 977 struct rte_flow_error *error); 978 979 static const struct rte_flow_ops mlx5_flow_ops = { 980 .validate = mlx5_flow_validate, 981 .create = mlx5_flow_create, 982 .destroy = mlx5_flow_destroy, 983 .flush = mlx5_flow_flush, 984 .isolate = mlx5_flow_isolate, 985 .query = mlx5_flow_query, 986 .dev_dump = mlx5_flow_dev_dump, 987 .get_aged_flows = mlx5_flow_get_aged_flows, 988 .action_handle_create = mlx5_action_handle_create, 989 .action_handle_destroy = mlx5_action_handle_destroy, 990 .action_handle_update = mlx5_action_handle_update, 991 .action_handle_query = mlx5_action_handle_query, 992 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 993 .tunnel_match = mlx5_flow_tunnel_match, 994 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 995 .tunnel_item_release = mlx5_flow_tunnel_item_release, 996 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 997 .flex_item_create = mlx5_flow_flex_item_create, 998 .flex_item_release = mlx5_flow_flex_item_release, 999 .info_get = mlx5_flow_info_get, 1000 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1001 .configure = mlx5_flow_port_configure, 1002 .pattern_template_create = mlx5_flow_pattern_template_create, 1003 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1004 .actions_template_create = mlx5_flow_actions_template_create, 1005 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1006 .template_table_create = mlx5_flow_table_create, 1007 .template_table_destroy = mlx5_flow_table_destroy, 1008 .async_create = mlx5_flow_async_flow_create, 1009 .async_destroy = mlx5_flow_async_flow_destroy, 1010 .pull = mlx5_flow_pull, 1011 .push = mlx5_flow_push, 1012 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1013 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1014 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1015 }; 1016 1017 /* Tunnel information. */ 1018 struct mlx5_flow_tunnel_info { 1019 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1020 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1021 }; 1022 1023 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1024 { 1025 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1026 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1027 }, 1028 { 1029 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1030 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1031 }, 1032 { 1033 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1034 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1035 }, 1036 { 1037 .tunnel = MLX5_FLOW_LAYER_GRE, 1038 .ptype = RTE_PTYPE_TUNNEL_GRE, 1039 }, 1040 { 1041 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1042 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1043 }, 1044 { 1045 .tunnel = MLX5_FLOW_LAYER_MPLS, 1046 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1047 }, 1048 { 1049 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1050 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1051 }, 1052 { 1053 .tunnel = MLX5_FLOW_LAYER_IPIP, 1054 .ptype = RTE_PTYPE_TUNNEL_IP, 1055 }, 1056 { 1057 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1058 .ptype = RTE_PTYPE_TUNNEL_IP, 1059 }, 1060 { 1061 .tunnel = MLX5_FLOW_LAYER_GTP, 1062 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1063 }, 1064 }; 1065 1066 1067 1068 /** 1069 * Translate tag ID to register. 1070 * 1071 * @param[in] dev 1072 * Pointer to the Ethernet device structure. 1073 * @param[in] feature 1074 * The feature that request the register. 1075 * @param[in] id 1076 * The request register ID. 1077 * @param[out] error 1078 * Error description in case of any. 1079 * 1080 * @return 1081 * The request register on success, a negative errno 1082 * value otherwise and rte_errno is set. 1083 */ 1084 int 1085 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1086 enum mlx5_feature_name feature, 1087 uint32_t id, 1088 struct rte_flow_error *error) 1089 { 1090 struct mlx5_priv *priv = dev->data->dev_private; 1091 struct mlx5_sh_config *config = &priv->sh->config; 1092 enum modify_reg start_reg; 1093 bool skip_mtr_reg = false; 1094 1095 switch (feature) { 1096 case MLX5_HAIRPIN_RX: 1097 return REG_B; 1098 case MLX5_HAIRPIN_TX: 1099 return REG_A; 1100 case MLX5_METADATA_RX: 1101 switch (config->dv_xmeta_en) { 1102 case MLX5_XMETA_MODE_LEGACY: 1103 return REG_B; 1104 case MLX5_XMETA_MODE_META16: 1105 return REG_C_0; 1106 case MLX5_XMETA_MODE_META32: 1107 return REG_C_1; 1108 case MLX5_XMETA_MODE_META32_HWS: 1109 return REG_C_1; 1110 } 1111 break; 1112 case MLX5_METADATA_TX: 1113 return REG_A; 1114 case MLX5_METADATA_FDB: 1115 switch (config->dv_xmeta_en) { 1116 case MLX5_XMETA_MODE_LEGACY: 1117 return REG_NON; 1118 case MLX5_XMETA_MODE_META16: 1119 return REG_C_0; 1120 case MLX5_XMETA_MODE_META32: 1121 return REG_C_1; 1122 case MLX5_XMETA_MODE_META32_HWS: 1123 return REG_C_1; 1124 } 1125 break; 1126 case MLX5_FLOW_MARK: 1127 switch (config->dv_xmeta_en) { 1128 case MLX5_XMETA_MODE_LEGACY: 1129 case MLX5_XMETA_MODE_META32_HWS: 1130 return REG_NON; 1131 case MLX5_XMETA_MODE_META16: 1132 return REG_C_1; 1133 case MLX5_XMETA_MODE_META32: 1134 return REG_C_0; 1135 } 1136 break; 1137 case MLX5_MTR_ID: 1138 /* 1139 * If meter color and meter id share one register, flow match 1140 * should use the meter color register for match. 1141 */ 1142 if (priv->mtr_reg_share) 1143 return priv->mtr_color_reg; 1144 else 1145 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1146 REG_C_3; 1147 case MLX5_MTR_COLOR: 1148 case MLX5_ASO_FLOW_HIT: 1149 case MLX5_ASO_CONNTRACK: 1150 case MLX5_SAMPLE_ID: 1151 /* All features use the same REG_C. */ 1152 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1153 return priv->mtr_color_reg; 1154 case MLX5_COPY_MARK: 1155 /* 1156 * Metadata COPY_MARK register using is in meter suffix sub 1157 * flow while with meter. It's safe to share the same register. 1158 */ 1159 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1160 case MLX5_APP_TAG: 1161 /* 1162 * If meter is enable, it will engage the register for color 1163 * match and flow match. If meter color match is not using the 1164 * REG_C_2, need to skip the REG_C_x be used by meter color 1165 * match. 1166 * If meter is disable, free to use all available registers. 1167 */ 1168 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1169 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1170 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1171 if (id > (uint32_t)(REG_C_7 - start_reg)) 1172 return rte_flow_error_set(error, EINVAL, 1173 RTE_FLOW_ERROR_TYPE_ITEM, 1174 NULL, "invalid tag id"); 1175 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1176 return rte_flow_error_set(error, ENOTSUP, 1177 RTE_FLOW_ERROR_TYPE_ITEM, 1178 NULL, "unsupported tag id"); 1179 /* 1180 * This case means meter is using the REG_C_x great than 2. 1181 * Take care not to conflict with meter color REG_C_x. 1182 * If the available index REG_C_y >= REG_C_x, skip the 1183 * color register. 1184 */ 1185 if (skip_mtr_reg && priv->sh->flow_mreg_c 1186 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1187 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1188 return rte_flow_error_set(error, EINVAL, 1189 RTE_FLOW_ERROR_TYPE_ITEM, 1190 NULL, "invalid tag id"); 1191 if (priv->sh->flow_mreg_c 1192 [id + 1 + start_reg - REG_C_0] != REG_NON) 1193 return priv->sh->flow_mreg_c 1194 [id + 1 + start_reg - REG_C_0]; 1195 return rte_flow_error_set(error, ENOTSUP, 1196 RTE_FLOW_ERROR_TYPE_ITEM, 1197 NULL, "unsupported tag id"); 1198 } 1199 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1200 } 1201 MLX5_ASSERT(false); 1202 return rte_flow_error_set(error, EINVAL, 1203 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1204 NULL, "invalid feature name"); 1205 } 1206 1207 /** 1208 * Check extensive flow metadata register support. 1209 * 1210 * @param dev 1211 * Pointer to rte_eth_dev structure. 1212 * 1213 * @return 1214 * True if device supports extensive flow metadata register, otherwise false. 1215 */ 1216 bool 1217 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1218 { 1219 struct mlx5_priv *priv = dev->data->dev_private; 1220 1221 /* 1222 * Having available reg_c can be regarded inclusively as supporting 1223 * extensive flow metadata register, which could mean, 1224 * - metadata register copy action by modify header. 1225 * - 16 modify header actions is supported. 1226 * - reg_c's are preserved across different domain (FDB and NIC) on 1227 * packet loopback by flow lookup miss. 1228 */ 1229 return priv->sh->flow_mreg_c[2] != REG_NON; 1230 } 1231 1232 /** 1233 * Get the lowest priority. 1234 * 1235 * @param[in] dev 1236 * Pointer to the Ethernet device structure. 1237 * @param[in] attributes 1238 * Pointer to device flow rule attributes. 1239 * 1240 * @return 1241 * The value of lowest priority of flow. 1242 */ 1243 uint32_t 1244 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1245 const struct rte_flow_attr *attr) 1246 { 1247 struct mlx5_priv *priv = dev->data->dev_private; 1248 1249 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1250 return priv->sh->flow_max_priority - 2; 1251 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1252 } 1253 1254 /** 1255 * Calculate matcher priority of the flow. 1256 * 1257 * @param[in] dev 1258 * Pointer to the Ethernet device structure. 1259 * @param[in] attr 1260 * Pointer to device flow rule attributes. 1261 * @param[in] subpriority 1262 * The priority based on the items. 1263 * @param[in] external 1264 * Flow is user flow. 1265 * @return 1266 * The matcher priority of the flow. 1267 */ 1268 uint16_t 1269 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1270 const struct rte_flow_attr *attr, 1271 uint32_t subpriority, bool external) 1272 { 1273 uint16_t priority = (uint16_t)attr->priority; 1274 struct mlx5_priv *priv = dev->data->dev_private; 1275 1276 /* NIC root rules */ 1277 if (!attr->group && !attr->transfer) { 1278 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1279 priority = priv->sh->flow_max_priority - 1; 1280 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1281 /* FDB root rules */ 1282 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1283 attr->group == 0 && 1284 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1285 return (priv->sh->flow_max_priority - 1) * 3; 1286 } 1287 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1288 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1289 return priority * 3 + subpriority; 1290 } 1291 1292 /** 1293 * Verify the @p item specifications (spec, last, mask) are compatible with the 1294 * NIC capabilities. 1295 * 1296 * @param[in] item 1297 * Item specification. 1298 * @param[in] mask 1299 * @p item->mask or flow default bit-masks. 1300 * @param[in] nic_mask 1301 * Bit-masks covering supported fields by the NIC to compare with user mask. 1302 * @param[in] size 1303 * Bit-masks size in bytes. 1304 * @param[in] range_accepted 1305 * True if range of values is accepted for specific fields, false otherwise. 1306 * @param[out] error 1307 * Pointer to error structure. 1308 * 1309 * @return 1310 * 0 on success, a negative errno value otherwise and rte_errno is set. 1311 */ 1312 int 1313 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1314 const uint8_t *mask, 1315 const uint8_t *nic_mask, 1316 unsigned int size, 1317 bool range_accepted, 1318 struct rte_flow_error *error) 1319 { 1320 unsigned int i; 1321 1322 MLX5_ASSERT(nic_mask); 1323 for (i = 0; i < size; ++i) 1324 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1325 return rte_flow_error_set(error, ENOTSUP, 1326 RTE_FLOW_ERROR_TYPE_ITEM, 1327 item, 1328 "mask enables non supported" 1329 " bits"); 1330 if (!item->spec && (item->mask || item->last)) 1331 return rte_flow_error_set(error, EINVAL, 1332 RTE_FLOW_ERROR_TYPE_ITEM, item, 1333 "mask/last without a spec is not" 1334 " supported"); 1335 if (item->spec && item->last && !range_accepted) { 1336 uint8_t spec[size]; 1337 uint8_t last[size]; 1338 unsigned int i; 1339 int ret; 1340 1341 for (i = 0; i < size; ++i) { 1342 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1343 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1344 } 1345 ret = memcmp(spec, last, size); 1346 if (ret != 0) 1347 return rte_flow_error_set(error, EINVAL, 1348 RTE_FLOW_ERROR_TYPE_ITEM, 1349 item, 1350 "range is not valid"); 1351 } 1352 return 0; 1353 } 1354 1355 /** 1356 * Adjust the hash fields according to the @p flow information. 1357 * 1358 * @param[in] dev_flow. 1359 * Pointer to the mlx5_flow. 1360 * @param[in] tunnel 1361 * 1 when the hash field is for a tunnel item. 1362 * @param[in] layer_types 1363 * RTE_ETH_RSS_* types. 1364 * @param[in] hash_fields 1365 * Item hash fields. 1366 * 1367 * @return 1368 * The hash fields that should be used. 1369 */ 1370 uint64_t 1371 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1372 int tunnel __rte_unused, uint64_t layer_types, 1373 uint64_t hash_fields) 1374 { 1375 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1376 int rss_request_inner = rss_desc->level >= 2; 1377 1378 /* Check RSS hash level for tunnel. */ 1379 if (tunnel && rss_request_inner) 1380 hash_fields |= IBV_RX_HASH_INNER; 1381 else if (tunnel || rss_request_inner) 1382 return 0; 1383 #endif 1384 /* Check if requested layer matches RSS hash fields. */ 1385 if (!(rss_desc->types & layer_types)) 1386 return 0; 1387 return hash_fields; 1388 } 1389 1390 /** 1391 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1392 * if several tunnel rules are used on this queue, the tunnel ptype will be 1393 * cleared. 1394 * 1395 * @param rxq_ctrl 1396 * Rx queue to update. 1397 */ 1398 static void 1399 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1400 { 1401 unsigned int i; 1402 uint32_t tunnel_ptype = 0; 1403 1404 /* Look up for the ptype to use. */ 1405 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1406 if (!rxq_ctrl->flow_tunnels_n[i]) 1407 continue; 1408 if (!tunnel_ptype) { 1409 tunnel_ptype = tunnels_info[i].ptype; 1410 } else { 1411 tunnel_ptype = 0; 1412 break; 1413 } 1414 } 1415 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1416 } 1417 1418 /** 1419 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1420 * flow. 1421 * 1422 * @param[in] dev 1423 * Pointer to the Ethernet device structure. 1424 * @param[in] dev_handle 1425 * Pointer to device flow handle structure. 1426 */ 1427 void 1428 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1429 struct mlx5_flow_handle *dev_handle) 1430 { 1431 struct mlx5_priv *priv = dev->data->dev_private; 1432 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1433 struct mlx5_ind_table_obj *ind_tbl = NULL; 1434 unsigned int i; 1435 1436 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1437 struct mlx5_hrxq *hrxq; 1438 1439 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1440 dev_handle->rix_hrxq); 1441 if (hrxq) 1442 ind_tbl = hrxq->ind_table; 1443 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1444 struct mlx5_shared_action_rss *shared_rss; 1445 1446 shared_rss = mlx5_ipool_get 1447 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1448 dev_handle->rix_srss); 1449 if (shared_rss) 1450 ind_tbl = shared_rss->ind_tbl; 1451 } 1452 if (!ind_tbl) 1453 return; 1454 for (i = 0; i != ind_tbl->queues_n; ++i) { 1455 int idx = ind_tbl->queues[i]; 1456 struct mlx5_rxq_ctrl *rxq_ctrl; 1457 1458 if (mlx5_is_external_rxq(dev, idx)) 1459 continue; 1460 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1461 MLX5_ASSERT(rxq_ctrl != NULL); 1462 if (rxq_ctrl == NULL) 1463 continue; 1464 /* 1465 * To support metadata register copy on Tx loopback, 1466 * this must be always enabled (metadata may arive 1467 * from other port - not from local flows only. 1468 */ 1469 if (tunnel) { 1470 unsigned int j; 1471 1472 /* Increase the counter matching the flow. */ 1473 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1474 if ((tunnels_info[j].tunnel & 1475 dev_handle->layers) == 1476 tunnels_info[j].tunnel) { 1477 rxq_ctrl->flow_tunnels_n[j]++; 1478 break; 1479 } 1480 } 1481 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1482 } 1483 } 1484 } 1485 1486 static void 1487 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1488 { 1489 struct mlx5_priv *priv = dev->data->dev_private; 1490 struct mlx5_rxq_ctrl *rxq_ctrl; 1491 uint16_t port_id; 1492 1493 if (priv->sh->shared_mark_enabled) 1494 return; 1495 if (priv->master || priv->representor) { 1496 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1497 struct mlx5_priv *opriv = 1498 rte_eth_devices[port_id].data->dev_private; 1499 1500 if (!opriv || 1501 opriv->sh != priv->sh || 1502 opriv->domain_id != priv->domain_id || 1503 opriv->mark_enabled) 1504 continue; 1505 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1506 rxq_ctrl->rxq.mark = 1; 1507 } 1508 opriv->mark_enabled = 1; 1509 } 1510 } else { 1511 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1512 rxq_ctrl->rxq.mark = 1; 1513 } 1514 priv->mark_enabled = 1; 1515 } 1516 priv->sh->shared_mark_enabled = 1; 1517 } 1518 1519 /** 1520 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1521 * 1522 * @param[in] dev 1523 * Pointer to the Ethernet device structure. 1524 * @param[in] flow 1525 * Pointer to flow structure. 1526 */ 1527 static void 1528 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1529 { 1530 struct mlx5_priv *priv = dev->data->dev_private; 1531 uint32_t handle_idx; 1532 struct mlx5_flow_handle *dev_handle; 1533 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1534 1535 MLX5_ASSERT(wks); 1536 if (wks->mark) 1537 flow_rxq_mark_flag_set(dev); 1538 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1539 handle_idx, dev_handle, next) 1540 flow_drv_rxq_flags_set(dev, dev_handle); 1541 } 1542 1543 /** 1544 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1545 * device flow if no other flow uses it with the same kind of request. 1546 * 1547 * @param dev 1548 * Pointer to Ethernet device. 1549 * @param[in] dev_handle 1550 * Pointer to the device flow handle structure. 1551 */ 1552 static void 1553 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1554 struct mlx5_flow_handle *dev_handle) 1555 { 1556 struct mlx5_priv *priv = dev->data->dev_private; 1557 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1558 struct mlx5_ind_table_obj *ind_tbl = NULL; 1559 unsigned int i; 1560 1561 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1562 struct mlx5_hrxq *hrxq; 1563 1564 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1565 dev_handle->rix_hrxq); 1566 if (hrxq) 1567 ind_tbl = hrxq->ind_table; 1568 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1569 struct mlx5_shared_action_rss *shared_rss; 1570 1571 shared_rss = mlx5_ipool_get 1572 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1573 dev_handle->rix_srss); 1574 if (shared_rss) 1575 ind_tbl = shared_rss->ind_tbl; 1576 } 1577 if (!ind_tbl) 1578 return; 1579 MLX5_ASSERT(dev->data->dev_started); 1580 for (i = 0; i != ind_tbl->queues_n; ++i) { 1581 int idx = ind_tbl->queues[i]; 1582 struct mlx5_rxq_ctrl *rxq_ctrl; 1583 1584 if (mlx5_is_external_rxq(dev, idx)) 1585 continue; 1586 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1587 MLX5_ASSERT(rxq_ctrl != NULL); 1588 if (rxq_ctrl == NULL) 1589 continue; 1590 if (tunnel) { 1591 unsigned int j; 1592 1593 /* Decrease the counter matching the flow. */ 1594 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1595 if ((tunnels_info[j].tunnel & 1596 dev_handle->layers) == 1597 tunnels_info[j].tunnel) { 1598 rxq_ctrl->flow_tunnels_n[j]--; 1599 break; 1600 } 1601 } 1602 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1603 } 1604 } 1605 } 1606 1607 /** 1608 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1609 * @p flow if no other flow uses it with the same kind of request. 1610 * 1611 * @param dev 1612 * Pointer to Ethernet device. 1613 * @param[in] flow 1614 * Pointer to the flow. 1615 */ 1616 static void 1617 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1618 { 1619 struct mlx5_priv *priv = dev->data->dev_private; 1620 uint32_t handle_idx; 1621 struct mlx5_flow_handle *dev_handle; 1622 1623 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1624 handle_idx, dev_handle, next) 1625 flow_drv_rxq_flags_trim(dev, dev_handle); 1626 } 1627 1628 /** 1629 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1630 * 1631 * @param dev 1632 * Pointer to Ethernet device. 1633 */ 1634 static void 1635 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1636 { 1637 struct mlx5_priv *priv = dev->data->dev_private; 1638 unsigned int i; 1639 1640 for (i = 0; i != priv->rxqs_n; ++i) { 1641 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1642 unsigned int j; 1643 1644 if (rxq == NULL || rxq->ctrl == NULL) 1645 continue; 1646 rxq->ctrl->rxq.mark = 0; 1647 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1648 rxq->ctrl->flow_tunnels_n[j] = 0; 1649 rxq->ctrl->rxq.tunnel = 0; 1650 } 1651 priv->mark_enabled = 0; 1652 priv->sh->shared_mark_enabled = 0; 1653 } 1654 1655 /** 1656 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1657 * 1658 * @param[in] dev 1659 * Pointer to the Ethernet device structure. 1660 */ 1661 void 1662 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1663 { 1664 struct mlx5_priv *priv = dev->data->dev_private; 1665 unsigned int i; 1666 1667 for (i = 0; i != priv->rxqs_n; ++i) { 1668 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1669 struct mlx5_rxq_data *data; 1670 1671 if (rxq == NULL || rxq->ctrl == NULL) 1672 continue; 1673 data = &rxq->ctrl->rxq; 1674 if (!rte_flow_dynf_metadata_avail()) { 1675 data->dynf_meta = 0; 1676 data->flow_meta_mask = 0; 1677 data->flow_meta_offset = -1; 1678 data->flow_meta_port_mask = 0; 1679 } else { 1680 data->dynf_meta = 1; 1681 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1682 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1683 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1684 } 1685 } 1686 } 1687 1688 /* 1689 * return a pointer to the desired action in the list of actions. 1690 * 1691 * @param[in] actions 1692 * The list of actions to search the action in. 1693 * @param[in] action 1694 * The action to find. 1695 * 1696 * @return 1697 * Pointer to the action in the list, if found. NULL otherwise. 1698 */ 1699 const struct rte_flow_action * 1700 mlx5_flow_find_action(const struct rte_flow_action *actions, 1701 enum rte_flow_action_type action) 1702 { 1703 if (actions == NULL) 1704 return NULL; 1705 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1706 if (actions->type == action) 1707 return actions; 1708 return NULL; 1709 } 1710 1711 /* 1712 * Validate the flag action. 1713 * 1714 * @param[in] action_flags 1715 * Bit-fields that holds the actions detected until now. 1716 * @param[in] attr 1717 * Attributes of flow that includes this action. 1718 * @param[out] error 1719 * Pointer to error structure. 1720 * 1721 * @return 1722 * 0 on success, a negative errno value otherwise and rte_errno is set. 1723 */ 1724 int 1725 mlx5_flow_validate_action_flag(uint64_t action_flags, 1726 const struct rte_flow_attr *attr, 1727 struct rte_flow_error *error) 1728 { 1729 if (action_flags & MLX5_FLOW_ACTION_MARK) 1730 return rte_flow_error_set(error, EINVAL, 1731 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1732 "can't mark and flag in same flow"); 1733 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1734 return rte_flow_error_set(error, EINVAL, 1735 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1736 "can't have 2 flag" 1737 " actions in same flow"); 1738 if (attr->egress) 1739 return rte_flow_error_set(error, ENOTSUP, 1740 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1741 "flag action not supported for " 1742 "egress"); 1743 return 0; 1744 } 1745 1746 /* 1747 * Validate the mark action. 1748 * 1749 * @param[in] action 1750 * Pointer to the queue action. 1751 * @param[in] action_flags 1752 * Bit-fields that holds the actions detected until now. 1753 * @param[in] attr 1754 * Attributes of flow that includes this action. 1755 * @param[out] error 1756 * Pointer to error structure. 1757 * 1758 * @return 1759 * 0 on success, a negative errno value otherwise and rte_errno is set. 1760 */ 1761 int 1762 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1763 uint64_t action_flags, 1764 const struct rte_flow_attr *attr, 1765 struct rte_flow_error *error) 1766 { 1767 const struct rte_flow_action_mark *mark = action->conf; 1768 1769 if (!mark) 1770 return rte_flow_error_set(error, EINVAL, 1771 RTE_FLOW_ERROR_TYPE_ACTION, 1772 action, 1773 "configuration cannot be null"); 1774 if (mark->id >= MLX5_FLOW_MARK_MAX) 1775 return rte_flow_error_set(error, EINVAL, 1776 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1777 &mark->id, 1778 "mark id must in 0 <= id < " 1779 RTE_STR(MLX5_FLOW_MARK_MAX)); 1780 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1781 return rte_flow_error_set(error, EINVAL, 1782 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1783 "can't flag and mark in same flow"); 1784 if (action_flags & MLX5_FLOW_ACTION_MARK) 1785 return rte_flow_error_set(error, EINVAL, 1786 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1787 "can't have 2 mark actions in same" 1788 " flow"); 1789 if (attr->egress) 1790 return rte_flow_error_set(error, ENOTSUP, 1791 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1792 "mark action not supported for " 1793 "egress"); 1794 return 0; 1795 } 1796 1797 /* 1798 * Validate the drop action. 1799 * 1800 * @param[in] action_flags 1801 * Bit-fields that holds the actions detected until now. 1802 * @param[in] attr 1803 * Attributes of flow that includes this action. 1804 * @param[out] error 1805 * Pointer to error structure. 1806 * 1807 * @return 1808 * 0 on success, a negative errno value otherwise and rte_errno is set. 1809 */ 1810 int 1811 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1812 const struct rte_flow_attr *attr, 1813 struct rte_flow_error *error) 1814 { 1815 if (attr->egress) 1816 return rte_flow_error_set(error, ENOTSUP, 1817 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1818 "drop action not supported for " 1819 "egress"); 1820 return 0; 1821 } 1822 1823 /* 1824 * Validate the queue action. 1825 * 1826 * @param[in] action 1827 * Pointer to the queue action. 1828 * @param[in] action_flags 1829 * Bit-fields that holds the actions detected until now. 1830 * @param[in] dev 1831 * Pointer to the Ethernet device structure. 1832 * @param[in] attr 1833 * Attributes of flow that includes this action. 1834 * @param[out] error 1835 * Pointer to error structure. 1836 * 1837 * @return 1838 * 0 on success, a negative errno value otherwise and rte_errno is set. 1839 */ 1840 int 1841 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1842 uint64_t action_flags, 1843 struct rte_eth_dev *dev, 1844 const struct rte_flow_attr *attr, 1845 struct rte_flow_error *error) 1846 { 1847 struct mlx5_priv *priv = dev->data->dev_private; 1848 const struct rte_flow_action_queue *queue = action->conf; 1849 1850 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1851 return rte_flow_error_set(error, EINVAL, 1852 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1853 "can't have 2 fate actions in" 1854 " same flow"); 1855 if (attr->egress) 1856 return rte_flow_error_set(error, ENOTSUP, 1857 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1858 "queue action not supported for egress."); 1859 if (mlx5_is_external_rxq(dev, queue->index)) 1860 return 0; 1861 if (!priv->rxqs_n) 1862 return rte_flow_error_set(error, EINVAL, 1863 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1864 NULL, "No Rx queues configured"); 1865 if (queue->index >= priv->rxqs_n) 1866 return rte_flow_error_set(error, EINVAL, 1867 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1868 &queue->index, 1869 "queue index out of range"); 1870 if (mlx5_rxq_get(dev, queue->index) == NULL) 1871 return rte_flow_error_set(error, EINVAL, 1872 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1873 &queue->index, 1874 "queue is not configured"); 1875 return 0; 1876 } 1877 1878 /** 1879 * Validate queue numbers for device RSS. 1880 * 1881 * @param[in] dev 1882 * Configured device. 1883 * @param[in] queues 1884 * Array of queue numbers. 1885 * @param[in] queues_n 1886 * Size of the @p queues array. 1887 * @param[out] error 1888 * On error, filled with a textual error description. 1889 * @param[out] queue_idx 1890 * On error, filled with an offending queue index in @p queues array. 1891 * 1892 * @return 1893 * 0 on success, a negative errno code on error. 1894 */ 1895 static int 1896 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1897 const uint16_t *queues, uint32_t queues_n, 1898 const char **error, uint32_t *queue_idx) 1899 { 1900 const struct mlx5_priv *priv = dev->data->dev_private; 1901 bool is_hairpin = false; 1902 bool is_ext_rss = false; 1903 uint32_t i; 1904 1905 for (i = 0; i != queues_n; ++i) { 1906 struct mlx5_rxq_ctrl *rxq_ctrl; 1907 1908 if (mlx5_is_external_rxq(dev, queues[0])) { 1909 is_ext_rss = true; 1910 continue; 1911 } 1912 if (is_ext_rss) { 1913 *error = "Combining external and regular RSS queues is not supported"; 1914 *queue_idx = i; 1915 return -ENOTSUP; 1916 } 1917 if (queues[i] >= priv->rxqs_n) { 1918 *error = "queue index out of range"; 1919 *queue_idx = i; 1920 return -EINVAL; 1921 } 1922 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 1923 if (rxq_ctrl == NULL) { 1924 *error = "queue is not configured"; 1925 *queue_idx = i; 1926 return -EINVAL; 1927 } 1928 if (i == 0 && rxq_ctrl->is_hairpin) 1929 is_hairpin = true; 1930 if (is_hairpin != rxq_ctrl->is_hairpin) { 1931 *error = "combining hairpin and regular RSS queues is not supported"; 1932 *queue_idx = i; 1933 return -ENOTSUP; 1934 } 1935 } 1936 return 0; 1937 } 1938 1939 /* 1940 * Validate the rss action. 1941 * 1942 * @param[in] dev 1943 * Pointer to the Ethernet device structure. 1944 * @param[in] action 1945 * Pointer to the queue action. 1946 * @param[out] error 1947 * Pointer to error structure. 1948 * 1949 * @return 1950 * 0 on success, a negative errno value otherwise and rte_errno is set. 1951 */ 1952 int 1953 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1954 const struct rte_flow_action *action, 1955 struct rte_flow_error *error) 1956 { 1957 struct mlx5_priv *priv = dev->data->dev_private; 1958 const struct rte_flow_action_rss *rss = action->conf; 1959 int ret; 1960 const char *message; 1961 uint32_t queue_idx; 1962 1963 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1964 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1965 return rte_flow_error_set(error, ENOTSUP, 1966 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1967 &rss->func, 1968 "RSS hash function not supported"); 1969 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1970 if (rss->level > 2) 1971 #else 1972 if (rss->level > 1) 1973 #endif 1974 return rte_flow_error_set(error, ENOTSUP, 1975 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1976 &rss->level, 1977 "tunnel RSS is not supported"); 1978 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1979 if (rss->key_len == 0 && rss->key != NULL) 1980 return rte_flow_error_set(error, ENOTSUP, 1981 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1982 &rss->key_len, 1983 "RSS hash key length 0"); 1984 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1985 return rte_flow_error_set(error, ENOTSUP, 1986 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1987 &rss->key_len, 1988 "RSS hash key too small"); 1989 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 1990 return rte_flow_error_set(error, ENOTSUP, 1991 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1992 &rss->key_len, 1993 "RSS hash key too large"); 1994 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 1995 return rte_flow_error_set(error, ENOTSUP, 1996 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1997 &rss->queue_num, 1998 "number of queues too large"); 1999 if (rss->types & MLX5_RSS_HF_MASK) 2000 return rte_flow_error_set(error, ENOTSUP, 2001 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2002 &rss->types, 2003 "some RSS protocols are not" 2004 " supported"); 2005 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2006 !(rss->types & RTE_ETH_RSS_IP)) 2007 return rte_flow_error_set(error, EINVAL, 2008 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2009 "L3 partial RSS requested but L3 RSS" 2010 " type not specified"); 2011 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2012 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2013 return rte_flow_error_set(error, EINVAL, 2014 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2015 "L4 partial RSS requested but L4 RSS" 2016 " type not specified"); 2017 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2018 return rte_flow_error_set(error, EINVAL, 2019 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2020 NULL, "No Rx queues configured"); 2021 if (!rss->queue_num) 2022 return rte_flow_error_set(error, EINVAL, 2023 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2024 NULL, "No queues configured"); 2025 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2026 &message, &queue_idx); 2027 if (ret != 0) { 2028 return rte_flow_error_set(error, -ret, 2029 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2030 &rss->queue[queue_idx], message); 2031 } 2032 return 0; 2033 } 2034 2035 /* 2036 * Validate the rss action. 2037 * 2038 * @param[in] action 2039 * Pointer to the queue action. 2040 * @param[in] action_flags 2041 * Bit-fields that holds the actions detected until now. 2042 * @param[in] dev 2043 * Pointer to the Ethernet device structure. 2044 * @param[in] attr 2045 * Attributes of flow that includes this action. 2046 * @param[in] item_flags 2047 * Items that were detected. 2048 * @param[out] error 2049 * Pointer to error structure. 2050 * 2051 * @return 2052 * 0 on success, a negative errno value otherwise and rte_errno is set. 2053 */ 2054 int 2055 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2056 uint64_t action_flags, 2057 struct rte_eth_dev *dev, 2058 const struct rte_flow_attr *attr, 2059 uint64_t item_flags, 2060 struct rte_flow_error *error) 2061 { 2062 const struct rte_flow_action_rss *rss = action->conf; 2063 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2064 int ret; 2065 2066 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2067 return rte_flow_error_set(error, EINVAL, 2068 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2069 "can't have 2 fate actions" 2070 " in same flow"); 2071 ret = mlx5_validate_action_rss(dev, action, error); 2072 if (ret) 2073 return ret; 2074 if (attr->egress) 2075 return rte_flow_error_set(error, ENOTSUP, 2076 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2077 "rss action not supported for " 2078 "egress"); 2079 if (rss->level > 1 && !tunnel) 2080 return rte_flow_error_set(error, EINVAL, 2081 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2082 "inner RSS is not supported for " 2083 "non-tunnel flows"); 2084 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2085 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2086 return rte_flow_error_set(error, EINVAL, 2087 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2088 "RSS on eCPRI is not supported now"); 2089 } 2090 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2091 !(item_flags & 2092 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2093 rss->level > 1) 2094 return rte_flow_error_set(error, EINVAL, 2095 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2096 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2097 return 0; 2098 } 2099 2100 /* 2101 * Validate the default miss action. 2102 * 2103 * @param[in] action_flags 2104 * Bit-fields that holds the actions detected until now. 2105 * @param[out] error 2106 * Pointer to error structure. 2107 * 2108 * @return 2109 * 0 on success, a negative errno value otherwise and rte_errno is set. 2110 */ 2111 int 2112 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2113 const struct rte_flow_attr *attr, 2114 struct rte_flow_error *error) 2115 { 2116 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2117 return rte_flow_error_set(error, EINVAL, 2118 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2119 "can't have 2 fate actions in" 2120 " same flow"); 2121 if (attr->egress) 2122 return rte_flow_error_set(error, ENOTSUP, 2123 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2124 "default miss action not supported " 2125 "for egress"); 2126 if (attr->group) 2127 return rte_flow_error_set(error, ENOTSUP, 2128 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2129 "only group 0 is supported"); 2130 if (attr->transfer) 2131 return rte_flow_error_set(error, ENOTSUP, 2132 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2133 NULL, "transfer is not supported"); 2134 return 0; 2135 } 2136 2137 /* 2138 * Validate the count action. 2139 * 2140 * @param[in] dev 2141 * Pointer to the Ethernet device structure. 2142 * @param[in] attr 2143 * Attributes of flow that includes this action. 2144 * @param[out] error 2145 * Pointer to error structure. 2146 * 2147 * @return 2148 * 0 on success, a negative errno value otherwise and rte_errno is set. 2149 */ 2150 int 2151 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2152 const struct rte_flow_attr *attr, 2153 struct rte_flow_error *error) 2154 { 2155 if (attr->egress) 2156 return rte_flow_error_set(error, ENOTSUP, 2157 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2158 "count action not supported for " 2159 "egress"); 2160 return 0; 2161 } 2162 2163 /* 2164 * Validate the ASO CT action. 2165 * 2166 * @param[in] dev 2167 * Pointer to the Ethernet device structure. 2168 * @param[in] conntrack 2169 * Pointer to the CT action profile. 2170 * @param[out] error 2171 * Pointer to error structure. 2172 * 2173 * @return 2174 * 0 on success, a negative errno value otherwise and rte_errno is set. 2175 */ 2176 int 2177 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2178 const struct rte_flow_action_conntrack *conntrack, 2179 struct rte_flow_error *error) 2180 { 2181 RTE_SET_USED(dev); 2182 2183 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2184 return rte_flow_error_set(error, EINVAL, 2185 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2186 "Invalid CT state"); 2187 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2188 return rte_flow_error_set(error, EINVAL, 2189 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2190 "Invalid last TCP packet flag"); 2191 return 0; 2192 } 2193 2194 /** 2195 * Verify the @p attributes will be correctly understood by the NIC and store 2196 * them in the @p flow if everything is correct. 2197 * 2198 * @param[in] dev 2199 * Pointer to the Ethernet device structure. 2200 * @param[in] attributes 2201 * Pointer to flow attributes 2202 * @param[out] error 2203 * Pointer to error structure. 2204 * 2205 * @return 2206 * 0 on success, a negative errno value otherwise and rte_errno is set. 2207 */ 2208 int 2209 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2210 const struct rte_flow_attr *attributes, 2211 struct rte_flow_error *error) 2212 { 2213 struct mlx5_priv *priv = dev->data->dev_private; 2214 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2215 2216 if (attributes->group) 2217 return rte_flow_error_set(error, ENOTSUP, 2218 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2219 NULL, "groups is not supported"); 2220 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2221 attributes->priority >= priority_max) 2222 return rte_flow_error_set(error, ENOTSUP, 2223 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2224 NULL, "priority out of range"); 2225 if (attributes->egress) 2226 return rte_flow_error_set(error, ENOTSUP, 2227 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2228 "egress is not supported"); 2229 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2230 return rte_flow_error_set(error, ENOTSUP, 2231 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2232 NULL, "transfer is not supported"); 2233 if (!attributes->ingress) 2234 return rte_flow_error_set(error, EINVAL, 2235 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2236 NULL, 2237 "ingress attribute is mandatory"); 2238 return 0; 2239 } 2240 2241 /** 2242 * Validate ICMP6 item. 2243 * 2244 * @param[in] item 2245 * Item specification. 2246 * @param[in] item_flags 2247 * Bit-fields that holds the items detected until now. 2248 * @param[in] ext_vlan_sup 2249 * Whether extended VLAN features are supported or not. 2250 * @param[out] error 2251 * Pointer to error structure. 2252 * 2253 * @return 2254 * 0 on success, a negative errno value otherwise and rte_errno is set. 2255 */ 2256 int 2257 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2258 uint64_t item_flags, 2259 uint8_t target_protocol, 2260 struct rte_flow_error *error) 2261 { 2262 const struct rte_flow_item_icmp6 *mask = item->mask; 2263 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2264 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2265 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2266 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2267 MLX5_FLOW_LAYER_OUTER_L4; 2268 int ret; 2269 2270 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2271 return rte_flow_error_set(error, EINVAL, 2272 RTE_FLOW_ERROR_TYPE_ITEM, item, 2273 "protocol filtering not compatible" 2274 " with ICMP6 layer"); 2275 if (!(item_flags & l3m)) 2276 return rte_flow_error_set(error, EINVAL, 2277 RTE_FLOW_ERROR_TYPE_ITEM, item, 2278 "IPv6 is mandatory to filter on" 2279 " ICMP6"); 2280 if (item_flags & l4m) 2281 return rte_flow_error_set(error, EINVAL, 2282 RTE_FLOW_ERROR_TYPE_ITEM, item, 2283 "multiple L4 layers not supported"); 2284 if (!mask) 2285 mask = &rte_flow_item_icmp6_mask; 2286 ret = mlx5_flow_item_acceptable 2287 (item, (const uint8_t *)mask, 2288 (const uint8_t *)&rte_flow_item_icmp6_mask, 2289 sizeof(struct rte_flow_item_icmp6), 2290 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2291 if (ret < 0) 2292 return ret; 2293 return 0; 2294 } 2295 2296 /** 2297 * Validate ICMP item. 2298 * 2299 * @param[in] item 2300 * Item specification. 2301 * @param[in] item_flags 2302 * Bit-fields that holds the items detected until now. 2303 * @param[out] error 2304 * Pointer to error structure. 2305 * 2306 * @return 2307 * 0 on success, a negative errno value otherwise and rte_errno is set. 2308 */ 2309 int 2310 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2311 uint64_t item_flags, 2312 uint8_t target_protocol, 2313 struct rte_flow_error *error) 2314 { 2315 const struct rte_flow_item_icmp *mask = item->mask; 2316 const struct rte_flow_item_icmp nic_mask = { 2317 .hdr.icmp_type = 0xff, 2318 .hdr.icmp_code = 0xff, 2319 .hdr.icmp_ident = RTE_BE16(0xffff), 2320 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2321 }; 2322 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2323 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2324 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2325 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2326 MLX5_FLOW_LAYER_OUTER_L4; 2327 int ret; 2328 2329 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2330 return rte_flow_error_set(error, EINVAL, 2331 RTE_FLOW_ERROR_TYPE_ITEM, item, 2332 "protocol filtering not compatible" 2333 " with ICMP layer"); 2334 if (!(item_flags & l3m)) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "IPv4 is mandatory to filter" 2338 " on ICMP"); 2339 if (item_flags & l4m) 2340 return rte_flow_error_set(error, EINVAL, 2341 RTE_FLOW_ERROR_TYPE_ITEM, item, 2342 "multiple L4 layers not supported"); 2343 if (!mask) 2344 mask = &nic_mask; 2345 ret = mlx5_flow_item_acceptable 2346 (item, (const uint8_t *)mask, 2347 (const uint8_t *)&nic_mask, 2348 sizeof(struct rte_flow_item_icmp), 2349 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2350 if (ret < 0) 2351 return ret; 2352 return 0; 2353 } 2354 2355 /** 2356 * Validate Ethernet item. 2357 * 2358 * @param[in] item 2359 * Item specification. 2360 * @param[in] item_flags 2361 * Bit-fields that holds the items detected until now. 2362 * @param[out] error 2363 * Pointer to error structure. 2364 * 2365 * @return 2366 * 0 on success, a negative errno value otherwise and rte_errno is set. 2367 */ 2368 int 2369 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2370 uint64_t item_flags, bool ext_vlan_sup, 2371 struct rte_flow_error *error) 2372 { 2373 const struct rte_flow_item_eth *mask = item->mask; 2374 const struct rte_flow_item_eth nic_mask = { 2375 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2376 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2377 .type = RTE_BE16(0xffff), 2378 .has_vlan = ext_vlan_sup ? 1 : 0, 2379 }; 2380 int ret; 2381 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2382 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2383 MLX5_FLOW_LAYER_OUTER_L2; 2384 2385 if (item_flags & ethm) 2386 return rte_flow_error_set(error, ENOTSUP, 2387 RTE_FLOW_ERROR_TYPE_ITEM, item, 2388 "multiple L2 layers not supported"); 2389 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2390 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2391 return rte_flow_error_set(error, EINVAL, 2392 RTE_FLOW_ERROR_TYPE_ITEM, item, 2393 "L2 layer should not follow " 2394 "L3 layers"); 2395 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2396 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2397 return rte_flow_error_set(error, EINVAL, 2398 RTE_FLOW_ERROR_TYPE_ITEM, item, 2399 "L2 layer should not follow VLAN"); 2400 if (item_flags & MLX5_FLOW_LAYER_GTP) 2401 return rte_flow_error_set(error, EINVAL, 2402 RTE_FLOW_ERROR_TYPE_ITEM, item, 2403 "L2 layer should not follow GTP"); 2404 if (!mask) 2405 mask = &rte_flow_item_eth_mask; 2406 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2407 (const uint8_t *)&nic_mask, 2408 sizeof(struct rte_flow_item_eth), 2409 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2410 return ret; 2411 } 2412 2413 /** 2414 * Validate VLAN item. 2415 * 2416 * @param[in] item 2417 * Item specification. 2418 * @param[in] item_flags 2419 * Bit-fields that holds the items detected until now. 2420 * @param[in] dev 2421 * Ethernet device flow is being created on. 2422 * @param[out] error 2423 * Pointer to error structure. 2424 * 2425 * @return 2426 * 0 on success, a negative errno value otherwise and rte_errno is set. 2427 */ 2428 int 2429 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2430 uint64_t item_flags, 2431 struct rte_eth_dev *dev, 2432 struct rte_flow_error *error) 2433 { 2434 const struct rte_flow_item_vlan *spec = item->spec; 2435 const struct rte_flow_item_vlan *mask = item->mask; 2436 const struct rte_flow_item_vlan nic_mask = { 2437 .tci = RTE_BE16(UINT16_MAX), 2438 .inner_type = RTE_BE16(UINT16_MAX), 2439 }; 2440 uint16_t vlan_tag = 0; 2441 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2442 int ret; 2443 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2444 MLX5_FLOW_LAYER_INNER_L4) : 2445 (MLX5_FLOW_LAYER_OUTER_L3 | 2446 MLX5_FLOW_LAYER_OUTER_L4); 2447 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2448 MLX5_FLOW_LAYER_OUTER_VLAN; 2449 2450 if (item_flags & vlanm) 2451 return rte_flow_error_set(error, EINVAL, 2452 RTE_FLOW_ERROR_TYPE_ITEM, item, 2453 "multiple VLAN layers not supported"); 2454 else if ((item_flags & l34m) != 0) 2455 return rte_flow_error_set(error, EINVAL, 2456 RTE_FLOW_ERROR_TYPE_ITEM, item, 2457 "VLAN cannot follow L3/L4 layer"); 2458 if (!mask) 2459 mask = &rte_flow_item_vlan_mask; 2460 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2461 (const uint8_t *)&nic_mask, 2462 sizeof(struct rte_flow_item_vlan), 2463 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2464 if (ret) 2465 return ret; 2466 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2467 struct mlx5_priv *priv = dev->data->dev_private; 2468 2469 if (priv->vmwa_context) { 2470 /* 2471 * Non-NULL context means we have a virtual machine 2472 * and SR-IOV enabled, we have to create VLAN interface 2473 * to make hypervisor to setup E-Switch vport 2474 * context correctly. We avoid creating the multiple 2475 * VLAN interfaces, so we cannot support VLAN tag mask. 2476 */ 2477 return rte_flow_error_set(error, EINVAL, 2478 RTE_FLOW_ERROR_TYPE_ITEM, 2479 item, 2480 "VLAN tag mask is not" 2481 " supported in virtual" 2482 " environment"); 2483 } 2484 } 2485 if (spec) { 2486 vlan_tag = spec->tci; 2487 vlan_tag &= mask->tci; 2488 } 2489 /* 2490 * From verbs perspective an empty VLAN is equivalent 2491 * to a packet without VLAN layer. 2492 */ 2493 if (!vlan_tag) 2494 return rte_flow_error_set(error, EINVAL, 2495 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2496 item->spec, 2497 "VLAN cannot be empty"); 2498 return 0; 2499 } 2500 2501 /** 2502 * Validate IPV4 item. 2503 * 2504 * @param[in] item 2505 * Item specification. 2506 * @param[in] item_flags 2507 * Bit-fields that holds the items detected until now. 2508 * @param[in] last_item 2509 * Previous validated item in the pattern items. 2510 * @param[in] ether_type 2511 * Type in the ethernet layer header (including dot1q). 2512 * @param[in] acc_mask 2513 * Acceptable mask, if NULL default internal default mask 2514 * will be used to check whether item fields are supported. 2515 * @param[in] range_accepted 2516 * True if range of values is accepted for specific fields, false otherwise. 2517 * @param[out] error 2518 * Pointer to error structure. 2519 * 2520 * @return 2521 * 0 on success, a negative errno value otherwise and rte_errno is set. 2522 */ 2523 int 2524 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2525 uint64_t item_flags, 2526 uint64_t last_item, 2527 uint16_t ether_type, 2528 const struct rte_flow_item_ipv4 *acc_mask, 2529 bool range_accepted, 2530 struct rte_flow_error *error) 2531 { 2532 const struct rte_flow_item_ipv4 *mask = item->mask; 2533 const struct rte_flow_item_ipv4 *spec = item->spec; 2534 const struct rte_flow_item_ipv4 nic_mask = { 2535 .hdr = { 2536 .src_addr = RTE_BE32(0xffffffff), 2537 .dst_addr = RTE_BE32(0xffffffff), 2538 .type_of_service = 0xff, 2539 .next_proto_id = 0xff, 2540 }, 2541 }; 2542 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2543 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2544 MLX5_FLOW_LAYER_OUTER_L3; 2545 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2546 MLX5_FLOW_LAYER_OUTER_L4; 2547 int ret; 2548 uint8_t next_proto = 0xFF; 2549 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2550 MLX5_FLOW_LAYER_OUTER_VLAN | 2551 MLX5_FLOW_LAYER_INNER_VLAN); 2552 2553 if ((last_item & l2_vlan) && ether_type && 2554 ether_type != RTE_ETHER_TYPE_IPV4) 2555 return rte_flow_error_set(error, EINVAL, 2556 RTE_FLOW_ERROR_TYPE_ITEM, item, 2557 "IPv4 cannot follow L2/VLAN layer " 2558 "which ether type is not IPv4"); 2559 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2560 if (mask && spec) 2561 next_proto = mask->hdr.next_proto_id & 2562 spec->hdr.next_proto_id; 2563 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2564 return rte_flow_error_set(error, EINVAL, 2565 RTE_FLOW_ERROR_TYPE_ITEM, 2566 item, 2567 "multiple tunnel " 2568 "not supported"); 2569 } 2570 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2571 return rte_flow_error_set(error, EINVAL, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "wrong tunnel type - IPv6 specified " 2574 "but IPv4 item provided"); 2575 if (item_flags & l3m) 2576 return rte_flow_error_set(error, ENOTSUP, 2577 RTE_FLOW_ERROR_TYPE_ITEM, item, 2578 "multiple L3 layers not supported"); 2579 else if (item_flags & l4m) 2580 return rte_flow_error_set(error, EINVAL, 2581 RTE_FLOW_ERROR_TYPE_ITEM, item, 2582 "L3 cannot follow an L4 layer."); 2583 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2584 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2585 return rte_flow_error_set(error, EINVAL, 2586 RTE_FLOW_ERROR_TYPE_ITEM, item, 2587 "L3 cannot follow an NVGRE layer."); 2588 if (!mask) 2589 mask = &rte_flow_item_ipv4_mask; 2590 else if (mask->hdr.next_proto_id != 0 && 2591 mask->hdr.next_proto_id != 0xff) 2592 return rte_flow_error_set(error, EINVAL, 2593 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2594 "partial mask is not supported" 2595 " for protocol"); 2596 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2597 acc_mask ? (const uint8_t *)acc_mask 2598 : (const uint8_t *)&nic_mask, 2599 sizeof(struct rte_flow_item_ipv4), 2600 range_accepted, error); 2601 if (ret < 0) 2602 return ret; 2603 return 0; 2604 } 2605 2606 /** 2607 * Validate IPV6 item. 2608 * 2609 * @param[in] item 2610 * Item specification. 2611 * @param[in] item_flags 2612 * Bit-fields that holds the items detected until now. 2613 * @param[in] last_item 2614 * Previous validated item in the pattern items. 2615 * @param[in] ether_type 2616 * Type in the ethernet layer header (including dot1q). 2617 * @param[in] acc_mask 2618 * Acceptable mask, if NULL default internal default mask 2619 * will be used to check whether item fields are supported. 2620 * @param[out] error 2621 * Pointer to error structure. 2622 * 2623 * @return 2624 * 0 on success, a negative errno value otherwise and rte_errno is set. 2625 */ 2626 int 2627 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2628 uint64_t item_flags, 2629 uint64_t last_item, 2630 uint16_t ether_type, 2631 const struct rte_flow_item_ipv6 *acc_mask, 2632 struct rte_flow_error *error) 2633 { 2634 const struct rte_flow_item_ipv6 *mask = item->mask; 2635 const struct rte_flow_item_ipv6 *spec = item->spec; 2636 const struct rte_flow_item_ipv6 nic_mask = { 2637 .hdr = { 2638 .src_addr = 2639 "\xff\xff\xff\xff\xff\xff\xff\xff" 2640 "\xff\xff\xff\xff\xff\xff\xff\xff", 2641 .dst_addr = 2642 "\xff\xff\xff\xff\xff\xff\xff\xff" 2643 "\xff\xff\xff\xff\xff\xff\xff\xff", 2644 .vtc_flow = RTE_BE32(0xffffffff), 2645 .proto = 0xff, 2646 }, 2647 }; 2648 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2649 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2650 MLX5_FLOW_LAYER_OUTER_L3; 2651 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2652 MLX5_FLOW_LAYER_OUTER_L4; 2653 int ret; 2654 uint8_t next_proto = 0xFF; 2655 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2656 MLX5_FLOW_LAYER_OUTER_VLAN | 2657 MLX5_FLOW_LAYER_INNER_VLAN); 2658 2659 if ((last_item & l2_vlan) && ether_type && 2660 ether_type != RTE_ETHER_TYPE_IPV6) 2661 return rte_flow_error_set(error, EINVAL, 2662 RTE_FLOW_ERROR_TYPE_ITEM, item, 2663 "IPv6 cannot follow L2/VLAN layer " 2664 "which ether type is not IPv6"); 2665 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2666 next_proto = spec->hdr.proto; 2667 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2668 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2669 return rte_flow_error_set(error, EINVAL, 2670 RTE_FLOW_ERROR_TYPE_ITEM, 2671 item, 2672 "multiple tunnel " 2673 "not supported"); 2674 } 2675 if (next_proto == IPPROTO_HOPOPTS || 2676 next_proto == IPPROTO_ROUTING || 2677 next_proto == IPPROTO_FRAGMENT || 2678 next_proto == IPPROTO_ESP || 2679 next_proto == IPPROTO_AH || 2680 next_proto == IPPROTO_DSTOPTS) 2681 return rte_flow_error_set(error, EINVAL, 2682 RTE_FLOW_ERROR_TYPE_ITEM, item, 2683 "IPv6 proto (next header) should " 2684 "not be set as extension header"); 2685 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2686 return rte_flow_error_set(error, EINVAL, 2687 RTE_FLOW_ERROR_TYPE_ITEM, item, 2688 "wrong tunnel type - IPv4 specified " 2689 "but IPv6 item provided"); 2690 if (item_flags & l3m) 2691 return rte_flow_error_set(error, ENOTSUP, 2692 RTE_FLOW_ERROR_TYPE_ITEM, item, 2693 "multiple L3 layers not supported"); 2694 else if (item_flags & l4m) 2695 return rte_flow_error_set(error, EINVAL, 2696 RTE_FLOW_ERROR_TYPE_ITEM, item, 2697 "L3 cannot follow an L4 layer."); 2698 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2699 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2700 return rte_flow_error_set(error, EINVAL, 2701 RTE_FLOW_ERROR_TYPE_ITEM, item, 2702 "L3 cannot follow an NVGRE layer."); 2703 if (!mask) 2704 mask = &rte_flow_item_ipv6_mask; 2705 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2706 acc_mask ? (const uint8_t *)acc_mask 2707 : (const uint8_t *)&nic_mask, 2708 sizeof(struct rte_flow_item_ipv6), 2709 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2710 if (ret < 0) 2711 return ret; 2712 return 0; 2713 } 2714 2715 /** 2716 * Validate UDP item. 2717 * 2718 * @param[in] item 2719 * Item specification. 2720 * @param[in] item_flags 2721 * Bit-fields that holds the items detected until now. 2722 * @param[in] target_protocol 2723 * The next protocol in the previous item. 2724 * @param[in] flow_mask 2725 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2726 * @param[out] error 2727 * Pointer to error structure. 2728 * 2729 * @return 2730 * 0 on success, a negative errno value otherwise and rte_errno is set. 2731 */ 2732 int 2733 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2734 uint64_t item_flags, 2735 uint8_t target_protocol, 2736 struct rte_flow_error *error) 2737 { 2738 const struct rte_flow_item_udp *mask = item->mask; 2739 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2740 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2741 MLX5_FLOW_LAYER_OUTER_L3; 2742 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2743 MLX5_FLOW_LAYER_OUTER_L4; 2744 int ret; 2745 2746 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2747 return rte_flow_error_set(error, EINVAL, 2748 RTE_FLOW_ERROR_TYPE_ITEM, item, 2749 "protocol filtering not compatible" 2750 " with UDP layer"); 2751 if (!(item_flags & l3m)) 2752 return rte_flow_error_set(error, EINVAL, 2753 RTE_FLOW_ERROR_TYPE_ITEM, item, 2754 "L3 is mandatory to filter on L4"); 2755 if (item_flags & l4m) 2756 return rte_flow_error_set(error, EINVAL, 2757 RTE_FLOW_ERROR_TYPE_ITEM, item, 2758 "multiple L4 layers not supported"); 2759 if (!mask) 2760 mask = &rte_flow_item_udp_mask; 2761 ret = mlx5_flow_item_acceptable 2762 (item, (const uint8_t *)mask, 2763 (const uint8_t *)&rte_flow_item_udp_mask, 2764 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2765 error); 2766 if (ret < 0) 2767 return ret; 2768 return 0; 2769 } 2770 2771 /** 2772 * Validate TCP item. 2773 * 2774 * @param[in] item 2775 * Item specification. 2776 * @param[in] item_flags 2777 * Bit-fields that holds the items detected until now. 2778 * @param[in] target_protocol 2779 * The next protocol in the previous item. 2780 * @param[out] error 2781 * Pointer to error structure. 2782 * 2783 * @return 2784 * 0 on success, a negative errno value otherwise and rte_errno is set. 2785 */ 2786 int 2787 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2788 uint64_t item_flags, 2789 uint8_t target_protocol, 2790 const struct rte_flow_item_tcp *flow_mask, 2791 struct rte_flow_error *error) 2792 { 2793 const struct rte_flow_item_tcp *mask = item->mask; 2794 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2795 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2796 MLX5_FLOW_LAYER_OUTER_L3; 2797 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2798 MLX5_FLOW_LAYER_OUTER_L4; 2799 int ret; 2800 2801 MLX5_ASSERT(flow_mask); 2802 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2803 return rte_flow_error_set(error, EINVAL, 2804 RTE_FLOW_ERROR_TYPE_ITEM, item, 2805 "protocol filtering not compatible" 2806 " with TCP layer"); 2807 if (!(item_flags & l3m)) 2808 return rte_flow_error_set(error, EINVAL, 2809 RTE_FLOW_ERROR_TYPE_ITEM, item, 2810 "L3 is mandatory to filter on L4"); 2811 if (item_flags & l4m) 2812 return rte_flow_error_set(error, EINVAL, 2813 RTE_FLOW_ERROR_TYPE_ITEM, item, 2814 "multiple L4 layers not supported"); 2815 if (!mask) 2816 mask = &rte_flow_item_tcp_mask; 2817 ret = mlx5_flow_item_acceptable 2818 (item, (const uint8_t *)mask, 2819 (const uint8_t *)flow_mask, 2820 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2821 error); 2822 if (ret < 0) 2823 return ret; 2824 return 0; 2825 } 2826 2827 /** 2828 * Validate VXLAN item. 2829 * 2830 * @param[in] dev 2831 * Pointer to the Ethernet device structure. 2832 * @param[in] udp_dport 2833 * UDP destination port 2834 * @param[in] item 2835 * Item specification. 2836 * @param[in] item_flags 2837 * Bit-fields that holds the items detected until now. 2838 * @param root 2839 * Whether action is on root table. 2840 * @param[out] error 2841 * Pointer to error structure. 2842 * 2843 * @return 2844 * 0 on success, a negative errno value otherwise and rte_errno is set. 2845 */ 2846 int 2847 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2848 uint16_t udp_dport, 2849 const struct rte_flow_item *item, 2850 uint64_t item_flags, 2851 bool root, 2852 struct rte_flow_error *error) 2853 { 2854 const struct rte_flow_item_vxlan *spec = item->spec; 2855 const struct rte_flow_item_vxlan *mask = item->mask; 2856 int ret; 2857 struct mlx5_priv *priv = dev->data->dev_private; 2858 union vni { 2859 uint32_t vlan_id; 2860 uint8_t vni[4]; 2861 } id = { .vlan_id = 0, }; 2862 const struct rte_flow_item_vxlan nic_mask = { 2863 .vni = "\xff\xff\xff", 2864 .rsvd1 = 0xff, 2865 }; 2866 const struct rte_flow_item_vxlan *valid_mask; 2867 2868 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2869 return rte_flow_error_set(error, ENOTSUP, 2870 RTE_FLOW_ERROR_TYPE_ITEM, item, 2871 "multiple tunnel layers not" 2872 " supported"); 2873 valid_mask = &rte_flow_item_vxlan_mask; 2874 /* 2875 * Verify only UDPv4 is present as defined in 2876 * https://tools.ietf.org/html/rfc7348 2877 */ 2878 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2879 return rte_flow_error_set(error, EINVAL, 2880 RTE_FLOW_ERROR_TYPE_ITEM, item, 2881 "no outer UDP layer found"); 2882 if (!mask) 2883 mask = &rte_flow_item_vxlan_mask; 2884 2885 if (priv->sh->steering_format_version != 2886 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2887 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2888 /* non-root table */ 2889 if (!root && priv->sh->misc5_cap) 2890 valid_mask = &nic_mask; 2891 /* Group zero in NIC domain */ 2892 if (!root && priv->sh->tunnel_header_0_1) 2893 valid_mask = &nic_mask; 2894 } 2895 ret = mlx5_flow_item_acceptable 2896 (item, (const uint8_t *)mask, 2897 (const uint8_t *)valid_mask, 2898 sizeof(struct rte_flow_item_vxlan), 2899 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2900 if (ret < 0) 2901 return ret; 2902 if (spec) { 2903 memcpy(&id.vni[1], spec->vni, 3); 2904 memcpy(&id.vni[1], mask->vni, 3); 2905 } 2906 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2907 return rte_flow_error_set(error, ENOTSUP, 2908 RTE_FLOW_ERROR_TYPE_ITEM, item, 2909 "VXLAN tunnel must be fully defined"); 2910 return 0; 2911 } 2912 2913 /** 2914 * Validate VXLAN_GPE item. 2915 * 2916 * @param[in] item 2917 * Item specification. 2918 * @param[in] item_flags 2919 * Bit-fields that holds the items detected until now. 2920 * @param[in] priv 2921 * Pointer to the private data structure. 2922 * @param[in] target_protocol 2923 * The next protocol in the previous item. 2924 * @param[out] error 2925 * Pointer to error structure. 2926 * 2927 * @return 2928 * 0 on success, a negative errno value otherwise and rte_errno is set. 2929 */ 2930 int 2931 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2932 uint64_t item_flags, 2933 struct rte_eth_dev *dev, 2934 struct rte_flow_error *error) 2935 { 2936 struct mlx5_priv *priv = dev->data->dev_private; 2937 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2938 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2939 int ret; 2940 union vni { 2941 uint32_t vlan_id; 2942 uint8_t vni[4]; 2943 } id = { .vlan_id = 0, }; 2944 2945 if (!priv->sh->config.l3_vxlan_en) 2946 return rte_flow_error_set(error, ENOTSUP, 2947 RTE_FLOW_ERROR_TYPE_ITEM, item, 2948 "L3 VXLAN is not enabled by device" 2949 " parameter and/or not configured in" 2950 " firmware"); 2951 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2952 return rte_flow_error_set(error, ENOTSUP, 2953 RTE_FLOW_ERROR_TYPE_ITEM, item, 2954 "multiple tunnel layers not" 2955 " supported"); 2956 /* 2957 * Verify only UDPv4 is present as defined in 2958 * https://tools.ietf.org/html/rfc7348 2959 */ 2960 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2961 return rte_flow_error_set(error, EINVAL, 2962 RTE_FLOW_ERROR_TYPE_ITEM, item, 2963 "no outer UDP layer found"); 2964 if (!mask) 2965 mask = &rte_flow_item_vxlan_gpe_mask; 2966 ret = mlx5_flow_item_acceptable 2967 (item, (const uint8_t *)mask, 2968 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2969 sizeof(struct rte_flow_item_vxlan_gpe), 2970 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2971 if (ret < 0) 2972 return ret; 2973 if (spec) { 2974 if (spec->protocol) 2975 return rte_flow_error_set(error, ENOTSUP, 2976 RTE_FLOW_ERROR_TYPE_ITEM, 2977 item, 2978 "VxLAN-GPE protocol" 2979 " not supported"); 2980 memcpy(&id.vni[1], spec->vni, 3); 2981 memcpy(&id.vni[1], mask->vni, 3); 2982 } 2983 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2984 return rte_flow_error_set(error, ENOTSUP, 2985 RTE_FLOW_ERROR_TYPE_ITEM, item, 2986 "VXLAN-GPE tunnel must be fully" 2987 " defined"); 2988 return 0; 2989 } 2990 /** 2991 * Validate GRE Key item. 2992 * 2993 * @param[in] item 2994 * Item specification. 2995 * @param[in] item_flags 2996 * Bit flags to mark detected items. 2997 * @param[in] gre_item 2998 * Pointer to gre_item 2999 * @param[out] error 3000 * Pointer to error structure. 3001 * 3002 * @return 3003 * 0 on success, a negative errno value otherwise and rte_errno is set. 3004 */ 3005 int 3006 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3007 uint64_t item_flags, 3008 const struct rte_flow_item *gre_item, 3009 struct rte_flow_error *error) 3010 { 3011 const rte_be32_t *mask = item->mask; 3012 int ret = 0; 3013 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3014 const struct rte_flow_item_gre *gre_spec; 3015 const struct rte_flow_item_gre *gre_mask; 3016 3017 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3018 return rte_flow_error_set(error, ENOTSUP, 3019 RTE_FLOW_ERROR_TYPE_ITEM, item, 3020 "Multiple GRE key not support"); 3021 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3022 return rte_flow_error_set(error, ENOTSUP, 3023 RTE_FLOW_ERROR_TYPE_ITEM, item, 3024 "No preceding GRE header"); 3025 if (item_flags & MLX5_FLOW_LAYER_INNER) 3026 return rte_flow_error_set(error, ENOTSUP, 3027 RTE_FLOW_ERROR_TYPE_ITEM, item, 3028 "GRE key following a wrong item"); 3029 gre_mask = gre_item->mask; 3030 if (!gre_mask) 3031 gre_mask = &rte_flow_item_gre_mask; 3032 gre_spec = gre_item->spec; 3033 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3034 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3035 return rte_flow_error_set(error, EINVAL, 3036 RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "Key bit must be on"); 3038 3039 if (!mask) 3040 mask = &gre_key_default_mask; 3041 ret = mlx5_flow_item_acceptable 3042 (item, (const uint8_t *)mask, 3043 (const uint8_t *)&gre_key_default_mask, 3044 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3045 return ret; 3046 } 3047 3048 /** 3049 * Validate GRE optional item. 3050 * 3051 * @param[in] dev 3052 * Pointer to the Ethernet device structure. 3053 * @param[in] item 3054 * Item specification. 3055 * @param[in] item_flags 3056 * Bit flags to mark detected items. 3057 * @param[in] attr 3058 * Flow rule attributes. 3059 * @param[in] gre_item 3060 * Pointer to gre_item 3061 * @param[out] error 3062 * Pointer to error structure. 3063 * 3064 * @return 3065 * 0 on success, a negative errno value otherwise and rte_errno is set. 3066 */ 3067 int 3068 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3069 const struct rte_flow_item *item, 3070 uint64_t item_flags, 3071 const struct rte_flow_attr *attr, 3072 const struct rte_flow_item *gre_item, 3073 struct rte_flow_error *error) 3074 { 3075 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3076 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3077 const struct rte_flow_item_gre_opt *spec = item->spec; 3078 const struct rte_flow_item_gre_opt *mask = item->mask; 3079 struct mlx5_priv *priv = dev->data->dev_private; 3080 int ret = 0; 3081 struct rte_flow_item_gre_opt nic_mask = { 3082 .checksum_rsvd = { 3083 .checksum = RTE_BE16(UINT16_MAX), 3084 .reserved1 = 0x0, 3085 }, 3086 .key = { 3087 .key = RTE_BE32(UINT32_MAX), 3088 }, 3089 .sequence = { 3090 .sequence = RTE_BE32(UINT32_MAX), 3091 }, 3092 }; 3093 3094 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3095 return rte_flow_error_set(error, ENOTSUP, 3096 RTE_FLOW_ERROR_TYPE_ITEM, item, 3097 "No preceding GRE header"); 3098 if (item_flags & MLX5_FLOW_LAYER_INNER) 3099 return rte_flow_error_set(error, ENOTSUP, 3100 RTE_FLOW_ERROR_TYPE_ITEM, item, 3101 "GRE option following a wrong item"); 3102 if (!spec || !mask) 3103 return rte_flow_error_set(error, EINVAL, 3104 RTE_FLOW_ERROR_TYPE_ITEM, item, 3105 "At least one field gre_option(checksum/key/sequence) must be specified"); 3106 if (!gre_mask) 3107 gre_mask = &rte_flow_item_gre_mask; 3108 if (mask->checksum_rsvd.checksum) 3109 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3110 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3111 return rte_flow_error_set(error, EINVAL, 3112 RTE_FLOW_ERROR_TYPE_ITEM, 3113 item, 3114 "Checksum bit must be on"); 3115 if (mask->key.key) 3116 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3117 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3118 return rte_flow_error_set(error, EINVAL, 3119 RTE_FLOW_ERROR_TYPE_ITEM, 3120 item, "Key bit must be on"); 3121 if (mask->sequence.sequence) 3122 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3123 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3124 return rte_flow_error_set(error, EINVAL, 3125 RTE_FLOW_ERROR_TYPE_ITEM, 3126 item, 3127 "Sequence bit must be on"); 3128 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3129 if (priv->sh->steering_format_version == 3130 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3131 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3132 !priv->sh->misc5_cap) || 3133 (!(priv->sh->tunnel_header_0_1 && 3134 priv->sh->tunnel_header_2_3) && 3135 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3136 return rte_flow_error_set(error, EINVAL, 3137 RTE_FLOW_ERROR_TYPE_ITEM, 3138 item, 3139 "Checksum/Sequence not supported"); 3140 } 3141 ret = mlx5_flow_item_acceptable 3142 (item, (const uint8_t *)mask, 3143 (const uint8_t *)&nic_mask, 3144 sizeof(struct rte_flow_item_gre_opt), 3145 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3146 return ret; 3147 } 3148 3149 /** 3150 * Validate GRE item. 3151 * 3152 * @param[in] item 3153 * Item specification. 3154 * @param[in] item_flags 3155 * Bit flags to mark detected items. 3156 * @param[in] target_protocol 3157 * The next protocol in the previous item. 3158 * @param[out] error 3159 * Pointer to error structure. 3160 * 3161 * @return 3162 * 0 on success, a negative errno value otherwise and rte_errno is set. 3163 */ 3164 int 3165 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3166 uint64_t item_flags, 3167 uint8_t target_protocol, 3168 struct rte_flow_error *error) 3169 { 3170 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3171 const struct rte_flow_item_gre *mask = item->mask; 3172 int ret; 3173 const struct rte_flow_item_gre nic_mask = { 3174 .c_rsvd0_ver = RTE_BE16(0xB000), 3175 .protocol = RTE_BE16(UINT16_MAX), 3176 }; 3177 3178 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3179 return rte_flow_error_set(error, EINVAL, 3180 RTE_FLOW_ERROR_TYPE_ITEM, item, 3181 "protocol filtering not compatible" 3182 " with this GRE layer"); 3183 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3184 return rte_flow_error_set(error, ENOTSUP, 3185 RTE_FLOW_ERROR_TYPE_ITEM, item, 3186 "multiple tunnel layers not" 3187 " supported"); 3188 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3189 return rte_flow_error_set(error, ENOTSUP, 3190 RTE_FLOW_ERROR_TYPE_ITEM, item, 3191 "L3 Layer is missing"); 3192 if (!mask) 3193 mask = &rte_flow_item_gre_mask; 3194 ret = mlx5_flow_item_acceptable 3195 (item, (const uint8_t *)mask, 3196 (const uint8_t *)&nic_mask, 3197 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3198 error); 3199 if (ret < 0) 3200 return ret; 3201 #ifndef HAVE_MLX5DV_DR 3202 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3203 if (spec && (spec->protocol & mask->protocol)) 3204 return rte_flow_error_set(error, ENOTSUP, 3205 RTE_FLOW_ERROR_TYPE_ITEM, item, 3206 "without MPLS support the" 3207 " specification cannot be used for" 3208 " filtering"); 3209 #endif 3210 #endif 3211 return 0; 3212 } 3213 3214 /** 3215 * Validate Geneve item. 3216 * 3217 * @param[in] item 3218 * Item specification. 3219 * @param[in] itemFlags 3220 * Bit-fields that holds the items detected until now. 3221 * @param[in] enPriv 3222 * Pointer to the private data structure. 3223 * @param[out] error 3224 * Pointer to error structure. 3225 * 3226 * @return 3227 * 0 on success, a negative errno value otherwise and rte_errno is set. 3228 */ 3229 3230 int 3231 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3232 uint64_t item_flags, 3233 struct rte_eth_dev *dev, 3234 struct rte_flow_error *error) 3235 { 3236 struct mlx5_priv *priv = dev->data->dev_private; 3237 const struct rte_flow_item_geneve *spec = item->spec; 3238 const struct rte_flow_item_geneve *mask = item->mask; 3239 int ret; 3240 uint16_t gbhdr; 3241 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3242 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3243 const struct rte_flow_item_geneve nic_mask = { 3244 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3245 .vni = "\xff\xff\xff", 3246 .protocol = RTE_BE16(UINT16_MAX), 3247 }; 3248 3249 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3250 return rte_flow_error_set(error, ENOTSUP, 3251 RTE_FLOW_ERROR_TYPE_ITEM, item, 3252 "L3 Geneve is not enabled by device" 3253 " parameter and/or not configured in" 3254 " firmware"); 3255 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3256 return rte_flow_error_set(error, ENOTSUP, 3257 RTE_FLOW_ERROR_TYPE_ITEM, item, 3258 "multiple tunnel layers not" 3259 " supported"); 3260 /* 3261 * Verify only UDPv4 is present as defined in 3262 * https://tools.ietf.org/html/rfc7348 3263 */ 3264 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3265 return rte_flow_error_set(error, EINVAL, 3266 RTE_FLOW_ERROR_TYPE_ITEM, item, 3267 "no outer UDP layer found"); 3268 if (!mask) 3269 mask = &rte_flow_item_geneve_mask; 3270 ret = mlx5_flow_item_acceptable 3271 (item, (const uint8_t *)mask, 3272 (const uint8_t *)&nic_mask, 3273 sizeof(struct rte_flow_item_geneve), 3274 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3275 if (ret) 3276 return ret; 3277 if (spec) { 3278 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3279 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3280 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3281 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3282 return rte_flow_error_set(error, ENOTSUP, 3283 RTE_FLOW_ERROR_TYPE_ITEM, 3284 item, 3285 "Geneve protocol unsupported" 3286 " fields are being used"); 3287 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3288 return rte_flow_error_set 3289 (error, ENOTSUP, 3290 RTE_FLOW_ERROR_TYPE_ITEM, 3291 item, 3292 "Unsupported Geneve options length"); 3293 } 3294 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3295 return rte_flow_error_set 3296 (error, ENOTSUP, 3297 RTE_FLOW_ERROR_TYPE_ITEM, item, 3298 "Geneve tunnel must be fully defined"); 3299 return 0; 3300 } 3301 3302 /** 3303 * Validate Geneve TLV option item. 3304 * 3305 * @param[in] item 3306 * Item specification. 3307 * @param[in] last_item 3308 * Previous validated item in the pattern items. 3309 * @param[in] geneve_item 3310 * Previous GENEVE item specification. 3311 * @param[in] dev 3312 * Pointer to the rte_eth_dev structure. 3313 * @param[out] error 3314 * Pointer to error structure. 3315 * 3316 * @return 3317 * 0 on success, a negative errno value otherwise and rte_errno is set. 3318 */ 3319 int 3320 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3321 uint64_t last_item, 3322 const struct rte_flow_item *geneve_item, 3323 struct rte_eth_dev *dev, 3324 struct rte_flow_error *error) 3325 { 3326 struct mlx5_priv *priv = dev->data->dev_private; 3327 struct mlx5_dev_ctx_shared *sh = priv->sh; 3328 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3329 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3330 uint8_t data_max_supported = 3331 hca_attr->max_geneve_tlv_option_data_len * 4; 3332 const struct rte_flow_item_geneve *geneve_spec; 3333 const struct rte_flow_item_geneve *geneve_mask; 3334 const struct rte_flow_item_geneve_opt *spec = item->spec; 3335 const struct rte_flow_item_geneve_opt *mask = item->mask; 3336 unsigned int i; 3337 unsigned int data_len; 3338 uint8_t tlv_option_len; 3339 uint16_t optlen_m, optlen_v; 3340 const struct rte_flow_item_geneve_opt full_mask = { 3341 .option_class = RTE_BE16(0xffff), 3342 .option_type = 0xff, 3343 .option_len = 0x1f, 3344 }; 3345 3346 if (!mask) 3347 mask = &rte_flow_item_geneve_opt_mask; 3348 if (!spec) 3349 return rte_flow_error_set 3350 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3351 "Geneve TLV opt class/type/length must be specified"); 3352 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3353 return rte_flow_error_set 3354 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3355 "Geneve TLV opt length exceeds the limit (31)"); 3356 /* Check if class type and length masks are full. */ 3357 if (full_mask.option_class != mask->option_class || 3358 full_mask.option_type != mask->option_type || 3359 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3360 return rte_flow_error_set 3361 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3362 "Geneve TLV opt class/type/length masks must be full"); 3363 /* Check if length is supported */ 3364 if ((uint32_t)spec->option_len > 3365 hca_attr->max_geneve_tlv_option_data_len) 3366 return rte_flow_error_set 3367 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3368 "Geneve TLV opt length not supported"); 3369 if (hca_attr->max_geneve_tlv_options > 1) 3370 DRV_LOG(DEBUG, 3371 "max_geneve_tlv_options supports more than 1 option"); 3372 /* Check GENEVE item preceding. */ 3373 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3374 return rte_flow_error_set 3375 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3376 "Geneve opt item must be preceded with Geneve item"); 3377 geneve_spec = geneve_item->spec; 3378 geneve_mask = geneve_item->mask ? geneve_item->mask : 3379 &rte_flow_item_geneve_mask; 3380 /* Check if GENEVE TLV option size doesn't exceed option length */ 3381 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3382 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3383 tlv_option_len = spec->option_len & mask->option_len; 3384 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3385 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3386 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3387 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3388 if ((optlen_v & optlen_m) <= tlv_option_len) 3389 return rte_flow_error_set 3390 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3391 "GENEVE TLV option length exceeds optlen"); 3392 } 3393 /* Check if length is 0 or data is 0. */ 3394 if (spec->data == NULL || spec->option_len == 0) 3395 return rte_flow_error_set 3396 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3397 "Geneve TLV opt with zero data/length not supported"); 3398 /* Check not all data & mask are 0. */ 3399 data_len = spec->option_len * 4; 3400 if (mask->data == NULL) { 3401 for (i = 0; i < data_len; i++) 3402 if (spec->data[i]) 3403 break; 3404 if (i == data_len) 3405 return rte_flow_error_set(error, ENOTSUP, 3406 RTE_FLOW_ERROR_TYPE_ITEM, item, 3407 "Can't match on Geneve option data 0"); 3408 } else { 3409 for (i = 0; i < data_len; i++) 3410 if (spec->data[i] & mask->data[i]) 3411 break; 3412 if (i == data_len) 3413 return rte_flow_error_set(error, ENOTSUP, 3414 RTE_FLOW_ERROR_TYPE_ITEM, item, 3415 "Can't match on Geneve option data and mask 0"); 3416 /* Check data mask supported. */ 3417 for (i = data_max_supported; i < data_len ; i++) 3418 if (mask->data[i]) 3419 return rte_flow_error_set(error, ENOTSUP, 3420 RTE_FLOW_ERROR_TYPE_ITEM, item, 3421 "Data mask is of unsupported size"); 3422 } 3423 /* Check GENEVE option is supported in NIC. */ 3424 if (!hca_attr->geneve_tlv_opt) 3425 return rte_flow_error_set 3426 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3427 "Geneve TLV opt not supported"); 3428 /* Check if we already have geneve option with different type/class. */ 3429 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3430 geneve_opt_resource = sh->geneve_tlv_option_resource; 3431 if (geneve_opt_resource != NULL) 3432 if (geneve_opt_resource->option_class != spec->option_class || 3433 geneve_opt_resource->option_type != spec->option_type || 3434 geneve_opt_resource->length != spec->option_len) { 3435 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3436 return rte_flow_error_set(error, ENOTSUP, 3437 RTE_FLOW_ERROR_TYPE_ITEM, item, 3438 "Only one Geneve TLV option supported"); 3439 } 3440 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3441 return 0; 3442 } 3443 3444 /** 3445 * Validate MPLS item. 3446 * 3447 * @param[in] dev 3448 * Pointer to the rte_eth_dev structure. 3449 * @param[in] item 3450 * Item specification. 3451 * @param[in] item_flags 3452 * Bit-fields that holds the items detected until now. 3453 * @param[in] prev_layer 3454 * The protocol layer indicated in previous item. 3455 * @param[out] error 3456 * Pointer to error structure. 3457 * 3458 * @return 3459 * 0 on success, a negative errno value otherwise and rte_errno is set. 3460 */ 3461 int 3462 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3463 const struct rte_flow_item *item __rte_unused, 3464 uint64_t item_flags __rte_unused, 3465 uint64_t prev_layer __rte_unused, 3466 struct rte_flow_error *error) 3467 { 3468 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3469 const struct rte_flow_item_mpls *mask = item->mask; 3470 struct mlx5_priv *priv = dev->data->dev_private; 3471 int ret; 3472 3473 if (!priv->sh->dev_cap.mpls_en) 3474 return rte_flow_error_set(error, ENOTSUP, 3475 RTE_FLOW_ERROR_TYPE_ITEM, item, 3476 "MPLS not supported or" 3477 " disabled in firmware" 3478 " configuration."); 3479 /* MPLS over UDP, GRE is allowed */ 3480 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3481 MLX5_FLOW_LAYER_GRE | 3482 MLX5_FLOW_LAYER_GRE_KEY))) 3483 return rte_flow_error_set(error, EINVAL, 3484 RTE_FLOW_ERROR_TYPE_ITEM, item, 3485 "protocol filtering not compatible" 3486 " with MPLS layer"); 3487 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3488 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3489 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3490 return rte_flow_error_set(error, ENOTSUP, 3491 RTE_FLOW_ERROR_TYPE_ITEM, item, 3492 "multiple tunnel layers not" 3493 " supported"); 3494 if (!mask) 3495 mask = &rte_flow_item_mpls_mask; 3496 ret = mlx5_flow_item_acceptable 3497 (item, (const uint8_t *)mask, 3498 (const uint8_t *)&rte_flow_item_mpls_mask, 3499 sizeof(struct rte_flow_item_mpls), 3500 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3501 if (ret < 0) 3502 return ret; 3503 return 0; 3504 #else 3505 return rte_flow_error_set(error, ENOTSUP, 3506 RTE_FLOW_ERROR_TYPE_ITEM, item, 3507 "MPLS is not supported by Verbs, please" 3508 " update."); 3509 #endif 3510 } 3511 3512 /** 3513 * Validate NVGRE item. 3514 * 3515 * @param[in] item 3516 * Item specification. 3517 * @param[in] item_flags 3518 * Bit flags to mark detected items. 3519 * @param[in] target_protocol 3520 * The next protocol in the previous item. 3521 * @param[out] error 3522 * Pointer to error structure. 3523 * 3524 * @return 3525 * 0 on success, a negative errno value otherwise and rte_errno is set. 3526 */ 3527 int 3528 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3529 uint64_t item_flags, 3530 uint8_t target_protocol, 3531 struct rte_flow_error *error) 3532 { 3533 const struct rte_flow_item_nvgre *mask = item->mask; 3534 int ret; 3535 3536 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3537 return rte_flow_error_set(error, EINVAL, 3538 RTE_FLOW_ERROR_TYPE_ITEM, item, 3539 "protocol filtering not compatible" 3540 " with this GRE layer"); 3541 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3542 return rte_flow_error_set(error, ENOTSUP, 3543 RTE_FLOW_ERROR_TYPE_ITEM, item, 3544 "multiple tunnel layers not" 3545 " supported"); 3546 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3547 return rte_flow_error_set(error, ENOTSUP, 3548 RTE_FLOW_ERROR_TYPE_ITEM, item, 3549 "L3 Layer is missing"); 3550 if (!mask) 3551 mask = &rte_flow_item_nvgre_mask; 3552 ret = mlx5_flow_item_acceptable 3553 (item, (const uint8_t *)mask, 3554 (const uint8_t *)&rte_flow_item_nvgre_mask, 3555 sizeof(struct rte_flow_item_nvgre), 3556 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3557 if (ret < 0) 3558 return ret; 3559 return 0; 3560 } 3561 3562 /** 3563 * Validate eCPRI item. 3564 * 3565 * @param[in] item 3566 * Item specification. 3567 * @param[in] item_flags 3568 * Bit-fields that holds the items detected until now. 3569 * @param[in] last_item 3570 * Previous validated item in the pattern items. 3571 * @param[in] ether_type 3572 * Type in the ethernet layer header (including dot1q). 3573 * @param[in] acc_mask 3574 * Acceptable mask, if NULL default internal default mask 3575 * will be used to check whether item fields are supported. 3576 * @param[out] error 3577 * Pointer to error structure. 3578 * 3579 * @return 3580 * 0 on success, a negative errno value otherwise and rte_errno is set. 3581 */ 3582 int 3583 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3584 uint64_t item_flags, 3585 uint64_t last_item, 3586 uint16_t ether_type, 3587 const struct rte_flow_item_ecpri *acc_mask, 3588 struct rte_flow_error *error) 3589 { 3590 const struct rte_flow_item_ecpri *mask = item->mask; 3591 const struct rte_flow_item_ecpri nic_mask = { 3592 .hdr = { 3593 .common = { 3594 .u32 = 3595 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3596 .type = 0xFF, 3597 }).u32), 3598 }, 3599 .dummy[0] = 0xFFFFFFFF, 3600 }, 3601 }; 3602 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3603 MLX5_FLOW_LAYER_OUTER_VLAN); 3604 struct rte_flow_item_ecpri mask_lo; 3605 3606 if (!(last_item & outer_l2_vlan) && 3607 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3608 return rte_flow_error_set(error, EINVAL, 3609 RTE_FLOW_ERROR_TYPE_ITEM, item, 3610 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3611 if ((last_item & outer_l2_vlan) && ether_type && 3612 ether_type != RTE_ETHER_TYPE_ECPRI) 3613 return rte_flow_error_set(error, EINVAL, 3614 RTE_FLOW_ERROR_TYPE_ITEM, item, 3615 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3616 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3617 return rte_flow_error_set(error, EINVAL, 3618 RTE_FLOW_ERROR_TYPE_ITEM, item, 3619 "eCPRI with tunnel is not supported right now"); 3620 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3621 return rte_flow_error_set(error, ENOTSUP, 3622 RTE_FLOW_ERROR_TYPE_ITEM, item, 3623 "multiple L3 layers not supported"); 3624 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3625 return rte_flow_error_set(error, EINVAL, 3626 RTE_FLOW_ERROR_TYPE_ITEM, item, 3627 "eCPRI cannot coexist with a TCP layer"); 3628 /* In specification, eCPRI could be over UDP layer. */ 3629 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3630 return rte_flow_error_set(error, EINVAL, 3631 RTE_FLOW_ERROR_TYPE_ITEM, item, 3632 "eCPRI over UDP layer is not yet supported right now"); 3633 /* Mask for type field in common header could be zero. */ 3634 if (!mask) 3635 mask = &rte_flow_item_ecpri_mask; 3636 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3637 /* Input mask is in big-endian format. */ 3638 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3639 return rte_flow_error_set(error, EINVAL, 3640 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3641 "partial mask is not supported for protocol"); 3642 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3643 return rte_flow_error_set(error, EINVAL, 3644 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3645 "message header mask must be after a type mask"); 3646 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3647 acc_mask ? (const uint8_t *)acc_mask 3648 : (const uint8_t *)&nic_mask, 3649 sizeof(struct rte_flow_item_ecpri), 3650 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3651 } 3652 3653 static int 3654 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3655 const struct rte_flow_attr *attr __rte_unused, 3656 const struct rte_flow_item items[] __rte_unused, 3657 const struct rte_flow_action actions[] __rte_unused, 3658 bool external __rte_unused, 3659 int hairpin __rte_unused, 3660 struct rte_flow_error *error) 3661 { 3662 return rte_flow_error_set(error, ENOTSUP, 3663 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3664 } 3665 3666 static struct mlx5_flow * 3667 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3668 const struct rte_flow_attr *attr __rte_unused, 3669 const struct rte_flow_item items[] __rte_unused, 3670 const struct rte_flow_action actions[] __rte_unused, 3671 struct rte_flow_error *error) 3672 { 3673 rte_flow_error_set(error, ENOTSUP, 3674 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3675 return NULL; 3676 } 3677 3678 static int 3679 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3680 struct mlx5_flow *dev_flow __rte_unused, 3681 const struct rte_flow_attr *attr __rte_unused, 3682 const struct rte_flow_item items[] __rte_unused, 3683 const struct rte_flow_action actions[] __rte_unused, 3684 struct rte_flow_error *error) 3685 { 3686 return rte_flow_error_set(error, ENOTSUP, 3687 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3688 } 3689 3690 static int 3691 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3692 struct rte_flow *flow __rte_unused, 3693 struct rte_flow_error *error) 3694 { 3695 return rte_flow_error_set(error, ENOTSUP, 3696 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3697 } 3698 3699 static void 3700 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3701 struct rte_flow *flow __rte_unused) 3702 { 3703 } 3704 3705 static void 3706 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3707 struct rte_flow *flow __rte_unused) 3708 { 3709 } 3710 3711 static int 3712 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3713 struct rte_flow *flow __rte_unused, 3714 const struct rte_flow_action *actions __rte_unused, 3715 void *data __rte_unused, 3716 struct rte_flow_error *error) 3717 { 3718 return rte_flow_error_set(error, ENOTSUP, 3719 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3720 } 3721 3722 static int 3723 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3724 uint32_t domains __rte_unused, 3725 uint32_t flags __rte_unused) 3726 { 3727 return 0; 3728 } 3729 3730 /* Void driver to protect from null pointer reference. */ 3731 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3732 .validate = flow_null_validate, 3733 .prepare = flow_null_prepare, 3734 .translate = flow_null_translate, 3735 .apply = flow_null_apply, 3736 .remove = flow_null_remove, 3737 .destroy = flow_null_destroy, 3738 .query = flow_null_query, 3739 .sync_domain = flow_null_sync_domain, 3740 }; 3741 3742 /** 3743 * Select flow driver type according to flow attributes and device 3744 * configuration. 3745 * 3746 * @param[in] dev 3747 * Pointer to the dev structure. 3748 * @param[in] attr 3749 * Pointer to the flow attributes. 3750 * 3751 * @return 3752 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3753 */ 3754 static enum mlx5_flow_drv_type 3755 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3756 { 3757 struct mlx5_priv *priv = dev->data->dev_private; 3758 /* The OS can determine first a specific flow type (DV, VERBS) */ 3759 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3760 3761 if (type != MLX5_FLOW_TYPE_MAX) 3762 return type; 3763 /* 3764 * Currently when dv_flow_en == 2, only HW steering engine is 3765 * supported. New engines can also be chosen here if ready. 3766 */ 3767 if (priv->sh->config.dv_flow_en == 2) 3768 return MLX5_FLOW_TYPE_HW; 3769 if (!attr) 3770 return MLX5_FLOW_TYPE_MIN; 3771 /* If no OS specific type - continue with DV/VERBS selection */ 3772 if (attr->transfer && priv->sh->config.dv_esw_en) 3773 type = MLX5_FLOW_TYPE_DV; 3774 if (!attr->transfer) 3775 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3776 MLX5_FLOW_TYPE_VERBS; 3777 return type; 3778 } 3779 3780 #define flow_get_drv_ops(type) flow_drv_ops[type] 3781 3782 /** 3783 * Flow driver validation API. This abstracts calling driver specific functions. 3784 * The type of flow driver is determined according to flow attributes. 3785 * 3786 * @param[in] dev 3787 * Pointer to the dev structure. 3788 * @param[in] attr 3789 * Pointer to the flow attributes. 3790 * @param[in] items 3791 * Pointer to the list of items. 3792 * @param[in] actions 3793 * Pointer to the list of actions. 3794 * @param[in] external 3795 * This flow rule is created by request external to PMD. 3796 * @param[in] hairpin 3797 * Number of hairpin TX actions, 0 means classic flow. 3798 * @param[out] error 3799 * Pointer to the error structure. 3800 * 3801 * @return 3802 * 0 on success, a negative errno value otherwise and rte_errno is set. 3803 */ 3804 static inline int 3805 flow_drv_validate(struct rte_eth_dev *dev, 3806 const struct rte_flow_attr *attr, 3807 const struct rte_flow_item items[], 3808 const struct rte_flow_action actions[], 3809 bool external, int hairpin, struct rte_flow_error *error) 3810 { 3811 const struct mlx5_flow_driver_ops *fops; 3812 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3813 3814 fops = flow_get_drv_ops(type); 3815 return fops->validate(dev, attr, items, actions, external, 3816 hairpin, error); 3817 } 3818 3819 /** 3820 * Flow driver preparation API. This abstracts calling driver specific 3821 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3822 * calculates the size of memory required for device flow, allocates the memory, 3823 * initializes the device flow and returns the pointer. 3824 * 3825 * @note 3826 * This function initializes device flow structure such as dv or verbs in 3827 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3828 * rest. For example, adding returning device flow to flow->dev_flow list and 3829 * setting backward reference to the flow should be done out of this function. 3830 * layers field is not filled either. 3831 * 3832 * @param[in] dev 3833 * Pointer to the dev structure. 3834 * @param[in] attr 3835 * Pointer to the flow attributes. 3836 * @param[in] items 3837 * Pointer to the list of items. 3838 * @param[in] actions 3839 * Pointer to the list of actions. 3840 * @param[in] flow_idx 3841 * This memory pool index to the flow. 3842 * @param[out] error 3843 * Pointer to the error structure. 3844 * 3845 * @return 3846 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3847 */ 3848 static inline struct mlx5_flow * 3849 flow_drv_prepare(struct rte_eth_dev *dev, 3850 const struct rte_flow *flow, 3851 const struct rte_flow_attr *attr, 3852 const struct rte_flow_item items[], 3853 const struct rte_flow_action actions[], 3854 uint32_t flow_idx, 3855 struct rte_flow_error *error) 3856 { 3857 const struct mlx5_flow_driver_ops *fops; 3858 enum mlx5_flow_drv_type type = flow->drv_type; 3859 struct mlx5_flow *mlx5_flow = NULL; 3860 3861 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3862 fops = flow_get_drv_ops(type); 3863 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3864 if (mlx5_flow) 3865 mlx5_flow->flow_idx = flow_idx; 3866 return mlx5_flow; 3867 } 3868 3869 /** 3870 * Flow driver translation API. This abstracts calling driver specific 3871 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3872 * translates a generic flow into a driver flow. flow_drv_prepare() must 3873 * precede. 3874 * 3875 * @note 3876 * dev_flow->layers could be filled as a result of parsing during translation 3877 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3878 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3879 * flow->actions could be overwritten even though all the expanded dev_flows 3880 * have the same actions. 3881 * 3882 * @param[in] dev 3883 * Pointer to the rte dev structure. 3884 * @param[in, out] dev_flow 3885 * Pointer to the mlx5 flow. 3886 * @param[in] attr 3887 * Pointer to the flow attributes. 3888 * @param[in] items 3889 * Pointer to the list of items. 3890 * @param[in] actions 3891 * Pointer to the list of actions. 3892 * @param[out] error 3893 * Pointer to the error structure. 3894 * 3895 * @return 3896 * 0 on success, a negative errno value otherwise and rte_errno is set. 3897 */ 3898 static inline int 3899 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3900 const struct rte_flow_attr *attr, 3901 const struct rte_flow_item items[], 3902 const struct rte_flow_action actions[], 3903 struct rte_flow_error *error) 3904 { 3905 const struct mlx5_flow_driver_ops *fops; 3906 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3907 3908 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3909 fops = flow_get_drv_ops(type); 3910 return fops->translate(dev, dev_flow, attr, items, actions, error); 3911 } 3912 3913 /** 3914 * Flow driver apply API. This abstracts calling driver specific functions. 3915 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3916 * translated driver flows on to device. flow_drv_translate() must precede. 3917 * 3918 * @param[in] dev 3919 * Pointer to Ethernet device structure. 3920 * @param[in, out] flow 3921 * Pointer to flow structure. 3922 * @param[out] error 3923 * Pointer to error structure. 3924 * 3925 * @return 3926 * 0 on success, a negative errno value otherwise and rte_errno is set. 3927 */ 3928 static inline int 3929 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3930 struct rte_flow_error *error) 3931 { 3932 const struct mlx5_flow_driver_ops *fops; 3933 enum mlx5_flow_drv_type type = flow->drv_type; 3934 3935 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3936 fops = flow_get_drv_ops(type); 3937 return fops->apply(dev, flow, error); 3938 } 3939 3940 /** 3941 * Flow driver destroy API. This abstracts calling driver specific functions. 3942 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3943 * on device and releases resources of the flow. 3944 * 3945 * @param[in] dev 3946 * Pointer to Ethernet device. 3947 * @param[in, out] flow 3948 * Pointer to flow structure. 3949 */ 3950 static inline void 3951 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3952 { 3953 const struct mlx5_flow_driver_ops *fops; 3954 enum mlx5_flow_drv_type type = flow->drv_type; 3955 3956 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3957 fops = flow_get_drv_ops(type); 3958 fops->destroy(dev, flow); 3959 } 3960 3961 /** 3962 * Flow driver find RSS policy tbl API. This abstracts calling driver 3963 * specific functions. Parent flow (rte_flow) should have driver 3964 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3965 * 3966 * @param[in] dev 3967 * Pointer to Ethernet device. 3968 * @param[in, out] flow 3969 * Pointer to flow structure. 3970 * @param[in] policy 3971 * Pointer to meter policy table. 3972 * @param[in] rss_desc 3973 * Pointer to rss_desc 3974 */ 3975 static struct mlx5_flow_meter_sub_policy * 3976 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3977 struct rte_flow *flow, 3978 struct mlx5_flow_meter_policy *policy, 3979 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3980 { 3981 const struct mlx5_flow_driver_ops *fops; 3982 enum mlx5_flow_drv_type type = flow->drv_type; 3983 3984 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3985 fops = flow_get_drv_ops(type); 3986 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 3987 } 3988 3989 /** 3990 * Flow driver color tag rule API. This abstracts calling driver 3991 * specific functions. Parent flow (rte_flow) should have driver 3992 * type (drv_type). It will create the color tag rules in hierarchy meter. 3993 * 3994 * @param[in] dev 3995 * Pointer to Ethernet device. 3996 * @param[in, out] flow 3997 * Pointer to flow structure. 3998 * @param[in] fm 3999 * Pointer to flow meter structure. 4000 * @param[in] src_port 4001 * The src port this extra rule should use. 4002 * @param[in] item 4003 * The src port id match item. 4004 * @param[out] error 4005 * Pointer to error structure. 4006 */ 4007 static int 4008 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4009 struct rte_flow *flow, 4010 struct mlx5_flow_meter_info *fm, 4011 int32_t src_port, 4012 const struct rte_flow_item *item, 4013 struct rte_flow_error *error) 4014 { 4015 const struct mlx5_flow_driver_ops *fops; 4016 enum mlx5_flow_drv_type type = flow->drv_type; 4017 4018 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4019 fops = flow_get_drv_ops(type); 4020 return fops->meter_hierarchy_rule_create(dev, fm, 4021 src_port, item, error); 4022 } 4023 4024 /** 4025 * Get RSS action from the action list. 4026 * 4027 * @param[in] dev 4028 * Pointer to Ethernet device. 4029 * @param[in] actions 4030 * Pointer to the list of actions. 4031 * @param[in] flow 4032 * Parent flow structure pointer. 4033 * 4034 * @return 4035 * Pointer to the RSS action if exist, else return NULL. 4036 */ 4037 static const struct rte_flow_action_rss* 4038 flow_get_rss_action(struct rte_eth_dev *dev, 4039 const struct rte_flow_action actions[]) 4040 { 4041 struct mlx5_priv *priv = dev->data->dev_private; 4042 const struct rte_flow_action_rss *rss = NULL; 4043 struct mlx5_meter_policy_action_container *acg; 4044 struct mlx5_meter_policy_action_container *acy; 4045 4046 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4047 switch (actions->type) { 4048 case RTE_FLOW_ACTION_TYPE_RSS: 4049 rss = actions->conf; 4050 break; 4051 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4052 { 4053 const struct rte_flow_action_sample *sample = 4054 actions->conf; 4055 const struct rte_flow_action *act = sample->actions; 4056 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4057 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4058 rss = act->conf; 4059 break; 4060 } 4061 case RTE_FLOW_ACTION_TYPE_METER: 4062 { 4063 uint32_t mtr_idx; 4064 struct mlx5_flow_meter_info *fm; 4065 struct mlx5_flow_meter_policy *policy; 4066 const struct rte_flow_action_meter *mtr = actions->conf; 4067 4068 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4069 if (fm && !fm->def_policy) { 4070 policy = mlx5_flow_meter_policy_find(dev, 4071 fm->policy_id, NULL); 4072 MLX5_ASSERT(policy); 4073 if (policy->is_hierarchy) { 4074 policy = 4075 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4076 policy); 4077 if (!policy) 4078 return NULL; 4079 } 4080 if (policy->is_rss) { 4081 acg = 4082 &policy->act_cnt[RTE_COLOR_GREEN]; 4083 acy = 4084 &policy->act_cnt[RTE_COLOR_YELLOW]; 4085 if (acg->fate_action == 4086 MLX5_FLOW_FATE_SHARED_RSS) 4087 rss = acg->rss->conf; 4088 else if (acy->fate_action == 4089 MLX5_FLOW_FATE_SHARED_RSS) 4090 rss = acy->rss->conf; 4091 } 4092 } 4093 break; 4094 } 4095 default: 4096 break; 4097 } 4098 } 4099 return rss; 4100 } 4101 4102 /** 4103 * Get ASO age action by index. 4104 * 4105 * @param[in] dev 4106 * Pointer to the Ethernet device structure. 4107 * @param[in] age_idx 4108 * Index to the ASO age action. 4109 * 4110 * @return 4111 * The specified ASO age action. 4112 */ 4113 struct mlx5_aso_age_action* 4114 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4115 { 4116 uint16_t pool_idx = age_idx & UINT16_MAX; 4117 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4118 struct mlx5_priv *priv = dev->data->dev_private; 4119 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4120 struct mlx5_aso_age_pool *pool; 4121 4122 rte_rwlock_read_lock(&mng->resize_rwl); 4123 pool = mng->pools[pool_idx]; 4124 rte_rwlock_read_unlock(&mng->resize_rwl); 4125 return &pool->actions[offset - 1]; 4126 } 4127 4128 /* maps indirect action to translated direct in some actions array */ 4129 struct mlx5_translated_action_handle { 4130 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4131 int index; /**< Index in related array of rte_flow_action. */ 4132 }; 4133 4134 /** 4135 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4136 * direct action if translation possible. 4137 * This functionality used to run same execution path for both direct and 4138 * indirect actions on flow create. All necessary preparations for indirect 4139 * action handling should be performed on *handle* actions list returned 4140 * from this call. 4141 * 4142 * @param[in] dev 4143 * Pointer to Ethernet device. 4144 * @param[in] actions 4145 * List of actions to translate. 4146 * @param[out] handle 4147 * List to store translated indirect action object handles. 4148 * @param[in, out] indir_n 4149 * Size of *handle* array. On return should be updated with number of 4150 * indirect actions retrieved from the *actions* list. 4151 * @param[out] translated_actions 4152 * List of actions where all indirect actions were translated to direct 4153 * if possible. NULL if no translation took place. 4154 * @param[out] error 4155 * Pointer to the error structure. 4156 * 4157 * @return 4158 * 0 on success, a negative errno value otherwise and rte_errno is set. 4159 */ 4160 static int 4161 flow_action_handles_translate(struct rte_eth_dev *dev, 4162 const struct rte_flow_action actions[], 4163 struct mlx5_translated_action_handle *handle, 4164 int *indir_n, 4165 struct rte_flow_action **translated_actions, 4166 struct rte_flow_error *error) 4167 { 4168 struct mlx5_priv *priv = dev->data->dev_private; 4169 struct rte_flow_action *translated = NULL; 4170 size_t actions_size; 4171 int n; 4172 int copied_n = 0; 4173 struct mlx5_translated_action_handle *handle_end = NULL; 4174 4175 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4176 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4177 continue; 4178 if (copied_n == *indir_n) { 4179 return rte_flow_error_set 4180 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4181 NULL, "too many shared actions"); 4182 } 4183 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4184 sizeof(actions[n].conf)); 4185 handle[copied_n].index = n; 4186 copied_n++; 4187 } 4188 n++; 4189 *indir_n = copied_n; 4190 if (!copied_n) 4191 return 0; 4192 actions_size = sizeof(struct rte_flow_action) * n; 4193 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4194 if (!translated) { 4195 rte_errno = ENOMEM; 4196 return -ENOMEM; 4197 } 4198 memcpy(translated, actions, actions_size); 4199 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4200 struct mlx5_shared_action_rss *shared_rss; 4201 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4202 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4203 uint32_t idx = act_idx & 4204 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4205 4206 switch (type) { 4207 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4208 shared_rss = mlx5_ipool_get 4209 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4210 translated[handle->index].type = 4211 RTE_FLOW_ACTION_TYPE_RSS; 4212 translated[handle->index].conf = 4213 &shared_rss->origin; 4214 break; 4215 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4216 translated[handle->index].type = 4217 (enum rte_flow_action_type) 4218 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4219 translated[handle->index].conf = (void *)(uintptr_t)idx; 4220 break; 4221 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4222 if (priv->sh->flow_hit_aso_en) { 4223 translated[handle->index].type = 4224 (enum rte_flow_action_type) 4225 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4226 translated[handle->index].conf = 4227 (void *)(uintptr_t)idx; 4228 break; 4229 } 4230 /* Fall-through */ 4231 case MLX5_INDIRECT_ACTION_TYPE_CT: 4232 if (priv->sh->ct_aso_en) { 4233 translated[handle->index].type = 4234 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4235 translated[handle->index].conf = 4236 (void *)(uintptr_t)idx; 4237 break; 4238 } 4239 /* Fall-through */ 4240 default: 4241 mlx5_free(translated); 4242 return rte_flow_error_set 4243 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4244 NULL, "invalid indirect action type"); 4245 } 4246 } 4247 *translated_actions = translated; 4248 return 0; 4249 } 4250 4251 /** 4252 * Get Shared RSS action from the action list. 4253 * 4254 * @param[in] dev 4255 * Pointer to Ethernet device. 4256 * @param[in] shared 4257 * Pointer to the list of actions. 4258 * @param[in] shared_n 4259 * Actions list length. 4260 * 4261 * @return 4262 * The MLX5 RSS action ID if exists, otherwise return 0. 4263 */ 4264 static uint32_t 4265 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4266 struct mlx5_translated_action_handle *handle, 4267 int shared_n) 4268 { 4269 struct mlx5_translated_action_handle *handle_end; 4270 struct mlx5_priv *priv = dev->data->dev_private; 4271 struct mlx5_shared_action_rss *shared_rss; 4272 4273 4274 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4275 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4276 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4277 uint32_t idx = act_idx & 4278 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4279 switch (type) { 4280 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4281 shared_rss = mlx5_ipool_get 4282 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4283 idx); 4284 __atomic_add_fetch(&shared_rss->refcnt, 1, 4285 __ATOMIC_RELAXED); 4286 return idx; 4287 default: 4288 break; 4289 } 4290 } 4291 return 0; 4292 } 4293 4294 static unsigned int 4295 find_graph_root(uint32_t rss_level) 4296 { 4297 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4298 MLX5_EXPANSION_ROOT_OUTER; 4299 } 4300 4301 /** 4302 * Get layer flags from the prefix flow. 4303 * 4304 * Some flows may be split to several subflows, the prefix subflow gets the 4305 * match items and the suffix sub flow gets the actions. 4306 * Some actions need the user defined match item flags to get the detail for 4307 * the action. 4308 * This function helps the suffix flow to get the item layer flags from prefix 4309 * subflow. 4310 * 4311 * @param[in] dev_flow 4312 * Pointer the created prefix subflow. 4313 * 4314 * @return 4315 * The layers get from prefix subflow. 4316 */ 4317 static inline uint64_t 4318 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4319 { 4320 uint64_t layers = 0; 4321 4322 /* 4323 * Layers bits could be localization, but usually the compiler will 4324 * help to do the optimization work for source code. 4325 * If no decap actions, use the layers directly. 4326 */ 4327 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4328 return dev_flow->handle->layers; 4329 /* Convert L3 layers with decap action. */ 4330 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4331 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4332 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4333 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4334 /* Convert L4 layers with decap action. */ 4335 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4336 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4337 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4338 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4339 return layers; 4340 } 4341 4342 /** 4343 * Get metadata split action information. 4344 * 4345 * @param[in] actions 4346 * Pointer to the list of actions. 4347 * @param[out] qrss 4348 * Pointer to the return pointer. 4349 * @param[out] qrss_type 4350 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4351 * if no QUEUE/RSS is found. 4352 * @param[out] encap_idx 4353 * Pointer to the index of the encap action if exists, otherwise the last 4354 * action index. 4355 * 4356 * @return 4357 * Total number of actions. 4358 */ 4359 static int 4360 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4361 const struct rte_flow_action **qrss, 4362 int *encap_idx) 4363 { 4364 const struct rte_flow_action_raw_encap *raw_encap; 4365 int actions_n = 0; 4366 int raw_decap_idx = -1; 4367 4368 *encap_idx = -1; 4369 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4370 switch (actions->type) { 4371 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4372 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4373 *encap_idx = actions_n; 4374 break; 4375 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4376 raw_decap_idx = actions_n; 4377 break; 4378 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4379 raw_encap = actions->conf; 4380 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4381 *encap_idx = raw_decap_idx != -1 ? 4382 raw_decap_idx : actions_n; 4383 break; 4384 case RTE_FLOW_ACTION_TYPE_QUEUE: 4385 case RTE_FLOW_ACTION_TYPE_RSS: 4386 *qrss = actions; 4387 break; 4388 default: 4389 break; 4390 } 4391 actions_n++; 4392 } 4393 if (*encap_idx == -1) 4394 *encap_idx = actions_n; 4395 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4396 return actions_n + 1; 4397 } 4398 4399 /** 4400 * Check if the action will change packet. 4401 * 4402 * @param dev 4403 * Pointer to Ethernet device. 4404 * @param[in] type 4405 * action type. 4406 * 4407 * @return 4408 * true if action will change packet, false otherwise. 4409 */ 4410 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4411 enum rte_flow_action_type type) 4412 { 4413 struct mlx5_priv *priv = dev->data->dev_private; 4414 4415 switch (type) { 4416 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4417 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4418 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4419 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4420 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4421 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4422 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4423 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4424 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4425 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4426 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4427 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4428 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4429 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4430 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4431 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4432 case RTE_FLOW_ACTION_TYPE_SET_META: 4433 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4434 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4435 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4436 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4437 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4438 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4439 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4440 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4441 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4442 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4443 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4444 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4445 return true; 4446 case RTE_FLOW_ACTION_TYPE_FLAG: 4447 case RTE_FLOW_ACTION_TYPE_MARK: 4448 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4449 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4450 return true; 4451 else 4452 return false; 4453 default: 4454 return false; 4455 } 4456 } 4457 4458 /** 4459 * Check meter action from the action list. 4460 * 4461 * @param dev 4462 * Pointer to Ethernet device. 4463 * @param[in] actions 4464 * Pointer to the list of actions. 4465 * @param[out] has_mtr 4466 * Pointer to the meter exist flag. 4467 * @param[out] has_modify 4468 * Pointer to the flag showing there's packet change action. 4469 * @param[out] meter_id 4470 * Pointer to the meter id. 4471 * 4472 * @return 4473 * Total number of actions. 4474 */ 4475 static int 4476 flow_check_meter_action(struct rte_eth_dev *dev, 4477 const struct rte_flow_action actions[], 4478 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4479 { 4480 const struct rte_flow_action_meter *mtr = NULL; 4481 int actions_n = 0; 4482 4483 MLX5_ASSERT(has_mtr); 4484 *has_mtr = false; 4485 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4486 switch (actions->type) { 4487 case RTE_FLOW_ACTION_TYPE_METER: 4488 mtr = actions->conf; 4489 *meter_id = mtr->mtr_id; 4490 *has_mtr = true; 4491 break; 4492 default: 4493 break; 4494 } 4495 if (!*has_mtr) 4496 *has_modify |= flow_check_modify_action_type(dev, 4497 actions->type); 4498 actions_n++; 4499 } 4500 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4501 return actions_n + 1; 4502 } 4503 4504 /** 4505 * Check if the flow should be split due to hairpin. 4506 * The reason for the split is that in current HW we can't 4507 * support encap and push-vlan on Rx, so if a flow contains 4508 * these actions we move it to Tx. 4509 * 4510 * @param dev 4511 * Pointer to Ethernet device. 4512 * @param[in] attr 4513 * Flow rule attributes. 4514 * @param[in] actions 4515 * Associated actions (list terminated by the END action). 4516 * 4517 * @return 4518 * > 0 the number of actions and the flow should be split, 4519 * 0 when no split required. 4520 */ 4521 static int 4522 flow_check_hairpin_split(struct rte_eth_dev *dev, 4523 const struct rte_flow_attr *attr, 4524 const struct rte_flow_action actions[]) 4525 { 4526 int queue_action = 0; 4527 int action_n = 0; 4528 int split = 0; 4529 const struct rte_flow_action_queue *queue; 4530 const struct rte_flow_action_rss *rss; 4531 const struct rte_flow_action_raw_encap *raw_encap; 4532 const struct rte_eth_hairpin_conf *conf; 4533 4534 if (!attr->ingress) 4535 return 0; 4536 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4537 switch (actions->type) { 4538 case RTE_FLOW_ACTION_TYPE_QUEUE: 4539 queue = actions->conf; 4540 if (queue == NULL) 4541 return 0; 4542 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4543 if (conf == NULL || conf->tx_explicit != 0) 4544 return 0; 4545 queue_action = 1; 4546 action_n++; 4547 break; 4548 case RTE_FLOW_ACTION_TYPE_RSS: 4549 rss = actions->conf; 4550 if (rss == NULL || rss->queue_num == 0) 4551 return 0; 4552 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4553 if (conf == NULL || conf->tx_explicit != 0) 4554 return 0; 4555 queue_action = 1; 4556 action_n++; 4557 break; 4558 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4559 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4560 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4561 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4562 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4563 split++; 4564 action_n++; 4565 break; 4566 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4567 raw_encap = actions->conf; 4568 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4569 split++; 4570 action_n++; 4571 break; 4572 default: 4573 action_n++; 4574 break; 4575 } 4576 } 4577 if (split && queue_action) 4578 return action_n; 4579 return 0; 4580 } 4581 4582 /* Declare flow create/destroy prototype in advance. */ 4583 static uint32_t 4584 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4585 const struct rte_flow_attr *attr, 4586 const struct rte_flow_item items[], 4587 const struct rte_flow_action actions[], 4588 bool external, struct rte_flow_error *error); 4589 4590 static void 4591 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4592 uint32_t flow_idx); 4593 4594 int 4595 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4596 struct mlx5_list_entry *entry, void *cb_ctx) 4597 { 4598 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4599 struct mlx5_flow_mreg_copy_resource *mcp_res = 4600 container_of(entry, typeof(*mcp_res), hlist_ent); 4601 4602 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4603 } 4604 4605 struct mlx5_list_entry * 4606 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4607 { 4608 struct rte_eth_dev *dev = tool_ctx; 4609 struct mlx5_priv *priv = dev->data->dev_private; 4610 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4611 struct mlx5_flow_mreg_copy_resource *mcp_res; 4612 struct rte_flow_error *error = ctx->error; 4613 uint32_t idx = 0; 4614 int ret; 4615 uint32_t mark_id = *(uint32_t *)(ctx->data); 4616 struct rte_flow_attr attr = { 4617 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4618 .ingress = 1, 4619 }; 4620 struct mlx5_rte_flow_item_tag tag_spec = { 4621 .data = mark_id, 4622 }; 4623 struct rte_flow_item items[] = { 4624 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4625 }; 4626 struct rte_flow_action_mark ftag = { 4627 .id = mark_id, 4628 }; 4629 struct mlx5_flow_action_copy_mreg cp_mreg = { 4630 .dst = REG_B, 4631 .src = REG_NON, 4632 }; 4633 struct rte_flow_action_jump jump = { 4634 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4635 }; 4636 struct rte_flow_action actions[] = { 4637 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4638 }; 4639 4640 /* Fill the register fields in the flow. */ 4641 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4642 if (ret < 0) 4643 return NULL; 4644 tag_spec.id = ret; 4645 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4646 if (ret < 0) 4647 return NULL; 4648 cp_mreg.src = ret; 4649 /* Provide the full width of FLAG specific value. */ 4650 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4651 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4652 /* Build a new flow. */ 4653 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4654 items[0] = (struct rte_flow_item){ 4655 .type = (enum rte_flow_item_type) 4656 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4657 .spec = &tag_spec, 4658 }; 4659 items[1] = (struct rte_flow_item){ 4660 .type = RTE_FLOW_ITEM_TYPE_END, 4661 }; 4662 actions[0] = (struct rte_flow_action){ 4663 .type = (enum rte_flow_action_type) 4664 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4665 .conf = &ftag, 4666 }; 4667 actions[1] = (struct rte_flow_action){ 4668 .type = (enum rte_flow_action_type) 4669 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4670 .conf = &cp_mreg, 4671 }; 4672 actions[2] = (struct rte_flow_action){ 4673 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4674 .conf = &jump, 4675 }; 4676 actions[3] = (struct rte_flow_action){ 4677 .type = RTE_FLOW_ACTION_TYPE_END, 4678 }; 4679 } else { 4680 /* Default rule, wildcard match. */ 4681 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4682 items[0] = (struct rte_flow_item){ 4683 .type = RTE_FLOW_ITEM_TYPE_END, 4684 }; 4685 actions[0] = (struct rte_flow_action){ 4686 .type = (enum rte_flow_action_type) 4687 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4688 .conf = &cp_mreg, 4689 }; 4690 actions[1] = (struct rte_flow_action){ 4691 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4692 .conf = &jump, 4693 }; 4694 actions[2] = (struct rte_flow_action){ 4695 .type = RTE_FLOW_ACTION_TYPE_END, 4696 }; 4697 } 4698 /* Build a new entry. */ 4699 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4700 if (!mcp_res) { 4701 rte_errno = ENOMEM; 4702 return NULL; 4703 } 4704 mcp_res->idx = idx; 4705 mcp_res->mark_id = mark_id; 4706 /* 4707 * The copy Flows are not included in any list. There 4708 * ones are referenced from other Flows and can not 4709 * be applied, removed, deleted in arbitrary order 4710 * by list traversing. 4711 */ 4712 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4713 &attr, items, actions, false, error); 4714 if (!mcp_res->rix_flow) { 4715 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4716 return NULL; 4717 } 4718 return &mcp_res->hlist_ent; 4719 } 4720 4721 struct mlx5_list_entry * 4722 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4723 void *cb_ctx __rte_unused) 4724 { 4725 struct rte_eth_dev *dev = tool_ctx; 4726 struct mlx5_priv *priv = dev->data->dev_private; 4727 struct mlx5_flow_mreg_copy_resource *mcp_res; 4728 uint32_t idx = 0; 4729 4730 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4731 if (!mcp_res) { 4732 rte_errno = ENOMEM; 4733 return NULL; 4734 } 4735 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4736 mcp_res->idx = idx; 4737 return &mcp_res->hlist_ent; 4738 } 4739 4740 void 4741 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4742 { 4743 struct mlx5_flow_mreg_copy_resource *mcp_res = 4744 container_of(entry, typeof(*mcp_res), hlist_ent); 4745 struct rte_eth_dev *dev = tool_ctx; 4746 struct mlx5_priv *priv = dev->data->dev_private; 4747 4748 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4749 } 4750 4751 /** 4752 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4753 * 4754 * As mark_id is unique, if there's already a registered flow for the mark_id, 4755 * return by increasing the reference counter of the resource. Otherwise, create 4756 * the resource (mcp_res) and flow. 4757 * 4758 * Flow looks like, 4759 * - If ingress port is ANY and reg_c[1] is mark_id, 4760 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4761 * 4762 * For default flow (zero mark_id), flow is like, 4763 * - If ingress port is ANY, 4764 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4765 * 4766 * @param dev 4767 * Pointer to Ethernet device. 4768 * @param mark_id 4769 * ID of MARK action, zero means default flow for META. 4770 * @param[out] error 4771 * Perform verbose error reporting if not NULL. 4772 * 4773 * @return 4774 * Associated resource on success, NULL otherwise and rte_errno is set. 4775 */ 4776 static struct mlx5_flow_mreg_copy_resource * 4777 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4778 struct rte_flow_error *error) 4779 { 4780 struct mlx5_priv *priv = dev->data->dev_private; 4781 struct mlx5_list_entry *entry; 4782 struct mlx5_flow_cb_ctx ctx = { 4783 .dev = dev, 4784 .error = error, 4785 .data = &mark_id, 4786 }; 4787 4788 /* Check if already registered. */ 4789 MLX5_ASSERT(priv->mreg_cp_tbl); 4790 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4791 if (!entry) 4792 return NULL; 4793 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4794 hlist_ent); 4795 } 4796 4797 void 4798 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4799 { 4800 struct mlx5_flow_mreg_copy_resource *mcp_res = 4801 container_of(entry, typeof(*mcp_res), hlist_ent); 4802 struct rte_eth_dev *dev = tool_ctx; 4803 struct mlx5_priv *priv = dev->data->dev_private; 4804 4805 MLX5_ASSERT(mcp_res->rix_flow); 4806 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4807 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4808 } 4809 4810 /** 4811 * Release flow in RX_CP_TBL. 4812 * 4813 * @param dev 4814 * Pointer to Ethernet device. 4815 * @flow 4816 * Parent flow for wich copying is provided. 4817 */ 4818 static void 4819 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4820 struct rte_flow *flow) 4821 { 4822 struct mlx5_flow_mreg_copy_resource *mcp_res; 4823 struct mlx5_priv *priv = dev->data->dev_private; 4824 4825 if (!flow->rix_mreg_copy) 4826 return; 4827 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4828 flow->rix_mreg_copy); 4829 if (!mcp_res || !priv->mreg_cp_tbl) 4830 return; 4831 MLX5_ASSERT(mcp_res->rix_flow); 4832 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4833 flow->rix_mreg_copy = 0; 4834 } 4835 4836 /** 4837 * Remove the default copy action from RX_CP_TBL. 4838 * 4839 * This functions is called in the mlx5_dev_start(). No thread safe 4840 * is guaranteed. 4841 * 4842 * @param dev 4843 * Pointer to Ethernet device. 4844 */ 4845 static void 4846 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4847 { 4848 struct mlx5_list_entry *entry; 4849 struct mlx5_priv *priv = dev->data->dev_private; 4850 struct mlx5_flow_cb_ctx ctx; 4851 uint32_t mark_id; 4852 4853 /* Check if default flow is registered. */ 4854 if (!priv->mreg_cp_tbl) 4855 return; 4856 mark_id = MLX5_DEFAULT_COPY_ID; 4857 ctx.data = &mark_id; 4858 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4859 if (!entry) 4860 return; 4861 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4862 } 4863 4864 /** 4865 * Add the default copy action in in RX_CP_TBL. 4866 * 4867 * This functions is called in the mlx5_dev_start(). No thread safe 4868 * is guaranteed. 4869 * 4870 * @param dev 4871 * Pointer to Ethernet device. 4872 * @param[out] error 4873 * Perform verbose error reporting if not NULL. 4874 * 4875 * @return 4876 * 0 for success, negative value otherwise and rte_errno is set. 4877 */ 4878 static int 4879 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4880 struct rte_flow_error *error) 4881 { 4882 struct mlx5_priv *priv = dev->data->dev_private; 4883 struct mlx5_flow_mreg_copy_resource *mcp_res; 4884 struct mlx5_flow_cb_ctx ctx; 4885 uint32_t mark_id; 4886 4887 /* Check whether extensive metadata feature is engaged. */ 4888 if (!priv->sh->config.dv_flow_en || 4889 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4890 !mlx5_flow_ext_mreg_supported(dev) || 4891 !priv->sh->dv_regc0_mask) 4892 return 0; 4893 /* 4894 * Add default mreg copy flow may be called multiple time, but 4895 * only be called once in stop. Avoid register it twice. 4896 */ 4897 mark_id = MLX5_DEFAULT_COPY_ID; 4898 ctx.data = &mark_id; 4899 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4900 return 0; 4901 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4902 if (!mcp_res) 4903 return -rte_errno; 4904 return 0; 4905 } 4906 4907 /** 4908 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4909 * 4910 * All the flow having Q/RSS action should be split by 4911 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4912 * performs the following, 4913 * - CQE->flow_tag := reg_c[1] (MARK) 4914 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4915 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4916 * but there should be a flow per each MARK ID set by MARK action. 4917 * 4918 * For the aforementioned reason, if there's a MARK action in flow's action 4919 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4920 * the MARK ID to CQE's flow_tag like, 4921 * - If reg_c[1] is mark_id, 4922 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4923 * 4924 * For SET_META action which stores value in reg_c[0], as the destination is 4925 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4926 * MARK ID means the default flow. The default flow looks like, 4927 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4928 * 4929 * @param dev 4930 * Pointer to Ethernet device. 4931 * @param flow 4932 * Pointer to flow structure. 4933 * @param[in] actions 4934 * Pointer to the list of actions. 4935 * @param[out] error 4936 * Perform verbose error reporting if not NULL. 4937 * 4938 * @return 4939 * 0 on success, negative value otherwise and rte_errno is set. 4940 */ 4941 static int 4942 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4943 struct rte_flow *flow, 4944 const struct rte_flow_action *actions, 4945 struct rte_flow_error *error) 4946 { 4947 struct mlx5_priv *priv = dev->data->dev_private; 4948 struct mlx5_sh_config *config = &priv->sh->config; 4949 struct mlx5_flow_mreg_copy_resource *mcp_res; 4950 const struct rte_flow_action_mark *mark; 4951 4952 /* Check whether extensive metadata feature is engaged. */ 4953 if (!config->dv_flow_en || 4954 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4955 !mlx5_flow_ext_mreg_supported(dev) || 4956 !priv->sh->dv_regc0_mask) 4957 return 0; 4958 /* Find MARK action. */ 4959 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4960 switch (actions->type) { 4961 case RTE_FLOW_ACTION_TYPE_FLAG: 4962 mcp_res = flow_mreg_add_copy_action 4963 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4964 if (!mcp_res) 4965 return -rte_errno; 4966 flow->rix_mreg_copy = mcp_res->idx; 4967 return 0; 4968 case RTE_FLOW_ACTION_TYPE_MARK: 4969 mark = (const struct rte_flow_action_mark *) 4970 actions->conf; 4971 mcp_res = 4972 flow_mreg_add_copy_action(dev, mark->id, error); 4973 if (!mcp_res) 4974 return -rte_errno; 4975 flow->rix_mreg_copy = mcp_res->idx; 4976 return 0; 4977 default: 4978 break; 4979 } 4980 } 4981 return 0; 4982 } 4983 4984 #define MLX5_MAX_SPLIT_ACTIONS 24 4985 #define MLX5_MAX_SPLIT_ITEMS 24 4986 4987 /** 4988 * Split the hairpin flow. 4989 * Since HW can't support encap and push-vlan on Rx, we move these 4990 * actions to Tx. 4991 * If the count action is after the encap then we also 4992 * move the count action. in this case the count will also measure 4993 * the outer bytes. 4994 * 4995 * @param dev 4996 * Pointer to Ethernet device. 4997 * @param[in] actions 4998 * Associated actions (list terminated by the END action). 4999 * @param[out] actions_rx 5000 * Rx flow actions. 5001 * @param[out] actions_tx 5002 * Tx flow actions.. 5003 * @param[out] pattern_tx 5004 * The pattern items for the Tx flow. 5005 * @param[out] flow_id 5006 * The flow ID connected to this flow. 5007 * 5008 * @return 5009 * 0 on success. 5010 */ 5011 static int 5012 flow_hairpin_split(struct rte_eth_dev *dev, 5013 const struct rte_flow_action actions[], 5014 struct rte_flow_action actions_rx[], 5015 struct rte_flow_action actions_tx[], 5016 struct rte_flow_item pattern_tx[], 5017 uint32_t flow_id) 5018 { 5019 const struct rte_flow_action_raw_encap *raw_encap; 5020 const struct rte_flow_action_raw_decap *raw_decap; 5021 struct mlx5_rte_flow_action_set_tag *set_tag; 5022 struct rte_flow_action *tag_action; 5023 struct mlx5_rte_flow_item_tag *tag_item; 5024 struct rte_flow_item *item; 5025 char *addr; 5026 int encap = 0; 5027 5028 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5029 switch (actions->type) { 5030 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5031 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5032 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5033 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5034 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5035 rte_memcpy(actions_tx, actions, 5036 sizeof(struct rte_flow_action)); 5037 actions_tx++; 5038 break; 5039 case RTE_FLOW_ACTION_TYPE_COUNT: 5040 if (encap) { 5041 rte_memcpy(actions_tx, actions, 5042 sizeof(struct rte_flow_action)); 5043 actions_tx++; 5044 } else { 5045 rte_memcpy(actions_rx, actions, 5046 sizeof(struct rte_flow_action)); 5047 actions_rx++; 5048 } 5049 break; 5050 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5051 raw_encap = actions->conf; 5052 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5053 memcpy(actions_tx, actions, 5054 sizeof(struct rte_flow_action)); 5055 actions_tx++; 5056 encap = 1; 5057 } else { 5058 rte_memcpy(actions_rx, actions, 5059 sizeof(struct rte_flow_action)); 5060 actions_rx++; 5061 } 5062 break; 5063 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5064 raw_decap = actions->conf; 5065 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5066 memcpy(actions_tx, actions, 5067 sizeof(struct rte_flow_action)); 5068 actions_tx++; 5069 } else { 5070 rte_memcpy(actions_rx, actions, 5071 sizeof(struct rte_flow_action)); 5072 actions_rx++; 5073 } 5074 break; 5075 default: 5076 rte_memcpy(actions_rx, actions, 5077 sizeof(struct rte_flow_action)); 5078 actions_rx++; 5079 break; 5080 } 5081 } 5082 /* Add set meta action and end action for the Rx flow. */ 5083 tag_action = actions_rx; 5084 tag_action->type = (enum rte_flow_action_type) 5085 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5086 actions_rx++; 5087 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5088 actions_rx++; 5089 set_tag = (void *)actions_rx; 5090 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5091 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5092 .data = flow_id, 5093 }; 5094 MLX5_ASSERT(set_tag->id > REG_NON); 5095 tag_action->conf = set_tag; 5096 /* Create Tx item list. */ 5097 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5098 addr = (void *)&pattern_tx[2]; 5099 item = pattern_tx; 5100 item->type = (enum rte_flow_item_type) 5101 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5102 tag_item = (void *)addr; 5103 tag_item->data = flow_id; 5104 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5105 MLX5_ASSERT(set_tag->id > REG_NON); 5106 item->spec = tag_item; 5107 addr += sizeof(struct mlx5_rte_flow_item_tag); 5108 tag_item = (void *)addr; 5109 tag_item->data = UINT32_MAX; 5110 tag_item->id = UINT16_MAX; 5111 item->mask = tag_item; 5112 item->last = NULL; 5113 item++; 5114 item->type = RTE_FLOW_ITEM_TYPE_END; 5115 return 0; 5116 } 5117 5118 /** 5119 * The last stage of splitting chain, just creates the subflow 5120 * without any modification. 5121 * 5122 * @param[in] dev 5123 * Pointer to Ethernet device. 5124 * @param[in] flow 5125 * Parent flow structure pointer. 5126 * @param[in, out] sub_flow 5127 * Pointer to return the created subflow, may be NULL. 5128 * @param[in] attr 5129 * Flow rule attributes. 5130 * @param[in] items 5131 * Pattern specification (list terminated by the END pattern item). 5132 * @param[in] actions 5133 * Associated actions (list terminated by the END action). 5134 * @param[in] flow_split_info 5135 * Pointer to flow split info structure. 5136 * @param[out] error 5137 * Perform verbose error reporting if not NULL. 5138 * @return 5139 * 0 on success, negative value otherwise 5140 */ 5141 static int 5142 flow_create_split_inner(struct rte_eth_dev *dev, 5143 struct rte_flow *flow, 5144 struct mlx5_flow **sub_flow, 5145 const struct rte_flow_attr *attr, 5146 const struct rte_flow_item items[], 5147 const struct rte_flow_action actions[], 5148 struct mlx5_flow_split_info *flow_split_info, 5149 struct rte_flow_error *error) 5150 { 5151 struct mlx5_flow *dev_flow; 5152 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5153 5154 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5155 flow_split_info->flow_idx, error); 5156 if (!dev_flow) 5157 return -rte_errno; 5158 dev_flow->flow = flow; 5159 dev_flow->external = flow_split_info->external; 5160 dev_flow->skip_scale = flow_split_info->skip_scale; 5161 /* Subflow object was created, we must include one in the list. */ 5162 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5163 dev_flow->handle, next); 5164 /* 5165 * If dev_flow is as one of the suffix flow, some actions in suffix 5166 * flow may need some user defined item layer flags, and pass the 5167 * Metadata rxq mark flag to suffix flow as well. 5168 */ 5169 if (flow_split_info->prefix_layers) 5170 dev_flow->handle->layers = flow_split_info->prefix_layers; 5171 if (flow_split_info->prefix_mark) { 5172 MLX5_ASSERT(wks); 5173 wks->mark = 1; 5174 } 5175 if (sub_flow) 5176 *sub_flow = dev_flow; 5177 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5178 dev_flow->dv.table_id = flow_split_info->table_id; 5179 #endif 5180 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5181 } 5182 5183 /** 5184 * Get the sub policy of a meter. 5185 * 5186 * @param[in] dev 5187 * Pointer to Ethernet device. 5188 * @param[in] flow 5189 * Parent flow structure pointer. 5190 * @param wks 5191 * Pointer to thread flow work space. 5192 * @param[in] attr 5193 * Flow rule attributes. 5194 * @param[in] items 5195 * Pattern specification (list terminated by the END pattern item). 5196 * @param[out] error 5197 * Perform verbose error reporting if not NULL. 5198 * 5199 * @return 5200 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5201 */ 5202 static struct mlx5_flow_meter_sub_policy * 5203 get_meter_sub_policy(struct rte_eth_dev *dev, 5204 struct rte_flow *flow, 5205 struct mlx5_flow_workspace *wks, 5206 const struct rte_flow_attr *attr, 5207 const struct rte_flow_item items[], 5208 struct rte_flow_error *error) 5209 { 5210 struct mlx5_flow_meter_policy *policy; 5211 struct mlx5_flow_meter_policy *final_policy; 5212 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5213 5214 policy = wks->policy; 5215 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5216 if (final_policy->is_rss || final_policy->is_queue) { 5217 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5218 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5219 uint32_t i; 5220 5221 /* 5222 * This is a tmp dev_flow, 5223 * no need to register any matcher for it in translate. 5224 */ 5225 wks->skip_matcher_reg = 1; 5226 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5227 struct mlx5_flow dev_flow = {0}; 5228 struct mlx5_flow_handle dev_handle = { {0} }; 5229 uint8_t fate = final_policy->act_cnt[i].fate_action; 5230 5231 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5232 const struct rte_flow_action_rss *rss_act = 5233 final_policy->act_cnt[i].rss->conf; 5234 struct rte_flow_action rss_actions[2] = { 5235 [0] = { 5236 .type = RTE_FLOW_ACTION_TYPE_RSS, 5237 .conf = rss_act, 5238 }, 5239 [1] = { 5240 .type = RTE_FLOW_ACTION_TYPE_END, 5241 .conf = NULL, 5242 } 5243 }; 5244 5245 dev_flow.handle = &dev_handle; 5246 dev_flow.ingress = attr->ingress; 5247 dev_flow.flow = flow; 5248 dev_flow.external = 0; 5249 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5250 dev_flow.dv.transfer = attr->transfer; 5251 #endif 5252 /** 5253 * Translate RSS action to get rss hash fields. 5254 */ 5255 if (flow_drv_translate(dev, &dev_flow, attr, 5256 items, rss_actions, error)) 5257 goto exit; 5258 rss_desc_v[i] = wks->rss_desc; 5259 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5260 rss_desc_v[i].hash_fields = 5261 dev_flow.hash_fields; 5262 rss_desc_v[i].queue_num = 5263 rss_desc_v[i].hash_fields ? 5264 rss_desc_v[i].queue_num : 1; 5265 rss_desc_v[i].tunnel = 5266 !!(dev_flow.handle->layers & 5267 MLX5_FLOW_LAYER_TUNNEL); 5268 /* Use the RSS queues in the containers. */ 5269 rss_desc_v[i].queue = 5270 (uint16_t *)(uintptr_t)rss_act->queue; 5271 rss_desc[i] = &rss_desc_v[i]; 5272 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5273 /* This is queue action. */ 5274 rss_desc_v[i] = wks->rss_desc; 5275 rss_desc_v[i].key_len = 0; 5276 rss_desc_v[i].hash_fields = 0; 5277 rss_desc_v[i].queue = 5278 &final_policy->act_cnt[i].queue; 5279 rss_desc_v[i].queue_num = 1; 5280 rss_desc[i] = &rss_desc_v[i]; 5281 } else { 5282 rss_desc[i] = NULL; 5283 } 5284 } 5285 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5286 flow, policy, rss_desc); 5287 } else { 5288 enum mlx5_meter_domain mtr_domain = 5289 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5290 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5291 MLX5_MTR_DOMAIN_INGRESS); 5292 sub_policy = policy->sub_policys[mtr_domain][0]; 5293 } 5294 if (!sub_policy) 5295 rte_flow_error_set(error, EINVAL, 5296 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5297 "Failed to get meter sub-policy."); 5298 exit: 5299 return sub_policy; 5300 } 5301 5302 /** 5303 * Split the meter flow. 5304 * 5305 * As meter flow will split to three sub flow, other than meter 5306 * action, the other actions make sense to only meter accepts 5307 * the packet. If it need to be dropped, no other additional 5308 * actions should be take. 5309 * 5310 * One kind of special action which decapsulates the L3 tunnel 5311 * header will be in the prefix sub flow, as not to take the 5312 * L3 tunnel header into account. 5313 * 5314 * @param[in] dev 5315 * Pointer to Ethernet device. 5316 * @param[in] flow 5317 * Parent flow structure pointer. 5318 * @param wks 5319 * Pointer to thread flow work space. 5320 * @param[in] attr 5321 * Flow rule attributes. 5322 * @param[in] items 5323 * Pattern specification (list terminated by the END pattern item). 5324 * @param[out] sfx_items 5325 * Suffix flow match items (list terminated by the END pattern item). 5326 * @param[in] actions 5327 * Associated actions (list terminated by the END action). 5328 * @param[out] actions_sfx 5329 * Suffix flow actions. 5330 * @param[out] actions_pre 5331 * Prefix flow actions. 5332 * @param[out] mtr_flow_id 5333 * Pointer to meter flow id. 5334 * @param[out] error 5335 * Perform verbose error reporting if not NULL. 5336 * 5337 * @return 5338 * 0 on success, a negative errno value otherwise and rte_errno is set. 5339 */ 5340 static int 5341 flow_meter_split_prep(struct rte_eth_dev *dev, 5342 struct rte_flow *flow, 5343 struct mlx5_flow_workspace *wks, 5344 const struct rte_flow_attr *attr, 5345 const struct rte_flow_item items[], 5346 struct rte_flow_item sfx_items[], 5347 const struct rte_flow_action actions[], 5348 struct rte_flow_action actions_sfx[], 5349 struct rte_flow_action actions_pre[], 5350 uint32_t *mtr_flow_id, 5351 struct rte_flow_error *error) 5352 { 5353 struct mlx5_priv *priv = dev->data->dev_private; 5354 struct mlx5_flow_meter_info *fm = wks->fm; 5355 struct rte_flow_action *tag_action = NULL; 5356 struct rte_flow_item *tag_item; 5357 struct mlx5_rte_flow_action_set_tag *set_tag; 5358 const struct rte_flow_action_raw_encap *raw_encap; 5359 const struct rte_flow_action_raw_decap *raw_decap; 5360 struct mlx5_rte_flow_item_tag *tag_item_spec; 5361 struct mlx5_rte_flow_item_tag *tag_item_mask; 5362 uint32_t tag_id = 0; 5363 struct rte_flow_item *vlan_item_dst = NULL; 5364 const struct rte_flow_item *vlan_item_src = NULL; 5365 const struct rte_flow_item *orig_items = items; 5366 struct rte_flow_action *hw_mtr_action; 5367 struct rte_flow_action *action_pre_head = NULL; 5368 uint16_t flow_src_port = priv->representor_id; 5369 bool mtr_first; 5370 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5371 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5372 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5373 uint32_t flow_id = 0; 5374 uint32_t flow_id_reversed = 0; 5375 uint8_t flow_id_bits = 0; 5376 bool after_meter = false; 5377 int shift; 5378 5379 /* Prepare the suffix subflow items. */ 5380 tag_item = sfx_items++; 5381 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5382 int item_type = items->type; 5383 5384 switch (item_type) { 5385 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5386 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5387 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5388 return -rte_errno; 5389 if (!fm->def_policy && wks->policy->is_hierarchy && 5390 flow_src_port != priv->representor_id) { 5391 if (flow_drv_mtr_hierarchy_rule_create(dev, 5392 flow, fm, 5393 flow_src_port, 5394 items, 5395 error)) 5396 return -rte_errno; 5397 } 5398 memcpy(sfx_items, items, sizeof(*sfx_items)); 5399 sfx_items++; 5400 break; 5401 case RTE_FLOW_ITEM_TYPE_VLAN: 5402 /* Determine if copy vlan item below. */ 5403 vlan_item_src = items; 5404 vlan_item_dst = sfx_items++; 5405 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5406 break; 5407 default: 5408 break; 5409 } 5410 } 5411 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5412 sfx_items++; 5413 mtr_first = priv->sh->meter_aso_en && 5414 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5415 /* For ASO meter, meter must be before tag in TX direction. */ 5416 if (mtr_first) { 5417 action_pre_head = actions_pre++; 5418 /* Leave space for tag action. */ 5419 tag_action = actions_pre++; 5420 } 5421 /* Prepare the actions for prefix and suffix flow. */ 5422 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5423 struct rte_flow_action *action_cur = NULL; 5424 5425 switch (actions->type) { 5426 case RTE_FLOW_ACTION_TYPE_METER: 5427 if (mtr_first) { 5428 action_cur = action_pre_head; 5429 } else { 5430 /* Leave space for tag action. */ 5431 tag_action = actions_pre++; 5432 action_cur = actions_pre++; 5433 } 5434 after_meter = true; 5435 break; 5436 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5437 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5438 action_cur = actions_pre++; 5439 break; 5440 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5441 raw_encap = actions->conf; 5442 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5443 action_cur = actions_pre++; 5444 break; 5445 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5446 raw_decap = actions->conf; 5447 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5448 action_cur = actions_pre++; 5449 break; 5450 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5451 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5452 if (vlan_item_dst && vlan_item_src) { 5453 memcpy(vlan_item_dst, vlan_item_src, 5454 sizeof(*vlan_item_dst)); 5455 /* 5456 * Convert to internal match item, it is used 5457 * for vlan push and set vid. 5458 */ 5459 vlan_item_dst->type = (enum rte_flow_item_type) 5460 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5461 } 5462 break; 5463 case RTE_FLOW_ACTION_TYPE_COUNT: 5464 if (fm->def_policy) 5465 action_cur = after_meter ? 5466 actions_sfx++ : actions_pre++; 5467 break; 5468 default: 5469 break; 5470 } 5471 if (!action_cur) 5472 action_cur = (fm->def_policy) ? 5473 actions_sfx++ : actions_pre++; 5474 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5475 } 5476 /* Add end action to the actions. */ 5477 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5478 if (priv->sh->meter_aso_en) { 5479 /** 5480 * For ASO meter, need to add an extra jump action explicitly, 5481 * to jump from meter to policer table. 5482 */ 5483 struct mlx5_flow_meter_sub_policy *sub_policy; 5484 struct mlx5_flow_tbl_data_entry *tbl_data; 5485 5486 if (!fm->def_policy) { 5487 sub_policy = get_meter_sub_policy(dev, flow, wks, 5488 attr, orig_items, 5489 error); 5490 if (!sub_policy) 5491 return -rte_errno; 5492 } else { 5493 enum mlx5_meter_domain mtr_domain = 5494 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5495 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5496 MLX5_MTR_DOMAIN_INGRESS); 5497 5498 sub_policy = 5499 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5500 } 5501 tbl_data = container_of(sub_policy->tbl_rsc, 5502 struct mlx5_flow_tbl_data_entry, tbl); 5503 hw_mtr_action = actions_pre++; 5504 hw_mtr_action->type = (enum rte_flow_action_type) 5505 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5506 hw_mtr_action->conf = tbl_data->jump.action; 5507 } 5508 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5509 actions_pre++; 5510 if (!tag_action) 5511 return rte_flow_error_set(error, ENOMEM, 5512 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5513 NULL, "No tag action space."); 5514 if (!mtr_flow_id) { 5515 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5516 goto exit; 5517 } 5518 /* Only default-policy Meter creates mtr flow id. */ 5519 if (fm->def_policy) { 5520 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5521 if (!tag_id) 5522 return rte_flow_error_set(error, ENOMEM, 5523 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5524 "Failed to allocate meter flow id."); 5525 flow_id = tag_id - 1; 5526 flow_id_bits = (!flow_id) ? 1 : 5527 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5528 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5529 mtr_reg_bits) { 5530 mlx5_ipool_free(fm->flow_ipool, tag_id); 5531 return rte_flow_error_set(error, EINVAL, 5532 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5533 "Meter flow id exceeds max limit."); 5534 } 5535 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5536 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5537 } 5538 /* Build tag actions and items for meter_id/meter flow_id. */ 5539 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5540 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5541 tag_item_mask = tag_item_spec + 1; 5542 /* Both flow_id and meter_id share the same register. */ 5543 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5544 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5545 0, error), 5546 .offset = mtr_id_offset, 5547 .length = mtr_reg_bits, 5548 .data = flow->meter, 5549 }; 5550 /* 5551 * The color Reg bits used by flow_id are growing from 5552 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5553 */ 5554 for (shift = 0; shift < flow_id_bits; shift++) 5555 flow_id_reversed = (flow_id_reversed << 1) | 5556 ((flow_id >> shift) & 0x1); 5557 set_tag->data |= 5558 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5559 tag_item_spec->id = set_tag->id; 5560 tag_item_spec->data = set_tag->data << mtr_id_offset; 5561 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5562 tag_action->type = (enum rte_flow_action_type) 5563 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5564 tag_action->conf = set_tag; 5565 tag_item->type = (enum rte_flow_item_type) 5566 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5567 tag_item->spec = tag_item_spec; 5568 tag_item->last = NULL; 5569 tag_item->mask = tag_item_mask; 5570 exit: 5571 if (mtr_flow_id) 5572 *mtr_flow_id = tag_id; 5573 return 0; 5574 } 5575 5576 /** 5577 * Split action list having QUEUE/RSS for metadata register copy. 5578 * 5579 * Once Q/RSS action is detected in user's action list, the flow action 5580 * should be split in order to copy metadata registers, which will happen in 5581 * RX_CP_TBL like, 5582 * - CQE->flow_tag := reg_c[1] (MARK) 5583 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5584 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5585 * This is because the last action of each flow must be a terminal action 5586 * (QUEUE, RSS or DROP). 5587 * 5588 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5589 * stored and kept in the mlx5_flow structure per each sub_flow. 5590 * 5591 * The Q/RSS action is replaced with, 5592 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5593 * And the following JUMP action is added at the end, 5594 * - JUMP, to RX_CP_TBL. 5595 * 5596 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5597 * flow_create_split_metadata() routine. The flow will look like, 5598 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5599 * 5600 * @param dev 5601 * Pointer to Ethernet device. 5602 * @param[out] split_actions 5603 * Pointer to store split actions to jump to CP_TBL. 5604 * @param[in] actions 5605 * Pointer to the list of original flow actions. 5606 * @param[in] qrss 5607 * Pointer to the Q/RSS action. 5608 * @param[in] actions_n 5609 * Number of original actions. 5610 * @param[in] mtr_sfx 5611 * Check if it is in meter suffix table. 5612 * @param[out] error 5613 * Perform verbose error reporting if not NULL. 5614 * 5615 * @return 5616 * non-zero unique flow_id on success, otherwise 0 and 5617 * error/rte_error are set. 5618 */ 5619 static uint32_t 5620 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5621 struct rte_flow_action *split_actions, 5622 const struct rte_flow_action *actions, 5623 const struct rte_flow_action *qrss, 5624 int actions_n, int mtr_sfx, 5625 struct rte_flow_error *error) 5626 { 5627 struct mlx5_priv *priv = dev->data->dev_private; 5628 struct mlx5_rte_flow_action_set_tag *set_tag; 5629 struct rte_flow_action_jump *jump; 5630 const int qrss_idx = qrss - actions; 5631 uint32_t flow_id = 0; 5632 int ret = 0; 5633 5634 /* 5635 * Given actions will be split 5636 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5637 * - Add jump to mreg CP_TBL. 5638 * As a result, there will be one more action. 5639 */ 5640 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5641 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5642 ++actions_n; 5643 set_tag = (void *)(split_actions + actions_n); 5644 /* 5645 * If we are not the meter suffix flow, add the tag action. 5646 * Since meter suffix flow already has the tag added. 5647 */ 5648 if (!mtr_sfx) { 5649 /* 5650 * Allocate the new subflow ID. This one is unique within 5651 * device and not shared with representors. Otherwise, 5652 * we would have to resolve multi-thread access synch 5653 * issue. Each flow on the shared device is appended 5654 * with source vport identifier, so the resulting 5655 * flows will be unique in the shared (by master and 5656 * representors) domain even if they have coinciding 5657 * IDs. 5658 */ 5659 mlx5_ipool_malloc(priv->sh->ipool 5660 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5661 if (!flow_id) 5662 return rte_flow_error_set(error, ENOMEM, 5663 RTE_FLOW_ERROR_TYPE_ACTION, 5664 NULL, "can't allocate id " 5665 "for split Q/RSS subflow"); 5666 /* Internal SET_TAG action to set flow ID. */ 5667 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5668 .data = flow_id, 5669 }; 5670 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5671 if (ret < 0) 5672 return ret; 5673 set_tag->id = ret; 5674 /* Construct new actions array. */ 5675 /* Replace QUEUE/RSS action. */ 5676 split_actions[qrss_idx] = (struct rte_flow_action){ 5677 .type = (enum rte_flow_action_type) 5678 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5679 .conf = set_tag, 5680 }; 5681 } else { 5682 /* 5683 * If we are the suffix flow of meter, tag already exist. 5684 * Set the QUEUE/RSS action to void. 5685 */ 5686 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5687 } 5688 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5689 jump = (void *)(set_tag + 1); 5690 *jump = (struct rte_flow_action_jump){ 5691 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5692 }; 5693 split_actions[actions_n - 2] = (struct rte_flow_action){ 5694 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5695 .conf = jump, 5696 }; 5697 split_actions[actions_n - 1] = (struct rte_flow_action){ 5698 .type = RTE_FLOW_ACTION_TYPE_END, 5699 }; 5700 return flow_id; 5701 } 5702 5703 /** 5704 * Extend the given action list for Tx metadata copy. 5705 * 5706 * Copy the given action list to the ext_actions and add flow metadata register 5707 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5708 * 5709 * @param[out] ext_actions 5710 * Pointer to the extended action list. 5711 * @param[in] actions 5712 * Pointer to the list of actions. 5713 * @param[in] actions_n 5714 * Number of actions in the list. 5715 * @param[out] error 5716 * Perform verbose error reporting if not NULL. 5717 * @param[in] encap_idx 5718 * The encap action index. 5719 * 5720 * @return 5721 * 0 on success, negative value otherwise 5722 */ 5723 static int 5724 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5725 struct rte_flow_action *ext_actions, 5726 const struct rte_flow_action *actions, 5727 int actions_n, struct rte_flow_error *error, 5728 int encap_idx) 5729 { 5730 struct mlx5_flow_action_copy_mreg *cp_mreg = 5731 (struct mlx5_flow_action_copy_mreg *) 5732 (ext_actions + actions_n + 1); 5733 int ret; 5734 5735 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5736 if (ret < 0) 5737 return ret; 5738 cp_mreg->dst = ret; 5739 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5740 if (ret < 0) 5741 return ret; 5742 cp_mreg->src = ret; 5743 if (encap_idx != 0) 5744 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5745 if (encap_idx == actions_n - 1) { 5746 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5747 .type = (enum rte_flow_action_type) 5748 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5749 .conf = cp_mreg, 5750 }; 5751 ext_actions[actions_n] = (struct rte_flow_action){ 5752 .type = RTE_FLOW_ACTION_TYPE_END, 5753 }; 5754 } else { 5755 ext_actions[encap_idx] = (struct rte_flow_action){ 5756 .type = (enum rte_flow_action_type) 5757 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5758 .conf = cp_mreg, 5759 }; 5760 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5761 sizeof(*ext_actions) * (actions_n - encap_idx)); 5762 } 5763 return 0; 5764 } 5765 5766 /** 5767 * Check the match action from the action list. 5768 * 5769 * @param[in] actions 5770 * Pointer to the list of actions. 5771 * @param[in] attr 5772 * Flow rule attributes. 5773 * @param[in] action 5774 * The action to be check if exist. 5775 * @param[out] match_action_pos 5776 * Pointer to the position of the matched action if exists, otherwise is -1. 5777 * @param[out] qrss_action_pos 5778 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5779 * @param[out] modify_after_mirror 5780 * Pointer to the flag of modify action after FDB mirroring. 5781 * 5782 * @return 5783 * > 0 the total number of actions. 5784 * 0 if not found match action in action list. 5785 */ 5786 static int 5787 flow_check_match_action(const struct rte_flow_action actions[], 5788 const struct rte_flow_attr *attr, 5789 enum rte_flow_action_type action, 5790 int *match_action_pos, int *qrss_action_pos, 5791 int *modify_after_mirror) 5792 { 5793 const struct rte_flow_action_sample *sample; 5794 const struct rte_flow_action_raw_decap *decap; 5795 int actions_n = 0; 5796 uint32_t ratio = 0; 5797 int sub_type = 0; 5798 int flag = 0; 5799 int fdb_mirror = 0; 5800 5801 *match_action_pos = -1; 5802 *qrss_action_pos = -1; 5803 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5804 if (actions->type == action) { 5805 flag = 1; 5806 *match_action_pos = actions_n; 5807 } 5808 switch (actions->type) { 5809 case RTE_FLOW_ACTION_TYPE_QUEUE: 5810 case RTE_FLOW_ACTION_TYPE_RSS: 5811 *qrss_action_pos = actions_n; 5812 break; 5813 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5814 sample = actions->conf; 5815 ratio = sample->ratio; 5816 sub_type = ((const struct rte_flow_action *) 5817 (sample->actions))->type; 5818 if (ratio == 1 && attr->transfer) 5819 fdb_mirror = 1; 5820 break; 5821 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5822 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5823 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5824 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5825 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5826 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5827 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5828 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5829 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5830 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5831 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5832 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5833 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5834 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5835 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5836 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5837 case RTE_FLOW_ACTION_TYPE_FLAG: 5838 case RTE_FLOW_ACTION_TYPE_MARK: 5839 case RTE_FLOW_ACTION_TYPE_SET_META: 5840 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5841 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5842 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5843 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5844 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5845 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5846 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5847 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5848 case RTE_FLOW_ACTION_TYPE_METER: 5849 if (fdb_mirror) 5850 *modify_after_mirror = 1; 5851 break; 5852 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5853 decap = actions->conf; 5854 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5855 ; 5856 actions_n++; 5857 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5858 const struct rte_flow_action_raw_encap *encap = 5859 actions->conf; 5860 if (decap->size <= 5861 MLX5_ENCAPSULATION_DECISION_SIZE && 5862 encap->size > 5863 MLX5_ENCAPSULATION_DECISION_SIZE) 5864 /* L3 encap. */ 5865 break; 5866 } 5867 if (fdb_mirror) 5868 *modify_after_mirror = 1; 5869 break; 5870 default: 5871 break; 5872 } 5873 actions_n++; 5874 } 5875 if (flag && fdb_mirror && !*modify_after_mirror) { 5876 /* FDB mirroring uses the destination array to implement 5877 * instead of FLOW_SAMPLER object. 5878 */ 5879 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5880 flag = 0; 5881 } 5882 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5883 return flag ? actions_n + 1 : 0; 5884 } 5885 5886 #define SAMPLE_SUFFIX_ITEM 3 5887 5888 /** 5889 * Split the sample flow. 5890 * 5891 * As sample flow will split to two sub flow, sample flow with 5892 * sample action, the other actions will move to new suffix flow. 5893 * 5894 * Also add unique tag id with tag action in the sample flow, 5895 * the same tag id will be as match in the suffix flow. 5896 * 5897 * @param dev 5898 * Pointer to Ethernet device. 5899 * @param[in] add_tag 5900 * Add extra tag action flag. 5901 * @param[out] sfx_items 5902 * Suffix flow match items (list terminated by the END pattern item). 5903 * @param[in] actions 5904 * Associated actions (list terminated by the END action). 5905 * @param[out] actions_sfx 5906 * Suffix flow actions. 5907 * @param[out] actions_pre 5908 * Prefix flow actions. 5909 * @param[in] actions_n 5910 * The total number of actions. 5911 * @param[in] sample_action_pos 5912 * The sample action position. 5913 * @param[in] qrss_action_pos 5914 * The Queue/RSS action position. 5915 * @param[in] jump_table 5916 * Add extra jump action flag. 5917 * @param[out] error 5918 * Perform verbose error reporting if not NULL. 5919 * 5920 * @return 5921 * 0 on success, or unique flow_id, a negative errno value 5922 * otherwise and rte_errno is set. 5923 */ 5924 static int 5925 flow_sample_split_prep(struct rte_eth_dev *dev, 5926 int add_tag, 5927 const struct rte_flow_item items[], 5928 struct rte_flow_item sfx_items[], 5929 const struct rte_flow_action actions[], 5930 struct rte_flow_action actions_sfx[], 5931 struct rte_flow_action actions_pre[], 5932 int actions_n, 5933 int sample_action_pos, 5934 int qrss_action_pos, 5935 int jump_table, 5936 struct rte_flow_error *error) 5937 { 5938 struct mlx5_priv *priv = dev->data->dev_private; 5939 struct mlx5_rte_flow_action_set_tag *set_tag; 5940 struct mlx5_rte_flow_item_tag *tag_spec; 5941 struct mlx5_rte_flow_item_tag *tag_mask; 5942 struct rte_flow_action_jump *jump_action; 5943 uint32_t tag_id = 0; 5944 int append_index = 0; 5945 int set_tag_idx = -1; 5946 int index; 5947 int ret; 5948 5949 if (sample_action_pos < 0) 5950 return rte_flow_error_set(error, EINVAL, 5951 RTE_FLOW_ERROR_TYPE_ACTION, 5952 NULL, "invalid position of sample " 5953 "action in list"); 5954 /* Prepare the actions for prefix and suffix flow. */ 5955 if (add_tag) { 5956 /* Update the new added tag action index preceding 5957 * the PUSH_VLAN or ENCAP action. 5958 */ 5959 const struct rte_flow_action_raw_encap *raw_encap; 5960 const struct rte_flow_action *action = actions; 5961 int encap_idx; 5962 int action_idx = 0; 5963 int raw_decap_idx = -1; 5964 int push_vlan_idx = -1; 5965 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 5966 switch (action->type) { 5967 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5968 raw_decap_idx = action_idx; 5969 break; 5970 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5971 raw_encap = action->conf; 5972 if (raw_encap->size > 5973 MLX5_ENCAPSULATION_DECISION_SIZE) { 5974 encap_idx = raw_decap_idx != -1 ? 5975 raw_decap_idx : action_idx; 5976 if (encap_idx < sample_action_pos && 5977 push_vlan_idx == -1) 5978 set_tag_idx = encap_idx; 5979 } 5980 break; 5981 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5982 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5983 encap_idx = action_idx; 5984 if (encap_idx < sample_action_pos && 5985 push_vlan_idx == -1) 5986 set_tag_idx = encap_idx; 5987 break; 5988 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5989 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5990 push_vlan_idx = action_idx; 5991 if (push_vlan_idx < sample_action_pos) 5992 set_tag_idx = action_idx; 5993 break; 5994 default: 5995 break; 5996 } 5997 action_idx++; 5998 } 5999 } 6000 /* Prepare the actions for prefix and suffix flow. */ 6001 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6002 index = qrss_action_pos; 6003 /* Put the preceding the Queue/RSS action into prefix flow. */ 6004 if (index != 0) 6005 memcpy(actions_pre, actions, 6006 sizeof(struct rte_flow_action) * index); 6007 /* Put others preceding the sample action into prefix flow. */ 6008 if (sample_action_pos > index + 1) 6009 memcpy(actions_pre + index, actions + index + 1, 6010 sizeof(struct rte_flow_action) * 6011 (sample_action_pos - index - 1)); 6012 index = sample_action_pos - 1; 6013 /* Put Queue/RSS action into Suffix flow. */ 6014 memcpy(actions_sfx, actions + qrss_action_pos, 6015 sizeof(struct rte_flow_action)); 6016 actions_sfx++; 6017 } else if (add_tag && set_tag_idx >= 0) { 6018 if (set_tag_idx > 0) 6019 memcpy(actions_pre, actions, 6020 sizeof(struct rte_flow_action) * set_tag_idx); 6021 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6022 sizeof(struct rte_flow_action) * 6023 (sample_action_pos - set_tag_idx)); 6024 index = sample_action_pos; 6025 } else { 6026 index = sample_action_pos; 6027 if (index != 0) 6028 memcpy(actions_pre, actions, 6029 sizeof(struct rte_flow_action) * index); 6030 } 6031 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6032 * For CX6DX and above, metadata registers Cx preserve their value, 6033 * add an extra tag action for NIC-RX and E-Switch Domain. 6034 */ 6035 if (add_tag) { 6036 /* Prepare the prefix tag action. */ 6037 append_index++; 6038 set_tag = (void *)(actions_pre + actions_n + append_index); 6039 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6040 /* Trust VF/SF on CX5 not supported meter so that the reserved 6041 * metadata regC is REG_NON, back to use application tag 6042 * index 0. 6043 */ 6044 if (unlikely(ret == REG_NON)) 6045 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6046 if (ret < 0) 6047 return ret; 6048 mlx5_ipool_malloc(priv->sh->ipool 6049 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6050 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6051 .id = ret, 6052 .data = tag_id, 6053 }; 6054 /* Prepare the suffix subflow items. */ 6055 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6056 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6057 memcpy(sfx_items, items, sizeof(*sfx_items)); 6058 sfx_items++; 6059 } 6060 } 6061 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6062 tag_spec->data = tag_id; 6063 tag_spec->id = set_tag->id; 6064 tag_mask = tag_spec + 1; 6065 tag_mask->data = UINT32_MAX; 6066 sfx_items[0] = (struct rte_flow_item){ 6067 .type = (enum rte_flow_item_type) 6068 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6069 .spec = tag_spec, 6070 .last = NULL, 6071 .mask = tag_mask, 6072 }; 6073 sfx_items[1] = (struct rte_flow_item){ 6074 .type = (enum rte_flow_item_type) 6075 RTE_FLOW_ITEM_TYPE_END, 6076 }; 6077 /* Prepare the tag action in prefix subflow. */ 6078 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6079 actions_pre[set_tag_idx] = 6080 (struct rte_flow_action){ 6081 .type = (enum rte_flow_action_type) 6082 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6083 .conf = set_tag, 6084 }; 6085 /* Update next sample position due to add one tag action */ 6086 index += 1; 6087 } 6088 /* Copy the sample action into prefix flow. */ 6089 memcpy(actions_pre + index, actions + sample_action_pos, 6090 sizeof(struct rte_flow_action)); 6091 index += 1; 6092 /* For the modify action after the sample action in E-Switch mirroring, 6093 * Add the extra jump action in prefix subflow and jump into the next 6094 * table, then do the modify action in the new table. 6095 */ 6096 if (jump_table) { 6097 /* Prepare the prefix jump action. */ 6098 append_index++; 6099 jump_action = (void *)(actions_pre + actions_n + append_index); 6100 jump_action->group = jump_table; 6101 actions_pre[index++] = 6102 (struct rte_flow_action){ 6103 .type = (enum rte_flow_action_type) 6104 RTE_FLOW_ACTION_TYPE_JUMP, 6105 .conf = jump_action, 6106 }; 6107 } 6108 actions_pre[index] = (struct rte_flow_action){ 6109 .type = (enum rte_flow_action_type) 6110 RTE_FLOW_ACTION_TYPE_END, 6111 }; 6112 /* Put the actions after sample into Suffix flow. */ 6113 memcpy(actions_sfx, actions + sample_action_pos + 1, 6114 sizeof(struct rte_flow_action) * 6115 (actions_n - sample_action_pos - 1)); 6116 return tag_id; 6117 } 6118 6119 /** 6120 * The splitting for metadata feature. 6121 * 6122 * - Q/RSS action on NIC Rx should be split in order to pass by 6123 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6124 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6125 * 6126 * - All the actions on NIC Tx should have a mreg copy action to 6127 * copy reg_a from WQE to reg_c[0]. 6128 * 6129 * @param dev 6130 * Pointer to Ethernet device. 6131 * @param[in] flow 6132 * Parent flow structure pointer. 6133 * @param[in] attr 6134 * Flow rule attributes. 6135 * @param[in] items 6136 * Pattern specification (list terminated by the END pattern item). 6137 * @param[in] actions 6138 * Associated actions (list terminated by the END action). 6139 * @param[in] flow_split_info 6140 * Pointer to flow split info structure. 6141 * @param[out] error 6142 * Perform verbose error reporting if not NULL. 6143 * @return 6144 * 0 on success, negative value otherwise 6145 */ 6146 static int 6147 flow_create_split_metadata(struct rte_eth_dev *dev, 6148 struct rte_flow *flow, 6149 const struct rte_flow_attr *attr, 6150 const struct rte_flow_item items[], 6151 const struct rte_flow_action actions[], 6152 struct mlx5_flow_split_info *flow_split_info, 6153 struct rte_flow_error *error) 6154 { 6155 struct mlx5_priv *priv = dev->data->dev_private; 6156 struct mlx5_sh_config *config = &priv->sh->config; 6157 const struct rte_flow_action *qrss = NULL; 6158 struct rte_flow_action *ext_actions = NULL; 6159 struct mlx5_flow *dev_flow = NULL; 6160 uint32_t qrss_id = 0; 6161 int mtr_sfx = 0; 6162 size_t act_size; 6163 int actions_n; 6164 int encap_idx; 6165 int ret; 6166 6167 /* Check whether extensive metadata feature is engaged. */ 6168 if (!config->dv_flow_en || 6169 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6170 !mlx5_flow_ext_mreg_supported(dev)) 6171 return flow_create_split_inner(dev, flow, NULL, attr, items, 6172 actions, flow_split_info, error); 6173 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6174 &encap_idx); 6175 if (qrss) { 6176 /* Exclude hairpin flows from splitting. */ 6177 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6178 const struct rte_flow_action_queue *queue; 6179 6180 queue = qrss->conf; 6181 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6182 qrss = NULL; 6183 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6184 const struct rte_flow_action_rss *rss; 6185 6186 rss = qrss->conf; 6187 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6188 qrss = NULL; 6189 } 6190 } 6191 if (qrss) { 6192 /* Check if it is in meter suffix table. */ 6193 mtr_sfx = attr->group == 6194 ((attr->transfer && priv->fdb_def_rule) ? 6195 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6196 MLX5_FLOW_TABLE_LEVEL_METER); 6197 /* 6198 * Q/RSS action on NIC Rx should be split in order to pass by 6199 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6200 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6201 */ 6202 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6203 sizeof(struct rte_flow_action_set_tag) + 6204 sizeof(struct rte_flow_action_jump); 6205 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6206 SOCKET_ID_ANY); 6207 if (!ext_actions) 6208 return rte_flow_error_set(error, ENOMEM, 6209 RTE_FLOW_ERROR_TYPE_ACTION, 6210 NULL, "no memory to split " 6211 "metadata flow"); 6212 /* 6213 * Create the new actions list with removed Q/RSS action 6214 * and appended set tag and jump to register copy table 6215 * (RX_CP_TBL). We should preallocate unique tag ID here 6216 * in advance, because it is needed for set tag action. 6217 */ 6218 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6219 qrss, actions_n, 6220 mtr_sfx, error); 6221 if (!mtr_sfx && !qrss_id) { 6222 ret = -rte_errno; 6223 goto exit; 6224 } 6225 } else if (attr->egress && !attr->transfer) { 6226 /* 6227 * All the actions on NIC Tx should have a metadata register 6228 * copy action to copy reg_a from WQE to reg_c[meta] 6229 */ 6230 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6231 sizeof(struct mlx5_flow_action_copy_mreg); 6232 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6233 SOCKET_ID_ANY); 6234 if (!ext_actions) 6235 return rte_flow_error_set(error, ENOMEM, 6236 RTE_FLOW_ERROR_TYPE_ACTION, 6237 NULL, "no memory to split " 6238 "metadata flow"); 6239 /* Create the action list appended with copy register. */ 6240 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6241 actions_n, error, encap_idx); 6242 if (ret < 0) 6243 goto exit; 6244 } 6245 /* Add the unmodified original or prefix subflow. */ 6246 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6247 items, ext_actions ? ext_actions : 6248 actions, flow_split_info, error); 6249 if (ret < 0) 6250 goto exit; 6251 MLX5_ASSERT(dev_flow); 6252 if (qrss) { 6253 const struct rte_flow_attr q_attr = { 6254 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6255 .ingress = 1, 6256 }; 6257 /* Internal PMD action to set register. */ 6258 struct mlx5_rte_flow_item_tag q_tag_spec = { 6259 .data = qrss_id, 6260 .id = REG_NON, 6261 }; 6262 struct rte_flow_item q_items[] = { 6263 { 6264 .type = (enum rte_flow_item_type) 6265 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6266 .spec = &q_tag_spec, 6267 .last = NULL, 6268 .mask = NULL, 6269 }, 6270 { 6271 .type = RTE_FLOW_ITEM_TYPE_END, 6272 }, 6273 }; 6274 struct rte_flow_action q_actions[] = { 6275 { 6276 .type = qrss->type, 6277 .conf = qrss->conf, 6278 }, 6279 { 6280 .type = RTE_FLOW_ACTION_TYPE_END, 6281 }, 6282 }; 6283 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6284 6285 /* 6286 * Configure the tag item only if there is no meter subflow. 6287 * Since tag is already marked in the meter suffix subflow 6288 * we can just use the meter suffix items as is. 6289 */ 6290 if (qrss_id) { 6291 /* Not meter subflow. */ 6292 MLX5_ASSERT(!mtr_sfx); 6293 /* 6294 * Put unique id in prefix flow due to it is destroyed 6295 * after suffix flow and id will be freed after there 6296 * is no actual flows with this id and identifier 6297 * reallocation becomes possible (for example, for 6298 * other flows in other threads). 6299 */ 6300 dev_flow->handle->split_flow_id = qrss_id; 6301 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6302 error); 6303 if (ret < 0) 6304 goto exit; 6305 q_tag_spec.id = ret; 6306 } 6307 dev_flow = NULL; 6308 /* Add suffix subflow to execute Q/RSS. */ 6309 flow_split_info->prefix_layers = layers; 6310 flow_split_info->prefix_mark = 0; 6311 flow_split_info->table_id = 0; 6312 ret = flow_create_split_inner(dev, flow, &dev_flow, 6313 &q_attr, mtr_sfx ? items : 6314 q_items, q_actions, 6315 flow_split_info, error); 6316 if (ret < 0) 6317 goto exit; 6318 /* qrss ID should be freed if failed. */ 6319 qrss_id = 0; 6320 MLX5_ASSERT(dev_flow); 6321 } 6322 6323 exit: 6324 /* 6325 * We do not destroy the partially created sub_flows in case of error. 6326 * These ones are included into parent flow list and will be destroyed 6327 * by flow_drv_destroy. 6328 */ 6329 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6330 qrss_id); 6331 mlx5_free(ext_actions); 6332 return ret; 6333 } 6334 6335 /** 6336 * Create meter internal drop flow with the original pattern. 6337 * 6338 * @param dev 6339 * Pointer to Ethernet device. 6340 * @param[in] flow 6341 * Parent flow structure pointer. 6342 * @param[in] attr 6343 * Flow rule attributes. 6344 * @param[in] items 6345 * Pattern specification (list terminated by the END pattern item). 6346 * @param[in] flow_split_info 6347 * Pointer to flow split info structure. 6348 * @param[in] fm 6349 * Pointer to flow meter structure. 6350 * @param[out] error 6351 * Perform verbose error reporting if not NULL. 6352 * @return 6353 * 0 on success, negative value otherwise 6354 */ 6355 static uint32_t 6356 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6357 struct rte_flow *flow, 6358 const struct rte_flow_attr *attr, 6359 const struct rte_flow_item items[], 6360 struct mlx5_flow_split_info *flow_split_info, 6361 struct mlx5_flow_meter_info *fm, 6362 struct rte_flow_error *error) 6363 { 6364 struct mlx5_flow *dev_flow = NULL; 6365 struct rte_flow_attr drop_attr = *attr; 6366 struct rte_flow_action drop_actions[3]; 6367 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6368 6369 MLX5_ASSERT(fm->drop_cnt); 6370 drop_actions[0].type = 6371 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6372 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6373 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6374 drop_actions[1].conf = NULL; 6375 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6376 drop_actions[2].conf = NULL; 6377 drop_split_info.external = false; 6378 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6379 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6380 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6381 return flow_create_split_inner(dev, flow, &dev_flow, 6382 &drop_attr, items, drop_actions, 6383 &drop_split_info, error); 6384 } 6385 6386 /** 6387 * The splitting for meter feature. 6388 * 6389 * - The meter flow will be split to two flows as prefix and 6390 * suffix flow. The packets make sense only it pass the prefix 6391 * meter action. 6392 * 6393 * - Reg_C_5 is used for the packet to match betweend prefix and 6394 * suffix flow. 6395 * 6396 * @param dev 6397 * Pointer to Ethernet device. 6398 * @param[in] flow 6399 * Parent flow structure pointer. 6400 * @param[in] attr 6401 * Flow rule attributes. 6402 * @param[in] items 6403 * Pattern specification (list terminated by the END pattern item). 6404 * @param[in] actions 6405 * Associated actions (list terminated by the END action). 6406 * @param[in] flow_split_info 6407 * Pointer to flow split info structure. 6408 * @param[out] error 6409 * Perform verbose error reporting if not NULL. 6410 * @return 6411 * 0 on success, negative value otherwise 6412 */ 6413 static int 6414 flow_create_split_meter(struct rte_eth_dev *dev, 6415 struct rte_flow *flow, 6416 const struct rte_flow_attr *attr, 6417 const struct rte_flow_item items[], 6418 const struct rte_flow_action actions[], 6419 struct mlx5_flow_split_info *flow_split_info, 6420 struct rte_flow_error *error) 6421 { 6422 struct mlx5_priv *priv = dev->data->dev_private; 6423 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6424 struct rte_flow_action *sfx_actions = NULL; 6425 struct rte_flow_action *pre_actions = NULL; 6426 struct rte_flow_item *sfx_items = NULL; 6427 struct mlx5_flow *dev_flow = NULL; 6428 struct rte_flow_attr sfx_attr = *attr; 6429 struct mlx5_flow_meter_info *fm = NULL; 6430 uint8_t skip_scale_restore; 6431 bool has_mtr = false; 6432 bool has_modify = false; 6433 bool set_mtr_reg = true; 6434 bool is_mtr_hierarchy = false; 6435 uint32_t meter_id = 0; 6436 uint32_t mtr_idx = 0; 6437 uint32_t mtr_flow_id = 0; 6438 size_t act_size; 6439 size_t item_size; 6440 int actions_n = 0; 6441 int ret = 0; 6442 6443 if (priv->mtr_en) 6444 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6445 &has_modify, &meter_id); 6446 if (has_mtr) { 6447 if (flow->meter) { 6448 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6449 if (!fm) 6450 return rte_flow_error_set(error, EINVAL, 6451 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6452 NULL, "Meter not found."); 6453 } else { 6454 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6455 if (!fm) 6456 return rte_flow_error_set(error, EINVAL, 6457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6458 NULL, "Meter not found."); 6459 ret = mlx5_flow_meter_attach(priv, fm, 6460 &sfx_attr, error); 6461 if (ret) 6462 return -rte_errno; 6463 flow->meter = mtr_idx; 6464 } 6465 MLX5_ASSERT(wks); 6466 wks->fm = fm; 6467 if (!fm->def_policy) { 6468 wks->policy = mlx5_flow_meter_policy_find(dev, 6469 fm->policy_id, 6470 NULL); 6471 MLX5_ASSERT(wks->policy); 6472 if (wks->policy->mark) 6473 wks->mark = 1; 6474 if (wks->policy->is_hierarchy) { 6475 wks->final_policy = 6476 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6477 wks->policy); 6478 if (!wks->final_policy) 6479 return rte_flow_error_set(error, 6480 EINVAL, 6481 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6482 "Failed to find terminal policy of hierarchy."); 6483 is_mtr_hierarchy = true; 6484 } 6485 } 6486 /* 6487 * If it isn't default-policy Meter, and 6488 * 1. Not meter hierarchy and there's no action in flow to change 6489 * packet (modify/encap/decap etc.), OR 6490 * 2. No drop count needed for this meter. 6491 * Then no need to use regC to save meter id anymore. 6492 */ 6493 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6494 set_mtr_reg = false; 6495 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6496 #define METER_PREFIX_ACTION 7 6497 act_size = (sizeof(struct rte_flow_action) * 6498 (actions_n + METER_PREFIX_ACTION)) + 6499 sizeof(struct mlx5_rte_flow_action_set_tag); 6500 /* Suffix items: tag, vlan, port id, end. */ 6501 #define METER_SUFFIX_ITEM 4 6502 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6503 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6504 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6505 0, SOCKET_ID_ANY); 6506 if (!sfx_actions) 6507 return rte_flow_error_set(error, ENOMEM, 6508 RTE_FLOW_ERROR_TYPE_ACTION, 6509 NULL, "no memory to split " 6510 "meter flow"); 6511 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6512 act_size); 6513 /* There's no suffix flow for meter of non-default policy. */ 6514 if (!fm->def_policy) 6515 pre_actions = sfx_actions + 1; 6516 else 6517 pre_actions = sfx_actions + actions_n; 6518 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6519 items, sfx_items, actions, 6520 sfx_actions, pre_actions, 6521 (set_mtr_reg ? &mtr_flow_id : NULL), 6522 error); 6523 if (ret) { 6524 ret = -rte_errno; 6525 goto exit; 6526 } 6527 /* Add the prefix subflow. */ 6528 skip_scale_restore = flow_split_info->skip_scale; 6529 flow_split_info->skip_scale |= 6530 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6531 ret = flow_create_split_inner(dev, flow, &dev_flow, 6532 attr, items, pre_actions, 6533 flow_split_info, error); 6534 flow_split_info->skip_scale = skip_scale_restore; 6535 if (ret) { 6536 if (mtr_flow_id) 6537 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6538 ret = -rte_errno; 6539 goto exit; 6540 } 6541 if (mtr_flow_id) { 6542 dev_flow->handle->split_flow_id = mtr_flow_id; 6543 dev_flow->handle->is_meter_flow_id = 1; 6544 } 6545 if (!fm->def_policy) { 6546 if (!set_mtr_reg && fm->drop_cnt) 6547 ret = 6548 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6549 &sfx_attr, items, 6550 flow_split_info, 6551 fm, error); 6552 goto exit; 6553 } 6554 /* Setting the sfx group atrr. */ 6555 sfx_attr.group = sfx_attr.transfer ? 6556 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6557 MLX5_FLOW_TABLE_LEVEL_METER; 6558 flow_split_info->prefix_layers = 6559 flow_get_prefix_layer_flags(dev_flow); 6560 flow_split_info->prefix_mark |= wks->mark; 6561 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6562 } 6563 /* Add the prefix subflow. */ 6564 ret = flow_create_split_metadata(dev, flow, 6565 &sfx_attr, sfx_items ? 6566 sfx_items : items, 6567 sfx_actions ? sfx_actions : actions, 6568 flow_split_info, error); 6569 exit: 6570 if (sfx_actions) 6571 mlx5_free(sfx_actions); 6572 return ret; 6573 } 6574 6575 /** 6576 * The splitting for sample feature. 6577 * 6578 * Once Sample action is detected in the action list, the flow actions should 6579 * be split into prefix sub flow and suffix sub flow. 6580 * 6581 * The original items remain in the prefix sub flow, all actions preceding the 6582 * sample action and the sample action itself will be copied to the prefix 6583 * sub flow, the actions following the sample action will be copied to the 6584 * suffix sub flow, Queue action always be located in the suffix sub flow. 6585 * 6586 * In order to make the packet from prefix sub flow matches with suffix sub 6587 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6588 * flow uses tag item with the unique flow id. 6589 * 6590 * @param dev 6591 * Pointer to Ethernet device. 6592 * @param[in] flow 6593 * Parent flow structure pointer. 6594 * @param[in] attr 6595 * Flow rule attributes. 6596 * @param[in] items 6597 * Pattern specification (list terminated by the END pattern item). 6598 * @param[in] actions 6599 * Associated actions (list terminated by the END action). 6600 * @param[in] flow_split_info 6601 * Pointer to flow split info structure. 6602 * @param[out] error 6603 * Perform verbose error reporting if not NULL. 6604 * @return 6605 * 0 on success, negative value otherwise 6606 */ 6607 static int 6608 flow_create_split_sample(struct rte_eth_dev *dev, 6609 struct rte_flow *flow, 6610 const struct rte_flow_attr *attr, 6611 const struct rte_flow_item items[], 6612 const struct rte_flow_action actions[], 6613 struct mlx5_flow_split_info *flow_split_info, 6614 struct rte_flow_error *error) 6615 { 6616 struct mlx5_priv *priv = dev->data->dev_private; 6617 struct rte_flow_action *sfx_actions = NULL; 6618 struct rte_flow_action *pre_actions = NULL; 6619 struct rte_flow_item *sfx_items = NULL; 6620 struct mlx5_flow *dev_flow = NULL; 6621 struct rte_flow_attr sfx_attr = *attr; 6622 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6623 struct mlx5_flow_dv_sample_resource *sample_res; 6624 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6625 struct mlx5_flow_tbl_resource *sfx_tbl; 6626 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6627 #endif 6628 size_t act_size; 6629 size_t item_size; 6630 uint32_t fdb_tx = 0; 6631 int32_t tag_id = 0; 6632 int actions_n = 0; 6633 int sample_action_pos; 6634 int qrss_action_pos; 6635 int add_tag = 0; 6636 int modify_after_mirror = 0; 6637 uint16_t jump_table = 0; 6638 const uint32_t next_ft_step = 1; 6639 int ret = 0; 6640 6641 if (priv->sampler_en) 6642 actions_n = flow_check_match_action(actions, attr, 6643 RTE_FLOW_ACTION_TYPE_SAMPLE, 6644 &sample_action_pos, &qrss_action_pos, 6645 &modify_after_mirror); 6646 if (actions_n) { 6647 /* The prefix actions must includes sample, tag, end. */ 6648 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6649 + sizeof(struct mlx5_rte_flow_action_set_tag); 6650 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6651 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6652 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6653 item_size), 0, SOCKET_ID_ANY); 6654 if (!sfx_actions) 6655 return rte_flow_error_set(error, ENOMEM, 6656 RTE_FLOW_ERROR_TYPE_ACTION, 6657 NULL, "no memory to split " 6658 "sample flow"); 6659 /* The representor_id is UINT16_MAX for uplink. */ 6660 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6661 /* 6662 * When reg_c_preserve is set, metadata registers Cx preserve 6663 * their value even through packet duplication. 6664 */ 6665 add_tag = (!fdb_tx || 6666 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6667 if (add_tag) 6668 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6669 + act_size); 6670 if (modify_after_mirror) 6671 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6672 next_ft_step; 6673 pre_actions = sfx_actions + actions_n; 6674 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6675 actions, sfx_actions, 6676 pre_actions, actions_n, 6677 sample_action_pos, 6678 qrss_action_pos, jump_table, 6679 error); 6680 if (tag_id < 0 || (add_tag && !tag_id)) { 6681 ret = -rte_errno; 6682 goto exit; 6683 } 6684 if (modify_after_mirror) 6685 flow_split_info->skip_scale = 6686 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6687 /* Add the prefix subflow. */ 6688 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6689 items, pre_actions, 6690 flow_split_info, error); 6691 if (ret) { 6692 ret = -rte_errno; 6693 goto exit; 6694 } 6695 dev_flow->handle->split_flow_id = tag_id; 6696 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6697 if (!modify_after_mirror) { 6698 /* Set the sfx group attr. */ 6699 sample_res = (struct mlx5_flow_dv_sample_resource *) 6700 dev_flow->dv.sample_res; 6701 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6702 sample_res->normal_path_tbl; 6703 sfx_tbl_data = container_of(sfx_tbl, 6704 struct mlx5_flow_tbl_data_entry, 6705 tbl); 6706 sfx_attr.group = sfx_attr.transfer ? 6707 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6708 } else { 6709 MLX5_ASSERT(attr->transfer); 6710 sfx_attr.group = jump_table; 6711 } 6712 flow_split_info->prefix_layers = 6713 flow_get_prefix_layer_flags(dev_flow); 6714 MLX5_ASSERT(wks); 6715 flow_split_info->prefix_mark |= wks->mark; 6716 /* Suffix group level already be scaled with factor, set 6717 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6718 * again in translation. 6719 */ 6720 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6721 #endif 6722 } 6723 /* Add the suffix subflow. */ 6724 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6725 sfx_items ? sfx_items : items, 6726 sfx_actions ? sfx_actions : actions, 6727 flow_split_info, error); 6728 exit: 6729 if (sfx_actions) 6730 mlx5_free(sfx_actions); 6731 return ret; 6732 } 6733 6734 /** 6735 * Split the flow to subflow set. The splitters might be linked 6736 * in the chain, like this: 6737 * flow_create_split_outer() calls: 6738 * flow_create_split_meter() calls: 6739 * flow_create_split_metadata(meter_subflow_0) calls: 6740 * flow_create_split_inner(metadata_subflow_0) 6741 * flow_create_split_inner(metadata_subflow_1) 6742 * flow_create_split_inner(metadata_subflow_2) 6743 * flow_create_split_metadata(meter_subflow_1) calls: 6744 * flow_create_split_inner(metadata_subflow_0) 6745 * flow_create_split_inner(metadata_subflow_1) 6746 * flow_create_split_inner(metadata_subflow_2) 6747 * 6748 * This provide flexible way to add new levels of flow splitting. 6749 * The all of successfully created subflows are included to the 6750 * parent flow dev_flow list. 6751 * 6752 * @param dev 6753 * Pointer to Ethernet device. 6754 * @param[in] flow 6755 * Parent flow structure pointer. 6756 * @param[in] attr 6757 * Flow rule attributes. 6758 * @param[in] items 6759 * Pattern specification (list terminated by the END pattern item). 6760 * @param[in] actions 6761 * Associated actions (list terminated by the END action). 6762 * @param[in] flow_split_info 6763 * Pointer to flow split info structure. 6764 * @param[out] error 6765 * Perform verbose error reporting if not NULL. 6766 * @return 6767 * 0 on success, negative value otherwise 6768 */ 6769 static int 6770 flow_create_split_outer(struct rte_eth_dev *dev, 6771 struct rte_flow *flow, 6772 const struct rte_flow_attr *attr, 6773 const struct rte_flow_item items[], 6774 const struct rte_flow_action actions[], 6775 struct mlx5_flow_split_info *flow_split_info, 6776 struct rte_flow_error *error) 6777 { 6778 int ret; 6779 6780 ret = flow_create_split_sample(dev, flow, attr, items, 6781 actions, flow_split_info, error); 6782 MLX5_ASSERT(ret <= 0); 6783 return ret; 6784 } 6785 6786 static inline struct mlx5_flow_tunnel * 6787 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6788 { 6789 struct mlx5_flow_tunnel *tunnel; 6790 6791 #pragma GCC diagnostic push 6792 #pragma GCC diagnostic ignored "-Wcast-qual" 6793 tunnel = (typeof(tunnel))flow->tunnel; 6794 #pragma GCC diagnostic pop 6795 6796 return tunnel; 6797 } 6798 6799 /** 6800 * Adjust flow RSS workspace if needed. 6801 * 6802 * @param wks 6803 * Pointer to thread flow work space. 6804 * @param rss_desc 6805 * Pointer to RSS descriptor. 6806 * @param[in] nrssq_num 6807 * New RSS queue number. 6808 * 6809 * @return 6810 * 0 on success, -1 otherwise and rte_errno is set. 6811 */ 6812 static int 6813 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6814 struct mlx5_flow_rss_desc *rss_desc, 6815 uint32_t nrssq_num) 6816 { 6817 if (likely(nrssq_num <= wks->rssq_num)) 6818 return 0; 6819 rss_desc->queue = realloc(rss_desc->queue, 6820 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6821 if (!rss_desc->queue) { 6822 rte_errno = ENOMEM; 6823 return -1; 6824 } 6825 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6826 return 0; 6827 } 6828 6829 /** 6830 * Create a flow and add it to @p list. 6831 * 6832 * @param dev 6833 * Pointer to Ethernet device. 6834 * @param list 6835 * Pointer to a TAILQ flow list. If this parameter NULL, 6836 * no list insertion occurred, flow is just created, 6837 * this is caller's responsibility to track the 6838 * created flow. 6839 * @param[in] attr 6840 * Flow rule attributes. 6841 * @param[in] items 6842 * Pattern specification (list terminated by the END pattern item). 6843 * @param[in] actions 6844 * Associated actions (list terminated by the END action). 6845 * @param[in] external 6846 * This flow rule is created by request external to PMD. 6847 * @param[out] error 6848 * Perform verbose error reporting if not NULL. 6849 * 6850 * @return 6851 * A flow index on success, 0 otherwise and rte_errno is set. 6852 */ 6853 static uint32_t 6854 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6855 const struct rte_flow_attr *attr, 6856 const struct rte_flow_item items[], 6857 const struct rte_flow_action original_actions[], 6858 bool external, struct rte_flow_error *error) 6859 { 6860 struct mlx5_priv *priv = dev->data->dev_private; 6861 struct rte_flow *flow = NULL; 6862 struct mlx5_flow *dev_flow; 6863 const struct rte_flow_action_rss *rss = NULL; 6864 struct mlx5_translated_action_handle 6865 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6866 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6867 union { 6868 struct mlx5_flow_expand_rss buf; 6869 uint8_t buffer[4096]; 6870 } expand_buffer; 6871 union { 6872 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6873 uint8_t buffer[2048]; 6874 } actions_rx; 6875 union { 6876 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6877 uint8_t buffer[2048]; 6878 } actions_hairpin_tx; 6879 union { 6880 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6881 uint8_t buffer[2048]; 6882 } items_tx; 6883 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6884 struct mlx5_flow_rss_desc *rss_desc; 6885 const struct rte_flow_action *p_actions_rx; 6886 uint32_t i; 6887 uint32_t idx = 0; 6888 int hairpin_flow; 6889 struct rte_flow_attr attr_tx = { .priority = 0 }; 6890 const struct rte_flow_action *actions; 6891 struct rte_flow_action *translated_actions = NULL; 6892 struct mlx5_flow_tunnel *tunnel; 6893 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6894 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6895 struct mlx5_flow_split_info flow_split_info = { 6896 .external = !!external, 6897 .skip_scale = 0, 6898 .flow_idx = 0, 6899 .prefix_mark = 0, 6900 .prefix_layers = 0, 6901 .table_id = 0 6902 }; 6903 int ret; 6904 6905 MLX5_ASSERT(wks); 6906 rss_desc = &wks->rss_desc; 6907 ret = flow_action_handles_translate(dev, original_actions, 6908 indir_actions, 6909 &indir_actions_n, 6910 &translated_actions, error); 6911 if (ret < 0) { 6912 MLX5_ASSERT(translated_actions == NULL); 6913 return 0; 6914 } 6915 actions = translated_actions ? translated_actions : original_actions; 6916 p_actions_rx = actions; 6917 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6918 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6919 external, hairpin_flow, error); 6920 if (ret < 0) 6921 goto error_before_hairpin_split; 6922 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6923 if (!flow) { 6924 rte_errno = ENOMEM; 6925 goto error_before_hairpin_split; 6926 } 6927 if (hairpin_flow > 0) { 6928 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6929 rte_errno = EINVAL; 6930 goto error_before_hairpin_split; 6931 } 6932 flow_hairpin_split(dev, actions, actions_rx.actions, 6933 actions_hairpin_tx.actions, items_tx.items, 6934 idx); 6935 p_actions_rx = actions_rx.actions; 6936 } 6937 flow_split_info.flow_idx = idx; 6938 flow->drv_type = flow_get_drv_type(dev, attr); 6939 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6940 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6941 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6942 /* RSS Action only works on NIC RX domain */ 6943 if (attr->ingress && !attr->transfer) 6944 rss = flow_get_rss_action(dev, p_actions_rx); 6945 if (rss) { 6946 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6947 return 0; 6948 /* 6949 * The following information is required by 6950 * mlx5_flow_hashfields_adjust() in advance. 6951 */ 6952 rss_desc->level = rss->level; 6953 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6954 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6955 } 6956 flow->dev_handles = 0; 6957 if (rss && rss->types) { 6958 unsigned int graph_root; 6959 6960 graph_root = find_graph_root(rss->level); 6961 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6962 items, rss->types, 6963 mlx5_support_expansion, graph_root); 6964 MLX5_ASSERT(ret > 0 && 6965 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6966 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6967 for (i = 0; i < buf->entries; ++i) 6968 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6969 } 6970 } else { 6971 buf->entries = 1; 6972 buf->entry[0].pattern = (void *)(uintptr_t)items; 6973 } 6974 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6975 indir_actions_n); 6976 for (i = 0; i < buf->entries; ++i) { 6977 /* Initialize flow split data. */ 6978 flow_split_info.prefix_layers = 0; 6979 flow_split_info.prefix_mark = 0; 6980 flow_split_info.skip_scale = 0; 6981 /* 6982 * The splitter may create multiple dev_flows, 6983 * depending on configuration. In the simplest 6984 * case it just creates unmodified original flow. 6985 */ 6986 ret = flow_create_split_outer(dev, flow, attr, 6987 buf->entry[i].pattern, 6988 p_actions_rx, &flow_split_info, 6989 error); 6990 if (ret < 0) 6991 goto error; 6992 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 6993 ret = flow_tunnel_add_default_miss(dev, flow, attr, 6994 p_actions_rx, 6995 idx, 6996 wks->flows[0].tunnel, 6997 &default_miss_ctx, 6998 error); 6999 if (ret < 0) { 7000 mlx5_free(default_miss_ctx.queue); 7001 goto error; 7002 } 7003 } 7004 } 7005 /* Create the tx flow. */ 7006 if (hairpin_flow) { 7007 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7008 attr_tx.ingress = 0; 7009 attr_tx.egress = 1; 7010 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7011 actions_hairpin_tx.actions, 7012 idx, error); 7013 if (!dev_flow) 7014 goto error; 7015 dev_flow->flow = flow; 7016 dev_flow->external = 0; 7017 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7018 dev_flow->handle, next); 7019 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7020 items_tx.items, 7021 actions_hairpin_tx.actions, error); 7022 if (ret < 0) 7023 goto error; 7024 } 7025 /* 7026 * Update the metadata register copy table. If extensive 7027 * metadata feature is enabled and registers are supported 7028 * we might create the extra rte_flow for each unique 7029 * MARK/FLAG action ID. 7030 * 7031 * The table is updated for ingress Flows only, because 7032 * the egress Flows belong to the different device and 7033 * copy table should be updated in peer NIC Rx domain. 7034 */ 7035 if (attr->ingress && 7036 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7037 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7038 if (ret) 7039 goto error; 7040 } 7041 /* 7042 * If the flow is external (from application) OR device is started, 7043 * OR mreg discover, then apply immediately. 7044 */ 7045 if (external || dev->data->dev_started || 7046 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7047 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7048 ret = flow_drv_apply(dev, flow, error); 7049 if (ret < 0) 7050 goto error; 7051 } 7052 flow->type = type; 7053 flow_rxq_flags_set(dev, flow); 7054 rte_free(translated_actions); 7055 tunnel = flow_tunnel_from_rule(wks->flows); 7056 if (tunnel) { 7057 flow->tunnel = 1; 7058 flow->tunnel_id = tunnel->tunnel_id; 7059 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7060 mlx5_free(default_miss_ctx.queue); 7061 } 7062 mlx5_flow_pop_thread_workspace(); 7063 return idx; 7064 error: 7065 MLX5_ASSERT(flow); 7066 ret = rte_errno; /* Save rte_errno before cleanup. */ 7067 flow_mreg_del_copy_action(dev, flow); 7068 flow_drv_destroy(dev, flow); 7069 if (rss_desc->shared_rss) 7070 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7071 mlx5_ipool_get 7072 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7073 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7074 mlx5_ipool_free(priv->flows[type], idx); 7075 rte_errno = ret; /* Restore rte_errno. */ 7076 ret = rte_errno; 7077 rte_errno = ret; 7078 error_before_hairpin_split: 7079 mlx5_flow_pop_thread_workspace(); 7080 rte_free(translated_actions); 7081 return 0; 7082 } 7083 7084 /** 7085 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7086 * incoming packets to table 1. 7087 * 7088 * Other flow rules, requested for group n, will be created in 7089 * e-switch table n+1. 7090 * Jump action to e-switch group n will be created to group n+1. 7091 * 7092 * Used when working in switchdev mode, to utilise advantages of table 1 7093 * and above. 7094 * 7095 * @param dev 7096 * Pointer to Ethernet device. 7097 * 7098 * @return 7099 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7100 */ 7101 struct rte_flow * 7102 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7103 { 7104 const struct rte_flow_attr attr = { 7105 .group = 0, 7106 .priority = 0, 7107 .ingress = 1, 7108 .egress = 0, 7109 .transfer = 1, 7110 }; 7111 const struct rte_flow_item pattern = { 7112 .type = RTE_FLOW_ITEM_TYPE_END, 7113 }; 7114 struct rte_flow_action_jump jump = { 7115 .group = 1, 7116 }; 7117 const struct rte_flow_action actions[] = { 7118 { 7119 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7120 .conf = &jump, 7121 }, 7122 { 7123 .type = RTE_FLOW_ACTION_TYPE_END, 7124 }, 7125 }; 7126 struct rte_flow_error error; 7127 7128 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7129 &attr, &pattern, 7130 actions, false, &error); 7131 } 7132 7133 /** 7134 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7135 * and sq number, directs all packets to peer vport. 7136 * 7137 * @param dev 7138 * Pointer to Ethernet device. 7139 * @param txq 7140 * Txq index. 7141 * 7142 * @return 7143 * Flow ID on success, 0 otherwise and rte_errno is set. 7144 */ 7145 uint32_t 7146 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq) 7147 { 7148 struct rte_flow_attr attr = { 7149 .group = 0, 7150 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7151 .ingress = 1, 7152 .egress = 0, 7153 .transfer = 1, 7154 }; 7155 struct rte_flow_item_port_id port_spec = { 7156 .id = MLX5_PORT_ESW_MGR, 7157 }; 7158 struct mlx5_rte_flow_item_sq txq_spec = { 7159 .queue = txq, 7160 }; 7161 struct rte_flow_item pattern[] = { 7162 { 7163 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7164 .spec = &port_spec, 7165 }, 7166 { 7167 .type = (enum rte_flow_item_type) 7168 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7169 .spec = &txq_spec, 7170 }, 7171 { 7172 .type = RTE_FLOW_ITEM_TYPE_END, 7173 }, 7174 }; 7175 struct rte_flow_action_jump jump = { 7176 .group = 1, 7177 }; 7178 struct rte_flow_action_port_id port = { 7179 .id = dev->data->port_id, 7180 }; 7181 struct rte_flow_action actions[] = { 7182 { 7183 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7184 .conf = &jump, 7185 }, 7186 { 7187 .type = RTE_FLOW_ACTION_TYPE_END, 7188 }, 7189 }; 7190 struct rte_flow_error error; 7191 7192 /* 7193 * Creates group 0, highest priority jump flow. 7194 * Matches txq to bypass kernel packets. 7195 */ 7196 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7197 false, &error) == 0) 7198 return 0; 7199 /* Create group 1, lowest priority redirect flow for txq. */ 7200 attr.group = 1; 7201 actions[0].conf = &port; 7202 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7203 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7204 actions, false, &error); 7205 } 7206 7207 /** 7208 * Validate a flow supported by the NIC. 7209 * 7210 * @see rte_flow_validate() 7211 * @see rte_flow_ops 7212 */ 7213 int 7214 mlx5_flow_validate(struct rte_eth_dev *dev, 7215 const struct rte_flow_attr *attr, 7216 const struct rte_flow_item items[], 7217 const struct rte_flow_action original_actions[], 7218 struct rte_flow_error *error) 7219 { 7220 int hairpin_flow; 7221 struct mlx5_translated_action_handle 7222 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7223 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7224 const struct rte_flow_action *actions; 7225 struct rte_flow_action *translated_actions = NULL; 7226 int ret = flow_action_handles_translate(dev, original_actions, 7227 indir_actions, 7228 &indir_actions_n, 7229 &translated_actions, error); 7230 7231 if (ret) 7232 return ret; 7233 actions = translated_actions ? translated_actions : original_actions; 7234 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7235 ret = flow_drv_validate(dev, attr, items, actions, 7236 true, hairpin_flow, error); 7237 rte_free(translated_actions); 7238 return ret; 7239 } 7240 7241 /** 7242 * Create a flow. 7243 * 7244 * @see rte_flow_create() 7245 * @see rte_flow_ops 7246 */ 7247 struct rte_flow * 7248 mlx5_flow_create(struct rte_eth_dev *dev, 7249 const struct rte_flow_attr *attr, 7250 const struct rte_flow_item items[], 7251 const struct rte_flow_action actions[], 7252 struct rte_flow_error *error) 7253 { 7254 struct mlx5_priv *priv = dev->data->dev_private; 7255 7256 if (priv->sh->config.dv_flow_en == 2) { 7257 rte_flow_error_set(error, ENOTSUP, 7258 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7259 NULL, 7260 "Flow non-Q creation not supported"); 7261 return NULL; 7262 } 7263 /* 7264 * If the device is not started yet, it is not allowed to created a 7265 * flow from application. PMD default flows and traffic control flows 7266 * are not affected. 7267 */ 7268 if (unlikely(!dev->data->dev_started)) { 7269 DRV_LOG(DEBUG, "port %u is not started when " 7270 "inserting a flow", dev->data->port_id); 7271 rte_flow_error_set(error, ENODEV, 7272 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7273 NULL, 7274 "port not started"); 7275 return NULL; 7276 } 7277 7278 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7279 attr, items, actions, 7280 true, error); 7281 } 7282 7283 /** 7284 * Destroy a flow in a list. 7285 * 7286 * @param dev 7287 * Pointer to Ethernet device. 7288 * @param[in] flow_idx 7289 * Index of flow to destroy. 7290 */ 7291 static void 7292 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7293 uint32_t flow_idx) 7294 { 7295 struct mlx5_priv *priv = dev->data->dev_private; 7296 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7297 7298 if (!flow) 7299 return; 7300 MLX5_ASSERT(flow->type == type); 7301 /* 7302 * Update RX queue flags only if port is started, otherwise it is 7303 * already clean. 7304 */ 7305 if (dev->data->dev_started) 7306 flow_rxq_flags_trim(dev, flow); 7307 flow_drv_destroy(dev, flow); 7308 if (flow->tunnel) { 7309 struct mlx5_flow_tunnel *tunnel; 7310 7311 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7312 RTE_VERIFY(tunnel); 7313 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7314 mlx5_flow_tunnel_free(dev, tunnel); 7315 } 7316 flow_mreg_del_copy_action(dev, flow); 7317 mlx5_ipool_free(priv->flows[type], flow_idx); 7318 } 7319 7320 /** 7321 * Destroy all flows. 7322 * 7323 * @param dev 7324 * Pointer to Ethernet device. 7325 * @param type 7326 * Flow type to be flushed. 7327 * @param active 7328 * If flushing is called actively. 7329 */ 7330 void 7331 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7332 bool active) 7333 { 7334 struct mlx5_priv *priv = dev->data->dev_private; 7335 uint32_t num_flushed = 0, fidx = 1; 7336 struct rte_flow *flow; 7337 7338 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7339 if (priv->sh->config.dv_flow_en == 2 && 7340 type == MLX5_FLOW_TYPE_GEN) { 7341 flow_hw_q_flow_flush(dev, NULL); 7342 return; 7343 } 7344 #endif 7345 7346 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7347 flow_list_destroy(dev, type, fidx); 7348 num_flushed++; 7349 } 7350 if (active) { 7351 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7352 dev->data->port_id, num_flushed); 7353 } 7354 } 7355 7356 /** 7357 * Stop all default actions for flows. 7358 * 7359 * @param dev 7360 * Pointer to Ethernet device. 7361 */ 7362 void 7363 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7364 { 7365 flow_mreg_del_default_copy_action(dev); 7366 flow_rxq_flags_clear(dev); 7367 } 7368 7369 /** 7370 * Start all default actions for flows. 7371 * 7372 * @param dev 7373 * Pointer to Ethernet device. 7374 * @return 7375 * 0 on success, a negative errno value otherwise and rte_errno is set. 7376 */ 7377 int 7378 mlx5_flow_start_default(struct rte_eth_dev *dev) 7379 { 7380 struct rte_flow_error error; 7381 7382 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7383 return flow_mreg_add_default_copy_action(dev, &error); 7384 } 7385 7386 /** 7387 * Release key of thread specific flow workspace data. 7388 */ 7389 void 7390 flow_release_workspace(void *data) 7391 { 7392 struct mlx5_flow_workspace *wks = data; 7393 struct mlx5_flow_workspace *next; 7394 7395 while (wks) { 7396 next = wks->next; 7397 free(wks->rss_desc.queue); 7398 free(wks); 7399 wks = next; 7400 } 7401 } 7402 7403 /** 7404 * Get thread specific current flow workspace. 7405 * 7406 * @return pointer to thread specific flow workspace data, NULL on error. 7407 */ 7408 struct mlx5_flow_workspace* 7409 mlx5_flow_get_thread_workspace(void) 7410 { 7411 struct mlx5_flow_workspace *data; 7412 7413 data = mlx5_flow_os_get_specific_workspace(); 7414 MLX5_ASSERT(data && data->inuse); 7415 if (!data || !data->inuse) 7416 DRV_LOG(ERR, "flow workspace not initialized."); 7417 return data; 7418 } 7419 7420 /** 7421 * Allocate and init new flow workspace. 7422 * 7423 * @return pointer to flow workspace data, NULL on error. 7424 */ 7425 static struct mlx5_flow_workspace* 7426 flow_alloc_thread_workspace(void) 7427 { 7428 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7429 7430 if (!data) { 7431 DRV_LOG(ERR, "Failed to allocate flow workspace " 7432 "memory."); 7433 return NULL; 7434 } 7435 data->rss_desc.queue = calloc(1, 7436 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7437 if (!data->rss_desc.queue) 7438 goto err; 7439 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7440 return data; 7441 err: 7442 free(data->rss_desc.queue); 7443 free(data); 7444 return NULL; 7445 } 7446 7447 /** 7448 * Get new thread specific flow workspace. 7449 * 7450 * If current workspace inuse, create new one and set as current. 7451 * 7452 * @return pointer to thread specific flow workspace data, NULL on error. 7453 */ 7454 struct mlx5_flow_workspace* 7455 mlx5_flow_push_thread_workspace(void) 7456 { 7457 struct mlx5_flow_workspace *curr; 7458 struct mlx5_flow_workspace *data; 7459 7460 curr = mlx5_flow_os_get_specific_workspace(); 7461 if (!curr) { 7462 data = flow_alloc_thread_workspace(); 7463 if (!data) 7464 return NULL; 7465 } else if (!curr->inuse) { 7466 data = curr; 7467 } else if (curr->next) { 7468 data = curr->next; 7469 } else { 7470 data = flow_alloc_thread_workspace(); 7471 if (!data) 7472 return NULL; 7473 curr->next = data; 7474 data->prev = curr; 7475 } 7476 data->inuse = 1; 7477 data->flow_idx = 0; 7478 /* Set as current workspace */ 7479 if (mlx5_flow_os_set_specific_workspace(data)) 7480 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7481 return data; 7482 } 7483 7484 /** 7485 * Close current thread specific flow workspace. 7486 * 7487 * If previous workspace available, set it as current. 7488 * 7489 * @return pointer to thread specific flow workspace data, NULL on error. 7490 */ 7491 void 7492 mlx5_flow_pop_thread_workspace(void) 7493 { 7494 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7495 7496 if (!data) 7497 return; 7498 if (!data->inuse) { 7499 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7500 return; 7501 } 7502 data->inuse = 0; 7503 if (!data->prev) 7504 return; 7505 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7506 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7507 } 7508 7509 /** 7510 * Verify the flow list is empty 7511 * 7512 * @param dev 7513 * Pointer to Ethernet device. 7514 * 7515 * @return the number of flows not released. 7516 */ 7517 int 7518 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7519 { 7520 struct mlx5_priv *priv = dev->data->dev_private; 7521 struct rte_flow *flow; 7522 uint32_t idx = 0; 7523 int ret = 0, i; 7524 7525 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7526 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7527 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7528 dev->data->port_id, (void *)flow); 7529 ret++; 7530 } 7531 } 7532 return ret; 7533 } 7534 7535 /** 7536 * Enable default hairpin egress flow. 7537 * 7538 * @param dev 7539 * Pointer to Ethernet device. 7540 * @param queue 7541 * The queue index. 7542 * 7543 * @return 7544 * 0 on success, a negative errno value otherwise and rte_errno is set. 7545 */ 7546 int 7547 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7548 uint32_t queue) 7549 { 7550 const struct rte_flow_attr attr = { 7551 .egress = 1, 7552 .priority = 0, 7553 }; 7554 struct mlx5_rte_flow_item_sq queue_spec = { 7555 .queue = queue, 7556 }; 7557 struct mlx5_rte_flow_item_sq queue_mask = { 7558 .queue = UINT32_MAX, 7559 }; 7560 struct rte_flow_item items[] = { 7561 { 7562 .type = (enum rte_flow_item_type) 7563 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7564 .spec = &queue_spec, 7565 .last = NULL, 7566 .mask = &queue_mask, 7567 }, 7568 { 7569 .type = RTE_FLOW_ITEM_TYPE_END, 7570 }, 7571 }; 7572 struct rte_flow_action_jump jump = { 7573 .group = MLX5_HAIRPIN_TX_TABLE, 7574 }; 7575 struct rte_flow_action actions[2]; 7576 uint32_t flow_idx; 7577 struct rte_flow_error error; 7578 7579 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7580 actions[0].conf = &jump; 7581 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7582 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7583 &attr, items, actions, false, &error); 7584 if (!flow_idx) { 7585 DRV_LOG(DEBUG, 7586 "Failed to create ctrl flow: rte_errno(%d)," 7587 " type(%d), message(%s)", 7588 rte_errno, error.type, 7589 error.message ? error.message : " (no stated reason)"); 7590 return -rte_errno; 7591 } 7592 return 0; 7593 } 7594 7595 /** 7596 * Enable a control flow configured from the control plane. 7597 * 7598 * @param dev 7599 * Pointer to Ethernet device. 7600 * @param eth_spec 7601 * An Ethernet flow spec to apply. 7602 * @param eth_mask 7603 * An Ethernet flow mask to apply. 7604 * @param vlan_spec 7605 * A VLAN flow spec to apply. 7606 * @param vlan_mask 7607 * A VLAN flow mask to apply. 7608 * 7609 * @return 7610 * 0 on success, a negative errno value otherwise and rte_errno is set. 7611 */ 7612 int 7613 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7614 struct rte_flow_item_eth *eth_spec, 7615 struct rte_flow_item_eth *eth_mask, 7616 struct rte_flow_item_vlan *vlan_spec, 7617 struct rte_flow_item_vlan *vlan_mask) 7618 { 7619 struct mlx5_priv *priv = dev->data->dev_private; 7620 const struct rte_flow_attr attr = { 7621 .ingress = 1, 7622 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7623 }; 7624 struct rte_flow_item items[] = { 7625 { 7626 .type = RTE_FLOW_ITEM_TYPE_ETH, 7627 .spec = eth_spec, 7628 .last = NULL, 7629 .mask = eth_mask, 7630 }, 7631 { 7632 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7633 RTE_FLOW_ITEM_TYPE_END, 7634 .spec = vlan_spec, 7635 .last = NULL, 7636 .mask = vlan_mask, 7637 }, 7638 { 7639 .type = RTE_FLOW_ITEM_TYPE_END, 7640 }, 7641 }; 7642 uint16_t queue[priv->reta_idx_n]; 7643 struct rte_flow_action_rss action_rss = { 7644 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7645 .level = 0, 7646 .types = priv->rss_conf.rss_hf, 7647 .key_len = priv->rss_conf.rss_key_len, 7648 .queue_num = priv->reta_idx_n, 7649 .key = priv->rss_conf.rss_key, 7650 .queue = queue, 7651 }; 7652 struct rte_flow_action actions[] = { 7653 { 7654 .type = RTE_FLOW_ACTION_TYPE_RSS, 7655 .conf = &action_rss, 7656 }, 7657 { 7658 .type = RTE_FLOW_ACTION_TYPE_END, 7659 }, 7660 }; 7661 uint32_t flow_idx; 7662 struct rte_flow_error error; 7663 unsigned int i; 7664 7665 if (!priv->reta_idx_n || !priv->rxqs_n) { 7666 return 0; 7667 } 7668 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7669 action_rss.types = 0; 7670 for (i = 0; i != priv->reta_idx_n; ++i) 7671 queue[i] = (*priv->reta_idx)[i]; 7672 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7673 &attr, items, actions, false, &error); 7674 if (!flow_idx) 7675 return -rte_errno; 7676 return 0; 7677 } 7678 7679 /** 7680 * Enable a flow control configured from the control plane. 7681 * 7682 * @param dev 7683 * Pointer to Ethernet device. 7684 * @param eth_spec 7685 * An Ethernet flow spec to apply. 7686 * @param eth_mask 7687 * An Ethernet flow mask to apply. 7688 * 7689 * @return 7690 * 0 on success, a negative errno value otherwise and rte_errno is set. 7691 */ 7692 int 7693 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7694 struct rte_flow_item_eth *eth_spec, 7695 struct rte_flow_item_eth *eth_mask) 7696 { 7697 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7698 } 7699 7700 /** 7701 * Create default miss flow rule matching lacp traffic 7702 * 7703 * @param dev 7704 * Pointer to Ethernet device. 7705 * @param eth_spec 7706 * An Ethernet flow spec to apply. 7707 * 7708 * @return 7709 * 0 on success, a negative errno value otherwise and rte_errno is set. 7710 */ 7711 int 7712 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7713 { 7714 /* 7715 * The LACP matching is done by only using ether type since using 7716 * a multicast dst mac causes kernel to give low priority to this flow. 7717 */ 7718 static const struct rte_flow_item_eth lacp_spec = { 7719 .type = RTE_BE16(0x8809), 7720 }; 7721 static const struct rte_flow_item_eth lacp_mask = { 7722 .type = 0xffff, 7723 }; 7724 const struct rte_flow_attr attr = { 7725 .ingress = 1, 7726 }; 7727 struct rte_flow_item items[] = { 7728 { 7729 .type = RTE_FLOW_ITEM_TYPE_ETH, 7730 .spec = &lacp_spec, 7731 .mask = &lacp_mask, 7732 }, 7733 { 7734 .type = RTE_FLOW_ITEM_TYPE_END, 7735 }, 7736 }; 7737 struct rte_flow_action actions[] = { 7738 { 7739 .type = (enum rte_flow_action_type) 7740 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7741 }, 7742 { 7743 .type = RTE_FLOW_ACTION_TYPE_END, 7744 }, 7745 }; 7746 struct rte_flow_error error; 7747 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7748 &attr, items, actions, 7749 false, &error); 7750 7751 if (!flow_idx) 7752 return -rte_errno; 7753 return 0; 7754 } 7755 7756 /** 7757 * Destroy a flow. 7758 * 7759 * @see rte_flow_destroy() 7760 * @see rte_flow_ops 7761 */ 7762 int 7763 mlx5_flow_destroy(struct rte_eth_dev *dev, 7764 struct rte_flow *flow, 7765 struct rte_flow_error *error __rte_unused) 7766 { 7767 struct mlx5_priv *priv = dev->data->dev_private; 7768 7769 if (priv->sh->config.dv_flow_en == 2) 7770 return rte_flow_error_set(error, ENOTSUP, 7771 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7772 NULL, 7773 "Flow non-Q destruction not supported"); 7774 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7775 (uintptr_t)(void *)flow); 7776 return 0; 7777 } 7778 7779 /** 7780 * Destroy all flows. 7781 * 7782 * @see rte_flow_flush() 7783 * @see rte_flow_ops 7784 */ 7785 int 7786 mlx5_flow_flush(struct rte_eth_dev *dev, 7787 struct rte_flow_error *error __rte_unused) 7788 { 7789 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7790 return 0; 7791 } 7792 7793 /** 7794 * Isolated mode. 7795 * 7796 * @see rte_flow_isolate() 7797 * @see rte_flow_ops 7798 */ 7799 int 7800 mlx5_flow_isolate(struct rte_eth_dev *dev, 7801 int enable, 7802 struct rte_flow_error *error) 7803 { 7804 struct mlx5_priv *priv = dev->data->dev_private; 7805 7806 if (dev->data->dev_started) { 7807 rte_flow_error_set(error, EBUSY, 7808 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7809 NULL, 7810 "port must be stopped first"); 7811 return -rte_errno; 7812 } 7813 priv->isolated = !!enable; 7814 if (enable) 7815 dev->dev_ops = &mlx5_dev_ops_isolate; 7816 else 7817 dev->dev_ops = &mlx5_dev_ops; 7818 7819 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7820 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7821 7822 return 0; 7823 } 7824 7825 /** 7826 * Query a flow. 7827 * 7828 * @see rte_flow_query() 7829 * @see rte_flow_ops 7830 */ 7831 static int 7832 flow_drv_query(struct rte_eth_dev *dev, 7833 uint32_t flow_idx, 7834 const struct rte_flow_action *actions, 7835 void *data, 7836 struct rte_flow_error *error) 7837 { 7838 struct mlx5_priv *priv = dev->data->dev_private; 7839 const struct mlx5_flow_driver_ops *fops; 7840 struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7841 flow_idx); 7842 enum mlx5_flow_drv_type ftype; 7843 7844 if (!flow) { 7845 return rte_flow_error_set(error, ENOENT, 7846 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7847 NULL, 7848 "invalid flow handle"); 7849 } 7850 ftype = flow->drv_type; 7851 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7852 fops = flow_get_drv_ops(ftype); 7853 7854 return fops->query(dev, flow, actions, data, error); 7855 } 7856 7857 /** 7858 * Query a flow. 7859 * 7860 * @see rte_flow_query() 7861 * @see rte_flow_ops 7862 */ 7863 int 7864 mlx5_flow_query(struct rte_eth_dev *dev, 7865 struct rte_flow *flow, 7866 const struct rte_flow_action *actions, 7867 void *data, 7868 struct rte_flow_error *error) 7869 { 7870 int ret; 7871 struct mlx5_priv *priv = dev->data->dev_private; 7872 7873 if (priv->sh->config.dv_flow_en == 2) 7874 return rte_flow_error_set(error, ENOTSUP, 7875 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7876 NULL, 7877 "Flow non-Q query not supported"); 7878 ret = flow_drv_query(dev, (uintptr_t)(void *)flow, actions, data, 7879 error); 7880 if (ret < 0) 7881 return ret; 7882 return 0; 7883 } 7884 7885 /** 7886 * Get rte_flow callbacks. 7887 * 7888 * @param dev 7889 * Pointer to Ethernet device structure. 7890 * @param ops 7891 * Pointer to operation-specific structure. 7892 * 7893 * @return 0 7894 */ 7895 int 7896 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7897 const struct rte_flow_ops **ops) 7898 { 7899 *ops = &mlx5_flow_ops; 7900 return 0; 7901 } 7902 7903 /** 7904 * Validate meter policy actions. 7905 * Dispatcher for action type specific validation. 7906 * 7907 * @param[in] dev 7908 * Pointer to the Ethernet device structure. 7909 * @param[in] action 7910 * The meter policy action object to validate. 7911 * @param[in] attr 7912 * Attributes of flow to determine steering domain. 7913 * @param[out] is_rss 7914 * Is RSS or not. 7915 * @param[out] domain_bitmap 7916 * Domain bitmap. 7917 * @param[out] is_def_policy 7918 * Is default policy or not. 7919 * @param[out] error 7920 * Perform verbose error reporting if not NULL. Initialized in case of 7921 * error only. 7922 * 7923 * @return 7924 * 0 on success, otherwise negative errno value. 7925 */ 7926 int 7927 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7928 const struct rte_flow_action *actions[RTE_COLORS], 7929 struct rte_flow_attr *attr, 7930 bool *is_rss, 7931 uint8_t *domain_bitmap, 7932 uint8_t *policy_mode, 7933 struct rte_mtr_error *error) 7934 { 7935 const struct mlx5_flow_driver_ops *fops; 7936 7937 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7938 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7939 domain_bitmap, policy_mode, error); 7940 } 7941 7942 /** 7943 * Destroy the meter table set. 7944 * 7945 * @param[in] dev 7946 * Pointer to Ethernet device. 7947 * @param[in] mtr_policy 7948 * Meter policy struct. 7949 */ 7950 void 7951 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7952 struct mlx5_flow_meter_policy *mtr_policy) 7953 { 7954 const struct mlx5_flow_driver_ops *fops; 7955 7956 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7957 fops->destroy_mtr_acts(dev, mtr_policy); 7958 } 7959 7960 /** 7961 * Create policy action, lock free, 7962 * (mutex should be acquired by caller). 7963 * Dispatcher for action type specific call. 7964 * 7965 * @param[in] dev 7966 * Pointer to the Ethernet device structure. 7967 * @param[in] mtr_policy 7968 * Meter policy struct. 7969 * @param[in] action 7970 * Action specification used to create meter actions. 7971 * @param[in] attr 7972 * Flow rule attributes. 7973 * @param[out] error 7974 * Perform verbose error reporting if not NULL. Initialized in case of 7975 * error only. 7976 * 7977 * @return 7978 * 0 on success, otherwise negative errno value. 7979 */ 7980 int 7981 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 7982 struct mlx5_flow_meter_policy *mtr_policy, 7983 const struct rte_flow_action *actions[RTE_COLORS], 7984 struct rte_flow_attr *attr, 7985 struct rte_mtr_error *error) 7986 { 7987 const struct mlx5_flow_driver_ops *fops; 7988 7989 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7990 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 7991 } 7992 7993 /** 7994 * Create policy rules, lock free, 7995 * (mutex should be acquired by caller). 7996 * Dispatcher for action type specific call. 7997 * 7998 * @param[in] dev 7999 * Pointer to the Ethernet device structure. 8000 * @param[in] mtr_policy 8001 * Meter policy struct. 8002 * 8003 * @return 8004 * 0 on success, -1 otherwise. 8005 */ 8006 int 8007 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8008 struct mlx5_flow_meter_policy *mtr_policy) 8009 { 8010 const struct mlx5_flow_driver_ops *fops; 8011 8012 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8013 return fops->create_policy_rules(dev, mtr_policy); 8014 } 8015 8016 /** 8017 * Destroy policy rules, lock free, 8018 * (mutex should be acquired by caller). 8019 * Dispatcher for action type specific call. 8020 * 8021 * @param[in] dev 8022 * Pointer to the Ethernet device structure. 8023 * @param[in] mtr_policy 8024 * Meter policy struct. 8025 */ 8026 void 8027 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8028 struct mlx5_flow_meter_policy *mtr_policy) 8029 { 8030 const struct mlx5_flow_driver_ops *fops; 8031 8032 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8033 fops->destroy_policy_rules(dev, mtr_policy); 8034 } 8035 8036 /** 8037 * Destroy the default policy table set. 8038 * 8039 * @param[in] dev 8040 * Pointer to Ethernet device. 8041 */ 8042 void 8043 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8044 { 8045 const struct mlx5_flow_driver_ops *fops; 8046 8047 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8048 fops->destroy_def_policy(dev); 8049 } 8050 8051 /** 8052 * Destroy the default policy table set. 8053 * 8054 * @param[in] dev 8055 * Pointer to Ethernet device. 8056 * 8057 * @return 8058 * 0 on success, -1 otherwise. 8059 */ 8060 int 8061 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8062 { 8063 const struct mlx5_flow_driver_ops *fops; 8064 8065 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8066 return fops->create_def_policy(dev); 8067 } 8068 8069 /** 8070 * Create the needed meter and suffix tables. 8071 * 8072 * @param[in] dev 8073 * Pointer to Ethernet device. 8074 * 8075 * @return 8076 * 0 on success, -1 otherwise. 8077 */ 8078 int 8079 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8080 struct mlx5_flow_meter_info *fm, 8081 uint32_t mtr_idx, 8082 uint8_t domain_bitmap) 8083 { 8084 const struct mlx5_flow_driver_ops *fops; 8085 8086 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8087 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8088 } 8089 8090 /** 8091 * Destroy the meter table set. 8092 * 8093 * @param[in] dev 8094 * Pointer to Ethernet device. 8095 * @param[in] tbl 8096 * Pointer to the meter table set. 8097 */ 8098 void 8099 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8100 struct mlx5_flow_meter_info *fm) 8101 { 8102 const struct mlx5_flow_driver_ops *fops; 8103 8104 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8105 fops->destroy_mtr_tbls(dev, fm); 8106 } 8107 8108 /** 8109 * Destroy the global meter drop table. 8110 * 8111 * @param[in] dev 8112 * Pointer to Ethernet device. 8113 */ 8114 void 8115 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8116 { 8117 const struct mlx5_flow_driver_ops *fops; 8118 8119 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8120 fops->destroy_mtr_drop_tbls(dev); 8121 } 8122 8123 /** 8124 * Destroy the sub policy table with RX queue. 8125 * 8126 * @param[in] dev 8127 * Pointer to Ethernet device. 8128 * @param[in] mtr_policy 8129 * Pointer to meter policy table. 8130 */ 8131 void 8132 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8133 struct mlx5_flow_meter_policy *mtr_policy) 8134 { 8135 const struct mlx5_flow_driver_ops *fops; 8136 8137 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8138 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8139 } 8140 8141 /** 8142 * Allocate the needed aso flow meter id. 8143 * 8144 * @param[in] dev 8145 * Pointer to Ethernet device. 8146 * 8147 * @return 8148 * Index to aso flow meter on success, NULL otherwise. 8149 */ 8150 uint32_t 8151 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8152 { 8153 const struct mlx5_flow_driver_ops *fops; 8154 8155 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8156 return fops->create_meter(dev); 8157 } 8158 8159 /** 8160 * Free the aso flow meter id. 8161 * 8162 * @param[in] dev 8163 * Pointer to Ethernet device. 8164 * @param[in] mtr_idx 8165 * Index to aso flow meter to be free. 8166 * 8167 * @return 8168 * 0 on success. 8169 */ 8170 void 8171 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8172 { 8173 const struct mlx5_flow_driver_ops *fops; 8174 8175 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8176 fops->free_meter(dev, mtr_idx); 8177 } 8178 8179 /** 8180 * Allocate a counter. 8181 * 8182 * @param[in] dev 8183 * Pointer to Ethernet device structure. 8184 * 8185 * @return 8186 * Index to allocated counter on success, 0 otherwise. 8187 */ 8188 uint32_t 8189 mlx5_counter_alloc(struct rte_eth_dev *dev) 8190 { 8191 const struct mlx5_flow_driver_ops *fops; 8192 struct rte_flow_attr attr = { .transfer = 0 }; 8193 8194 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8195 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8196 return fops->counter_alloc(dev); 8197 } 8198 DRV_LOG(ERR, 8199 "port %u counter allocate is not supported.", 8200 dev->data->port_id); 8201 return 0; 8202 } 8203 8204 /** 8205 * Free a counter. 8206 * 8207 * @param[in] dev 8208 * Pointer to Ethernet device structure. 8209 * @param[in] cnt 8210 * Index to counter to be free. 8211 */ 8212 void 8213 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8214 { 8215 const struct mlx5_flow_driver_ops *fops; 8216 struct rte_flow_attr attr = { .transfer = 0 }; 8217 8218 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8219 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8220 fops->counter_free(dev, cnt); 8221 return; 8222 } 8223 DRV_LOG(ERR, 8224 "port %u counter free is not supported.", 8225 dev->data->port_id); 8226 } 8227 8228 /** 8229 * Query counter statistics. 8230 * 8231 * @param[in] dev 8232 * Pointer to Ethernet device structure. 8233 * @param[in] cnt 8234 * Index to counter to query. 8235 * @param[in] clear 8236 * Set to clear counter statistics. 8237 * @param[out] pkts 8238 * The counter hits packets number to save. 8239 * @param[out] bytes 8240 * The counter hits bytes number to save. 8241 * 8242 * @return 8243 * 0 on success, a negative errno value otherwise. 8244 */ 8245 int 8246 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8247 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8248 { 8249 const struct mlx5_flow_driver_ops *fops; 8250 struct rte_flow_attr attr = { .transfer = 0 }; 8251 8252 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8253 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8254 return fops->counter_query(dev, cnt, clear, pkts, 8255 bytes, action); 8256 } 8257 DRV_LOG(ERR, 8258 "port %u counter query is not supported.", 8259 dev->data->port_id); 8260 return -ENOTSUP; 8261 } 8262 8263 /** 8264 * Get information about HWS pre-configurable resources. 8265 * 8266 * @param[in] dev 8267 * Pointer to the rte_eth_dev structure. 8268 * @param[out] port_info 8269 * Pointer to port information. 8270 * @param[out] queue_info 8271 * Pointer to queue information. 8272 * @param[out] error 8273 * Pointer to error structure. 8274 * 8275 * @return 8276 * 0 on success, a negative errno value otherwise and rte_errno is set. 8277 */ 8278 static int 8279 mlx5_flow_info_get(struct rte_eth_dev *dev, 8280 struct rte_flow_port_info *port_info, 8281 struct rte_flow_queue_info *queue_info, 8282 struct rte_flow_error *error) 8283 { 8284 const struct mlx5_flow_driver_ops *fops; 8285 struct rte_flow_attr attr = {0}; 8286 8287 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8288 return rte_flow_error_set(error, ENOTSUP, 8289 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8290 NULL, 8291 "info get with incorrect steering mode"); 8292 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8293 return fops->info_get(dev, port_info, queue_info, error); 8294 } 8295 8296 /** 8297 * Configure port HWS resources. 8298 * 8299 * @param[in] dev 8300 * Pointer to the rte_eth_dev structure. 8301 * @param[in] port_attr 8302 * Port configuration attributes. 8303 * @param[in] nb_queue 8304 * Number of queue. 8305 * @param[in] queue_attr 8306 * Array that holds attributes for each flow queue. 8307 * @param[out] error 8308 * Pointer to error structure. 8309 * 8310 * @return 8311 * 0 on success, a negative errno value otherwise and rte_errno is set. 8312 */ 8313 static int 8314 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8315 const struct rte_flow_port_attr *port_attr, 8316 uint16_t nb_queue, 8317 const struct rte_flow_queue_attr *queue_attr[], 8318 struct rte_flow_error *error) 8319 { 8320 const struct mlx5_flow_driver_ops *fops; 8321 struct rte_flow_attr attr = {0}; 8322 8323 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8324 return rte_flow_error_set(error, ENOTSUP, 8325 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8326 NULL, 8327 "port configure with incorrect steering mode"); 8328 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8329 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8330 } 8331 8332 /** 8333 * Validate item template. 8334 * 8335 * @param[in] dev 8336 * Pointer to the rte_eth_dev structure. 8337 * @param[in] attr 8338 * Pointer to the item template attributes. 8339 * @param[in] items 8340 * The template item pattern. 8341 * @param[out] error 8342 * Pointer to error structure. 8343 * 8344 * @return 8345 * 0 on success, a negative errno value otherwise and rte_errno is set. 8346 */ 8347 int 8348 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8349 const struct rte_flow_pattern_template_attr *attr, 8350 const struct rte_flow_item items[], 8351 struct rte_flow_error *error) 8352 { 8353 const struct mlx5_flow_driver_ops *fops; 8354 struct rte_flow_attr fattr = {0}; 8355 8356 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8357 rte_flow_error_set(error, ENOTSUP, 8358 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8359 "pattern validate with incorrect steering mode"); 8360 return -ENOTSUP; 8361 } 8362 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8363 return fops->pattern_validate(dev, attr, items, error); 8364 } 8365 8366 /** 8367 * Create flow item template. 8368 * 8369 * @param[in] dev 8370 * Pointer to the rte_eth_dev structure. 8371 * @param[in] attr 8372 * Pointer to the item template attributes. 8373 * @param[in] items 8374 * The template item pattern. 8375 * @param[out] error 8376 * Pointer to error structure. 8377 * 8378 * @return 8379 * 0 on success, a negative errno value otherwise and rte_errno is set. 8380 */ 8381 static struct rte_flow_pattern_template * 8382 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8383 const struct rte_flow_pattern_template_attr *attr, 8384 const struct rte_flow_item items[], 8385 struct rte_flow_error *error) 8386 { 8387 const struct mlx5_flow_driver_ops *fops; 8388 struct rte_flow_attr fattr = {0}; 8389 8390 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8391 rte_flow_error_set(error, ENOTSUP, 8392 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8393 NULL, 8394 "pattern create with incorrect steering mode"); 8395 return NULL; 8396 } 8397 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8398 return fops->pattern_template_create(dev, attr, items, error); 8399 } 8400 8401 /** 8402 * Destroy flow item template. 8403 * 8404 * @param[in] dev 8405 * Pointer to the rte_eth_dev structure. 8406 * @param[in] template 8407 * Pointer to the item template to be destroyed. 8408 * @param[out] error 8409 * Pointer to error structure. 8410 * 8411 * @return 8412 * 0 on success, a negative errno value otherwise and rte_errno is set. 8413 */ 8414 static int 8415 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8416 struct rte_flow_pattern_template *template, 8417 struct rte_flow_error *error) 8418 { 8419 const struct mlx5_flow_driver_ops *fops; 8420 struct rte_flow_attr attr = {0}; 8421 8422 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8423 return rte_flow_error_set(error, ENOTSUP, 8424 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8425 NULL, 8426 "pattern destroy with incorrect steering mode"); 8427 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8428 return fops->pattern_template_destroy(dev, template, error); 8429 } 8430 8431 /** 8432 * Validate flow actions template. 8433 * 8434 * @param[in] dev 8435 * Pointer to the rte_eth_dev structure. 8436 * @param[in] attr 8437 * Pointer to the action template attributes. 8438 * @param[in] actions 8439 * Associated actions (list terminated by the END action). 8440 * @param[in] masks 8441 * List of actions that marks which of the action's member is constant. 8442 * @param[out] error 8443 * Pointer to error structure. 8444 * 8445 * @return 8446 * 0 on success, a negative errno value otherwise and rte_errno is set. 8447 */ 8448 int 8449 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8450 const struct rte_flow_actions_template_attr *attr, 8451 const struct rte_flow_action actions[], 8452 const struct rte_flow_action masks[], 8453 struct rte_flow_error *error) 8454 { 8455 const struct mlx5_flow_driver_ops *fops; 8456 struct rte_flow_attr fattr = {0}; 8457 8458 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8459 rte_flow_error_set(error, ENOTSUP, 8460 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8461 "actions validate with incorrect steering mode"); 8462 return -ENOTSUP; 8463 } 8464 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8465 return fops->actions_validate(dev, attr, actions, masks, error); 8466 } 8467 8468 /** 8469 * Create flow item template. 8470 * 8471 * @param[in] dev 8472 * Pointer to the rte_eth_dev structure. 8473 * @param[in] attr 8474 * Pointer to the action template attributes. 8475 * @param[in] actions 8476 * Associated actions (list terminated by the END action). 8477 * @param[in] masks 8478 * List of actions that marks which of the action's member is constant. 8479 * @param[out] error 8480 * Pointer to error structure. 8481 * 8482 * @return 8483 * 0 on success, a negative errno value otherwise and rte_errno is set. 8484 */ 8485 static struct rte_flow_actions_template * 8486 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8487 const struct rte_flow_actions_template_attr *attr, 8488 const struct rte_flow_action actions[], 8489 const struct rte_flow_action masks[], 8490 struct rte_flow_error *error) 8491 { 8492 const struct mlx5_flow_driver_ops *fops; 8493 struct rte_flow_attr fattr = {0}; 8494 8495 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8496 rte_flow_error_set(error, ENOTSUP, 8497 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8498 NULL, 8499 "action create with incorrect steering mode"); 8500 return NULL; 8501 } 8502 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8503 return fops->actions_template_create(dev, attr, actions, masks, error); 8504 } 8505 8506 /** 8507 * Destroy flow action template. 8508 * 8509 * @param[in] dev 8510 * Pointer to the rte_eth_dev structure. 8511 * @param[in] template 8512 * Pointer to the action template to be destroyed. 8513 * @param[out] error 8514 * Pointer to error structure. 8515 * 8516 * @return 8517 * 0 on success, a negative errno value otherwise and rte_errno is set. 8518 */ 8519 static int 8520 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8521 struct rte_flow_actions_template *template, 8522 struct rte_flow_error *error) 8523 { 8524 const struct mlx5_flow_driver_ops *fops; 8525 struct rte_flow_attr attr = {0}; 8526 8527 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8528 return rte_flow_error_set(error, ENOTSUP, 8529 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8530 NULL, 8531 "action destroy with incorrect steering mode"); 8532 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8533 return fops->actions_template_destroy(dev, template, error); 8534 } 8535 8536 /** 8537 * Create flow table. 8538 * 8539 * @param[in] dev 8540 * Pointer to the rte_eth_dev structure. 8541 * @param[in] attr 8542 * Pointer to the table attributes. 8543 * @param[in] item_templates 8544 * Item template array to be binded to the table. 8545 * @param[in] nb_item_templates 8546 * Number of item template. 8547 * @param[in] action_templates 8548 * Action template array to be binded to the table. 8549 * @param[in] nb_action_templates 8550 * Number of action template. 8551 * @param[out] error 8552 * Pointer to error structure. 8553 * 8554 * @return 8555 * Table on success, NULL otherwise and rte_errno is set. 8556 */ 8557 static struct rte_flow_template_table * 8558 mlx5_flow_table_create(struct rte_eth_dev *dev, 8559 const struct rte_flow_template_table_attr *attr, 8560 struct rte_flow_pattern_template *item_templates[], 8561 uint8_t nb_item_templates, 8562 struct rte_flow_actions_template *action_templates[], 8563 uint8_t nb_action_templates, 8564 struct rte_flow_error *error) 8565 { 8566 const struct mlx5_flow_driver_ops *fops; 8567 struct rte_flow_attr fattr = {0}; 8568 8569 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8570 rte_flow_error_set(error, ENOTSUP, 8571 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8572 NULL, 8573 "table create with incorrect steering mode"); 8574 return NULL; 8575 } 8576 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8577 return fops->template_table_create(dev, 8578 attr, 8579 item_templates, 8580 nb_item_templates, 8581 action_templates, 8582 nb_action_templates, 8583 error); 8584 } 8585 8586 /** 8587 * PMD destroy flow table. 8588 * 8589 * @param[in] dev 8590 * Pointer to the rte_eth_dev structure. 8591 * @param[in] table 8592 * Pointer to the table to be destroyed. 8593 * @param[out] error 8594 * Pointer to error structure. 8595 * 8596 * @return 8597 * 0 on success, a negative errno value otherwise and rte_errno is set. 8598 */ 8599 static int 8600 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8601 struct rte_flow_template_table *table, 8602 struct rte_flow_error *error) 8603 { 8604 const struct mlx5_flow_driver_ops *fops; 8605 struct rte_flow_attr attr = {0}; 8606 8607 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8608 return rte_flow_error_set(error, ENOTSUP, 8609 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8610 NULL, 8611 "table destroy with incorrect steering mode"); 8612 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8613 return fops->template_table_destroy(dev, table, error); 8614 } 8615 8616 /** 8617 * Enqueue flow creation. 8618 * 8619 * @param[in] dev 8620 * Pointer to the rte_eth_dev structure. 8621 * @param[in] queue_id 8622 * The queue to create the flow. 8623 * @param[in] attr 8624 * Pointer to the flow operation attributes. 8625 * @param[in] items 8626 * Items with flow spec value. 8627 * @param[in] pattern_template_index 8628 * The item pattern flow follows from the table. 8629 * @param[in] actions 8630 * Action with flow spec value. 8631 * @param[in] action_template_index 8632 * The action pattern flow follows from the table. 8633 * @param[in] user_data 8634 * Pointer to the user_data. 8635 * @param[out] error 8636 * Pointer to error structure. 8637 * 8638 * @return 8639 * Flow pointer on success, NULL otherwise and rte_errno is set. 8640 */ 8641 static struct rte_flow * 8642 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8643 uint32_t queue_id, 8644 const struct rte_flow_op_attr *attr, 8645 struct rte_flow_template_table *table, 8646 const struct rte_flow_item items[], 8647 uint8_t pattern_template_index, 8648 const struct rte_flow_action actions[], 8649 uint8_t action_template_index, 8650 void *user_data, 8651 struct rte_flow_error *error) 8652 { 8653 const struct mlx5_flow_driver_ops *fops; 8654 struct rte_flow_attr fattr = {0}; 8655 8656 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8657 rte_flow_error_set(error, ENOTSUP, 8658 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8659 NULL, 8660 "flow_q create with incorrect steering mode"); 8661 return NULL; 8662 } 8663 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8664 return fops->async_flow_create(dev, queue_id, attr, table, 8665 items, pattern_template_index, 8666 actions, action_template_index, 8667 user_data, error); 8668 } 8669 8670 /** 8671 * Enqueue flow destruction. 8672 * 8673 * @param[in] dev 8674 * Pointer to the rte_eth_dev structure. 8675 * @param[in] queue 8676 * The queue to destroy the flow. 8677 * @param[in] attr 8678 * Pointer to the flow operation attributes. 8679 * @param[in] flow 8680 * Pointer to the flow to be destroyed. 8681 * @param[in] user_data 8682 * Pointer to the user_data. 8683 * @param[out] error 8684 * Pointer to error structure. 8685 * 8686 * @return 8687 * 0 on success, negative value otherwise and rte_errno is set. 8688 */ 8689 static int 8690 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8691 uint32_t queue, 8692 const struct rte_flow_op_attr *attr, 8693 struct rte_flow *flow, 8694 void *user_data, 8695 struct rte_flow_error *error) 8696 { 8697 const struct mlx5_flow_driver_ops *fops; 8698 struct rte_flow_attr fattr = {0}; 8699 8700 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8701 return rte_flow_error_set(error, ENOTSUP, 8702 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8703 NULL, 8704 "flow_q destroy with incorrect steering mode"); 8705 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8706 return fops->async_flow_destroy(dev, queue, attr, flow, 8707 user_data, error); 8708 } 8709 8710 /** 8711 * Pull the enqueued flows. 8712 * 8713 * @param[in] dev 8714 * Pointer to the rte_eth_dev structure. 8715 * @param[in] queue 8716 * The queue to pull the result. 8717 * @param[in/out] res 8718 * Array to save the results. 8719 * @param[in] n_res 8720 * Available result with the array. 8721 * @param[out] error 8722 * Pointer to error structure. 8723 * 8724 * @return 8725 * Result number on success, negative value otherwise and rte_errno is set. 8726 */ 8727 static int 8728 mlx5_flow_pull(struct rte_eth_dev *dev, 8729 uint32_t queue, 8730 struct rte_flow_op_result res[], 8731 uint16_t n_res, 8732 struct rte_flow_error *error) 8733 { 8734 const struct mlx5_flow_driver_ops *fops; 8735 struct rte_flow_attr attr = {0}; 8736 8737 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8738 return rte_flow_error_set(error, ENOTSUP, 8739 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8740 NULL, 8741 "flow_q pull with incorrect steering mode"); 8742 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8743 return fops->pull(dev, queue, res, n_res, error); 8744 } 8745 8746 /** 8747 * Push the enqueued flows. 8748 * 8749 * @param[in] dev 8750 * Pointer to the rte_eth_dev structure. 8751 * @param[in] queue 8752 * The queue to push the flows. 8753 * @param[out] error 8754 * Pointer to error structure. 8755 * 8756 * @return 8757 * 0 on success, negative value otherwise and rte_errno is set. 8758 */ 8759 static int 8760 mlx5_flow_push(struct rte_eth_dev *dev, 8761 uint32_t queue, 8762 struct rte_flow_error *error) 8763 { 8764 const struct mlx5_flow_driver_ops *fops; 8765 struct rte_flow_attr attr = {0}; 8766 8767 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8768 return rte_flow_error_set(error, ENOTSUP, 8769 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8770 NULL, 8771 "flow_q push with incorrect steering mode"); 8772 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8773 return fops->push(dev, queue, error); 8774 } 8775 8776 /** 8777 * Create shared action. 8778 * 8779 * @param[in] dev 8780 * Pointer to the rte_eth_dev structure. 8781 * @param[in] queue 8782 * Which queue to be used.. 8783 * @param[in] attr 8784 * Operation attribute. 8785 * @param[in] conf 8786 * Indirect action configuration. 8787 * @param[in] action 8788 * rte_flow action detail. 8789 * @param[in] user_data 8790 * Pointer to the user_data. 8791 * @param[out] error 8792 * Pointer to error structure. 8793 * 8794 * @return 8795 * Action handle on success, NULL otherwise and rte_errno is set. 8796 */ 8797 static struct rte_flow_action_handle * 8798 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8799 const struct rte_flow_op_attr *attr, 8800 const struct rte_flow_indir_action_conf *conf, 8801 const struct rte_flow_action *action, 8802 void *user_data, 8803 struct rte_flow_error *error) 8804 { 8805 const struct mlx5_flow_driver_ops *fops = 8806 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8807 8808 return fops->async_action_create(dev, queue, attr, conf, action, 8809 user_data, error); 8810 } 8811 8812 /** 8813 * Update shared action. 8814 * 8815 * @param[in] dev 8816 * Pointer to the rte_eth_dev structure. 8817 * @param[in] queue 8818 * Which queue to be used.. 8819 * @param[in] attr 8820 * Operation attribute. 8821 * @param[in] handle 8822 * Action handle to be updated. 8823 * @param[in] update 8824 * Update value. 8825 * @param[in] user_data 8826 * Pointer to the user_data. 8827 * @param[out] error 8828 * Pointer to error structure. 8829 * 8830 * @return 8831 * 0 on success, negative value otherwise and rte_errno is set. 8832 */ 8833 static int 8834 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8835 const struct rte_flow_op_attr *attr, 8836 struct rte_flow_action_handle *handle, 8837 const void *update, 8838 void *user_data, 8839 struct rte_flow_error *error) 8840 { 8841 const struct mlx5_flow_driver_ops *fops = 8842 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8843 8844 return fops->async_action_update(dev, queue, attr, handle, 8845 update, user_data, error); 8846 } 8847 8848 /** 8849 * Destroy shared action. 8850 * 8851 * @param[in] dev 8852 * Pointer to the rte_eth_dev structure. 8853 * @param[in] queue 8854 * Which queue to be used.. 8855 * @param[in] attr 8856 * Operation attribute. 8857 * @param[in] handle 8858 * Action handle to be destroyed. 8859 * @param[in] user_data 8860 * Pointer to the user_data. 8861 * @param[out] error 8862 * Pointer to error structure. 8863 * 8864 * @return 8865 * 0 on success, negative value otherwise and rte_errno is set. 8866 */ 8867 static int 8868 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 8869 const struct rte_flow_op_attr *attr, 8870 struct rte_flow_action_handle *handle, 8871 void *user_data, 8872 struct rte_flow_error *error) 8873 { 8874 const struct mlx5_flow_driver_ops *fops = 8875 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8876 8877 return fops->async_action_destroy(dev, queue, attr, handle, 8878 user_data, error); 8879 } 8880 8881 /** 8882 * Allocate a new memory for the counter values wrapped by all the needed 8883 * management. 8884 * 8885 * @param[in] sh 8886 * Pointer to mlx5_dev_ctx_shared object. 8887 * 8888 * @return 8889 * 0 on success, a negative errno value otherwise. 8890 */ 8891 static int 8892 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8893 { 8894 struct mlx5_counter_stats_mem_mng *mem_mng; 8895 volatile struct flow_counter_stats *raw_data; 8896 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8897 int size = (sizeof(struct flow_counter_stats) * 8898 MLX5_COUNTERS_PER_POOL + 8899 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8900 sizeof(struct mlx5_counter_stats_mem_mng); 8901 size_t pgsize = rte_mem_page_size(); 8902 uint8_t *mem; 8903 int ret; 8904 int i; 8905 8906 if (pgsize == (size_t)-1) { 8907 DRV_LOG(ERR, "Failed to get mem page size"); 8908 rte_errno = ENOMEM; 8909 return -ENOMEM; 8910 } 8911 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8912 if (!mem) { 8913 rte_errno = ENOMEM; 8914 return -ENOMEM; 8915 } 8916 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8917 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8918 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8919 sh->cdev->pdn, mem, size, 8920 &mem_mng->wm); 8921 if (ret) { 8922 rte_errno = errno; 8923 mlx5_free(mem); 8924 return -rte_errno; 8925 } 8926 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8927 raw_data = (volatile struct flow_counter_stats *)mem; 8928 for (i = 0; i < raws_n; ++i) { 8929 mem_mng->raws[i].mem_mng = mem_mng; 8930 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8931 } 8932 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8933 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, 8934 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8935 next); 8936 LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next); 8937 sh->cmng.mem_mng = mem_mng; 8938 return 0; 8939 } 8940 8941 /** 8942 * Set the statistic memory to the new counter pool. 8943 * 8944 * @param[in] sh 8945 * Pointer to mlx5_dev_ctx_shared object. 8946 * @param[in] pool 8947 * Pointer to the pool to set the statistic memory. 8948 * 8949 * @return 8950 * 0 on success, a negative errno value otherwise. 8951 */ 8952 static int 8953 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 8954 struct mlx5_flow_counter_pool *pool) 8955 { 8956 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 8957 /* Resize statistic memory once used out. */ 8958 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 8959 mlx5_flow_create_counter_stat_mem_mng(sh)) { 8960 DRV_LOG(ERR, "Cannot resize counter stat mem."); 8961 return -1; 8962 } 8963 rte_spinlock_lock(&pool->sl); 8964 pool->raw = cmng->mem_mng->raws + pool->index % 8965 MLX5_CNT_CONTAINER_RESIZE; 8966 rte_spinlock_unlock(&pool->sl); 8967 pool->raw_hw = NULL; 8968 return 0; 8969 } 8970 8971 #define MLX5_POOL_QUERY_FREQ_US 1000000 8972 8973 /** 8974 * Set the periodic procedure for triggering asynchronous batch queries for all 8975 * the counter pools. 8976 * 8977 * @param[in] sh 8978 * Pointer to mlx5_dev_ctx_shared object. 8979 */ 8980 void 8981 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 8982 { 8983 uint32_t pools_n, us; 8984 8985 pools_n = __atomic_load_n(&sh->cmng.n_valid, __ATOMIC_RELAXED); 8986 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 8987 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 8988 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 8989 sh->cmng.query_thread_on = 0; 8990 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 8991 } else { 8992 sh->cmng.query_thread_on = 1; 8993 } 8994 } 8995 8996 /** 8997 * The periodic procedure for triggering asynchronous batch queries for all the 8998 * counter pools. This function is probably called by the host thread. 8999 * 9000 * @param[in] arg 9001 * The parameter for the alarm process. 9002 */ 9003 void 9004 mlx5_flow_query_alarm(void *arg) 9005 { 9006 struct mlx5_dev_ctx_shared *sh = arg; 9007 int ret; 9008 uint16_t pool_index = sh->cmng.pool_index; 9009 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 9010 struct mlx5_flow_counter_pool *pool; 9011 uint16_t n_valid; 9012 9013 if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 9014 goto set_alarm; 9015 rte_spinlock_lock(&cmng->pool_update_sl); 9016 pool = cmng->pools[pool_index]; 9017 n_valid = cmng->n_valid; 9018 rte_spinlock_unlock(&cmng->pool_update_sl); 9019 /* Set the statistic memory to the new created pool. */ 9020 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9021 goto set_alarm; 9022 if (pool->raw_hw) 9023 /* There is a pool query in progress. */ 9024 goto set_alarm; 9025 pool->raw_hw = 9026 LIST_FIRST(&sh->cmng.free_stat_raws); 9027 if (!pool->raw_hw) 9028 /* No free counter statistics raw memory. */ 9029 goto set_alarm; 9030 /* 9031 * Identify the counters released between query trigger and query 9032 * handle more efficiently. The counter released in this gap period 9033 * should wait for a new round of query as the new arrived packets 9034 * will not be taken into account. 9035 */ 9036 pool->query_gen++; 9037 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9038 MLX5_COUNTERS_PER_POOL, 9039 NULL, NULL, 9040 pool->raw_hw->mem_mng->wm.lkey, 9041 (void *)(uintptr_t) 9042 pool->raw_hw->data, 9043 sh->devx_comp, 9044 (uint64_t)(uintptr_t)pool); 9045 if (ret) { 9046 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9047 " %d", pool->min_dcs->id); 9048 pool->raw_hw = NULL; 9049 goto set_alarm; 9050 } 9051 LIST_REMOVE(pool->raw_hw, next); 9052 sh->cmng.pending_queries++; 9053 pool_index++; 9054 if (pool_index >= n_valid) 9055 pool_index = 0; 9056 set_alarm: 9057 sh->cmng.pool_index = pool_index; 9058 mlx5_set_query_alarm(sh); 9059 } 9060 9061 /** 9062 * Check and callback event for new aged flow in the counter pool 9063 * 9064 * @param[in] sh 9065 * Pointer to mlx5_dev_ctx_shared object. 9066 * @param[in] pool 9067 * Pointer to Current counter pool. 9068 */ 9069 static void 9070 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9071 struct mlx5_flow_counter_pool *pool) 9072 { 9073 struct mlx5_priv *priv; 9074 struct mlx5_flow_counter *cnt; 9075 struct mlx5_age_info *age_info; 9076 struct mlx5_age_param *age_param; 9077 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9078 struct mlx5_counter_stats_raw *prev = pool->raw; 9079 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9080 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9081 uint16_t expected = AGE_CANDIDATE; 9082 uint32_t i; 9083 9084 pool->time_of_last_age_check = curr_time; 9085 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9086 cnt = MLX5_POOL_GET_CNT(pool, i); 9087 age_param = MLX5_CNT_TO_AGE(cnt); 9088 if (__atomic_load_n(&age_param->state, 9089 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9090 continue; 9091 if (cur->data[i].hits != prev->data[i].hits) { 9092 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9093 __ATOMIC_RELAXED); 9094 continue; 9095 } 9096 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9097 time_delta, 9098 __ATOMIC_RELAXED) <= age_param->timeout) 9099 continue; 9100 /** 9101 * Hold the lock first, or if between the 9102 * state AGE_TMOUT and tailq operation the 9103 * release happened, the release procedure 9104 * may delete a non-existent tailq node. 9105 */ 9106 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9107 age_info = GET_PORT_AGE_INFO(priv); 9108 rte_spinlock_lock(&age_info->aged_sl); 9109 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9110 AGE_TMOUT, false, 9111 __ATOMIC_RELAXED, 9112 __ATOMIC_RELAXED)) { 9113 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9114 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9115 } 9116 rte_spinlock_unlock(&age_info->aged_sl); 9117 } 9118 mlx5_age_event_prepare(sh); 9119 } 9120 9121 /** 9122 * Handler for the HW respond about ready values from an asynchronous batch 9123 * query. This function is probably called by the host thread. 9124 * 9125 * @param[in] sh 9126 * The pointer to the shared device context. 9127 * @param[in] async_id 9128 * The Devx async ID. 9129 * @param[in] status 9130 * The status of the completion. 9131 */ 9132 void 9133 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9134 uint64_t async_id, int status) 9135 { 9136 struct mlx5_flow_counter_pool *pool = 9137 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9138 struct mlx5_counter_stats_raw *raw_to_free; 9139 uint8_t query_gen = pool->query_gen ^ 1; 9140 struct mlx5_flow_counter_mng *cmng = &sh->cmng; 9141 enum mlx5_counter_type cnt_type = 9142 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9143 MLX5_COUNTER_TYPE_ORIGIN; 9144 9145 if (unlikely(status)) { 9146 raw_to_free = pool->raw_hw; 9147 } else { 9148 raw_to_free = pool->raw; 9149 if (pool->is_aged) 9150 mlx5_flow_aging_check(sh, pool); 9151 rte_spinlock_lock(&pool->sl); 9152 pool->raw = pool->raw_hw; 9153 rte_spinlock_unlock(&pool->sl); 9154 /* Be sure the new raw counters data is updated in memory. */ 9155 rte_io_wmb(); 9156 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9157 rte_spinlock_lock(&cmng->csl[cnt_type]); 9158 TAILQ_CONCAT(&cmng->counters[cnt_type], 9159 &pool->counters[query_gen], next); 9160 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9161 } 9162 } 9163 LIST_INSERT_HEAD(&sh->cmng.free_stat_raws, raw_to_free, next); 9164 pool->raw_hw = NULL; 9165 sh->cmng.pending_queries--; 9166 } 9167 9168 static int 9169 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9170 const struct flow_grp_info *grp_info, 9171 struct rte_flow_error *error) 9172 { 9173 if (grp_info->transfer && grp_info->external && 9174 grp_info->fdb_def_rule) { 9175 if (group == UINT32_MAX) 9176 return rte_flow_error_set 9177 (error, EINVAL, 9178 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9179 NULL, 9180 "group index not supported"); 9181 *table = group + 1; 9182 } else { 9183 *table = group; 9184 } 9185 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9186 return 0; 9187 } 9188 9189 /** 9190 * Translate the rte_flow group index to HW table value. 9191 * 9192 * If tunnel offload is disabled, all group ids converted to flow table 9193 * id using the standard method. 9194 * If tunnel offload is enabled, group id can be converted using the 9195 * standard or tunnel conversion method. Group conversion method 9196 * selection depends on flags in `grp_info` parameter: 9197 * - Internal (grp_info.external == 0) groups conversion uses the 9198 * standard method. 9199 * - Group ids in JUMP action converted with the tunnel conversion. 9200 * - Group id in rule attribute conversion depends on a rule type and 9201 * group id value: 9202 * ** non zero group attributes converted with the tunnel method 9203 * ** zero group attribute in non-tunnel rule is converted using the 9204 * standard method - there's only one root table 9205 * ** zero group attribute in steer tunnel rule is converted with the 9206 * standard method - single root table 9207 * ** zero group attribute in match tunnel rule is a special OvS 9208 * case: that value is used for portability reasons. That group 9209 * id is converted with the tunnel conversion method. 9210 * 9211 * @param[in] dev 9212 * Port device 9213 * @param[in] tunnel 9214 * PMD tunnel offload object 9215 * @param[in] group 9216 * rte_flow group index value. 9217 * @param[out] table 9218 * HW table value. 9219 * @param[in] grp_info 9220 * flags used for conversion 9221 * @param[out] error 9222 * Pointer to error structure. 9223 * 9224 * @return 9225 * 0 on success, a negative errno value otherwise and rte_errno is set. 9226 */ 9227 int 9228 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9229 const struct mlx5_flow_tunnel *tunnel, 9230 uint32_t group, uint32_t *table, 9231 const struct flow_grp_info *grp_info, 9232 struct rte_flow_error *error) 9233 { 9234 int ret; 9235 bool standard_translation; 9236 9237 if (!grp_info->skip_scale && grp_info->external && 9238 group < MLX5_MAX_TABLES_EXTERNAL) 9239 group *= MLX5_FLOW_TABLE_FACTOR; 9240 if (is_tunnel_offload_active(dev)) { 9241 standard_translation = !grp_info->external || 9242 grp_info->std_tbl_fix; 9243 } else { 9244 standard_translation = true; 9245 } 9246 DRV_LOG(DEBUG, 9247 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9248 dev->data->port_id, group, grp_info->transfer, 9249 grp_info->external, grp_info->fdb_def_rule, 9250 standard_translation ? "STANDARD" : "TUNNEL"); 9251 if (standard_translation) 9252 ret = flow_group_to_table(dev->data->port_id, group, table, 9253 grp_info, error); 9254 else 9255 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9256 table, error); 9257 9258 return ret; 9259 } 9260 9261 /** 9262 * Discover availability of metadata reg_c's. 9263 * 9264 * Iteratively use test flows to check availability. 9265 * 9266 * @param[in] dev 9267 * Pointer to the Ethernet device structure. 9268 * 9269 * @return 9270 * 0 on success, a negative errno value otherwise and rte_errno is set. 9271 */ 9272 int 9273 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9274 { 9275 struct mlx5_priv *priv = dev->data->dev_private; 9276 enum modify_reg idx; 9277 int n = 0; 9278 9279 /* reg_c[0] and reg_c[1] are reserved. */ 9280 priv->sh->flow_mreg_c[n++] = REG_C_0; 9281 priv->sh->flow_mreg_c[n++] = REG_C_1; 9282 /* Discover availability of other reg_c's. */ 9283 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9284 struct rte_flow_attr attr = { 9285 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9286 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9287 .ingress = 1, 9288 }; 9289 struct rte_flow_item items[] = { 9290 [0] = { 9291 .type = RTE_FLOW_ITEM_TYPE_END, 9292 }, 9293 }; 9294 struct rte_flow_action actions[] = { 9295 [0] = { 9296 .type = (enum rte_flow_action_type) 9297 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9298 .conf = &(struct mlx5_flow_action_copy_mreg){ 9299 .src = REG_C_1, 9300 .dst = idx, 9301 }, 9302 }, 9303 [1] = { 9304 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9305 .conf = &(struct rte_flow_action_jump){ 9306 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9307 }, 9308 }, 9309 [2] = { 9310 .type = RTE_FLOW_ACTION_TYPE_END, 9311 }, 9312 }; 9313 uint32_t flow_idx; 9314 struct rte_flow *flow; 9315 struct rte_flow_error error; 9316 9317 if (!priv->sh->config.dv_flow_en) 9318 break; 9319 /* Create internal flow, validation skips copy action. */ 9320 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9321 items, actions, false, &error); 9322 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9323 flow_idx); 9324 if (!flow) 9325 continue; 9326 priv->sh->flow_mreg_c[n++] = idx; 9327 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9328 } 9329 for (; n < MLX5_MREG_C_NUM; ++n) 9330 priv->sh->flow_mreg_c[n] = REG_NON; 9331 priv->sh->metadata_regc_check_flag = 1; 9332 return 0; 9333 } 9334 9335 int 9336 save_dump_file(const uint8_t *data, uint32_t size, 9337 uint32_t type, uint64_t id, void *arg, FILE *file) 9338 { 9339 char line[BUF_SIZE]; 9340 uint32_t out = 0; 9341 uint32_t k; 9342 uint32_t actions_num; 9343 struct rte_flow_query_count *count; 9344 9345 memset(line, 0, BUF_SIZE); 9346 switch (type) { 9347 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9348 actions_num = *(uint32_t *)(arg); 9349 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9350 type, id, actions_num); 9351 break; 9352 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9353 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9354 type, id); 9355 break; 9356 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9357 count = (struct rte_flow_query_count *)arg; 9358 fprintf(file, 9359 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9360 type, id, count->hits, count->bytes); 9361 return 0; 9362 default: 9363 return -1; 9364 } 9365 9366 for (k = 0; k < size; k++) { 9367 /* Make sure we do not overrun the line buffer length. */ 9368 if (out >= BUF_SIZE - 4) { 9369 line[out] = '\0'; 9370 break; 9371 } 9372 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9373 (data[k]) & 0xff); 9374 } 9375 fprintf(file, "%s\n", line); 9376 return 0; 9377 } 9378 9379 int 9380 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9381 struct rte_flow_query_count *count, struct rte_flow_error *error) 9382 { 9383 struct rte_flow_action action[2]; 9384 enum mlx5_flow_drv_type ftype; 9385 const struct mlx5_flow_driver_ops *fops; 9386 9387 if (!flow) { 9388 return rte_flow_error_set(error, ENOENT, 9389 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9390 NULL, 9391 "invalid flow handle"); 9392 } 9393 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9394 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9395 if (flow->counter) { 9396 memset(count, 0, sizeof(struct rte_flow_query_count)); 9397 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9398 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9399 ftype < MLX5_FLOW_TYPE_MAX); 9400 fops = flow_get_drv_ops(ftype); 9401 return fops->query(dev, flow, action, count, error); 9402 } 9403 return -1; 9404 } 9405 9406 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9407 /** 9408 * Dump flow ipool data to file 9409 * 9410 * @param[in] dev 9411 * The pointer to Ethernet device. 9412 * @param[in] file 9413 * A pointer to a file for output. 9414 * @param[out] error 9415 * Perform verbose error reporting if not NULL. PMDs initialize this 9416 * structure in case of error only. 9417 * @return 9418 * 0 on success, a negative value otherwise. 9419 */ 9420 int 9421 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9422 struct rte_flow *flow, FILE *file, 9423 struct rte_flow_error *error) 9424 { 9425 struct mlx5_priv *priv = dev->data->dev_private; 9426 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9427 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9428 uint32_t handle_idx; 9429 struct mlx5_flow_handle *dh; 9430 struct rte_flow_query_count count; 9431 uint32_t actions_num; 9432 const uint8_t *data; 9433 size_t size; 9434 uint64_t id; 9435 uint32_t type; 9436 void *action = NULL; 9437 9438 if (!flow) { 9439 return rte_flow_error_set(error, ENOENT, 9440 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9441 NULL, 9442 "invalid flow handle"); 9443 } 9444 handle_idx = flow->dev_handles; 9445 /* query counter */ 9446 if (flow->counter && 9447 (!mlx5_counter_query(dev, flow->counter, false, 9448 &count.hits, &count.bytes, &action)) && action) { 9449 id = (uint64_t)(uintptr_t)action; 9450 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9451 save_dump_file(NULL, 0, type, 9452 id, (void *)&count, file); 9453 } 9454 9455 while (handle_idx) { 9456 dh = mlx5_ipool_get(priv->sh->ipool 9457 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9458 if (!dh) 9459 continue; 9460 handle_idx = dh->next.next; 9461 9462 /* Get modify_hdr and encap_decap buf from ipools. */ 9463 encap_decap = NULL; 9464 modify_hdr = dh->dvh.modify_hdr; 9465 9466 if (dh->dvh.rix_encap_decap) { 9467 encap_decap = mlx5_ipool_get(priv->sh->ipool 9468 [MLX5_IPOOL_DECAP_ENCAP], 9469 dh->dvh.rix_encap_decap); 9470 } 9471 if (modify_hdr) { 9472 data = (const uint8_t *)modify_hdr->actions; 9473 size = (size_t)(modify_hdr->actions_num) * 8; 9474 id = (uint64_t)(uintptr_t)modify_hdr->action; 9475 actions_num = modify_hdr->actions_num; 9476 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9477 save_dump_file(data, size, type, id, 9478 (void *)(&actions_num), file); 9479 } 9480 if (encap_decap) { 9481 data = encap_decap->buf; 9482 size = encap_decap->size; 9483 id = (uint64_t)(uintptr_t)encap_decap->action; 9484 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9485 save_dump_file(data, size, type, 9486 id, NULL, file); 9487 } 9488 } 9489 return 0; 9490 } 9491 9492 /** 9493 * Dump all flow's encap_decap/modify_hdr/counter data to file 9494 * 9495 * @param[in] dev 9496 * The pointer to Ethernet device. 9497 * @param[in] file 9498 * A pointer to a file for output. 9499 * @param[out] error 9500 * Perform verbose error reporting if not NULL. PMDs initialize this 9501 * structure in case of error only. 9502 * @return 9503 * 0 on success, a negative value otherwise. 9504 */ 9505 static int 9506 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9507 FILE *file, struct rte_flow_error *error __rte_unused) 9508 { 9509 struct mlx5_priv *priv = dev->data->dev_private; 9510 struct mlx5_dev_ctx_shared *sh = priv->sh; 9511 struct mlx5_hlist *h; 9512 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9513 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9514 struct rte_flow_query_count count; 9515 uint32_t actions_num; 9516 const uint8_t *data; 9517 size_t size; 9518 uint64_t id; 9519 uint32_t type; 9520 uint32_t i; 9521 uint32_t j; 9522 struct mlx5_list_inconst *l_inconst; 9523 struct mlx5_list_entry *e; 9524 int lcore_index; 9525 struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; 9526 uint32_t max; 9527 void *action; 9528 9529 /* encap_decap hlist is lcore_share, get global core cache. */ 9530 i = MLX5_LIST_GLOBAL; 9531 h = sh->encaps_decaps; 9532 if (h) { 9533 for (j = 0; j <= h->mask; j++) { 9534 l_inconst = &h->buckets[j].l; 9535 if (!l_inconst || !l_inconst->cache[i]) 9536 continue; 9537 9538 e = LIST_FIRST(&l_inconst->cache[i]->h); 9539 while (e) { 9540 encap_decap = 9541 (struct mlx5_flow_dv_encap_decap_resource *)e; 9542 data = encap_decap->buf; 9543 size = encap_decap->size; 9544 id = (uint64_t)(uintptr_t)encap_decap->action; 9545 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9546 save_dump_file(data, size, type, 9547 id, NULL, file); 9548 e = LIST_NEXT(e, next); 9549 } 9550 } 9551 } 9552 9553 /* get modify_hdr */ 9554 h = sh->modify_cmds; 9555 if (h) { 9556 lcore_index = rte_lcore_index(rte_lcore_id()); 9557 if (unlikely(lcore_index == -1)) { 9558 lcore_index = MLX5_LIST_NLCORE; 9559 rte_spinlock_lock(&h->l_const.lcore_lock); 9560 } 9561 i = lcore_index; 9562 9563 for (j = 0; j <= h->mask; j++) { 9564 l_inconst = &h->buckets[j].l; 9565 if (!l_inconst || !l_inconst->cache[i]) 9566 continue; 9567 9568 e = LIST_FIRST(&l_inconst->cache[i]->h); 9569 while (e) { 9570 modify_hdr = 9571 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9572 data = (const uint8_t *)modify_hdr->actions; 9573 size = (size_t)(modify_hdr->actions_num) * 8; 9574 actions_num = modify_hdr->actions_num; 9575 id = (uint64_t)(uintptr_t)modify_hdr->action; 9576 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9577 save_dump_file(data, size, type, id, 9578 (void *)(&actions_num), file); 9579 e = LIST_NEXT(e, next); 9580 } 9581 } 9582 9583 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9584 rte_spinlock_unlock(&h->l_const.lcore_lock); 9585 } 9586 9587 /* get counter */ 9588 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9589 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9590 for (j = 1; j <= max; j++) { 9591 action = NULL; 9592 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9593 &count.bytes, &action)) && action) { 9594 id = (uint64_t)(uintptr_t)action; 9595 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9596 save_dump_file(NULL, 0, type, 9597 id, (void *)&count, file); 9598 } 9599 } 9600 return 0; 9601 } 9602 #endif 9603 9604 /** 9605 * Dump flow raw hw data to file 9606 * 9607 * @param[in] dev 9608 * The pointer to Ethernet device. 9609 * @param[in] file 9610 * A pointer to a file for output. 9611 * @param[out] error 9612 * Perform verbose error reporting if not NULL. PMDs initialize this 9613 * structure in case of error only. 9614 * @return 9615 * 0 on success, a negative value otherwise. 9616 */ 9617 int 9618 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9619 FILE *file, 9620 struct rte_flow_error *error __rte_unused) 9621 { 9622 struct mlx5_priv *priv = dev->data->dev_private; 9623 struct mlx5_dev_ctx_shared *sh = priv->sh; 9624 uint32_t handle_idx; 9625 int ret; 9626 struct mlx5_flow_handle *dh; 9627 struct rte_flow *flow; 9628 9629 if (!sh->config.dv_flow_en) { 9630 if (fputs("device dv flow disabled\n", file) <= 0) 9631 return -errno; 9632 return -ENOTSUP; 9633 } 9634 9635 /* dump all */ 9636 if (!flow_idx) { 9637 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9638 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9639 return -EINVAL; 9640 #endif 9641 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9642 sh->rx_domain, 9643 sh->tx_domain, file); 9644 } 9645 /* dump one */ 9646 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9647 (uintptr_t)(void *)flow_idx); 9648 if (!flow) 9649 return -EINVAL; 9650 9651 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9652 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9653 #endif 9654 handle_idx = flow->dev_handles; 9655 while (handle_idx) { 9656 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9657 handle_idx); 9658 if (!dh) 9659 return -ENOENT; 9660 if (dh->drv_flow) { 9661 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9662 file); 9663 if (ret) 9664 return -ENOENT; 9665 } 9666 handle_idx = dh->next.next; 9667 } 9668 return 0; 9669 } 9670 9671 /** 9672 * Get aged-out flows. 9673 * 9674 * @param[in] dev 9675 * Pointer to the Ethernet device structure. 9676 * @param[in] context 9677 * The address of an array of pointers to the aged-out flows contexts. 9678 * @param[in] nb_countexts 9679 * The length of context array pointers. 9680 * @param[out] error 9681 * Perform verbose error reporting if not NULL. Initialized in case of 9682 * error only. 9683 * 9684 * @return 9685 * how many contexts get in success, otherwise negative errno value. 9686 * if nb_contexts is 0, return the amount of all aged contexts. 9687 * if nb_contexts is not 0 , return the amount of aged flows reported 9688 * in the context array. 9689 */ 9690 int 9691 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9692 uint32_t nb_contexts, struct rte_flow_error *error) 9693 { 9694 const struct mlx5_flow_driver_ops *fops; 9695 struct rte_flow_attr attr = { .transfer = 0 }; 9696 9697 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 9698 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 9699 return fops->get_aged_flows(dev, contexts, nb_contexts, 9700 error); 9701 } 9702 DRV_LOG(ERR, 9703 "port %u get aged flows is not supported.", 9704 dev->data->port_id); 9705 return -ENOTSUP; 9706 } 9707 9708 /* Wrapper for driver action_validate op callback */ 9709 static int 9710 flow_drv_action_validate(struct rte_eth_dev *dev, 9711 const struct rte_flow_indir_action_conf *conf, 9712 const struct rte_flow_action *action, 9713 const struct mlx5_flow_driver_ops *fops, 9714 struct rte_flow_error *error) 9715 { 9716 static const char err_msg[] = "indirect action validation unsupported"; 9717 9718 if (!fops->action_validate) { 9719 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9720 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9721 NULL, err_msg); 9722 return -rte_errno; 9723 } 9724 return fops->action_validate(dev, conf, action, error); 9725 } 9726 9727 /** 9728 * Destroys the shared action by handle. 9729 * 9730 * @param dev 9731 * Pointer to Ethernet device structure. 9732 * @param[in] handle 9733 * Handle for the indirect action object to be destroyed. 9734 * @param[out] error 9735 * Perform verbose error reporting if not NULL. PMDs initialize this 9736 * structure in case of error only. 9737 * 9738 * @return 9739 * 0 on success, a negative errno value otherwise and rte_errno is set. 9740 * 9741 * @note: wrapper for driver action_create op callback. 9742 */ 9743 static int 9744 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9745 struct rte_flow_action_handle *handle, 9746 struct rte_flow_error *error) 9747 { 9748 static const char err_msg[] = "indirect action destruction unsupported"; 9749 struct rte_flow_attr attr = { .transfer = 0 }; 9750 const struct mlx5_flow_driver_ops *fops = 9751 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9752 9753 if (!fops->action_destroy) { 9754 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9755 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9756 NULL, err_msg); 9757 return -rte_errno; 9758 } 9759 return fops->action_destroy(dev, handle, error); 9760 } 9761 9762 /* Wrapper for driver action_destroy op callback */ 9763 static int 9764 flow_drv_action_update(struct rte_eth_dev *dev, 9765 struct rte_flow_action_handle *handle, 9766 const void *update, 9767 const struct mlx5_flow_driver_ops *fops, 9768 struct rte_flow_error *error) 9769 { 9770 static const char err_msg[] = "indirect action update unsupported"; 9771 9772 if (!fops->action_update) { 9773 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9774 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9775 NULL, err_msg); 9776 return -rte_errno; 9777 } 9778 return fops->action_update(dev, handle, update, error); 9779 } 9780 9781 /* Wrapper for driver action_destroy op callback */ 9782 static int 9783 flow_drv_action_query(struct rte_eth_dev *dev, 9784 const struct rte_flow_action_handle *handle, 9785 void *data, 9786 const struct mlx5_flow_driver_ops *fops, 9787 struct rte_flow_error *error) 9788 { 9789 static const char err_msg[] = "indirect action query unsupported"; 9790 9791 if (!fops->action_query) { 9792 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9793 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9794 NULL, err_msg); 9795 return -rte_errno; 9796 } 9797 return fops->action_query(dev, handle, data, error); 9798 } 9799 9800 /** 9801 * Create indirect action for reuse in multiple flow rules. 9802 * 9803 * @param dev 9804 * Pointer to Ethernet device structure. 9805 * @param conf 9806 * Pointer to indirect action object configuration. 9807 * @param[in] action 9808 * Action configuration for indirect action object creation. 9809 * @param[out] error 9810 * Perform verbose error reporting if not NULL. PMDs initialize this 9811 * structure in case of error only. 9812 * @return 9813 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9814 */ 9815 static struct rte_flow_action_handle * 9816 mlx5_action_handle_create(struct rte_eth_dev *dev, 9817 const struct rte_flow_indir_action_conf *conf, 9818 const struct rte_flow_action *action, 9819 struct rte_flow_error *error) 9820 { 9821 static const char err_msg[] = "indirect action creation unsupported"; 9822 struct rte_flow_attr attr = { .transfer = 0 }; 9823 const struct mlx5_flow_driver_ops *fops = 9824 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9825 9826 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9827 return NULL; 9828 if (!fops->action_create) { 9829 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9830 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9831 NULL, err_msg); 9832 return NULL; 9833 } 9834 return fops->action_create(dev, conf, action, error); 9835 } 9836 9837 /** 9838 * Updates inplace the indirect action configuration pointed by *handle* 9839 * with the configuration provided as *update* argument. 9840 * The update of the indirect action configuration effects all flow rules 9841 * reusing the action via handle. 9842 * 9843 * @param dev 9844 * Pointer to Ethernet device structure. 9845 * @param[in] handle 9846 * Handle for the indirect action to be updated. 9847 * @param[in] update 9848 * Action specification used to modify the action pointed by handle. 9849 * *update* could be of same type with the action pointed by the *handle* 9850 * handle argument, or some other structures like a wrapper, depending on 9851 * the indirect action type. 9852 * @param[out] error 9853 * Perform verbose error reporting if not NULL. PMDs initialize this 9854 * structure in case of error only. 9855 * 9856 * @return 9857 * 0 on success, a negative errno value otherwise and rte_errno is set. 9858 */ 9859 static int 9860 mlx5_action_handle_update(struct rte_eth_dev *dev, 9861 struct rte_flow_action_handle *handle, 9862 const void *update, 9863 struct rte_flow_error *error) 9864 { 9865 struct rte_flow_attr attr = { .transfer = 0 }; 9866 const struct mlx5_flow_driver_ops *fops = 9867 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9868 int ret; 9869 9870 ret = flow_drv_action_validate(dev, NULL, 9871 (const struct rte_flow_action *)update, fops, error); 9872 if (ret) 9873 return ret; 9874 return flow_drv_action_update(dev, handle, update, fops, 9875 error); 9876 } 9877 9878 /** 9879 * Query the indirect action by handle. 9880 * 9881 * This function allows retrieving action-specific data such as counters. 9882 * Data is gathered by special action which may be present/referenced in 9883 * more than one flow rule definition. 9884 * 9885 * see @RTE_FLOW_ACTION_TYPE_COUNT 9886 * 9887 * @param dev 9888 * Pointer to Ethernet device structure. 9889 * @param[in] handle 9890 * Handle for the indirect action to query. 9891 * @param[in, out] data 9892 * Pointer to storage for the associated query data type. 9893 * @param[out] error 9894 * Perform verbose error reporting if not NULL. PMDs initialize this 9895 * structure in case of error only. 9896 * 9897 * @return 9898 * 0 on success, a negative errno value otherwise and rte_errno is set. 9899 */ 9900 static int 9901 mlx5_action_handle_query(struct rte_eth_dev *dev, 9902 const struct rte_flow_action_handle *handle, 9903 void *data, 9904 struct rte_flow_error *error) 9905 { 9906 struct rte_flow_attr attr = { .transfer = 0 }; 9907 const struct mlx5_flow_driver_ops *fops = 9908 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9909 9910 return flow_drv_action_query(dev, handle, data, fops, error); 9911 } 9912 9913 /** 9914 * Destroy all indirect actions (shared RSS). 9915 * 9916 * @param dev 9917 * Pointer to Ethernet device. 9918 * 9919 * @return 9920 * 0 on success, a negative errno value otherwise and rte_errno is set. 9921 */ 9922 int 9923 mlx5_action_handle_flush(struct rte_eth_dev *dev) 9924 { 9925 struct rte_flow_error error; 9926 struct mlx5_priv *priv = dev->data->dev_private; 9927 struct mlx5_shared_action_rss *shared_rss; 9928 int ret = 0; 9929 uint32_t idx; 9930 9931 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 9932 priv->rss_shared_actions, idx, shared_rss, next) { 9933 ret |= mlx5_action_handle_destroy(dev, 9934 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 9935 } 9936 return ret; 9937 } 9938 9939 /** 9940 * Validate existing indirect actions against current device configuration 9941 * and attach them to device resources. 9942 * 9943 * @param dev 9944 * Pointer to Ethernet device. 9945 * 9946 * @return 9947 * 0 on success, a negative errno value otherwise and rte_errno is set. 9948 */ 9949 int 9950 mlx5_action_handle_attach(struct rte_eth_dev *dev) 9951 { 9952 struct mlx5_priv *priv = dev->data->dev_private; 9953 int ret = 0; 9954 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 9955 9956 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9957 const char *message; 9958 uint32_t queue_idx; 9959 9960 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 9961 ind_tbl->queues_n, 9962 &message, &queue_idx); 9963 if (ret != 0) { 9964 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 9965 dev->data->port_id, ind_tbl->queues[queue_idx], 9966 message); 9967 break; 9968 } 9969 } 9970 if (ret != 0) 9971 return ret; 9972 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9973 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 9974 if (ret != 0) { 9975 DRV_LOG(ERR, "Port %u could not attach " 9976 "indirection table obj %p", 9977 dev->data->port_id, (void *)ind_tbl); 9978 goto error; 9979 } 9980 } 9981 9982 return 0; 9983 error: 9984 ind_tbl_last = ind_tbl; 9985 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 9986 if (ind_tbl == ind_tbl_last) 9987 break; 9988 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 9989 DRV_LOG(CRIT, "Port %u could not detach " 9990 "indirection table obj %p on rollback", 9991 dev->data->port_id, (void *)ind_tbl); 9992 } 9993 return ret; 9994 } 9995 9996 /** 9997 * Detach indirect actions of the device from its resources. 9998 * 9999 * @param dev 10000 * Pointer to Ethernet device. 10001 * 10002 * @return 10003 * 0 on success, a negative errno value otherwise and rte_errno is set. 10004 */ 10005 int 10006 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10007 { 10008 struct mlx5_priv *priv = dev->data->dev_private; 10009 int ret = 0; 10010 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10011 10012 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10013 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10014 if (ret != 0) { 10015 DRV_LOG(ERR, "Port %u could not detach " 10016 "indirection table obj %p", 10017 dev->data->port_id, (void *)ind_tbl); 10018 goto error; 10019 } 10020 } 10021 return 0; 10022 error: 10023 ind_tbl_last = ind_tbl; 10024 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10025 if (ind_tbl == ind_tbl_last) 10026 break; 10027 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10028 DRV_LOG(CRIT, "Port %u could not attach " 10029 "indirection table obj %p on rollback", 10030 dev->data->port_id, (void *)ind_tbl); 10031 } 10032 return ret; 10033 } 10034 10035 #ifndef HAVE_MLX5DV_DR 10036 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10037 #else 10038 #define MLX5_DOMAIN_SYNC_FLOW \ 10039 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10040 #endif 10041 10042 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10043 { 10044 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10045 const struct mlx5_flow_driver_ops *fops; 10046 int ret; 10047 struct rte_flow_attr attr = { .transfer = 0 }; 10048 10049 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10050 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10051 if (ret > 0) 10052 ret = -ret; 10053 return ret; 10054 } 10055 10056 const struct mlx5_flow_tunnel * 10057 mlx5_get_tof(const struct rte_flow_item *item, 10058 const struct rte_flow_action *action, 10059 enum mlx5_tof_rule_type *rule_type) 10060 { 10061 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10062 if (item->type == (typeof(item->type)) 10063 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10064 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10065 return flow_items_to_tunnel(item); 10066 } 10067 } 10068 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10069 if (action->type == (typeof(action->type)) 10070 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10071 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10072 return flow_actions_to_tunnel(action); 10073 } 10074 } 10075 return NULL; 10076 } 10077 10078 /** 10079 * tunnel offload functionality is defined for DV environment only 10080 */ 10081 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10082 __extension__ 10083 union tunnel_offload_mark { 10084 uint32_t val; 10085 struct { 10086 uint32_t app_reserve:8; 10087 uint32_t table_id:15; 10088 uint32_t transfer:1; 10089 uint32_t _unused_:8; 10090 }; 10091 }; 10092 10093 static bool 10094 mlx5_access_tunnel_offload_db 10095 (struct rte_eth_dev *dev, 10096 bool (*match)(struct rte_eth_dev *, 10097 struct mlx5_flow_tunnel *, const void *), 10098 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10099 void (*miss)(struct rte_eth_dev *, void *), 10100 void *ctx, bool lock_op); 10101 10102 static int 10103 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10104 struct rte_flow *flow, 10105 const struct rte_flow_attr *attr, 10106 const struct rte_flow_action *app_actions, 10107 uint32_t flow_idx, 10108 const struct mlx5_flow_tunnel *tunnel, 10109 struct tunnel_default_miss_ctx *ctx, 10110 struct rte_flow_error *error) 10111 { 10112 struct mlx5_priv *priv = dev->data->dev_private; 10113 struct mlx5_flow *dev_flow; 10114 struct rte_flow_attr miss_attr = *attr; 10115 const struct rte_flow_item miss_items[2] = { 10116 { 10117 .type = RTE_FLOW_ITEM_TYPE_ETH, 10118 .spec = NULL, 10119 .last = NULL, 10120 .mask = NULL 10121 }, 10122 { 10123 .type = RTE_FLOW_ITEM_TYPE_END, 10124 .spec = NULL, 10125 .last = NULL, 10126 .mask = NULL 10127 } 10128 }; 10129 union tunnel_offload_mark mark_id; 10130 struct rte_flow_action_mark miss_mark; 10131 struct rte_flow_action miss_actions[3] = { 10132 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10133 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10134 }; 10135 const struct rte_flow_action_jump *jump_data; 10136 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10137 struct flow_grp_info grp_info = { 10138 .external = 1, 10139 .transfer = attr->transfer, 10140 .fdb_def_rule = !!priv->fdb_def_rule, 10141 .std_tbl_fix = 0, 10142 }; 10143 int ret; 10144 10145 if (!attr->transfer) { 10146 uint32_t q_size; 10147 10148 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10149 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10150 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10151 0, SOCKET_ID_ANY); 10152 if (!ctx->queue) 10153 return rte_flow_error_set 10154 (error, ENOMEM, 10155 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10156 NULL, "invalid default miss RSS"); 10157 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10158 ctx->action_rss.level = 0, 10159 ctx->action_rss.types = priv->rss_conf.rss_hf, 10160 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10161 ctx->action_rss.queue_num = priv->reta_idx_n, 10162 ctx->action_rss.key = priv->rss_conf.rss_key, 10163 ctx->action_rss.queue = ctx->queue; 10164 if (!priv->reta_idx_n || !priv->rxqs_n) 10165 return rte_flow_error_set 10166 (error, EINVAL, 10167 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10168 NULL, "invalid port configuration"); 10169 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10170 ctx->action_rss.types = 0; 10171 for (i = 0; i != priv->reta_idx_n; ++i) 10172 ctx->queue[i] = (*priv->reta_idx)[i]; 10173 } else { 10174 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10175 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10176 } 10177 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10178 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10179 jump_data = app_actions->conf; 10180 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10181 miss_attr.group = jump_data->group; 10182 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10183 &flow_table, &grp_info, error); 10184 if (ret) 10185 return rte_flow_error_set(error, EINVAL, 10186 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10187 NULL, "invalid tunnel id"); 10188 mark_id.app_reserve = 0; 10189 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10190 mark_id.transfer = !!attr->transfer; 10191 mark_id._unused_ = 0; 10192 miss_mark.id = mark_id.val; 10193 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10194 miss_items, miss_actions, flow_idx, error); 10195 if (!dev_flow) 10196 return -rte_errno; 10197 dev_flow->flow = flow; 10198 dev_flow->external = true; 10199 dev_flow->tunnel = tunnel; 10200 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10201 /* Subflow object was created, we must include one in the list. */ 10202 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10203 dev_flow->handle, next); 10204 DRV_LOG(DEBUG, 10205 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10206 dev->data->port_id, tunnel->app_tunnel.type, 10207 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10208 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10209 miss_actions, error); 10210 if (!ret) 10211 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10212 error); 10213 10214 return ret; 10215 } 10216 10217 static const struct mlx5_flow_tbl_data_entry * 10218 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10219 { 10220 struct mlx5_priv *priv = dev->data->dev_private; 10221 struct mlx5_dev_ctx_shared *sh = priv->sh; 10222 struct mlx5_list_entry *he; 10223 union tunnel_offload_mark mbits = { .val = mark }; 10224 union mlx5_flow_tbl_key table_key = { 10225 { 10226 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10227 .id = 0, 10228 .reserved = 0, 10229 .dummy = 0, 10230 .is_fdb = !!mbits.transfer, 10231 .is_egress = 0, 10232 } 10233 }; 10234 struct mlx5_flow_cb_ctx ctx = { 10235 .data = &table_key.v64, 10236 }; 10237 10238 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10239 return he ? 10240 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10241 } 10242 10243 static void 10244 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10245 struct mlx5_list_entry *entry) 10246 { 10247 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10248 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10249 10250 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10251 tunnel_flow_tbl_to_id(tte->flow_table)); 10252 mlx5_free(tte); 10253 } 10254 10255 static int 10256 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10257 struct mlx5_list_entry *entry, void *cb_ctx) 10258 { 10259 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10260 union tunnel_tbl_key tbl = { 10261 .val = *(uint64_t *)(ctx->data), 10262 }; 10263 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10264 10265 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10266 } 10267 10268 static struct mlx5_list_entry * 10269 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10270 { 10271 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10272 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10273 struct tunnel_tbl_entry *tte; 10274 union tunnel_tbl_key tbl = { 10275 .val = *(uint64_t *)(ctx->data), 10276 }; 10277 10278 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10279 sizeof(*tte), 0, 10280 SOCKET_ID_ANY); 10281 if (!tte) 10282 goto err; 10283 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10284 &tte->flow_table); 10285 if (tte->flow_table >= MLX5_MAX_TABLES) { 10286 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10287 tte->flow_table); 10288 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10289 tte->flow_table); 10290 goto err; 10291 } else if (!tte->flow_table) { 10292 goto err; 10293 } 10294 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10295 tte->tunnel_id = tbl.tunnel_id; 10296 tte->group = tbl.group; 10297 return &tte->hash; 10298 err: 10299 if (tte) 10300 mlx5_free(tte); 10301 return NULL; 10302 } 10303 10304 static struct mlx5_list_entry * 10305 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10306 struct mlx5_list_entry *oentry, 10307 void *cb_ctx __rte_unused) 10308 { 10309 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10310 0, SOCKET_ID_ANY); 10311 10312 if (!tte) 10313 return NULL; 10314 memcpy(tte, oentry, sizeof(*tte)); 10315 return &tte->hash; 10316 } 10317 10318 static void 10319 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10320 struct mlx5_list_entry *entry) 10321 { 10322 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10323 10324 mlx5_free(tte); 10325 } 10326 10327 static uint32_t 10328 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10329 const struct mlx5_flow_tunnel *tunnel, 10330 uint32_t group, uint32_t *table, 10331 struct rte_flow_error *error) 10332 { 10333 struct mlx5_list_entry *he; 10334 struct tunnel_tbl_entry *tte; 10335 union tunnel_tbl_key key = { 10336 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10337 .group = group 10338 }; 10339 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10340 struct mlx5_hlist *group_hash; 10341 struct mlx5_flow_cb_ctx ctx = { 10342 .data = &key.val, 10343 }; 10344 10345 group_hash = tunnel ? tunnel->groups : thub->groups; 10346 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10347 if (!he) 10348 return rte_flow_error_set(error, EINVAL, 10349 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10350 NULL, 10351 "tunnel group index not supported"); 10352 tte = container_of(he, typeof(*tte), hash); 10353 *table = tte->flow_table; 10354 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10355 dev->data->port_id, key.tunnel_id, group, *table); 10356 return 0; 10357 } 10358 10359 static void 10360 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10361 struct mlx5_flow_tunnel *tunnel) 10362 { 10363 struct mlx5_priv *priv = dev->data->dev_private; 10364 struct mlx5_indexed_pool *ipool; 10365 10366 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10367 dev->data->port_id, tunnel->tunnel_id); 10368 LIST_REMOVE(tunnel, chain); 10369 mlx5_hlist_destroy(tunnel->groups); 10370 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10371 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10372 } 10373 10374 static bool 10375 mlx5_access_tunnel_offload_db 10376 (struct rte_eth_dev *dev, 10377 bool (*match)(struct rte_eth_dev *, 10378 struct mlx5_flow_tunnel *, const void *), 10379 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10380 void (*miss)(struct rte_eth_dev *, void *), 10381 void *ctx, bool lock_op) 10382 { 10383 bool verdict = false; 10384 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10385 struct mlx5_flow_tunnel *tunnel; 10386 10387 rte_spinlock_lock(&thub->sl); 10388 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10389 verdict = match(dev, tunnel, (const void *)ctx); 10390 if (verdict) 10391 break; 10392 } 10393 if (!lock_op) 10394 rte_spinlock_unlock(&thub->sl); 10395 if (verdict && hit) 10396 hit(dev, tunnel, ctx); 10397 if (!verdict && miss) 10398 miss(dev, ctx); 10399 if (lock_op) 10400 rte_spinlock_unlock(&thub->sl); 10401 10402 return verdict; 10403 } 10404 10405 struct tunnel_db_find_tunnel_id_ctx { 10406 uint32_t tunnel_id; 10407 struct mlx5_flow_tunnel *tunnel; 10408 }; 10409 10410 static bool 10411 find_tunnel_id_match(struct rte_eth_dev *dev, 10412 struct mlx5_flow_tunnel *tunnel, const void *x) 10413 { 10414 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10415 10416 RTE_SET_USED(dev); 10417 return tunnel->tunnel_id == ctx->tunnel_id; 10418 } 10419 10420 static void 10421 find_tunnel_id_hit(struct rte_eth_dev *dev, 10422 struct mlx5_flow_tunnel *tunnel, void *x) 10423 { 10424 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10425 RTE_SET_USED(dev); 10426 ctx->tunnel = tunnel; 10427 } 10428 10429 static struct mlx5_flow_tunnel * 10430 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10431 { 10432 struct tunnel_db_find_tunnel_id_ctx ctx = { 10433 .tunnel_id = id, 10434 }; 10435 10436 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10437 find_tunnel_id_hit, NULL, &ctx, true); 10438 10439 return ctx.tunnel; 10440 } 10441 10442 static struct mlx5_flow_tunnel * 10443 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10444 const struct rte_flow_tunnel *app_tunnel) 10445 { 10446 struct mlx5_priv *priv = dev->data->dev_private; 10447 struct mlx5_indexed_pool *ipool; 10448 struct mlx5_flow_tunnel *tunnel; 10449 uint32_t id; 10450 10451 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10452 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10453 if (!tunnel) 10454 return NULL; 10455 if (id >= MLX5_MAX_TUNNELS) { 10456 mlx5_ipool_free(ipool, id); 10457 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10458 return NULL; 10459 } 10460 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10461 priv->sh, 10462 mlx5_flow_tunnel_grp2tbl_create_cb, 10463 mlx5_flow_tunnel_grp2tbl_match_cb, 10464 mlx5_flow_tunnel_grp2tbl_remove_cb, 10465 mlx5_flow_tunnel_grp2tbl_clone_cb, 10466 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10467 if (!tunnel->groups) { 10468 mlx5_ipool_free(ipool, id); 10469 return NULL; 10470 } 10471 /* initiate new PMD tunnel */ 10472 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10473 tunnel->tunnel_id = id; 10474 tunnel->action.type = (typeof(tunnel->action.type)) 10475 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10476 tunnel->action.conf = tunnel; 10477 tunnel->item.type = (typeof(tunnel->item.type)) 10478 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10479 tunnel->item.spec = tunnel; 10480 tunnel->item.last = NULL; 10481 tunnel->item.mask = NULL; 10482 10483 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10484 dev->data->port_id, tunnel->tunnel_id); 10485 10486 return tunnel; 10487 } 10488 10489 struct tunnel_db_get_tunnel_ctx { 10490 const struct rte_flow_tunnel *app_tunnel; 10491 struct mlx5_flow_tunnel *tunnel; 10492 }; 10493 10494 static bool get_tunnel_match(struct rte_eth_dev *dev, 10495 struct mlx5_flow_tunnel *tunnel, const void *x) 10496 { 10497 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10498 10499 RTE_SET_USED(dev); 10500 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10501 sizeof(*ctx->app_tunnel)); 10502 } 10503 10504 static void get_tunnel_hit(struct rte_eth_dev *dev, 10505 struct mlx5_flow_tunnel *tunnel, void *x) 10506 { 10507 /* called under tunnel spinlock protection */ 10508 struct tunnel_db_get_tunnel_ctx *ctx = x; 10509 10510 RTE_SET_USED(dev); 10511 tunnel->refctn++; 10512 ctx->tunnel = tunnel; 10513 } 10514 10515 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10516 { 10517 /* called under tunnel spinlock protection */ 10518 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10519 struct tunnel_db_get_tunnel_ctx *ctx = x; 10520 10521 rte_spinlock_unlock(&thub->sl); 10522 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10523 rte_spinlock_lock(&thub->sl); 10524 if (ctx->tunnel) { 10525 ctx->tunnel->refctn = 1; 10526 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10527 } 10528 } 10529 10530 10531 static int 10532 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10533 const struct rte_flow_tunnel *app_tunnel, 10534 struct mlx5_flow_tunnel **tunnel) 10535 { 10536 struct tunnel_db_get_tunnel_ctx ctx = { 10537 .app_tunnel = app_tunnel, 10538 }; 10539 10540 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10541 get_tunnel_miss, &ctx, true); 10542 *tunnel = ctx.tunnel; 10543 return ctx.tunnel ? 0 : -ENOMEM; 10544 } 10545 10546 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10547 { 10548 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10549 10550 if (!thub) 10551 return; 10552 if (!LIST_EMPTY(&thub->tunnels)) 10553 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10554 mlx5_hlist_destroy(thub->groups); 10555 mlx5_free(thub); 10556 } 10557 10558 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10559 { 10560 int err; 10561 struct mlx5_flow_tunnel_hub *thub; 10562 10563 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10564 0, SOCKET_ID_ANY); 10565 if (!thub) 10566 return -ENOMEM; 10567 LIST_INIT(&thub->tunnels); 10568 rte_spinlock_init(&thub->sl); 10569 thub->groups = mlx5_hlist_create("flow groups", 64, 10570 false, true, sh, 10571 mlx5_flow_tunnel_grp2tbl_create_cb, 10572 mlx5_flow_tunnel_grp2tbl_match_cb, 10573 mlx5_flow_tunnel_grp2tbl_remove_cb, 10574 mlx5_flow_tunnel_grp2tbl_clone_cb, 10575 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10576 if (!thub->groups) { 10577 err = -rte_errno; 10578 goto err; 10579 } 10580 sh->tunnel_hub = thub; 10581 10582 return 0; 10583 10584 err: 10585 if (thub->groups) 10586 mlx5_hlist_destroy(thub->groups); 10587 if (thub) 10588 mlx5_free(thub); 10589 return err; 10590 } 10591 10592 static inline int 10593 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10594 struct rte_flow_tunnel *tunnel, 10595 struct rte_flow_error *error) 10596 { 10597 struct mlx5_priv *priv = dev->data->dev_private; 10598 10599 if (!priv->sh->config.dv_flow_en) 10600 return rte_flow_error_set(error, ENOTSUP, 10601 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10602 "flow DV interface is off"); 10603 if (!is_tunnel_offload_active(dev)) 10604 return rte_flow_error_set(error, ENOTSUP, 10605 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10606 "tunnel offload was not activated"); 10607 if (!tunnel) 10608 return rte_flow_error_set(error, EINVAL, 10609 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10610 "no application tunnel"); 10611 switch (tunnel->type) { 10612 default: 10613 return rte_flow_error_set(error, EINVAL, 10614 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10615 "unsupported tunnel type"); 10616 case RTE_FLOW_ITEM_TYPE_VXLAN: 10617 case RTE_FLOW_ITEM_TYPE_GRE: 10618 case RTE_FLOW_ITEM_TYPE_NVGRE: 10619 case RTE_FLOW_ITEM_TYPE_GENEVE: 10620 break; 10621 } 10622 return 0; 10623 } 10624 10625 static int 10626 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10627 struct rte_flow_tunnel *app_tunnel, 10628 struct rte_flow_action **actions, 10629 uint32_t *num_of_actions, 10630 struct rte_flow_error *error) 10631 { 10632 struct mlx5_flow_tunnel *tunnel; 10633 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10634 10635 if (ret) 10636 return ret; 10637 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10638 if (ret < 0) { 10639 return rte_flow_error_set(error, ret, 10640 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10641 "failed to initialize pmd tunnel"); 10642 } 10643 *actions = &tunnel->action; 10644 *num_of_actions = 1; 10645 return 0; 10646 } 10647 10648 static int 10649 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10650 struct rte_flow_tunnel *app_tunnel, 10651 struct rte_flow_item **items, 10652 uint32_t *num_of_items, 10653 struct rte_flow_error *error) 10654 { 10655 struct mlx5_flow_tunnel *tunnel; 10656 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10657 10658 if (ret) 10659 return ret; 10660 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10661 if (ret < 0) { 10662 return rte_flow_error_set(error, ret, 10663 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10664 "failed to initialize pmd tunnel"); 10665 } 10666 *items = &tunnel->item; 10667 *num_of_items = 1; 10668 return 0; 10669 } 10670 10671 struct tunnel_db_element_release_ctx { 10672 struct rte_flow_item *items; 10673 struct rte_flow_action *actions; 10674 uint32_t num_elements; 10675 struct rte_flow_error *error; 10676 int ret; 10677 }; 10678 10679 static bool 10680 tunnel_element_release_match(struct rte_eth_dev *dev, 10681 struct mlx5_flow_tunnel *tunnel, const void *x) 10682 { 10683 const struct tunnel_db_element_release_ctx *ctx = x; 10684 10685 RTE_SET_USED(dev); 10686 if (ctx->num_elements != 1) 10687 return false; 10688 else if (ctx->items) 10689 return ctx->items == &tunnel->item; 10690 else if (ctx->actions) 10691 return ctx->actions == &tunnel->action; 10692 10693 return false; 10694 } 10695 10696 static void 10697 tunnel_element_release_hit(struct rte_eth_dev *dev, 10698 struct mlx5_flow_tunnel *tunnel, void *x) 10699 { 10700 struct tunnel_db_element_release_ctx *ctx = x; 10701 ctx->ret = 0; 10702 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10703 mlx5_flow_tunnel_free(dev, tunnel); 10704 } 10705 10706 static void 10707 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10708 { 10709 struct tunnel_db_element_release_ctx *ctx = x; 10710 RTE_SET_USED(dev); 10711 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10712 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10713 "invalid argument"); 10714 } 10715 10716 static int 10717 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10718 struct rte_flow_item *pmd_items, 10719 uint32_t num_items, struct rte_flow_error *err) 10720 { 10721 struct tunnel_db_element_release_ctx ctx = { 10722 .items = pmd_items, 10723 .actions = NULL, 10724 .num_elements = num_items, 10725 .error = err, 10726 }; 10727 10728 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10729 tunnel_element_release_hit, 10730 tunnel_element_release_miss, &ctx, false); 10731 10732 return ctx.ret; 10733 } 10734 10735 static int 10736 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10737 struct rte_flow_action *pmd_actions, 10738 uint32_t num_actions, struct rte_flow_error *err) 10739 { 10740 struct tunnel_db_element_release_ctx ctx = { 10741 .items = NULL, 10742 .actions = pmd_actions, 10743 .num_elements = num_actions, 10744 .error = err, 10745 }; 10746 10747 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10748 tunnel_element_release_hit, 10749 tunnel_element_release_miss, &ctx, false); 10750 10751 return ctx.ret; 10752 } 10753 10754 static int 10755 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10756 struct rte_mbuf *m, 10757 struct rte_flow_restore_info *info, 10758 struct rte_flow_error *err) 10759 { 10760 uint64_t ol_flags = m->ol_flags; 10761 const struct mlx5_flow_tbl_data_entry *tble; 10762 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10763 10764 if (!is_tunnel_offload_active(dev)) { 10765 info->flags = 0; 10766 return 0; 10767 } 10768 10769 if ((ol_flags & mask) != mask) 10770 goto err; 10771 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10772 if (!tble) { 10773 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10774 dev->data->port_id, m->hash.fdir.hi); 10775 goto err; 10776 } 10777 MLX5_ASSERT(tble->tunnel); 10778 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10779 info->group_id = tble->group_id; 10780 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10781 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10782 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10783 10784 return 0; 10785 10786 err: 10787 return rte_flow_error_set(err, EINVAL, 10788 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10789 "failed to get restore info"); 10790 } 10791 10792 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10793 static int 10794 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10795 __rte_unused struct rte_flow_tunnel *app_tunnel, 10796 __rte_unused struct rte_flow_action **actions, 10797 __rte_unused uint32_t *num_of_actions, 10798 __rte_unused struct rte_flow_error *error) 10799 { 10800 return -ENOTSUP; 10801 } 10802 10803 static int 10804 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10805 __rte_unused struct rte_flow_tunnel *app_tunnel, 10806 __rte_unused struct rte_flow_item **items, 10807 __rte_unused uint32_t *num_of_items, 10808 __rte_unused struct rte_flow_error *error) 10809 { 10810 return -ENOTSUP; 10811 } 10812 10813 static int 10814 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10815 __rte_unused struct rte_flow_item *pmd_items, 10816 __rte_unused uint32_t num_items, 10817 __rte_unused struct rte_flow_error *err) 10818 { 10819 return -ENOTSUP; 10820 } 10821 10822 static int 10823 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10824 __rte_unused struct rte_flow_action *pmd_action, 10825 __rte_unused uint32_t num_actions, 10826 __rte_unused struct rte_flow_error *err) 10827 { 10828 return -ENOTSUP; 10829 } 10830 10831 static int 10832 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10833 __rte_unused struct rte_mbuf *m, 10834 __rte_unused struct rte_flow_restore_info *i, 10835 __rte_unused struct rte_flow_error *err) 10836 { 10837 return -ENOTSUP; 10838 } 10839 10840 static int 10841 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10842 __rte_unused struct rte_flow *flow, 10843 __rte_unused const struct rte_flow_attr *attr, 10844 __rte_unused const struct rte_flow_action *actions, 10845 __rte_unused uint32_t flow_idx, 10846 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10847 __rte_unused struct tunnel_default_miss_ctx *ctx, 10848 __rte_unused struct rte_flow_error *error) 10849 { 10850 return -ENOTSUP; 10851 } 10852 10853 static struct mlx5_flow_tunnel * 10854 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10855 __rte_unused uint32_t id) 10856 { 10857 return NULL; 10858 } 10859 10860 static void 10861 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10862 __rte_unused struct mlx5_flow_tunnel *tunnel) 10863 { 10864 } 10865 10866 static uint32_t 10867 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10868 __rte_unused const struct mlx5_flow_tunnel *t, 10869 __rte_unused uint32_t group, 10870 __rte_unused uint32_t *table, 10871 struct rte_flow_error *error) 10872 { 10873 return rte_flow_error_set(error, ENOTSUP, 10874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10875 "tunnel offload requires DV support"); 10876 } 10877 10878 void 10879 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10880 __rte_unused uint16_t port_id) 10881 { 10882 } 10883 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10884 10885 /* Flex flow item API */ 10886 static struct rte_flow_item_flex_handle * 10887 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10888 const struct rte_flow_item_flex_conf *conf, 10889 struct rte_flow_error *error) 10890 { 10891 static const char err_msg[] = "flex item creation unsupported"; 10892 struct mlx5_priv *priv = dev->data->dev_private; 10893 struct rte_flow_attr attr = { .transfer = 0 }; 10894 const struct mlx5_flow_driver_ops *fops = 10895 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10896 10897 if (!priv->pci_dev) { 10898 rte_flow_error_set(error, ENOTSUP, 10899 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10900 "create flex item on PF only"); 10901 return NULL; 10902 } 10903 switch (priv->pci_dev->id.device_id) { 10904 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 10905 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 10906 break; 10907 default: 10908 rte_flow_error_set(error, ENOTSUP, 10909 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10910 "flex item available on BlueField ports only"); 10911 return NULL; 10912 } 10913 if (!fops->item_create) { 10914 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10915 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10916 NULL, err_msg); 10917 return NULL; 10918 } 10919 return fops->item_create(dev, conf, error); 10920 } 10921 10922 static int 10923 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 10924 const struct rte_flow_item_flex_handle *handle, 10925 struct rte_flow_error *error) 10926 { 10927 static const char err_msg[] = "flex item release unsupported"; 10928 struct rte_flow_attr attr = { .transfer = 0 }; 10929 const struct mlx5_flow_driver_ops *fops = 10930 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10931 10932 if (!fops->item_release) { 10933 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10934 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10935 NULL, err_msg); 10936 return -rte_errno; 10937 } 10938 return fops->item_release(dev, handle, error); 10939 } 10940 10941 static void 10942 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 10943 { 10944 int ret; 10945 struct rte_flow_error error; 10946 10947 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10948 char *item_name; 10949 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 10950 sizeof(item_name), 10951 (void *)(uintptr_t)item->type, &error); 10952 if (ret > 0) 10953 printf("%s ", item_name); 10954 else 10955 printf("%d\n", (int)item->type); 10956 } 10957 printf("END\n"); 10958 } 10959 10960 static int 10961 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 10962 { 10963 const struct rte_flow_item_udp *spec = udp_item->spec; 10964 const struct rte_flow_item_udp *mask = udp_item->mask; 10965 uint16_t udp_dport = 0; 10966 10967 if (spec != NULL) { 10968 if (!mask) 10969 mask = &rte_flow_item_udp_mask; 10970 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 10971 mask->hdr.dst_port); 10972 } 10973 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 10974 } 10975 10976 static const struct mlx5_flow_expand_node * 10977 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 10978 unsigned int item_idx, 10979 const struct mlx5_flow_expand_node graph[], 10980 const struct mlx5_flow_expand_node *node) 10981 { 10982 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 10983 10984 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 10985 node != NULL && 10986 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 10987 /* 10988 * The expansion node is VXLAN and it is also the last 10989 * expandable item in the pattern, so need to continue 10990 * expansion of the inner tunnel. 10991 */ 10992 MLX5_ASSERT(item_idx > 0); 10993 prev_item = pattern + item_idx - 1; 10994 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 10995 if (mlx5_flow_is_std_vxlan_port(prev_item)) 10996 return &graph[MLX5_EXPANSION_STD_VXLAN]; 10997 return &graph[MLX5_EXPANSION_L3_VXLAN]; 10998 } 10999 return node; 11000 } 11001 11002 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11003 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11004 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11005 }; 11006 11007 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11008 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11009 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11010 { 9, 10, 11 }, { 12, 13, 14 }, 11011 }; 11012 11013 /** 11014 * Discover the number of available flow priorities. 11015 * 11016 * @param dev 11017 * Ethernet device. 11018 * 11019 * @return 11020 * On success, number of available flow priorities. 11021 * On failure, a negative errno-style code and rte_errno is set. 11022 */ 11023 int 11024 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11025 { 11026 static const uint16_t vprio[] = {8, 16}; 11027 const struct mlx5_priv *priv = dev->data->dev_private; 11028 const struct mlx5_flow_driver_ops *fops; 11029 enum mlx5_flow_drv_type type; 11030 int ret; 11031 11032 type = mlx5_flow_os_get_type(); 11033 if (type == MLX5_FLOW_TYPE_MAX) { 11034 type = MLX5_FLOW_TYPE_VERBS; 11035 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11036 type = MLX5_FLOW_TYPE_DV; 11037 } 11038 fops = flow_get_drv_ops(type); 11039 if (fops->discover_priorities == NULL) { 11040 DRV_LOG(ERR, "Priority discovery not supported"); 11041 rte_errno = ENOTSUP; 11042 return -rte_errno; 11043 } 11044 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11045 if (ret < 0) 11046 return ret; 11047 switch (ret) { 11048 case 8: 11049 ret = RTE_DIM(priority_map_3); 11050 break; 11051 case 16: 11052 ret = RTE_DIM(priority_map_5); 11053 break; 11054 default: 11055 rte_errno = ENOTSUP; 11056 DRV_LOG(ERR, 11057 "port %u maximum priority: %d expected 8/16", 11058 dev->data->port_id, ret); 11059 return -rte_errno; 11060 } 11061 DRV_LOG(INFO, "port %u supported flow priorities:" 11062 " 0-%d for ingress or egress root table," 11063 " 0-%d for non-root table or transfer root table.", 11064 dev->data->port_id, ret - 2, 11065 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11066 return ret; 11067 } 11068 11069 /** 11070 * Adjust flow priority based on the highest layer and the request priority. 11071 * 11072 * @param[in] dev 11073 * Pointer to the Ethernet device structure. 11074 * @param[in] priority 11075 * The rule base priority. 11076 * @param[in] subpriority 11077 * The priority based on the items. 11078 * 11079 * @return 11080 * The new priority. 11081 */ 11082 uint32_t 11083 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11084 uint32_t subpriority) 11085 { 11086 uint32_t res = 0; 11087 struct mlx5_priv *priv = dev->data->dev_private; 11088 11089 switch (priv->sh->flow_max_priority) { 11090 case RTE_DIM(priority_map_3): 11091 res = priority_map_3[priority][subpriority]; 11092 break; 11093 case RTE_DIM(priority_map_5): 11094 res = priority_map_5[priority][subpriority]; 11095 break; 11096 } 11097 return res; 11098 } 11099 11100 /** 11101 * Get the priority for sending traffic to kernel table. 11102 * 11103 * @param[in] dev 11104 * Pointer to the Ethernet device structure. 11105 * 11106 * @return 11107 * On success: the value of priority for sending traffic to kernel table 11108 * On failure: -1 11109 */ 11110 uint32_t 11111 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11112 { 11113 struct mlx5_priv *priv = dev->data->dev_private; 11114 uint32_t res; 11115 11116 switch (priv->sh->flow_max_priority) { 11117 case RTE_DIM(priority_map_5): 11118 res = 15; 11119 break; 11120 case RTE_DIM(priority_map_3): 11121 res = 7; 11122 break; 11123 default: 11124 DRV_LOG(ERR, 11125 "port %u maximum priority: %d expected 8/16", 11126 dev->data->port_id, priv->sh->flow_max_priority); 11127 res = (uint32_t)-1; 11128 } 11129 return res; 11130 } 11131 11132 /** 11133 * Get the E-Switch Manager vport id. 11134 * 11135 * @param[in] dev 11136 * Pointer to the Ethernet device structure. 11137 * 11138 * @return 11139 * The vport id. 11140 */ 11141 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11142 { 11143 struct mlx5_priv *priv = dev->data->dev_private; 11144 struct mlx5_common_device *cdev = priv->sh->cdev; 11145 11146 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11147 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11148 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11149 11150 if (priv->pci_dev == NULL) 11151 return 0; 11152 switch (priv->pci_dev->id.device_id) { 11153 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 11154 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11155 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11156 /* 11157 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11158 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11159 * ECPF vport ID is selected and not the host port (0) in any BF case. 11160 */ 11161 return (int16_t)MLX5_ECPF_VPORT_ID; 11162 default: 11163 return MLX5_PF_VPORT_ID; 11164 } 11165 } 11166 11167 /** 11168 * Parse item to get the vport id. 11169 * 11170 * @param[in] dev 11171 * Pointer to the Ethernet device structure. 11172 * @param[in] item 11173 * The src port id match item. 11174 * @param[out] vport_id 11175 * Pointer to put the vport id. 11176 * @param[out] all_ports 11177 * Indicate if the item matches all ports. 11178 * @param[out] error 11179 * Pointer to error structure. 11180 * 11181 * @return 11182 * 0 on success, a negative errno value otherwise and rte_errno is set. 11183 */ 11184 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11185 const struct rte_flow_item *item, 11186 uint16_t *vport_id, 11187 bool *all_ports, 11188 struct rte_flow_error *error) 11189 { 11190 struct mlx5_priv *port_priv; 11191 const struct rte_flow_item_port_id *pid_v; 11192 uint32_t esw_mgr_port; 11193 11194 if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID && 11195 item->type != RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) 11196 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11197 NULL, "Incorrect item type."); 11198 pid_v = item->spec; 11199 if (!pid_v) { 11200 if (all_ports) 11201 *all_ports = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT); 11202 return 0; 11203 } 11204 if (all_ports) 11205 *all_ports = false; 11206 esw_mgr_port = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) ? 11207 MLX5_REPRESENTED_PORT_ESW_MGR : MLX5_PORT_ESW_MGR; 11208 if (pid_v->id == esw_mgr_port) { 11209 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11210 } else { 11211 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 11212 if (!port_priv) 11213 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11214 NULL, "Failed to get port info."); 11215 *vport_id = port_priv->representor_id; 11216 } 11217 11218 return 0; 11219 } 11220 11221 int 11222 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11223 uint16_t *proxy_port_id, 11224 struct rte_flow_error *error) 11225 { 11226 const struct mlx5_priv *priv = dev->data->dev_private; 11227 uint16_t port_id; 11228 11229 if (!priv->sh->config.dv_esw_en) 11230 return rte_flow_error_set(error, EINVAL, 11231 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11232 NULL, 11233 "unable to provide a proxy port" 11234 " without E-Switch configured"); 11235 if (!priv->master && !priv->representor) 11236 return rte_flow_error_set(error, EINVAL, 11237 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11238 NULL, 11239 "unable to provide a proxy port" 11240 " for port which is not a master" 11241 " or a representor port"); 11242 if (priv->master) { 11243 *proxy_port_id = dev->data->port_id; 11244 return 0; 11245 } 11246 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11247 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11248 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11249 11250 if (port_priv->master && 11251 port_priv->domain_id == priv->domain_id) { 11252 *proxy_port_id = port_id; 11253 return 0; 11254 } 11255 } 11256 return rte_flow_error_set(error, EINVAL, 11257 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11258 NULL, "unable to find a proxy port"); 11259 } 11260