1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /* Optional expand field. The expansion alg will not go deeper. */ 128 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 129 130 /* The node is not added implicitly as expansion to the flow pattern. 131 * If the node type does not match the flow pattern item type, the 132 * expansion alg will go deeper to its next items. 133 * In the current implementation, the list of next nodes indexes can 134 * have up to one node with this flag set and it has to be the last 135 * node index (before the list terminator). 136 */ 137 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 138 139 /** Object returned by mlx5_flow_expand_rss(). */ 140 struct mlx5_flow_expand_rss { 141 uint32_t entries; 142 /**< Number of entries @p patterns and @p priorities. */ 143 struct { 144 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 145 uint32_t priority; /**< Priority offset for each expansion. */ 146 } entry[]; 147 }; 148 149 static void 150 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 151 152 static const struct mlx5_flow_expand_node * 153 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 154 unsigned int item_idx, 155 const struct mlx5_flow_expand_node graph[], 156 const struct mlx5_flow_expand_node *node); 157 158 static bool 159 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 160 { 161 switch (item->type) { 162 case RTE_FLOW_ITEM_TYPE_ETH: 163 case RTE_FLOW_ITEM_TYPE_VLAN: 164 case RTE_FLOW_ITEM_TYPE_IPV4: 165 case RTE_FLOW_ITEM_TYPE_IPV6: 166 case RTE_FLOW_ITEM_TYPE_UDP: 167 case RTE_FLOW_ITEM_TYPE_TCP: 168 case RTE_FLOW_ITEM_TYPE_ESP: 169 case RTE_FLOW_ITEM_TYPE_ICMP: 170 case RTE_FLOW_ITEM_TYPE_ICMP6: 171 case RTE_FLOW_ITEM_TYPE_VXLAN: 172 case RTE_FLOW_ITEM_TYPE_NVGRE: 173 case RTE_FLOW_ITEM_TYPE_GRE: 174 case RTE_FLOW_ITEM_TYPE_GENEVE: 175 case RTE_FLOW_ITEM_TYPE_MPLS: 176 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 177 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 178 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 179 case RTE_FLOW_ITEM_TYPE_GTP: 180 return true; 181 default: 182 break; 183 } 184 return false; 185 } 186 187 /** 188 * Network Service Header (NSH) and its next protocol values 189 * are described in RFC-8393. 190 */ 191 static enum rte_flow_item_type 192 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 193 { 194 enum rte_flow_item_type type; 195 196 switch (proto_mask & proto_spec) { 197 case 0: 198 type = RTE_FLOW_ITEM_TYPE_VOID; 199 break; 200 case RTE_VXLAN_GPE_TYPE_IPV4: 201 type = RTE_FLOW_ITEM_TYPE_IPV4; 202 break; 203 case RTE_VXLAN_GPE_TYPE_IPV6: 204 type = RTE_VXLAN_GPE_TYPE_IPV6; 205 break; 206 case RTE_VXLAN_GPE_TYPE_ETH: 207 type = RTE_FLOW_ITEM_TYPE_ETH; 208 break; 209 default: 210 type = RTE_FLOW_ITEM_TYPE_END; 211 } 212 return type; 213 } 214 215 static enum rte_flow_item_type 216 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 217 { 218 enum rte_flow_item_type type; 219 220 switch (proto_mask & proto_spec) { 221 case 0: 222 type = RTE_FLOW_ITEM_TYPE_VOID; 223 break; 224 case IPPROTO_UDP: 225 type = RTE_FLOW_ITEM_TYPE_UDP; 226 break; 227 case IPPROTO_TCP: 228 type = RTE_FLOW_ITEM_TYPE_TCP; 229 break; 230 case IPPROTO_IPIP: 231 type = RTE_FLOW_ITEM_TYPE_IPV4; 232 break; 233 case IPPROTO_IPV6: 234 type = RTE_FLOW_ITEM_TYPE_IPV6; 235 break; 236 case IPPROTO_ESP: 237 type = RTE_FLOW_ITEM_TYPE_ESP; 238 break; 239 default: 240 type = RTE_FLOW_ITEM_TYPE_END; 241 } 242 return type; 243 } 244 245 static enum rte_flow_item_type 246 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 247 rte_be16_t type_mask, bool is_tunnel) 248 { 249 enum rte_flow_item_type type; 250 251 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 252 case 0: 253 type = RTE_FLOW_ITEM_TYPE_VOID; 254 break; 255 case RTE_ETHER_TYPE_TEB: 256 type = is_tunnel ? 257 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 258 break; 259 case RTE_ETHER_TYPE_VLAN: 260 type = !is_tunnel ? 261 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 262 break; 263 case RTE_ETHER_TYPE_IPV4: 264 type = RTE_FLOW_ITEM_TYPE_IPV4; 265 break; 266 case RTE_ETHER_TYPE_IPV6: 267 type = RTE_FLOW_ITEM_TYPE_IPV6; 268 break; 269 default: 270 type = RTE_FLOW_ITEM_TYPE_END; 271 } 272 return type; 273 } 274 275 static enum rte_flow_item_type 276 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 277 { 278 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 279 do { \ 280 const void *m = item->mask; \ 281 const void *s = item->spec; \ 282 mask = m ? \ 283 ((const struct rte_flow_item_##type *)m)->fld : \ 284 rte_flow_item_##type##_mask.fld; \ 285 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 286 } while (0) 287 288 enum rte_flow_item_type ret; 289 uint16_t spec, mask; 290 291 if (item == NULL || item->spec == NULL) 292 return RTE_FLOW_ITEM_TYPE_VOID; 293 switch (item->type) { 294 case RTE_FLOW_ITEM_TYPE_ETH: 295 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 296 if (!mask) 297 return RTE_FLOW_ITEM_TYPE_VOID; 298 ret = mlx5_ethertype_to_item_type(spec, mask, false); 299 break; 300 case RTE_FLOW_ITEM_TYPE_VLAN: 301 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 302 if (!mask) 303 return RTE_FLOW_ITEM_TYPE_VOID; 304 ret = mlx5_ethertype_to_item_type(spec, mask, false); 305 break; 306 case RTE_FLOW_ITEM_TYPE_IPV4: 307 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 308 if (!mask) 309 return RTE_FLOW_ITEM_TYPE_VOID; 310 ret = mlx5_inet_proto_to_item_type(spec, mask); 311 break; 312 case RTE_FLOW_ITEM_TYPE_IPV6: 313 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 314 if (!mask) 315 return RTE_FLOW_ITEM_TYPE_VOID; 316 ret = mlx5_inet_proto_to_item_type(spec, mask); 317 break; 318 case RTE_FLOW_ITEM_TYPE_GENEVE: 319 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 320 ret = mlx5_ethertype_to_item_type(spec, mask, true); 321 break; 322 case RTE_FLOW_ITEM_TYPE_GRE: 323 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 324 ret = mlx5_ethertype_to_item_type(spec, mask, true); 325 break; 326 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 327 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 328 ret = mlx5_nsh_proto_to_item_type(spec, mask); 329 break; 330 default: 331 ret = RTE_FLOW_ITEM_TYPE_VOID; 332 break; 333 } 334 return ret; 335 #undef MLX5_XSET_ITEM_MASK_SPEC 336 } 337 338 static const int * 339 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 340 const int *next_node) 341 { 342 const struct mlx5_flow_expand_node *node = NULL; 343 const int *next = next_node; 344 345 while (next && *next) { 346 /* 347 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 348 * flag set, because they were not found in the flow pattern. 349 */ 350 node = &graph[*next]; 351 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 352 break; 353 next = node->next; 354 } 355 return next; 356 } 357 358 #define MLX5_RSS_EXP_ELT_N 16 359 360 /** 361 * Expand RSS flows into several possible flows according to the RSS hash 362 * fields requested and the driver capabilities. 363 * 364 * @param[out] buf 365 * Buffer to store the result expansion. 366 * @param[in] size 367 * Buffer size in bytes. If 0, @p buf can be NULL. 368 * @param[in] pattern 369 * User flow pattern. 370 * @param[in] types 371 * RSS types to expand (see RTE_ETH_RSS_* definitions). 372 * @param[in] graph 373 * Input graph to expand @p pattern according to @p types. 374 * @param[in] graph_root_index 375 * Index of root node in @p graph, typically 0. 376 * 377 * @return 378 * A positive value representing the size of @p buf in bytes regardless of 379 * @p size on success, a negative errno value otherwise and rte_errno is 380 * set, the following errors are defined: 381 * 382 * -E2BIG: graph-depth @p graph is too deep. 383 * -EINVAL: @p size has not enough space for expanded pattern. 384 */ 385 static int 386 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 387 const struct rte_flow_item *pattern, uint64_t types, 388 const struct mlx5_flow_expand_node graph[], 389 int graph_root_index) 390 { 391 const struct rte_flow_item *item; 392 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 393 const int *next_node; 394 const int *stack[MLX5_RSS_EXP_ELT_N]; 395 int stack_pos = 0; 396 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 397 unsigned int i, item_idx, last_expand_item_idx = 0; 398 size_t lsize; 399 size_t user_pattern_size = 0; 400 void *addr = NULL; 401 const struct mlx5_flow_expand_node *next = NULL; 402 struct rte_flow_item missed_item; 403 int missed = 0; 404 int elt = 0; 405 const struct rte_flow_item *last_expand_item = NULL; 406 407 memset(&missed_item, 0, sizeof(missed_item)); 408 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 409 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 410 if (lsize > size) 411 return -EINVAL; 412 buf->entry[0].priority = 0; 413 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 414 buf->entries = 0; 415 addr = buf->entry[0].pattern; 416 for (item = pattern, item_idx = 0; 417 item->type != RTE_FLOW_ITEM_TYPE_END; 418 item++, item_idx++) { 419 if (!mlx5_flow_is_rss_expandable_item(item)) { 420 user_pattern_size += sizeof(*item); 421 continue; 422 } 423 last_expand_item = item; 424 last_expand_item_idx = item_idx; 425 i = 0; 426 while (node->next && node->next[i]) { 427 next = &graph[node->next[i]]; 428 if (next->type == item->type) 429 break; 430 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 431 node = next; 432 i = 0; 433 } else { 434 ++i; 435 } 436 } 437 if (next) 438 node = next; 439 user_pattern_size += sizeof(*item); 440 } 441 user_pattern_size += sizeof(*item); /* Handle END item. */ 442 lsize += user_pattern_size; 443 if (lsize > size) 444 return -EINVAL; 445 /* Copy the user pattern in the first entry of the buffer. */ 446 rte_memcpy(addr, pattern, user_pattern_size); 447 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 448 buf->entries = 1; 449 /* Start expanding. */ 450 memset(flow_items, 0, sizeof(flow_items)); 451 user_pattern_size -= sizeof(*item); 452 /* 453 * Check if the last valid item has spec set, need complete pattern, 454 * and the pattern can be used for expansion. 455 */ 456 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 457 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 458 /* Item type END indicates expansion is not required. */ 459 return lsize; 460 } 461 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 462 next = NULL; 463 missed = 1; 464 i = 0; 465 while (node->next && node->next[i]) { 466 next = &graph[node->next[i]]; 467 if (next->type == missed_item.type) { 468 flow_items[0].type = missed_item.type; 469 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 470 break; 471 } 472 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 473 node = next; 474 i = 0; 475 } else { 476 ++i; 477 } 478 next = NULL; 479 } 480 } 481 if (next && missed) { 482 elt = 2; /* missed item + item end. */ 483 node = next; 484 lsize += elt * sizeof(*item) + user_pattern_size; 485 if (lsize > size) 486 return -EINVAL; 487 if (node->rss_types & types) { 488 buf->entry[buf->entries].priority = 1; 489 buf->entry[buf->entries].pattern = addr; 490 buf->entries++; 491 rte_memcpy(addr, buf->entry[0].pattern, 492 user_pattern_size); 493 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 494 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 495 addr = (void *)(((uintptr_t)addr) + 496 elt * sizeof(*item)); 497 } 498 } else if (last_expand_item != NULL) { 499 node = mlx5_flow_expand_rss_adjust_node(pattern, 500 last_expand_item_idx, graph, node); 501 } 502 memset(flow_items, 0, sizeof(flow_items)); 503 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 504 node->next); 505 stack[stack_pos] = next_node; 506 node = next_node ? &graph[*next_node] : NULL; 507 while (node) { 508 flow_items[stack_pos].type = node->type; 509 if (node->rss_types & types) { 510 size_t n; 511 /* 512 * compute the number of items to copy from the 513 * expansion and copy it. 514 * When the stack_pos is 0, there are 1 element in it, 515 * plus the addition END item. 516 */ 517 elt = stack_pos + 2; 518 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 519 lsize += elt * sizeof(*item) + user_pattern_size; 520 if (lsize > size) 521 return -EINVAL; 522 n = elt * sizeof(*item); 523 buf->entry[buf->entries].priority = 524 stack_pos + 1 + missed; 525 buf->entry[buf->entries].pattern = addr; 526 buf->entries++; 527 rte_memcpy(addr, buf->entry[0].pattern, 528 user_pattern_size); 529 addr = (void *)(((uintptr_t)addr) + 530 user_pattern_size); 531 rte_memcpy(addr, &missed_item, 532 missed * sizeof(*item)); 533 addr = (void *)(((uintptr_t)addr) + 534 missed * sizeof(*item)); 535 rte_memcpy(addr, flow_items, n); 536 addr = (void *)(((uintptr_t)addr) + n); 537 } 538 /* Go deeper. */ 539 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 540 node->next) { 541 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 542 node->next); 543 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 544 rte_errno = E2BIG; 545 return -rte_errno; 546 } 547 stack[stack_pos] = next_node; 548 } else if (*(next_node + 1)) { 549 /* Follow up with the next possibility. */ 550 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 551 ++next_node); 552 } else if (!stack_pos) { 553 /* 554 * Completing the traverse over the different paths. 555 * The next_node is advanced to the terminator. 556 */ 557 ++next_node; 558 } else { 559 /* Move to the next path. */ 560 while (stack_pos) { 561 next_node = stack[--stack_pos]; 562 next_node++; 563 if (*next_node) 564 break; 565 } 566 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 567 next_node); 568 stack[stack_pos] = next_node; 569 } 570 node = next_node && *next_node ? &graph[*next_node] : NULL; 571 }; 572 return lsize; 573 } 574 575 enum mlx5_expansion { 576 MLX5_EXPANSION_ROOT, 577 MLX5_EXPANSION_ROOT_OUTER, 578 MLX5_EXPANSION_OUTER_ETH, 579 MLX5_EXPANSION_OUTER_VLAN, 580 MLX5_EXPANSION_OUTER_IPV4, 581 MLX5_EXPANSION_OUTER_IPV4_UDP, 582 MLX5_EXPANSION_OUTER_IPV4_TCP, 583 MLX5_EXPANSION_OUTER_IPV4_ESP, 584 MLX5_EXPANSION_OUTER_IPV4_ICMP, 585 MLX5_EXPANSION_OUTER_IPV6, 586 MLX5_EXPANSION_OUTER_IPV6_UDP, 587 MLX5_EXPANSION_OUTER_IPV6_TCP, 588 MLX5_EXPANSION_OUTER_IPV6_ESP, 589 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 590 MLX5_EXPANSION_VXLAN, 591 MLX5_EXPANSION_STD_VXLAN, 592 MLX5_EXPANSION_L3_VXLAN, 593 MLX5_EXPANSION_VXLAN_GPE, 594 MLX5_EXPANSION_GRE, 595 MLX5_EXPANSION_NVGRE, 596 MLX5_EXPANSION_GRE_KEY, 597 MLX5_EXPANSION_MPLS, 598 MLX5_EXPANSION_ETH, 599 MLX5_EXPANSION_VLAN, 600 MLX5_EXPANSION_IPV4, 601 MLX5_EXPANSION_IPV4_UDP, 602 MLX5_EXPANSION_IPV4_TCP, 603 MLX5_EXPANSION_IPV4_ESP, 604 MLX5_EXPANSION_IPV4_ICMP, 605 MLX5_EXPANSION_IPV6, 606 MLX5_EXPANSION_IPV6_UDP, 607 MLX5_EXPANSION_IPV6_TCP, 608 MLX5_EXPANSION_IPV6_ESP, 609 MLX5_EXPANSION_IPV6_ICMP6, 610 MLX5_EXPANSION_IPV6_FRAG_EXT, 611 MLX5_EXPANSION_GTP, 612 MLX5_EXPANSION_GENEVE, 613 }; 614 615 /** Supported expansion of items. */ 616 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 617 [MLX5_EXPANSION_ROOT] = { 618 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 619 MLX5_EXPANSION_IPV4, 620 MLX5_EXPANSION_IPV6), 621 .type = RTE_FLOW_ITEM_TYPE_END, 622 }, 623 [MLX5_EXPANSION_ROOT_OUTER] = { 624 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 625 MLX5_EXPANSION_OUTER_IPV4, 626 MLX5_EXPANSION_OUTER_IPV6), 627 .type = RTE_FLOW_ITEM_TYPE_END, 628 }, 629 [MLX5_EXPANSION_OUTER_ETH] = { 630 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 631 .type = RTE_FLOW_ITEM_TYPE_ETH, 632 .rss_types = 0, 633 }, 634 [MLX5_EXPANSION_OUTER_VLAN] = { 635 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 636 MLX5_EXPANSION_OUTER_IPV6), 637 .type = RTE_FLOW_ITEM_TYPE_VLAN, 638 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 639 }, 640 [MLX5_EXPANSION_OUTER_IPV4] = { 641 .next = MLX5_FLOW_EXPAND_RSS_NEXT 642 (MLX5_EXPANSION_OUTER_IPV4_UDP, 643 MLX5_EXPANSION_OUTER_IPV4_TCP, 644 MLX5_EXPANSION_OUTER_IPV4_ESP, 645 MLX5_EXPANSION_OUTER_IPV4_ICMP, 646 MLX5_EXPANSION_GRE, 647 MLX5_EXPANSION_NVGRE, 648 MLX5_EXPANSION_IPV4, 649 MLX5_EXPANSION_IPV6), 650 .type = RTE_FLOW_ITEM_TYPE_IPV4, 651 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 652 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 653 }, 654 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 655 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 656 MLX5_EXPANSION_VXLAN_GPE, 657 MLX5_EXPANSION_MPLS, 658 MLX5_EXPANSION_GENEVE, 659 MLX5_EXPANSION_GTP), 660 .type = RTE_FLOW_ITEM_TYPE_UDP, 661 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 662 }, 663 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 664 .type = RTE_FLOW_ITEM_TYPE_TCP, 665 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 666 }, 667 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 668 .type = RTE_FLOW_ITEM_TYPE_ESP, 669 .rss_types = RTE_ETH_RSS_ESP, 670 }, 671 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 672 .type = RTE_FLOW_ITEM_TYPE_ICMP, 673 }, 674 [MLX5_EXPANSION_OUTER_IPV6] = { 675 .next = MLX5_FLOW_EXPAND_RSS_NEXT 676 (MLX5_EXPANSION_OUTER_IPV6_UDP, 677 MLX5_EXPANSION_OUTER_IPV6_TCP, 678 MLX5_EXPANSION_OUTER_IPV6_ESP, 679 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 680 MLX5_EXPANSION_IPV4, 681 MLX5_EXPANSION_IPV6, 682 MLX5_EXPANSION_GRE, 683 MLX5_EXPANSION_NVGRE), 684 .type = RTE_FLOW_ITEM_TYPE_IPV6, 685 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 686 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 687 }, 688 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 689 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 690 MLX5_EXPANSION_VXLAN_GPE, 691 MLX5_EXPANSION_MPLS, 692 MLX5_EXPANSION_GENEVE, 693 MLX5_EXPANSION_GTP), 694 .type = RTE_FLOW_ITEM_TYPE_UDP, 695 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 696 }, 697 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 698 .type = RTE_FLOW_ITEM_TYPE_TCP, 699 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 700 }, 701 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 702 .type = RTE_FLOW_ITEM_TYPE_ESP, 703 .rss_types = RTE_ETH_RSS_ESP, 704 }, 705 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 706 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 707 }, 708 [MLX5_EXPANSION_VXLAN] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 710 MLX5_EXPANSION_IPV4, 711 MLX5_EXPANSION_IPV6), 712 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 713 }, 714 [MLX5_EXPANSION_STD_VXLAN] = { 715 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 716 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 717 }, 718 [MLX5_EXPANSION_L3_VXLAN] = { 719 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 720 MLX5_EXPANSION_IPV6), 721 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 722 }, 723 [MLX5_EXPANSION_VXLAN_GPE] = { 724 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 725 MLX5_EXPANSION_IPV4, 726 MLX5_EXPANSION_IPV6), 727 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 728 }, 729 [MLX5_EXPANSION_GRE] = { 730 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 731 MLX5_EXPANSION_IPV4, 732 MLX5_EXPANSION_IPV6, 733 MLX5_EXPANSION_GRE_KEY, 734 MLX5_EXPANSION_MPLS), 735 .type = RTE_FLOW_ITEM_TYPE_GRE, 736 }, 737 [MLX5_EXPANSION_GRE_KEY] = { 738 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 739 MLX5_EXPANSION_IPV6, 740 MLX5_EXPANSION_MPLS), 741 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 742 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 743 }, 744 [MLX5_EXPANSION_NVGRE] = { 745 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 746 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 747 }, 748 [MLX5_EXPANSION_MPLS] = { 749 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6, 751 MLX5_EXPANSION_ETH), 752 .type = RTE_FLOW_ITEM_TYPE_MPLS, 753 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 754 }, 755 [MLX5_EXPANSION_ETH] = { 756 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 757 .type = RTE_FLOW_ITEM_TYPE_ETH, 758 }, 759 [MLX5_EXPANSION_VLAN] = { 760 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 761 MLX5_EXPANSION_IPV6), 762 .type = RTE_FLOW_ITEM_TYPE_VLAN, 763 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 764 }, 765 [MLX5_EXPANSION_IPV4] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 767 MLX5_EXPANSION_IPV4_TCP, 768 MLX5_EXPANSION_IPV4_ESP, 769 MLX5_EXPANSION_IPV4_ICMP), 770 .type = RTE_FLOW_ITEM_TYPE_IPV4, 771 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 772 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 773 }, 774 [MLX5_EXPANSION_IPV4_UDP] = { 775 .type = RTE_FLOW_ITEM_TYPE_UDP, 776 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 777 }, 778 [MLX5_EXPANSION_IPV4_TCP] = { 779 .type = RTE_FLOW_ITEM_TYPE_TCP, 780 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 781 }, 782 [MLX5_EXPANSION_IPV4_ESP] = { 783 .type = RTE_FLOW_ITEM_TYPE_ESP, 784 .rss_types = RTE_ETH_RSS_ESP, 785 }, 786 [MLX5_EXPANSION_IPV4_ICMP] = { 787 .type = RTE_FLOW_ITEM_TYPE_ICMP, 788 }, 789 [MLX5_EXPANSION_IPV6] = { 790 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 791 MLX5_EXPANSION_IPV6_TCP, 792 MLX5_EXPANSION_IPV6_ESP, 793 MLX5_EXPANSION_IPV6_ICMP6, 794 MLX5_EXPANSION_IPV6_FRAG_EXT), 795 .type = RTE_FLOW_ITEM_TYPE_IPV6, 796 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 797 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 798 }, 799 [MLX5_EXPANSION_IPV6_UDP] = { 800 .type = RTE_FLOW_ITEM_TYPE_UDP, 801 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 802 }, 803 [MLX5_EXPANSION_IPV6_TCP] = { 804 .type = RTE_FLOW_ITEM_TYPE_TCP, 805 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 806 }, 807 [MLX5_EXPANSION_IPV6_ESP] = { 808 .type = RTE_FLOW_ITEM_TYPE_ESP, 809 .rss_types = RTE_ETH_RSS_ESP, 810 }, 811 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 812 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 813 }, 814 [MLX5_EXPANSION_IPV6_ICMP6] = { 815 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 816 }, 817 [MLX5_EXPANSION_GTP] = { 818 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 819 MLX5_EXPANSION_IPV6), 820 .type = RTE_FLOW_ITEM_TYPE_GTP, 821 }, 822 [MLX5_EXPANSION_GENEVE] = { 823 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 824 MLX5_EXPANSION_IPV4, 825 MLX5_EXPANSION_IPV6), 826 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 827 }, 828 }; 829 830 static struct rte_flow_action_handle * 831 mlx5_action_handle_create(struct rte_eth_dev *dev, 832 const struct rte_flow_indir_action_conf *conf, 833 const struct rte_flow_action *action, 834 struct rte_flow_error *error); 835 static int mlx5_action_handle_destroy 836 (struct rte_eth_dev *dev, 837 struct rte_flow_action_handle *handle, 838 struct rte_flow_error *error); 839 static int mlx5_action_handle_update 840 (struct rte_eth_dev *dev, 841 struct rte_flow_action_handle *handle, 842 const void *update, 843 struct rte_flow_error *error); 844 static int mlx5_action_handle_query 845 (struct rte_eth_dev *dev, 846 const struct rte_flow_action_handle *handle, 847 void *data, 848 struct rte_flow_error *error); 849 static int 850 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 851 struct rte_flow_tunnel *app_tunnel, 852 struct rte_flow_action **actions, 853 uint32_t *num_of_actions, 854 struct rte_flow_error *error); 855 static int 856 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 857 struct rte_flow_tunnel *app_tunnel, 858 struct rte_flow_item **items, 859 uint32_t *num_of_items, 860 struct rte_flow_error *error); 861 static int 862 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 863 struct rte_flow_item *pmd_items, 864 uint32_t num_items, struct rte_flow_error *err); 865 static int 866 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 867 struct rte_flow_action *pmd_actions, 868 uint32_t num_actions, 869 struct rte_flow_error *err); 870 static int 871 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 872 struct rte_mbuf *m, 873 struct rte_flow_restore_info *info, 874 struct rte_flow_error *err); 875 static struct rte_flow_item_flex_handle * 876 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 877 const struct rte_flow_item_flex_conf *conf, 878 struct rte_flow_error *error); 879 static int 880 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 881 const struct rte_flow_item_flex_handle *handle, 882 struct rte_flow_error *error); 883 static int 884 mlx5_flow_info_get(struct rte_eth_dev *dev, 885 struct rte_flow_port_info *port_info, 886 struct rte_flow_queue_info *queue_info, 887 struct rte_flow_error *error); 888 static int 889 mlx5_flow_port_configure(struct rte_eth_dev *dev, 890 const struct rte_flow_port_attr *port_attr, 891 uint16_t nb_queue, 892 const struct rte_flow_queue_attr *queue_attr[], 893 struct rte_flow_error *err); 894 895 static struct rte_flow_pattern_template * 896 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 897 const struct rte_flow_pattern_template_attr *attr, 898 const struct rte_flow_item items[], 899 struct rte_flow_error *error); 900 901 static int 902 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 903 struct rte_flow_pattern_template *template, 904 struct rte_flow_error *error); 905 static struct rte_flow_actions_template * 906 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 907 const struct rte_flow_actions_template_attr *attr, 908 const struct rte_flow_action actions[], 909 const struct rte_flow_action masks[], 910 struct rte_flow_error *error); 911 static int 912 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 913 struct rte_flow_actions_template *template, 914 struct rte_flow_error *error); 915 916 static struct rte_flow_template_table * 917 mlx5_flow_table_create(struct rte_eth_dev *dev, 918 const struct rte_flow_template_table_attr *attr, 919 struct rte_flow_pattern_template *item_templates[], 920 uint8_t nb_item_templates, 921 struct rte_flow_actions_template *action_templates[], 922 uint8_t nb_action_templates, 923 struct rte_flow_error *error); 924 static int 925 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 926 struct rte_flow_template_table *table, 927 struct rte_flow_error *error); 928 static struct rte_flow * 929 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 930 uint32_t queue, 931 const struct rte_flow_op_attr *attr, 932 struct rte_flow_template_table *table, 933 const struct rte_flow_item items[], 934 uint8_t pattern_template_index, 935 const struct rte_flow_action actions[], 936 uint8_t action_template_index, 937 void *user_data, 938 struct rte_flow_error *error); 939 static int 940 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 941 uint32_t queue, 942 const struct rte_flow_op_attr *attr, 943 struct rte_flow *flow, 944 void *user_data, 945 struct rte_flow_error *error); 946 static int 947 mlx5_flow_pull(struct rte_eth_dev *dev, 948 uint32_t queue, 949 struct rte_flow_op_result res[], 950 uint16_t n_res, 951 struct rte_flow_error *error); 952 static int 953 mlx5_flow_push(struct rte_eth_dev *dev, 954 uint32_t queue, 955 struct rte_flow_error *error); 956 957 static struct rte_flow_action_handle * 958 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 959 const struct rte_flow_op_attr *attr, 960 const struct rte_flow_indir_action_conf *conf, 961 const struct rte_flow_action *action, 962 void *user_data, 963 struct rte_flow_error *error); 964 965 static int 966 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 967 const struct rte_flow_op_attr *attr, 968 struct rte_flow_action_handle *handle, 969 const void *update, 970 void *user_data, 971 struct rte_flow_error *error); 972 973 static int 974 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 975 const struct rte_flow_op_attr *attr, 976 struct rte_flow_action_handle *handle, 977 void *user_data, 978 struct rte_flow_error *error); 979 980 static int 981 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 982 const struct rte_flow_op_attr *attr, 983 const struct rte_flow_action_handle *handle, 984 void *data, 985 void *user_data, 986 struct rte_flow_error *error); 987 988 static const struct rte_flow_ops mlx5_flow_ops = { 989 .validate = mlx5_flow_validate, 990 .create = mlx5_flow_create, 991 .destroy = mlx5_flow_destroy, 992 .flush = mlx5_flow_flush, 993 .isolate = mlx5_flow_isolate, 994 .query = mlx5_flow_query, 995 .dev_dump = mlx5_flow_dev_dump, 996 #ifdef MLX5_HAVE_RTE_FLOW_Q_AGE 997 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 998 #endif 999 .get_aged_flows = mlx5_flow_get_aged_flows, 1000 .action_handle_create = mlx5_action_handle_create, 1001 .action_handle_destroy = mlx5_action_handle_destroy, 1002 .action_handle_update = mlx5_action_handle_update, 1003 .action_handle_query = mlx5_action_handle_query, 1004 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1005 .tunnel_match = mlx5_flow_tunnel_match, 1006 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1007 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1008 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1009 .flex_item_create = mlx5_flow_flex_item_create, 1010 .flex_item_release = mlx5_flow_flex_item_release, 1011 .info_get = mlx5_flow_info_get, 1012 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1013 .configure = mlx5_flow_port_configure, 1014 .pattern_template_create = mlx5_flow_pattern_template_create, 1015 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1016 .actions_template_create = mlx5_flow_actions_template_create, 1017 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1018 .template_table_create = mlx5_flow_table_create, 1019 .template_table_destroy = mlx5_flow_table_destroy, 1020 .async_create = mlx5_flow_async_flow_create, 1021 .async_destroy = mlx5_flow_async_flow_destroy, 1022 .pull = mlx5_flow_pull, 1023 .push = mlx5_flow_push, 1024 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1025 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1026 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1027 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1028 }; 1029 1030 /* Tunnel information. */ 1031 struct mlx5_flow_tunnel_info { 1032 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1033 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1034 }; 1035 1036 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1037 { 1038 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1039 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1040 }, 1041 { 1042 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1043 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1044 }, 1045 { 1046 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1047 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1048 }, 1049 { 1050 .tunnel = MLX5_FLOW_LAYER_GRE, 1051 .ptype = RTE_PTYPE_TUNNEL_GRE, 1052 }, 1053 { 1054 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1055 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1056 }, 1057 { 1058 .tunnel = MLX5_FLOW_LAYER_MPLS, 1059 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1060 }, 1061 { 1062 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1063 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1064 }, 1065 { 1066 .tunnel = MLX5_FLOW_LAYER_IPIP, 1067 .ptype = RTE_PTYPE_TUNNEL_IP, 1068 }, 1069 { 1070 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1071 .ptype = RTE_PTYPE_TUNNEL_IP, 1072 }, 1073 { 1074 .tunnel = MLX5_FLOW_LAYER_GTP, 1075 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1076 }, 1077 }; 1078 1079 1080 1081 /** 1082 * Translate tag ID to register. 1083 * 1084 * @param[in] dev 1085 * Pointer to the Ethernet device structure. 1086 * @param[in] feature 1087 * The feature that request the register. 1088 * @param[in] id 1089 * The request register ID. 1090 * @param[out] error 1091 * Error description in case of any. 1092 * 1093 * @return 1094 * The request register on success, a negative errno 1095 * value otherwise and rte_errno is set. 1096 */ 1097 int 1098 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1099 enum mlx5_feature_name feature, 1100 uint32_t id, 1101 struct rte_flow_error *error) 1102 { 1103 struct mlx5_priv *priv = dev->data->dev_private; 1104 struct mlx5_sh_config *config = &priv->sh->config; 1105 enum modify_reg start_reg; 1106 bool skip_mtr_reg = false; 1107 1108 switch (feature) { 1109 case MLX5_HAIRPIN_RX: 1110 return REG_B; 1111 case MLX5_HAIRPIN_TX: 1112 return REG_A; 1113 case MLX5_METADATA_RX: 1114 switch (config->dv_xmeta_en) { 1115 case MLX5_XMETA_MODE_LEGACY: 1116 return REG_B; 1117 case MLX5_XMETA_MODE_META16: 1118 return REG_C_0; 1119 case MLX5_XMETA_MODE_META32: 1120 return REG_C_1; 1121 case MLX5_XMETA_MODE_META32_HWS: 1122 return REG_C_1; 1123 } 1124 break; 1125 case MLX5_METADATA_TX: 1126 return REG_A; 1127 case MLX5_METADATA_FDB: 1128 switch (config->dv_xmeta_en) { 1129 case MLX5_XMETA_MODE_LEGACY: 1130 return REG_NON; 1131 case MLX5_XMETA_MODE_META16: 1132 return REG_C_0; 1133 case MLX5_XMETA_MODE_META32: 1134 return REG_C_1; 1135 case MLX5_XMETA_MODE_META32_HWS: 1136 return REG_C_1; 1137 } 1138 break; 1139 case MLX5_FLOW_MARK: 1140 switch (config->dv_xmeta_en) { 1141 case MLX5_XMETA_MODE_LEGACY: 1142 case MLX5_XMETA_MODE_META32_HWS: 1143 return REG_NON; 1144 case MLX5_XMETA_MODE_META16: 1145 return REG_C_1; 1146 case MLX5_XMETA_MODE_META32: 1147 return REG_C_0; 1148 } 1149 break; 1150 case MLX5_MTR_ID: 1151 /* 1152 * If meter color and meter id share one register, flow match 1153 * should use the meter color register for match. 1154 */ 1155 if (priv->mtr_reg_share) 1156 return priv->mtr_color_reg; 1157 else 1158 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1159 REG_C_3; 1160 case MLX5_MTR_COLOR: 1161 case MLX5_ASO_FLOW_HIT: 1162 case MLX5_ASO_CONNTRACK: 1163 case MLX5_SAMPLE_ID: 1164 /* All features use the same REG_C. */ 1165 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1166 return priv->mtr_color_reg; 1167 case MLX5_COPY_MARK: 1168 /* 1169 * Metadata COPY_MARK register using is in meter suffix sub 1170 * flow while with meter. It's safe to share the same register. 1171 */ 1172 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1173 case MLX5_APP_TAG: 1174 /* 1175 * If meter is enable, it will engage the register for color 1176 * match and flow match. If meter color match is not using the 1177 * REG_C_2, need to skip the REG_C_x be used by meter color 1178 * match. 1179 * If meter is disable, free to use all available registers. 1180 */ 1181 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1182 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1183 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1184 if (id > (uint32_t)(REG_C_7 - start_reg)) 1185 return rte_flow_error_set(error, EINVAL, 1186 RTE_FLOW_ERROR_TYPE_ITEM, 1187 NULL, "invalid tag id"); 1188 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1189 return rte_flow_error_set(error, ENOTSUP, 1190 RTE_FLOW_ERROR_TYPE_ITEM, 1191 NULL, "unsupported tag id"); 1192 /* 1193 * This case means meter is using the REG_C_x great than 2. 1194 * Take care not to conflict with meter color REG_C_x. 1195 * If the available index REG_C_y >= REG_C_x, skip the 1196 * color register. 1197 */ 1198 if (skip_mtr_reg && priv->sh->flow_mreg_c 1199 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1200 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1201 return rte_flow_error_set(error, EINVAL, 1202 RTE_FLOW_ERROR_TYPE_ITEM, 1203 NULL, "invalid tag id"); 1204 if (priv->sh->flow_mreg_c 1205 [id + 1 + start_reg - REG_C_0] != REG_NON) 1206 return priv->sh->flow_mreg_c 1207 [id + 1 + start_reg - REG_C_0]; 1208 return rte_flow_error_set(error, ENOTSUP, 1209 RTE_FLOW_ERROR_TYPE_ITEM, 1210 NULL, "unsupported tag id"); 1211 } 1212 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1213 } 1214 MLX5_ASSERT(false); 1215 return rte_flow_error_set(error, EINVAL, 1216 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1217 NULL, "invalid feature name"); 1218 } 1219 1220 /** 1221 * Check extensive flow metadata register support. 1222 * 1223 * @param dev 1224 * Pointer to rte_eth_dev structure. 1225 * 1226 * @return 1227 * True if device supports extensive flow metadata register, otherwise false. 1228 */ 1229 bool 1230 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1231 { 1232 struct mlx5_priv *priv = dev->data->dev_private; 1233 1234 /* 1235 * Having available reg_c can be regarded inclusively as supporting 1236 * extensive flow metadata register, which could mean, 1237 * - metadata register copy action by modify header. 1238 * - 16 modify header actions is supported. 1239 * - reg_c's are preserved across different domain (FDB and NIC) on 1240 * packet loopback by flow lookup miss. 1241 */ 1242 return priv->sh->flow_mreg_c[2] != REG_NON; 1243 } 1244 1245 /** 1246 * Get the lowest priority. 1247 * 1248 * @param[in] dev 1249 * Pointer to the Ethernet device structure. 1250 * @param[in] attributes 1251 * Pointer to device flow rule attributes. 1252 * 1253 * @return 1254 * The value of lowest priority of flow. 1255 */ 1256 uint32_t 1257 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1258 const struct rte_flow_attr *attr) 1259 { 1260 struct mlx5_priv *priv = dev->data->dev_private; 1261 1262 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1263 return priv->sh->flow_max_priority - 2; 1264 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1265 } 1266 1267 /** 1268 * Calculate matcher priority of the flow. 1269 * 1270 * @param[in] dev 1271 * Pointer to the Ethernet device structure. 1272 * @param[in] attr 1273 * Pointer to device flow rule attributes. 1274 * @param[in] subpriority 1275 * The priority based on the items. 1276 * @param[in] external 1277 * Flow is user flow. 1278 * @return 1279 * The matcher priority of the flow. 1280 */ 1281 uint16_t 1282 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1283 const struct rte_flow_attr *attr, 1284 uint32_t subpriority, bool external) 1285 { 1286 uint16_t priority = (uint16_t)attr->priority; 1287 struct mlx5_priv *priv = dev->data->dev_private; 1288 1289 /* NIC root rules */ 1290 if (!attr->group && !attr->transfer) { 1291 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1292 priority = priv->sh->flow_max_priority - 1; 1293 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1294 /* FDB root rules */ 1295 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1296 attr->group == 0 && 1297 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1298 return (priv->sh->flow_max_priority - 1) * 3; 1299 } 1300 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1301 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1302 return priority * 3 + subpriority; 1303 } 1304 1305 /** 1306 * Verify the @p item specifications (spec, last, mask) are compatible with the 1307 * NIC capabilities. 1308 * 1309 * @param[in] item 1310 * Item specification. 1311 * @param[in] mask 1312 * @p item->mask or flow default bit-masks. 1313 * @param[in] nic_mask 1314 * Bit-masks covering supported fields by the NIC to compare with user mask. 1315 * @param[in] size 1316 * Bit-masks size in bytes. 1317 * @param[in] range_accepted 1318 * True if range of values is accepted for specific fields, false otherwise. 1319 * @param[out] error 1320 * Pointer to error structure. 1321 * 1322 * @return 1323 * 0 on success, a negative errno value otherwise and rte_errno is set. 1324 */ 1325 int 1326 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1327 const uint8_t *mask, 1328 const uint8_t *nic_mask, 1329 unsigned int size, 1330 bool range_accepted, 1331 struct rte_flow_error *error) 1332 { 1333 unsigned int i; 1334 1335 MLX5_ASSERT(nic_mask); 1336 for (i = 0; i < size; ++i) 1337 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1338 return rte_flow_error_set(error, ENOTSUP, 1339 RTE_FLOW_ERROR_TYPE_ITEM, 1340 item, 1341 "mask enables non supported" 1342 " bits"); 1343 if (!item->spec && (item->mask || item->last)) 1344 return rte_flow_error_set(error, EINVAL, 1345 RTE_FLOW_ERROR_TYPE_ITEM, item, 1346 "mask/last without a spec is not" 1347 " supported"); 1348 if (item->spec && item->last && !range_accepted) { 1349 uint8_t spec[size]; 1350 uint8_t last[size]; 1351 unsigned int i; 1352 int ret; 1353 1354 for (i = 0; i < size; ++i) { 1355 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1356 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1357 } 1358 ret = memcmp(spec, last, size); 1359 if (ret != 0) 1360 return rte_flow_error_set(error, EINVAL, 1361 RTE_FLOW_ERROR_TYPE_ITEM, 1362 item, 1363 "range is not valid"); 1364 } 1365 return 0; 1366 } 1367 1368 /** 1369 * Adjust the hash fields according to the @p flow information. 1370 * 1371 * @param[in] dev_flow. 1372 * Pointer to the mlx5_flow. 1373 * @param[in] tunnel 1374 * 1 when the hash field is for a tunnel item. 1375 * @param[in] layer_types 1376 * RTE_ETH_RSS_* types. 1377 * @param[in] hash_fields 1378 * Item hash fields. 1379 * 1380 * @return 1381 * The hash fields that should be used. 1382 */ 1383 uint64_t 1384 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1385 int tunnel __rte_unused, uint64_t layer_types, 1386 uint64_t hash_fields) 1387 { 1388 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1389 int rss_request_inner = rss_desc->level >= 2; 1390 1391 /* Check RSS hash level for tunnel. */ 1392 if (tunnel && rss_request_inner) 1393 hash_fields |= IBV_RX_HASH_INNER; 1394 else if (tunnel || rss_request_inner) 1395 return 0; 1396 #endif 1397 /* Check if requested layer matches RSS hash fields. */ 1398 if (!(rss_desc->types & layer_types)) 1399 return 0; 1400 return hash_fields; 1401 } 1402 1403 /** 1404 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1405 * if several tunnel rules are used on this queue, the tunnel ptype will be 1406 * cleared. 1407 * 1408 * @param rxq_ctrl 1409 * Rx queue to update. 1410 */ 1411 static void 1412 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1413 { 1414 unsigned int i; 1415 uint32_t tunnel_ptype = 0; 1416 1417 /* Look up for the ptype to use. */ 1418 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1419 if (!rxq_ctrl->flow_tunnels_n[i]) 1420 continue; 1421 if (!tunnel_ptype) { 1422 tunnel_ptype = tunnels_info[i].ptype; 1423 } else { 1424 tunnel_ptype = 0; 1425 break; 1426 } 1427 } 1428 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1429 } 1430 1431 /** 1432 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1433 * flow. 1434 * 1435 * @param[in] dev 1436 * Pointer to the Ethernet device structure. 1437 * @param[in] dev_handle 1438 * Pointer to device flow handle structure. 1439 */ 1440 void 1441 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1442 struct mlx5_flow_handle *dev_handle) 1443 { 1444 struct mlx5_priv *priv = dev->data->dev_private; 1445 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1446 struct mlx5_ind_table_obj *ind_tbl = NULL; 1447 unsigned int i; 1448 1449 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1450 struct mlx5_hrxq *hrxq; 1451 1452 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1453 dev_handle->rix_hrxq); 1454 if (hrxq) 1455 ind_tbl = hrxq->ind_table; 1456 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1457 struct mlx5_shared_action_rss *shared_rss; 1458 1459 shared_rss = mlx5_ipool_get 1460 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1461 dev_handle->rix_srss); 1462 if (shared_rss) 1463 ind_tbl = shared_rss->ind_tbl; 1464 } 1465 if (!ind_tbl) 1466 return; 1467 for (i = 0; i != ind_tbl->queues_n; ++i) { 1468 int idx = ind_tbl->queues[i]; 1469 struct mlx5_rxq_ctrl *rxq_ctrl; 1470 1471 if (mlx5_is_external_rxq(dev, idx)) 1472 continue; 1473 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1474 MLX5_ASSERT(rxq_ctrl != NULL); 1475 if (rxq_ctrl == NULL) 1476 continue; 1477 /* 1478 * To support metadata register copy on Tx loopback, 1479 * this must be always enabled (metadata may arive 1480 * from other port - not from local flows only. 1481 */ 1482 if (tunnel) { 1483 unsigned int j; 1484 1485 /* Increase the counter matching the flow. */ 1486 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1487 if ((tunnels_info[j].tunnel & 1488 dev_handle->layers) == 1489 tunnels_info[j].tunnel) { 1490 rxq_ctrl->flow_tunnels_n[j]++; 1491 break; 1492 } 1493 } 1494 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1495 } 1496 } 1497 } 1498 1499 static void 1500 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1501 { 1502 struct mlx5_priv *priv = dev->data->dev_private; 1503 struct mlx5_rxq_ctrl *rxq_ctrl; 1504 uint16_t port_id; 1505 1506 if (priv->sh->shared_mark_enabled) 1507 return; 1508 if (priv->master || priv->representor) { 1509 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1510 struct mlx5_priv *opriv = 1511 rte_eth_devices[port_id].data->dev_private; 1512 1513 if (!opriv || 1514 opriv->sh != priv->sh || 1515 opriv->domain_id != priv->domain_id || 1516 opriv->mark_enabled) 1517 continue; 1518 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1519 rxq_ctrl->rxq.mark = 1; 1520 } 1521 opriv->mark_enabled = 1; 1522 } 1523 } else { 1524 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1525 rxq_ctrl->rxq.mark = 1; 1526 } 1527 priv->mark_enabled = 1; 1528 } 1529 priv->sh->shared_mark_enabled = 1; 1530 } 1531 1532 /** 1533 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1534 * 1535 * @param[in] dev 1536 * Pointer to the Ethernet device structure. 1537 * @param[in] flow 1538 * Pointer to flow structure. 1539 */ 1540 static void 1541 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1542 { 1543 struct mlx5_priv *priv = dev->data->dev_private; 1544 uint32_t handle_idx; 1545 struct mlx5_flow_handle *dev_handle; 1546 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1547 1548 MLX5_ASSERT(wks); 1549 if (wks->mark) 1550 flow_rxq_mark_flag_set(dev); 1551 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1552 handle_idx, dev_handle, next) 1553 flow_drv_rxq_flags_set(dev, dev_handle); 1554 } 1555 1556 /** 1557 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1558 * device flow if no other flow uses it with the same kind of request. 1559 * 1560 * @param dev 1561 * Pointer to Ethernet device. 1562 * @param[in] dev_handle 1563 * Pointer to the device flow handle structure. 1564 */ 1565 static void 1566 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1567 struct mlx5_flow_handle *dev_handle) 1568 { 1569 struct mlx5_priv *priv = dev->data->dev_private; 1570 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1571 struct mlx5_ind_table_obj *ind_tbl = NULL; 1572 unsigned int i; 1573 1574 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1575 struct mlx5_hrxq *hrxq; 1576 1577 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1578 dev_handle->rix_hrxq); 1579 if (hrxq) 1580 ind_tbl = hrxq->ind_table; 1581 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1582 struct mlx5_shared_action_rss *shared_rss; 1583 1584 shared_rss = mlx5_ipool_get 1585 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1586 dev_handle->rix_srss); 1587 if (shared_rss) 1588 ind_tbl = shared_rss->ind_tbl; 1589 } 1590 if (!ind_tbl) 1591 return; 1592 MLX5_ASSERT(dev->data->dev_started); 1593 for (i = 0; i != ind_tbl->queues_n; ++i) { 1594 int idx = ind_tbl->queues[i]; 1595 struct mlx5_rxq_ctrl *rxq_ctrl; 1596 1597 if (mlx5_is_external_rxq(dev, idx)) 1598 continue; 1599 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1600 MLX5_ASSERT(rxq_ctrl != NULL); 1601 if (rxq_ctrl == NULL) 1602 continue; 1603 if (tunnel) { 1604 unsigned int j; 1605 1606 /* Decrease the counter matching the flow. */ 1607 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1608 if ((tunnels_info[j].tunnel & 1609 dev_handle->layers) == 1610 tunnels_info[j].tunnel) { 1611 rxq_ctrl->flow_tunnels_n[j]--; 1612 break; 1613 } 1614 } 1615 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1616 } 1617 } 1618 } 1619 1620 /** 1621 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1622 * @p flow if no other flow uses it with the same kind of request. 1623 * 1624 * @param dev 1625 * Pointer to Ethernet device. 1626 * @param[in] flow 1627 * Pointer to the flow. 1628 */ 1629 static void 1630 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1631 { 1632 struct mlx5_priv *priv = dev->data->dev_private; 1633 uint32_t handle_idx; 1634 struct mlx5_flow_handle *dev_handle; 1635 1636 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1637 handle_idx, dev_handle, next) 1638 flow_drv_rxq_flags_trim(dev, dev_handle); 1639 } 1640 1641 /** 1642 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1643 * 1644 * @param dev 1645 * Pointer to Ethernet device. 1646 */ 1647 static void 1648 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1649 { 1650 struct mlx5_priv *priv = dev->data->dev_private; 1651 unsigned int i; 1652 1653 for (i = 0; i != priv->rxqs_n; ++i) { 1654 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1655 unsigned int j; 1656 1657 if (rxq == NULL || rxq->ctrl == NULL) 1658 continue; 1659 rxq->ctrl->rxq.mark = 0; 1660 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1661 rxq->ctrl->flow_tunnels_n[j] = 0; 1662 rxq->ctrl->rxq.tunnel = 0; 1663 } 1664 priv->mark_enabled = 0; 1665 priv->sh->shared_mark_enabled = 0; 1666 } 1667 1668 /** 1669 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1670 * 1671 * @param[in] dev 1672 * Pointer to the Ethernet device structure. 1673 */ 1674 void 1675 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1676 { 1677 struct mlx5_priv *priv = dev->data->dev_private; 1678 unsigned int i; 1679 1680 for (i = 0; i != priv->rxqs_n; ++i) { 1681 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1682 struct mlx5_rxq_data *data; 1683 1684 if (rxq == NULL || rxq->ctrl == NULL) 1685 continue; 1686 data = &rxq->ctrl->rxq; 1687 if (!rte_flow_dynf_metadata_avail()) { 1688 data->dynf_meta = 0; 1689 data->flow_meta_mask = 0; 1690 data->flow_meta_offset = -1; 1691 data->flow_meta_port_mask = 0; 1692 } else { 1693 data->dynf_meta = 1; 1694 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1695 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1696 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1697 } 1698 } 1699 } 1700 1701 /* 1702 * return a pointer to the desired action in the list of actions. 1703 * 1704 * @param[in] actions 1705 * The list of actions to search the action in. 1706 * @param[in] action 1707 * The action to find. 1708 * 1709 * @return 1710 * Pointer to the action in the list, if found. NULL otherwise. 1711 */ 1712 const struct rte_flow_action * 1713 mlx5_flow_find_action(const struct rte_flow_action *actions, 1714 enum rte_flow_action_type action) 1715 { 1716 if (actions == NULL) 1717 return NULL; 1718 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1719 if (actions->type == action) 1720 return actions; 1721 return NULL; 1722 } 1723 1724 /* 1725 * Validate the flag action. 1726 * 1727 * @param[in] action_flags 1728 * Bit-fields that holds the actions detected until now. 1729 * @param[in] attr 1730 * Attributes of flow that includes this action. 1731 * @param[out] error 1732 * Pointer to error structure. 1733 * 1734 * @return 1735 * 0 on success, a negative errno value otherwise and rte_errno is set. 1736 */ 1737 int 1738 mlx5_flow_validate_action_flag(uint64_t action_flags, 1739 const struct rte_flow_attr *attr, 1740 struct rte_flow_error *error) 1741 { 1742 if (action_flags & MLX5_FLOW_ACTION_MARK) 1743 return rte_flow_error_set(error, EINVAL, 1744 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1745 "can't mark and flag in same flow"); 1746 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1747 return rte_flow_error_set(error, EINVAL, 1748 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1749 "can't have 2 flag" 1750 " actions in same flow"); 1751 if (attr->egress) 1752 return rte_flow_error_set(error, ENOTSUP, 1753 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1754 "flag action not supported for " 1755 "egress"); 1756 return 0; 1757 } 1758 1759 /* 1760 * Validate the mark action. 1761 * 1762 * @param[in] action 1763 * Pointer to the queue action. 1764 * @param[in] action_flags 1765 * Bit-fields that holds the actions detected until now. 1766 * @param[in] attr 1767 * Attributes of flow that includes this action. 1768 * @param[out] error 1769 * Pointer to error structure. 1770 * 1771 * @return 1772 * 0 on success, a negative errno value otherwise and rte_errno is set. 1773 */ 1774 int 1775 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1776 uint64_t action_flags, 1777 const struct rte_flow_attr *attr, 1778 struct rte_flow_error *error) 1779 { 1780 const struct rte_flow_action_mark *mark = action->conf; 1781 1782 if (!mark) 1783 return rte_flow_error_set(error, EINVAL, 1784 RTE_FLOW_ERROR_TYPE_ACTION, 1785 action, 1786 "configuration cannot be null"); 1787 if (mark->id >= MLX5_FLOW_MARK_MAX) 1788 return rte_flow_error_set(error, EINVAL, 1789 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1790 &mark->id, 1791 "mark id must in 0 <= id < " 1792 RTE_STR(MLX5_FLOW_MARK_MAX)); 1793 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1794 return rte_flow_error_set(error, EINVAL, 1795 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1796 "can't flag and mark in same flow"); 1797 if (action_flags & MLX5_FLOW_ACTION_MARK) 1798 return rte_flow_error_set(error, EINVAL, 1799 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1800 "can't have 2 mark actions in same" 1801 " flow"); 1802 if (attr->egress) 1803 return rte_flow_error_set(error, ENOTSUP, 1804 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1805 "mark action not supported for " 1806 "egress"); 1807 return 0; 1808 } 1809 1810 /* 1811 * Validate the drop action. 1812 * 1813 * @param[in] action_flags 1814 * Bit-fields that holds the actions detected until now. 1815 * @param[in] attr 1816 * Attributes of flow that includes this action. 1817 * @param[out] error 1818 * Pointer to error structure. 1819 * 1820 * @return 1821 * 0 on success, a negative errno value otherwise and rte_errno is set. 1822 */ 1823 int 1824 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1825 const struct rte_flow_attr *attr, 1826 struct rte_flow_error *error) 1827 { 1828 if (attr->egress) 1829 return rte_flow_error_set(error, ENOTSUP, 1830 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1831 "drop action not supported for " 1832 "egress"); 1833 return 0; 1834 } 1835 1836 /* 1837 * Validate the queue action. 1838 * 1839 * @param[in] action 1840 * Pointer to the queue action. 1841 * @param[in] action_flags 1842 * Bit-fields that holds the actions detected until now. 1843 * @param[in] dev 1844 * Pointer to the Ethernet device structure. 1845 * @param[in] attr 1846 * Attributes of flow that includes this action. 1847 * @param[out] error 1848 * Pointer to error structure. 1849 * 1850 * @return 1851 * 0 on success, a negative errno value otherwise and rte_errno is set. 1852 */ 1853 int 1854 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1855 uint64_t action_flags, 1856 struct rte_eth_dev *dev, 1857 const struct rte_flow_attr *attr, 1858 struct rte_flow_error *error) 1859 { 1860 struct mlx5_priv *priv = dev->data->dev_private; 1861 const struct rte_flow_action_queue *queue = action->conf; 1862 1863 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1864 return rte_flow_error_set(error, EINVAL, 1865 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1866 "can't have 2 fate actions in" 1867 " same flow"); 1868 if (attr->egress) 1869 return rte_flow_error_set(error, ENOTSUP, 1870 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1871 "queue action not supported for egress."); 1872 if (mlx5_is_external_rxq(dev, queue->index)) 1873 return 0; 1874 if (!priv->rxqs_n) 1875 return rte_flow_error_set(error, EINVAL, 1876 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1877 NULL, "No Rx queues configured"); 1878 if (queue->index >= priv->rxqs_n) 1879 return rte_flow_error_set(error, EINVAL, 1880 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1881 &queue->index, 1882 "queue index out of range"); 1883 if (mlx5_rxq_get(dev, queue->index) == NULL) 1884 return rte_flow_error_set(error, EINVAL, 1885 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1886 &queue->index, 1887 "queue is not configured"); 1888 return 0; 1889 } 1890 1891 /** 1892 * Validate queue numbers for device RSS. 1893 * 1894 * @param[in] dev 1895 * Configured device. 1896 * @param[in] queues 1897 * Array of queue numbers. 1898 * @param[in] queues_n 1899 * Size of the @p queues array. 1900 * @param[out] error 1901 * On error, filled with a textual error description. 1902 * @param[out] queue_idx 1903 * On error, filled with an offending queue index in @p queues array. 1904 * 1905 * @return 1906 * 0 on success, a negative errno code on error. 1907 */ 1908 static int 1909 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 1910 const uint16_t *queues, uint32_t queues_n, 1911 const char **error, uint32_t *queue_idx) 1912 { 1913 const struct mlx5_priv *priv = dev->data->dev_private; 1914 bool is_hairpin = false; 1915 bool is_ext_rss = false; 1916 uint32_t i; 1917 1918 for (i = 0; i != queues_n; ++i) { 1919 struct mlx5_rxq_ctrl *rxq_ctrl; 1920 1921 if (mlx5_is_external_rxq(dev, queues[0])) { 1922 is_ext_rss = true; 1923 continue; 1924 } 1925 if (is_ext_rss) { 1926 *error = "Combining external and regular RSS queues is not supported"; 1927 *queue_idx = i; 1928 return -ENOTSUP; 1929 } 1930 if (queues[i] >= priv->rxqs_n) { 1931 *error = "queue index out of range"; 1932 *queue_idx = i; 1933 return -EINVAL; 1934 } 1935 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 1936 if (rxq_ctrl == NULL) { 1937 *error = "queue is not configured"; 1938 *queue_idx = i; 1939 return -EINVAL; 1940 } 1941 if (i == 0 && rxq_ctrl->is_hairpin) 1942 is_hairpin = true; 1943 if (is_hairpin != rxq_ctrl->is_hairpin) { 1944 *error = "combining hairpin and regular RSS queues is not supported"; 1945 *queue_idx = i; 1946 return -ENOTSUP; 1947 } 1948 } 1949 return 0; 1950 } 1951 1952 /* 1953 * Validate the rss action. 1954 * 1955 * @param[in] dev 1956 * Pointer to the Ethernet device structure. 1957 * @param[in] action 1958 * Pointer to the queue action. 1959 * @param[out] error 1960 * Pointer to error structure. 1961 * 1962 * @return 1963 * 0 on success, a negative errno value otherwise and rte_errno is set. 1964 */ 1965 int 1966 mlx5_validate_action_rss(struct rte_eth_dev *dev, 1967 const struct rte_flow_action *action, 1968 struct rte_flow_error *error) 1969 { 1970 struct mlx5_priv *priv = dev->data->dev_private; 1971 const struct rte_flow_action_rss *rss = action->conf; 1972 int ret; 1973 const char *message; 1974 uint32_t queue_idx; 1975 1976 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 1977 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 1978 return rte_flow_error_set(error, ENOTSUP, 1979 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1980 &rss->func, 1981 "RSS hash function not supported"); 1982 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1983 if (rss->level > 2) 1984 #else 1985 if (rss->level > 1) 1986 #endif 1987 return rte_flow_error_set(error, ENOTSUP, 1988 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1989 &rss->level, 1990 "tunnel RSS is not supported"); 1991 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 1992 if (rss->key_len == 0 && rss->key != NULL) 1993 return rte_flow_error_set(error, ENOTSUP, 1994 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1995 &rss->key_len, 1996 "RSS hash key length 0"); 1997 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 1998 return rte_flow_error_set(error, ENOTSUP, 1999 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2000 &rss->key_len, 2001 "RSS hash key too small"); 2002 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2003 return rte_flow_error_set(error, ENOTSUP, 2004 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2005 &rss->key_len, 2006 "RSS hash key too large"); 2007 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2008 return rte_flow_error_set(error, ENOTSUP, 2009 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2010 &rss->queue_num, 2011 "number of queues too large"); 2012 if (rss->types & MLX5_RSS_HF_MASK) 2013 return rte_flow_error_set(error, ENOTSUP, 2014 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2015 &rss->types, 2016 "some RSS protocols are not" 2017 " supported"); 2018 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2019 !(rss->types & RTE_ETH_RSS_IP)) 2020 return rte_flow_error_set(error, EINVAL, 2021 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2022 "L3 partial RSS requested but L3 RSS" 2023 " type not specified"); 2024 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2025 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2026 return rte_flow_error_set(error, EINVAL, 2027 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2028 "L4 partial RSS requested but L4 RSS" 2029 " type not specified"); 2030 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2031 return rte_flow_error_set(error, EINVAL, 2032 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2033 NULL, "No Rx queues configured"); 2034 if (!rss->queue_num) 2035 return rte_flow_error_set(error, EINVAL, 2036 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2037 NULL, "No queues configured"); 2038 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2039 &message, &queue_idx); 2040 if (ret != 0) { 2041 return rte_flow_error_set(error, -ret, 2042 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2043 &rss->queue[queue_idx], message); 2044 } 2045 return 0; 2046 } 2047 2048 /* 2049 * Validate the rss action. 2050 * 2051 * @param[in] action 2052 * Pointer to the queue action. 2053 * @param[in] action_flags 2054 * Bit-fields that holds the actions detected until now. 2055 * @param[in] dev 2056 * Pointer to the Ethernet device structure. 2057 * @param[in] attr 2058 * Attributes of flow that includes this action. 2059 * @param[in] item_flags 2060 * Items that were detected. 2061 * @param[out] error 2062 * Pointer to error structure. 2063 * 2064 * @return 2065 * 0 on success, a negative errno value otherwise and rte_errno is set. 2066 */ 2067 int 2068 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2069 uint64_t action_flags, 2070 struct rte_eth_dev *dev, 2071 const struct rte_flow_attr *attr, 2072 uint64_t item_flags, 2073 struct rte_flow_error *error) 2074 { 2075 const struct rte_flow_action_rss *rss = action->conf; 2076 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2077 int ret; 2078 2079 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2080 return rte_flow_error_set(error, EINVAL, 2081 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2082 "can't have 2 fate actions" 2083 " in same flow"); 2084 ret = mlx5_validate_action_rss(dev, action, error); 2085 if (ret) 2086 return ret; 2087 if (attr->egress) 2088 return rte_flow_error_set(error, ENOTSUP, 2089 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2090 "rss action not supported for " 2091 "egress"); 2092 if (rss->level > 1 && !tunnel) 2093 return rte_flow_error_set(error, EINVAL, 2094 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2095 "inner RSS is not supported for " 2096 "non-tunnel flows"); 2097 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2098 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2099 return rte_flow_error_set(error, EINVAL, 2100 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2101 "RSS on eCPRI is not supported now"); 2102 } 2103 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2104 !(item_flags & 2105 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2106 rss->level > 1) 2107 return rte_flow_error_set(error, EINVAL, 2108 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2109 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2110 return 0; 2111 } 2112 2113 /* 2114 * Validate the default miss action. 2115 * 2116 * @param[in] action_flags 2117 * Bit-fields that holds the actions detected until now. 2118 * @param[out] error 2119 * Pointer to error structure. 2120 * 2121 * @return 2122 * 0 on success, a negative errno value otherwise and rte_errno is set. 2123 */ 2124 int 2125 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2126 const struct rte_flow_attr *attr, 2127 struct rte_flow_error *error) 2128 { 2129 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2130 return rte_flow_error_set(error, EINVAL, 2131 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2132 "can't have 2 fate actions in" 2133 " same flow"); 2134 if (attr->egress) 2135 return rte_flow_error_set(error, ENOTSUP, 2136 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2137 "default miss action not supported " 2138 "for egress"); 2139 if (attr->group) 2140 return rte_flow_error_set(error, ENOTSUP, 2141 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2142 "only group 0 is supported"); 2143 if (attr->transfer) 2144 return rte_flow_error_set(error, ENOTSUP, 2145 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2146 NULL, "transfer is not supported"); 2147 return 0; 2148 } 2149 2150 /* 2151 * Validate the count action. 2152 * 2153 * @param[in] dev 2154 * Pointer to the Ethernet device structure. 2155 * @param[in] attr 2156 * Attributes of flow that includes this action. 2157 * @param[out] error 2158 * Pointer to error structure. 2159 * 2160 * @return 2161 * 0 on success, a negative errno value otherwise and rte_errno is set. 2162 */ 2163 int 2164 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2165 const struct rte_flow_attr *attr, 2166 struct rte_flow_error *error) 2167 { 2168 if (attr->egress) 2169 return rte_flow_error_set(error, ENOTSUP, 2170 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2171 "count action not supported for " 2172 "egress"); 2173 return 0; 2174 } 2175 2176 /* 2177 * Validate the ASO CT action. 2178 * 2179 * @param[in] dev 2180 * Pointer to the Ethernet device structure. 2181 * @param[in] conntrack 2182 * Pointer to the CT action profile. 2183 * @param[out] error 2184 * Pointer to error structure. 2185 * 2186 * @return 2187 * 0 on success, a negative errno value otherwise and rte_errno is set. 2188 */ 2189 int 2190 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2191 const struct rte_flow_action_conntrack *conntrack, 2192 struct rte_flow_error *error) 2193 { 2194 RTE_SET_USED(dev); 2195 2196 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2197 return rte_flow_error_set(error, EINVAL, 2198 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2199 "Invalid CT state"); 2200 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2201 return rte_flow_error_set(error, EINVAL, 2202 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2203 "Invalid last TCP packet flag"); 2204 return 0; 2205 } 2206 2207 /** 2208 * Verify the @p attributes will be correctly understood by the NIC and store 2209 * them in the @p flow if everything is correct. 2210 * 2211 * @param[in] dev 2212 * Pointer to the Ethernet device structure. 2213 * @param[in] attributes 2214 * Pointer to flow attributes 2215 * @param[out] error 2216 * Pointer to error structure. 2217 * 2218 * @return 2219 * 0 on success, a negative errno value otherwise and rte_errno is set. 2220 */ 2221 int 2222 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2223 const struct rte_flow_attr *attributes, 2224 struct rte_flow_error *error) 2225 { 2226 struct mlx5_priv *priv = dev->data->dev_private; 2227 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2228 2229 if (attributes->group) 2230 return rte_flow_error_set(error, ENOTSUP, 2231 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2232 NULL, "groups is not supported"); 2233 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2234 attributes->priority >= priority_max) 2235 return rte_flow_error_set(error, ENOTSUP, 2236 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2237 NULL, "priority out of range"); 2238 if (attributes->egress) 2239 return rte_flow_error_set(error, ENOTSUP, 2240 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2241 "egress is not supported"); 2242 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2243 return rte_flow_error_set(error, ENOTSUP, 2244 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2245 NULL, "transfer is not supported"); 2246 if (!attributes->ingress) 2247 return rte_flow_error_set(error, EINVAL, 2248 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2249 NULL, 2250 "ingress attribute is mandatory"); 2251 return 0; 2252 } 2253 2254 /** 2255 * Validate ICMP6 item. 2256 * 2257 * @param[in] item 2258 * Item specification. 2259 * @param[in] item_flags 2260 * Bit-fields that holds the items detected until now. 2261 * @param[in] ext_vlan_sup 2262 * Whether extended VLAN features are supported or not. 2263 * @param[out] error 2264 * Pointer to error structure. 2265 * 2266 * @return 2267 * 0 on success, a negative errno value otherwise and rte_errno is set. 2268 */ 2269 int 2270 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2271 uint64_t item_flags, 2272 uint8_t target_protocol, 2273 struct rte_flow_error *error) 2274 { 2275 const struct rte_flow_item_icmp6 *mask = item->mask; 2276 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2277 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2278 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2279 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2280 MLX5_FLOW_LAYER_OUTER_L4; 2281 int ret; 2282 2283 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2284 return rte_flow_error_set(error, EINVAL, 2285 RTE_FLOW_ERROR_TYPE_ITEM, item, 2286 "protocol filtering not compatible" 2287 " with ICMP6 layer"); 2288 if (!(item_flags & l3m)) 2289 return rte_flow_error_set(error, EINVAL, 2290 RTE_FLOW_ERROR_TYPE_ITEM, item, 2291 "IPv6 is mandatory to filter on" 2292 " ICMP6"); 2293 if (item_flags & l4m) 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ITEM, item, 2296 "multiple L4 layers not supported"); 2297 if (!mask) 2298 mask = &rte_flow_item_icmp6_mask; 2299 ret = mlx5_flow_item_acceptable 2300 (item, (const uint8_t *)mask, 2301 (const uint8_t *)&rte_flow_item_icmp6_mask, 2302 sizeof(struct rte_flow_item_icmp6), 2303 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2304 if (ret < 0) 2305 return ret; 2306 return 0; 2307 } 2308 2309 /** 2310 * Validate ICMP item. 2311 * 2312 * @param[in] item 2313 * Item specification. 2314 * @param[in] item_flags 2315 * Bit-fields that holds the items detected until now. 2316 * @param[out] error 2317 * Pointer to error structure. 2318 * 2319 * @return 2320 * 0 on success, a negative errno value otherwise and rte_errno is set. 2321 */ 2322 int 2323 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2324 uint64_t item_flags, 2325 uint8_t target_protocol, 2326 struct rte_flow_error *error) 2327 { 2328 const struct rte_flow_item_icmp *mask = item->mask; 2329 const struct rte_flow_item_icmp nic_mask = { 2330 .hdr.icmp_type = 0xff, 2331 .hdr.icmp_code = 0xff, 2332 .hdr.icmp_ident = RTE_BE16(0xffff), 2333 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2334 }; 2335 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2336 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2337 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2338 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2339 MLX5_FLOW_LAYER_OUTER_L4; 2340 int ret; 2341 2342 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2343 return rte_flow_error_set(error, EINVAL, 2344 RTE_FLOW_ERROR_TYPE_ITEM, item, 2345 "protocol filtering not compatible" 2346 " with ICMP layer"); 2347 if (!(item_flags & l3m)) 2348 return rte_flow_error_set(error, EINVAL, 2349 RTE_FLOW_ERROR_TYPE_ITEM, item, 2350 "IPv4 is mandatory to filter" 2351 " on ICMP"); 2352 if (item_flags & l4m) 2353 return rte_flow_error_set(error, EINVAL, 2354 RTE_FLOW_ERROR_TYPE_ITEM, item, 2355 "multiple L4 layers not supported"); 2356 if (!mask) 2357 mask = &nic_mask; 2358 ret = mlx5_flow_item_acceptable 2359 (item, (const uint8_t *)mask, 2360 (const uint8_t *)&nic_mask, 2361 sizeof(struct rte_flow_item_icmp), 2362 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2363 if (ret < 0) 2364 return ret; 2365 return 0; 2366 } 2367 2368 /** 2369 * Validate Ethernet item. 2370 * 2371 * @param[in] item 2372 * Item specification. 2373 * @param[in] item_flags 2374 * Bit-fields that holds the items detected until now. 2375 * @param[out] error 2376 * Pointer to error structure. 2377 * 2378 * @return 2379 * 0 on success, a negative errno value otherwise and rte_errno is set. 2380 */ 2381 int 2382 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2383 uint64_t item_flags, bool ext_vlan_sup, 2384 struct rte_flow_error *error) 2385 { 2386 const struct rte_flow_item_eth *mask = item->mask; 2387 const struct rte_flow_item_eth nic_mask = { 2388 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2389 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2390 .type = RTE_BE16(0xffff), 2391 .has_vlan = ext_vlan_sup ? 1 : 0, 2392 }; 2393 int ret; 2394 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2395 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2396 MLX5_FLOW_LAYER_OUTER_L2; 2397 2398 if (item_flags & ethm) 2399 return rte_flow_error_set(error, ENOTSUP, 2400 RTE_FLOW_ERROR_TYPE_ITEM, item, 2401 "multiple L2 layers not supported"); 2402 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2403 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2404 return rte_flow_error_set(error, EINVAL, 2405 RTE_FLOW_ERROR_TYPE_ITEM, item, 2406 "L2 layer should not follow " 2407 "L3 layers"); 2408 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2409 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2410 return rte_flow_error_set(error, EINVAL, 2411 RTE_FLOW_ERROR_TYPE_ITEM, item, 2412 "L2 layer should not follow VLAN"); 2413 if (item_flags & MLX5_FLOW_LAYER_GTP) 2414 return rte_flow_error_set(error, EINVAL, 2415 RTE_FLOW_ERROR_TYPE_ITEM, item, 2416 "L2 layer should not follow GTP"); 2417 if (!mask) 2418 mask = &rte_flow_item_eth_mask; 2419 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2420 (const uint8_t *)&nic_mask, 2421 sizeof(struct rte_flow_item_eth), 2422 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2423 return ret; 2424 } 2425 2426 /** 2427 * Validate VLAN item. 2428 * 2429 * @param[in] item 2430 * Item specification. 2431 * @param[in] item_flags 2432 * Bit-fields that holds the items detected until now. 2433 * @param[in] dev 2434 * Ethernet device flow is being created on. 2435 * @param[out] error 2436 * Pointer to error structure. 2437 * 2438 * @return 2439 * 0 on success, a negative errno value otherwise and rte_errno is set. 2440 */ 2441 int 2442 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2443 uint64_t item_flags, 2444 struct rte_eth_dev *dev, 2445 struct rte_flow_error *error) 2446 { 2447 const struct rte_flow_item_vlan *spec = item->spec; 2448 const struct rte_flow_item_vlan *mask = item->mask; 2449 const struct rte_flow_item_vlan nic_mask = { 2450 .tci = RTE_BE16(UINT16_MAX), 2451 .inner_type = RTE_BE16(UINT16_MAX), 2452 }; 2453 uint16_t vlan_tag = 0; 2454 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2455 int ret; 2456 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2457 MLX5_FLOW_LAYER_INNER_L4) : 2458 (MLX5_FLOW_LAYER_OUTER_L3 | 2459 MLX5_FLOW_LAYER_OUTER_L4); 2460 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2461 MLX5_FLOW_LAYER_OUTER_VLAN; 2462 2463 if (item_flags & vlanm) 2464 return rte_flow_error_set(error, EINVAL, 2465 RTE_FLOW_ERROR_TYPE_ITEM, item, 2466 "multiple VLAN layers not supported"); 2467 else if ((item_flags & l34m) != 0) 2468 return rte_flow_error_set(error, EINVAL, 2469 RTE_FLOW_ERROR_TYPE_ITEM, item, 2470 "VLAN cannot follow L3/L4 layer"); 2471 if (!mask) 2472 mask = &rte_flow_item_vlan_mask; 2473 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2474 (const uint8_t *)&nic_mask, 2475 sizeof(struct rte_flow_item_vlan), 2476 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2477 if (ret) 2478 return ret; 2479 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2480 struct mlx5_priv *priv = dev->data->dev_private; 2481 2482 if (priv->vmwa_context) { 2483 /* 2484 * Non-NULL context means we have a virtual machine 2485 * and SR-IOV enabled, we have to create VLAN interface 2486 * to make hypervisor to setup E-Switch vport 2487 * context correctly. We avoid creating the multiple 2488 * VLAN interfaces, so we cannot support VLAN tag mask. 2489 */ 2490 return rte_flow_error_set(error, EINVAL, 2491 RTE_FLOW_ERROR_TYPE_ITEM, 2492 item, 2493 "VLAN tag mask is not" 2494 " supported in virtual" 2495 " environment"); 2496 } 2497 } 2498 if (spec) { 2499 vlan_tag = spec->tci; 2500 vlan_tag &= mask->tci; 2501 } 2502 /* 2503 * From verbs perspective an empty VLAN is equivalent 2504 * to a packet without VLAN layer. 2505 */ 2506 if (!vlan_tag) 2507 return rte_flow_error_set(error, EINVAL, 2508 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2509 item->spec, 2510 "VLAN cannot be empty"); 2511 return 0; 2512 } 2513 2514 /** 2515 * Validate IPV4 item. 2516 * 2517 * @param[in] item 2518 * Item specification. 2519 * @param[in] item_flags 2520 * Bit-fields that holds the items detected until now. 2521 * @param[in] last_item 2522 * Previous validated item in the pattern items. 2523 * @param[in] ether_type 2524 * Type in the ethernet layer header (including dot1q). 2525 * @param[in] acc_mask 2526 * Acceptable mask, if NULL default internal default mask 2527 * will be used to check whether item fields are supported. 2528 * @param[in] range_accepted 2529 * True if range of values is accepted for specific fields, false otherwise. 2530 * @param[out] error 2531 * Pointer to error structure. 2532 * 2533 * @return 2534 * 0 on success, a negative errno value otherwise and rte_errno is set. 2535 */ 2536 int 2537 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2538 uint64_t item_flags, 2539 uint64_t last_item, 2540 uint16_t ether_type, 2541 const struct rte_flow_item_ipv4 *acc_mask, 2542 bool range_accepted, 2543 struct rte_flow_error *error) 2544 { 2545 const struct rte_flow_item_ipv4 *mask = item->mask; 2546 const struct rte_flow_item_ipv4 *spec = item->spec; 2547 const struct rte_flow_item_ipv4 nic_mask = { 2548 .hdr = { 2549 .src_addr = RTE_BE32(0xffffffff), 2550 .dst_addr = RTE_BE32(0xffffffff), 2551 .type_of_service = 0xff, 2552 .next_proto_id = 0xff, 2553 }, 2554 }; 2555 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2556 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2557 MLX5_FLOW_LAYER_OUTER_L3; 2558 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2559 MLX5_FLOW_LAYER_OUTER_L4; 2560 int ret; 2561 uint8_t next_proto = 0xFF; 2562 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2563 MLX5_FLOW_LAYER_OUTER_VLAN | 2564 MLX5_FLOW_LAYER_INNER_VLAN); 2565 2566 if ((last_item & l2_vlan) && ether_type && 2567 ether_type != RTE_ETHER_TYPE_IPV4) 2568 return rte_flow_error_set(error, EINVAL, 2569 RTE_FLOW_ERROR_TYPE_ITEM, item, 2570 "IPv4 cannot follow L2/VLAN layer " 2571 "which ether type is not IPv4"); 2572 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2573 if (mask && spec) 2574 next_proto = mask->hdr.next_proto_id & 2575 spec->hdr.next_proto_id; 2576 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2577 return rte_flow_error_set(error, EINVAL, 2578 RTE_FLOW_ERROR_TYPE_ITEM, 2579 item, 2580 "multiple tunnel " 2581 "not supported"); 2582 } 2583 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2584 return rte_flow_error_set(error, EINVAL, 2585 RTE_FLOW_ERROR_TYPE_ITEM, item, 2586 "wrong tunnel type - IPv6 specified " 2587 "but IPv4 item provided"); 2588 if (item_flags & l3m) 2589 return rte_flow_error_set(error, ENOTSUP, 2590 RTE_FLOW_ERROR_TYPE_ITEM, item, 2591 "multiple L3 layers not supported"); 2592 else if (item_flags & l4m) 2593 return rte_flow_error_set(error, EINVAL, 2594 RTE_FLOW_ERROR_TYPE_ITEM, item, 2595 "L3 cannot follow an L4 layer."); 2596 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2597 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2598 return rte_flow_error_set(error, EINVAL, 2599 RTE_FLOW_ERROR_TYPE_ITEM, item, 2600 "L3 cannot follow an NVGRE layer."); 2601 if (!mask) 2602 mask = &rte_flow_item_ipv4_mask; 2603 else if (mask->hdr.next_proto_id != 0 && 2604 mask->hdr.next_proto_id != 0xff) 2605 return rte_flow_error_set(error, EINVAL, 2606 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2607 "partial mask is not supported" 2608 " for protocol"); 2609 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2610 acc_mask ? (const uint8_t *)acc_mask 2611 : (const uint8_t *)&nic_mask, 2612 sizeof(struct rte_flow_item_ipv4), 2613 range_accepted, error); 2614 if (ret < 0) 2615 return ret; 2616 return 0; 2617 } 2618 2619 /** 2620 * Validate IPV6 item. 2621 * 2622 * @param[in] item 2623 * Item specification. 2624 * @param[in] item_flags 2625 * Bit-fields that holds the items detected until now. 2626 * @param[in] last_item 2627 * Previous validated item in the pattern items. 2628 * @param[in] ether_type 2629 * Type in the ethernet layer header (including dot1q). 2630 * @param[in] acc_mask 2631 * Acceptable mask, if NULL default internal default mask 2632 * will be used to check whether item fields are supported. 2633 * @param[out] error 2634 * Pointer to error structure. 2635 * 2636 * @return 2637 * 0 on success, a negative errno value otherwise and rte_errno is set. 2638 */ 2639 int 2640 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2641 uint64_t item_flags, 2642 uint64_t last_item, 2643 uint16_t ether_type, 2644 const struct rte_flow_item_ipv6 *acc_mask, 2645 struct rte_flow_error *error) 2646 { 2647 const struct rte_flow_item_ipv6 *mask = item->mask; 2648 const struct rte_flow_item_ipv6 *spec = item->spec; 2649 const struct rte_flow_item_ipv6 nic_mask = { 2650 .hdr = { 2651 .src_addr = 2652 "\xff\xff\xff\xff\xff\xff\xff\xff" 2653 "\xff\xff\xff\xff\xff\xff\xff\xff", 2654 .dst_addr = 2655 "\xff\xff\xff\xff\xff\xff\xff\xff" 2656 "\xff\xff\xff\xff\xff\xff\xff\xff", 2657 .vtc_flow = RTE_BE32(0xffffffff), 2658 .proto = 0xff, 2659 }, 2660 }; 2661 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2662 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2663 MLX5_FLOW_LAYER_OUTER_L3; 2664 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2665 MLX5_FLOW_LAYER_OUTER_L4; 2666 int ret; 2667 uint8_t next_proto = 0xFF; 2668 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2669 MLX5_FLOW_LAYER_OUTER_VLAN | 2670 MLX5_FLOW_LAYER_INNER_VLAN); 2671 2672 if ((last_item & l2_vlan) && ether_type && 2673 ether_type != RTE_ETHER_TYPE_IPV6) 2674 return rte_flow_error_set(error, EINVAL, 2675 RTE_FLOW_ERROR_TYPE_ITEM, item, 2676 "IPv6 cannot follow L2/VLAN layer " 2677 "which ether type is not IPv6"); 2678 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2679 next_proto = spec->hdr.proto; 2680 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) { 2681 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2682 return rte_flow_error_set(error, EINVAL, 2683 RTE_FLOW_ERROR_TYPE_ITEM, 2684 item, 2685 "multiple tunnel " 2686 "not supported"); 2687 } 2688 if (next_proto == IPPROTO_HOPOPTS || 2689 next_proto == IPPROTO_ROUTING || 2690 next_proto == IPPROTO_FRAGMENT || 2691 next_proto == IPPROTO_ESP || 2692 next_proto == IPPROTO_AH || 2693 next_proto == IPPROTO_DSTOPTS) 2694 return rte_flow_error_set(error, EINVAL, 2695 RTE_FLOW_ERROR_TYPE_ITEM, item, 2696 "IPv6 proto (next header) should " 2697 "not be set as extension header"); 2698 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2699 return rte_flow_error_set(error, EINVAL, 2700 RTE_FLOW_ERROR_TYPE_ITEM, item, 2701 "wrong tunnel type - IPv4 specified " 2702 "but IPv6 item provided"); 2703 if (item_flags & l3m) 2704 return rte_flow_error_set(error, ENOTSUP, 2705 RTE_FLOW_ERROR_TYPE_ITEM, item, 2706 "multiple L3 layers not supported"); 2707 else if (item_flags & l4m) 2708 return rte_flow_error_set(error, EINVAL, 2709 RTE_FLOW_ERROR_TYPE_ITEM, item, 2710 "L3 cannot follow an L4 layer."); 2711 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2712 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2713 return rte_flow_error_set(error, EINVAL, 2714 RTE_FLOW_ERROR_TYPE_ITEM, item, 2715 "L3 cannot follow an NVGRE layer."); 2716 if (!mask) 2717 mask = &rte_flow_item_ipv6_mask; 2718 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2719 acc_mask ? (const uint8_t *)acc_mask 2720 : (const uint8_t *)&nic_mask, 2721 sizeof(struct rte_flow_item_ipv6), 2722 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2723 if (ret < 0) 2724 return ret; 2725 return 0; 2726 } 2727 2728 /** 2729 * Validate UDP item. 2730 * 2731 * @param[in] item 2732 * Item specification. 2733 * @param[in] item_flags 2734 * Bit-fields that holds the items detected until now. 2735 * @param[in] target_protocol 2736 * The next protocol in the previous item. 2737 * @param[in] flow_mask 2738 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2739 * @param[out] error 2740 * Pointer to error structure. 2741 * 2742 * @return 2743 * 0 on success, a negative errno value otherwise and rte_errno is set. 2744 */ 2745 int 2746 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2747 uint64_t item_flags, 2748 uint8_t target_protocol, 2749 struct rte_flow_error *error) 2750 { 2751 const struct rte_flow_item_udp *mask = item->mask; 2752 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2753 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2754 MLX5_FLOW_LAYER_OUTER_L3; 2755 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2756 MLX5_FLOW_LAYER_OUTER_L4; 2757 int ret; 2758 2759 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2760 return rte_flow_error_set(error, EINVAL, 2761 RTE_FLOW_ERROR_TYPE_ITEM, item, 2762 "protocol filtering not compatible" 2763 " with UDP layer"); 2764 if (!(item_flags & l3m)) 2765 return rte_flow_error_set(error, EINVAL, 2766 RTE_FLOW_ERROR_TYPE_ITEM, item, 2767 "L3 is mandatory to filter on L4"); 2768 if (item_flags & l4m) 2769 return rte_flow_error_set(error, EINVAL, 2770 RTE_FLOW_ERROR_TYPE_ITEM, item, 2771 "multiple L4 layers not supported"); 2772 if (!mask) 2773 mask = &rte_flow_item_udp_mask; 2774 ret = mlx5_flow_item_acceptable 2775 (item, (const uint8_t *)mask, 2776 (const uint8_t *)&rte_flow_item_udp_mask, 2777 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2778 error); 2779 if (ret < 0) 2780 return ret; 2781 return 0; 2782 } 2783 2784 /** 2785 * Validate TCP item. 2786 * 2787 * @param[in] item 2788 * Item specification. 2789 * @param[in] item_flags 2790 * Bit-fields that holds the items detected until now. 2791 * @param[in] target_protocol 2792 * The next protocol in the previous item. 2793 * @param[out] error 2794 * Pointer to error structure. 2795 * 2796 * @return 2797 * 0 on success, a negative errno value otherwise and rte_errno is set. 2798 */ 2799 int 2800 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2801 uint64_t item_flags, 2802 uint8_t target_protocol, 2803 const struct rte_flow_item_tcp *flow_mask, 2804 struct rte_flow_error *error) 2805 { 2806 const struct rte_flow_item_tcp *mask = item->mask; 2807 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2808 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2809 MLX5_FLOW_LAYER_OUTER_L3; 2810 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2811 MLX5_FLOW_LAYER_OUTER_L4; 2812 int ret; 2813 2814 MLX5_ASSERT(flow_mask); 2815 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2816 return rte_flow_error_set(error, EINVAL, 2817 RTE_FLOW_ERROR_TYPE_ITEM, item, 2818 "protocol filtering not compatible" 2819 " with TCP layer"); 2820 if (!(item_flags & l3m)) 2821 return rte_flow_error_set(error, EINVAL, 2822 RTE_FLOW_ERROR_TYPE_ITEM, item, 2823 "L3 is mandatory to filter on L4"); 2824 if (item_flags & l4m) 2825 return rte_flow_error_set(error, EINVAL, 2826 RTE_FLOW_ERROR_TYPE_ITEM, item, 2827 "multiple L4 layers not supported"); 2828 if (!mask) 2829 mask = &rte_flow_item_tcp_mask; 2830 ret = mlx5_flow_item_acceptable 2831 (item, (const uint8_t *)mask, 2832 (const uint8_t *)flow_mask, 2833 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2834 error); 2835 if (ret < 0) 2836 return ret; 2837 return 0; 2838 } 2839 2840 /** 2841 * Validate VXLAN item. 2842 * 2843 * @param[in] dev 2844 * Pointer to the Ethernet device structure. 2845 * @param[in] udp_dport 2846 * UDP destination port 2847 * @param[in] item 2848 * Item specification. 2849 * @param[in] item_flags 2850 * Bit-fields that holds the items detected until now. 2851 * @param root 2852 * Whether action is on root table. 2853 * @param[out] error 2854 * Pointer to error structure. 2855 * 2856 * @return 2857 * 0 on success, a negative errno value otherwise and rte_errno is set. 2858 */ 2859 int 2860 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2861 uint16_t udp_dport, 2862 const struct rte_flow_item *item, 2863 uint64_t item_flags, 2864 bool root, 2865 struct rte_flow_error *error) 2866 { 2867 const struct rte_flow_item_vxlan *spec = item->spec; 2868 const struct rte_flow_item_vxlan *mask = item->mask; 2869 int ret; 2870 struct mlx5_priv *priv = dev->data->dev_private; 2871 union vni { 2872 uint32_t vlan_id; 2873 uint8_t vni[4]; 2874 } id = { .vlan_id = 0, }; 2875 const struct rte_flow_item_vxlan nic_mask = { 2876 .vni = "\xff\xff\xff", 2877 .rsvd1 = 0xff, 2878 }; 2879 const struct rte_flow_item_vxlan *valid_mask; 2880 2881 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2882 return rte_flow_error_set(error, ENOTSUP, 2883 RTE_FLOW_ERROR_TYPE_ITEM, item, 2884 "multiple tunnel layers not" 2885 " supported"); 2886 valid_mask = &rte_flow_item_vxlan_mask; 2887 /* 2888 * Verify only UDPv4 is present as defined in 2889 * https://tools.ietf.org/html/rfc7348 2890 */ 2891 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2892 return rte_flow_error_set(error, EINVAL, 2893 RTE_FLOW_ERROR_TYPE_ITEM, item, 2894 "no outer UDP layer found"); 2895 if (!mask) 2896 mask = &rte_flow_item_vxlan_mask; 2897 2898 if (priv->sh->steering_format_version != 2899 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2900 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2901 /* non-root table */ 2902 if (!root && priv->sh->misc5_cap) 2903 valid_mask = &nic_mask; 2904 /* Group zero in NIC domain */ 2905 if (!root && priv->sh->tunnel_header_0_1) 2906 valid_mask = &nic_mask; 2907 } 2908 ret = mlx5_flow_item_acceptable 2909 (item, (const uint8_t *)mask, 2910 (const uint8_t *)valid_mask, 2911 sizeof(struct rte_flow_item_vxlan), 2912 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2913 if (ret < 0) 2914 return ret; 2915 if (spec) { 2916 memcpy(&id.vni[1], spec->vni, 3); 2917 memcpy(&id.vni[1], mask->vni, 3); 2918 } 2919 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2920 return rte_flow_error_set(error, ENOTSUP, 2921 RTE_FLOW_ERROR_TYPE_ITEM, item, 2922 "VXLAN tunnel must be fully defined"); 2923 return 0; 2924 } 2925 2926 /** 2927 * Validate VXLAN_GPE item. 2928 * 2929 * @param[in] item 2930 * Item specification. 2931 * @param[in] item_flags 2932 * Bit-fields that holds the items detected until now. 2933 * @param[in] priv 2934 * Pointer to the private data structure. 2935 * @param[in] target_protocol 2936 * The next protocol in the previous item. 2937 * @param[out] error 2938 * Pointer to error structure. 2939 * 2940 * @return 2941 * 0 on success, a negative errno value otherwise and rte_errno is set. 2942 */ 2943 int 2944 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2945 uint64_t item_flags, 2946 struct rte_eth_dev *dev, 2947 struct rte_flow_error *error) 2948 { 2949 struct mlx5_priv *priv = dev->data->dev_private; 2950 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2951 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2952 int ret; 2953 union vni { 2954 uint32_t vlan_id; 2955 uint8_t vni[4]; 2956 } id = { .vlan_id = 0, }; 2957 2958 if (!priv->sh->config.l3_vxlan_en) 2959 return rte_flow_error_set(error, ENOTSUP, 2960 RTE_FLOW_ERROR_TYPE_ITEM, item, 2961 "L3 VXLAN is not enabled by device" 2962 " parameter and/or not configured in" 2963 " firmware"); 2964 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2965 return rte_flow_error_set(error, ENOTSUP, 2966 RTE_FLOW_ERROR_TYPE_ITEM, item, 2967 "multiple tunnel layers not" 2968 " supported"); 2969 /* 2970 * Verify only UDPv4 is present as defined in 2971 * https://tools.ietf.org/html/rfc7348 2972 */ 2973 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2974 return rte_flow_error_set(error, EINVAL, 2975 RTE_FLOW_ERROR_TYPE_ITEM, item, 2976 "no outer UDP layer found"); 2977 if (!mask) 2978 mask = &rte_flow_item_vxlan_gpe_mask; 2979 ret = mlx5_flow_item_acceptable 2980 (item, (const uint8_t *)mask, 2981 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 2982 sizeof(struct rte_flow_item_vxlan_gpe), 2983 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2984 if (ret < 0) 2985 return ret; 2986 if (spec) { 2987 if (spec->protocol) 2988 return rte_flow_error_set(error, ENOTSUP, 2989 RTE_FLOW_ERROR_TYPE_ITEM, 2990 item, 2991 "VxLAN-GPE protocol" 2992 " not supported"); 2993 memcpy(&id.vni[1], spec->vni, 3); 2994 memcpy(&id.vni[1], mask->vni, 3); 2995 } 2996 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2997 return rte_flow_error_set(error, ENOTSUP, 2998 RTE_FLOW_ERROR_TYPE_ITEM, item, 2999 "VXLAN-GPE tunnel must be fully" 3000 " defined"); 3001 return 0; 3002 } 3003 /** 3004 * Validate GRE Key item. 3005 * 3006 * @param[in] item 3007 * Item specification. 3008 * @param[in] item_flags 3009 * Bit flags to mark detected items. 3010 * @param[in] gre_item 3011 * Pointer to gre_item 3012 * @param[out] error 3013 * Pointer to error structure. 3014 * 3015 * @return 3016 * 0 on success, a negative errno value otherwise and rte_errno is set. 3017 */ 3018 int 3019 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3020 uint64_t item_flags, 3021 const struct rte_flow_item *gre_item, 3022 struct rte_flow_error *error) 3023 { 3024 const rte_be32_t *mask = item->mask; 3025 int ret = 0; 3026 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3027 const struct rte_flow_item_gre *gre_spec; 3028 const struct rte_flow_item_gre *gre_mask; 3029 3030 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3031 return rte_flow_error_set(error, ENOTSUP, 3032 RTE_FLOW_ERROR_TYPE_ITEM, item, 3033 "Multiple GRE key not support"); 3034 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3035 return rte_flow_error_set(error, ENOTSUP, 3036 RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "No preceding GRE header"); 3038 if (item_flags & MLX5_FLOW_LAYER_INNER) 3039 return rte_flow_error_set(error, ENOTSUP, 3040 RTE_FLOW_ERROR_TYPE_ITEM, item, 3041 "GRE key following a wrong item"); 3042 gre_mask = gre_item->mask; 3043 if (!gre_mask) 3044 gre_mask = &rte_flow_item_gre_mask; 3045 gre_spec = gre_item->spec; 3046 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3047 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3048 return rte_flow_error_set(error, EINVAL, 3049 RTE_FLOW_ERROR_TYPE_ITEM, item, 3050 "Key bit must be on"); 3051 3052 if (!mask) 3053 mask = &gre_key_default_mask; 3054 ret = mlx5_flow_item_acceptable 3055 (item, (const uint8_t *)mask, 3056 (const uint8_t *)&gre_key_default_mask, 3057 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3058 return ret; 3059 } 3060 3061 /** 3062 * Validate GRE optional item. 3063 * 3064 * @param[in] dev 3065 * Pointer to the Ethernet device structure. 3066 * @param[in] item 3067 * Item specification. 3068 * @param[in] item_flags 3069 * Bit flags to mark detected items. 3070 * @param[in] attr 3071 * Flow rule attributes. 3072 * @param[in] gre_item 3073 * Pointer to gre_item 3074 * @param[out] error 3075 * Pointer to error structure. 3076 * 3077 * @return 3078 * 0 on success, a negative errno value otherwise and rte_errno is set. 3079 */ 3080 int 3081 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3082 const struct rte_flow_item *item, 3083 uint64_t item_flags, 3084 const struct rte_flow_attr *attr, 3085 const struct rte_flow_item *gre_item, 3086 struct rte_flow_error *error) 3087 { 3088 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3089 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3090 const struct rte_flow_item_gre_opt *spec = item->spec; 3091 const struct rte_flow_item_gre_opt *mask = item->mask; 3092 struct mlx5_priv *priv = dev->data->dev_private; 3093 int ret = 0; 3094 struct rte_flow_item_gre_opt nic_mask = { 3095 .checksum_rsvd = { 3096 .checksum = RTE_BE16(UINT16_MAX), 3097 .reserved1 = 0x0, 3098 }, 3099 .key = { 3100 .key = RTE_BE32(UINT32_MAX), 3101 }, 3102 .sequence = { 3103 .sequence = RTE_BE32(UINT32_MAX), 3104 }, 3105 }; 3106 3107 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3108 return rte_flow_error_set(error, ENOTSUP, 3109 RTE_FLOW_ERROR_TYPE_ITEM, item, 3110 "No preceding GRE header"); 3111 if (item_flags & MLX5_FLOW_LAYER_INNER) 3112 return rte_flow_error_set(error, ENOTSUP, 3113 RTE_FLOW_ERROR_TYPE_ITEM, item, 3114 "GRE option following a wrong item"); 3115 if (!spec || !mask) 3116 return rte_flow_error_set(error, EINVAL, 3117 RTE_FLOW_ERROR_TYPE_ITEM, item, 3118 "At least one field gre_option(checksum/key/sequence) must be specified"); 3119 if (!gre_mask) 3120 gre_mask = &rte_flow_item_gre_mask; 3121 if (mask->checksum_rsvd.checksum) 3122 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3123 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3124 return rte_flow_error_set(error, EINVAL, 3125 RTE_FLOW_ERROR_TYPE_ITEM, 3126 item, 3127 "Checksum bit must be on"); 3128 if (mask->key.key) 3129 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3130 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3131 return rte_flow_error_set(error, EINVAL, 3132 RTE_FLOW_ERROR_TYPE_ITEM, 3133 item, "Key bit must be on"); 3134 if (mask->sequence.sequence) 3135 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3136 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3137 return rte_flow_error_set(error, EINVAL, 3138 RTE_FLOW_ERROR_TYPE_ITEM, 3139 item, 3140 "Sequence bit must be on"); 3141 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3142 if (priv->sh->steering_format_version == 3143 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3144 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3145 !priv->sh->misc5_cap) || 3146 (!(priv->sh->tunnel_header_0_1 && 3147 priv->sh->tunnel_header_2_3) && 3148 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3149 return rte_flow_error_set(error, EINVAL, 3150 RTE_FLOW_ERROR_TYPE_ITEM, 3151 item, 3152 "Checksum/Sequence not supported"); 3153 } 3154 ret = mlx5_flow_item_acceptable 3155 (item, (const uint8_t *)mask, 3156 (const uint8_t *)&nic_mask, 3157 sizeof(struct rte_flow_item_gre_opt), 3158 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3159 return ret; 3160 } 3161 3162 /** 3163 * Validate GRE item. 3164 * 3165 * @param[in] item 3166 * Item specification. 3167 * @param[in] item_flags 3168 * Bit flags to mark detected items. 3169 * @param[in] target_protocol 3170 * The next protocol in the previous item. 3171 * @param[out] error 3172 * Pointer to error structure. 3173 * 3174 * @return 3175 * 0 on success, a negative errno value otherwise and rte_errno is set. 3176 */ 3177 int 3178 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3179 uint64_t item_flags, 3180 uint8_t target_protocol, 3181 struct rte_flow_error *error) 3182 { 3183 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3184 const struct rte_flow_item_gre *mask = item->mask; 3185 int ret; 3186 const struct rte_flow_item_gre nic_mask = { 3187 .c_rsvd0_ver = RTE_BE16(0xB000), 3188 .protocol = RTE_BE16(UINT16_MAX), 3189 }; 3190 3191 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3192 return rte_flow_error_set(error, EINVAL, 3193 RTE_FLOW_ERROR_TYPE_ITEM, item, 3194 "protocol filtering not compatible" 3195 " with this GRE layer"); 3196 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3197 return rte_flow_error_set(error, ENOTSUP, 3198 RTE_FLOW_ERROR_TYPE_ITEM, item, 3199 "multiple tunnel layers not" 3200 " supported"); 3201 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3202 return rte_flow_error_set(error, ENOTSUP, 3203 RTE_FLOW_ERROR_TYPE_ITEM, item, 3204 "L3 Layer is missing"); 3205 if (!mask) 3206 mask = &rte_flow_item_gre_mask; 3207 ret = mlx5_flow_item_acceptable 3208 (item, (const uint8_t *)mask, 3209 (const uint8_t *)&nic_mask, 3210 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3211 error); 3212 if (ret < 0) 3213 return ret; 3214 #ifndef HAVE_MLX5DV_DR 3215 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3216 if (spec && (spec->protocol & mask->protocol)) 3217 return rte_flow_error_set(error, ENOTSUP, 3218 RTE_FLOW_ERROR_TYPE_ITEM, item, 3219 "without MPLS support the" 3220 " specification cannot be used for" 3221 " filtering"); 3222 #endif 3223 #endif 3224 return 0; 3225 } 3226 3227 /** 3228 * Validate Geneve item. 3229 * 3230 * @param[in] item 3231 * Item specification. 3232 * @param[in] itemFlags 3233 * Bit-fields that holds the items detected until now. 3234 * @param[in] enPriv 3235 * Pointer to the private data structure. 3236 * @param[out] error 3237 * Pointer to error structure. 3238 * 3239 * @return 3240 * 0 on success, a negative errno value otherwise and rte_errno is set. 3241 */ 3242 3243 int 3244 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3245 uint64_t item_flags, 3246 struct rte_eth_dev *dev, 3247 struct rte_flow_error *error) 3248 { 3249 struct mlx5_priv *priv = dev->data->dev_private; 3250 const struct rte_flow_item_geneve *spec = item->spec; 3251 const struct rte_flow_item_geneve *mask = item->mask; 3252 int ret; 3253 uint16_t gbhdr; 3254 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3255 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3256 const struct rte_flow_item_geneve nic_mask = { 3257 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3258 .vni = "\xff\xff\xff", 3259 .protocol = RTE_BE16(UINT16_MAX), 3260 }; 3261 3262 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3263 return rte_flow_error_set(error, ENOTSUP, 3264 RTE_FLOW_ERROR_TYPE_ITEM, item, 3265 "L3 Geneve is not enabled by device" 3266 " parameter and/or not configured in" 3267 " firmware"); 3268 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3269 return rte_flow_error_set(error, ENOTSUP, 3270 RTE_FLOW_ERROR_TYPE_ITEM, item, 3271 "multiple tunnel layers not" 3272 " supported"); 3273 /* 3274 * Verify only UDPv4 is present as defined in 3275 * https://tools.ietf.org/html/rfc7348 3276 */ 3277 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3278 return rte_flow_error_set(error, EINVAL, 3279 RTE_FLOW_ERROR_TYPE_ITEM, item, 3280 "no outer UDP layer found"); 3281 if (!mask) 3282 mask = &rte_flow_item_geneve_mask; 3283 ret = mlx5_flow_item_acceptable 3284 (item, (const uint8_t *)mask, 3285 (const uint8_t *)&nic_mask, 3286 sizeof(struct rte_flow_item_geneve), 3287 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3288 if (ret) 3289 return ret; 3290 if (spec) { 3291 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3292 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3293 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3294 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3295 return rte_flow_error_set(error, ENOTSUP, 3296 RTE_FLOW_ERROR_TYPE_ITEM, 3297 item, 3298 "Geneve protocol unsupported" 3299 " fields are being used"); 3300 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3301 return rte_flow_error_set 3302 (error, ENOTSUP, 3303 RTE_FLOW_ERROR_TYPE_ITEM, 3304 item, 3305 "Unsupported Geneve options length"); 3306 } 3307 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3308 return rte_flow_error_set 3309 (error, ENOTSUP, 3310 RTE_FLOW_ERROR_TYPE_ITEM, item, 3311 "Geneve tunnel must be fully defined"); 3312 return 0; 3313 } 3314 3315 /** 3316 * Validate Geneve TLV option item. 3317 * 3318 * @param[in] item 3319 * Item specification. 3320 * @param[in] last_item 3321 * Previous validated item in the pattern items. 3322 * @param[in] geneve_item 3323 * Previous GENEVE item specification. 3324 * @param[in] dev 3325 * Pointer to the rte_eth_dev structure. 3326 * @param[out] error 3327 * Pointer to error structure. 3328 * 3329 * @return 3330 * 0 on success, a negative errno value otherwise and rte_errno is set. 3331 */ 3332 int 3333 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3334 uint64_t last_item, 3335 const struct rte_flow_item *geneve_item, 3336 struct rte_eth_dev *dev, 3337 struct rte_flow_error *error) 3338 { 3339 struct mlx5_priv *priv = dev->data->dev_private; 3340 struct mlx5_dev_ctx_shared *sh = priv->sh; 3341 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3342 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3343 uint8_t data_max_supported = 3344 hca_attr->max_geneve_tlv_option_data_len * 4; 3345 const struct rte_flow_item_geneve *geneve_spec; 3346 const struct rte_flow_item_geneve *geneve_mask; 3347 const struct rte_flow_item_geneve_opt *spec = item->spec; 3348 const struct rte_flow_item_geneve_opt *mask = item->mask; 3349 unsigned int i; 3350 unsigned int data_len; 3351 uint8_t tlv_option_len; 3352 uint16_t optlen_m, optlen_v; 3353 const struct rte_flow_item_geneve_opt full_mask = { 3354 .option_class = RTE_BE16(0xffff), 3355 .option_type = 0xff, 3356 .option_len = 0x1f, 3357 }; 3358 3359 if (!mask) 3360 mask = &rte_flow_item_geneve_opt_mask; 3361 if (!spec) 3362 return rte_flow_error_set 3363 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3364 "Geneve TLV opt class/type/length must be specified"); 3365 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3366 return rte_flow_error_set 3367 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3368 "Geneve TLV opt length exceeds the limit (31)"); 3369 /* Check if class type and length masks are full. */ 3370 if (full_mask.option_class != mask->option_class || 3371 full_mask.option_type != mask->option_type || 3372 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3373 return rte_flow_error_set 3374 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3375 "Geneve TLV opt class/type/length masks must be full"); 3376 /* Check if length is supported */ 3377 if ((uint32_t)spec->option_len > 3378 hca_attr->max_geneve_tlv_option_data_len) 3379 return rte_flow_error_set 3380 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3381 "Geneve TLV opt length not supported"); 3382 if (hca_attr->max_geneve_tlv_options > 1) 3383 DRV_LOG(DEBUG, 3384 "max_geneve_tlv_options supports more than 1 option"); 3385 /* Check GENEVE item preceding. */ 3386 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3387 return rte_flow_error_set 3388 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3389 "Geneve opt item must be preceded with Geneve item"); 3390 geneve_spec = geneve_item->spec; 3391 geneve_mask = geneve_item->mask ? geneve_item->mask : 3392 &rte_flow_item_geneve_mask; 3393 /* Check if GENEVE TLV option size doesn't exceed option length */ 3394 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3395 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3396 tlv_option_len = spec->option_len & mask->option_len; 3397 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3398 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3399 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3400 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3401 if ((optlen_v & optlen_m) <= tlv_option_len) 3402 return rte_flow_error_set 3403 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3404 "GENEVE TLV option length exceeds optlen"); 3405 } 3406 /* Check if length is 0 or data is 0. */ 3407 if (spec->data == NULL || spec->option_len == 0) 3408 return rte_flow_error_set 3409 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3410 "Geneve TLV opt with zero data/length not supported"); 3411 /* Check not all data & mask are 0. */ 3412 data_len = spec->option_len * 4; 3413 if (mask->data == NULL) { 3414 for (i = 0; i < data_len; i++) 3415 if (spec->data[i]) 3416 break; 3417 if (i == data_len) 3418 return rte_flow_error_set(error, ENOTSUP, 3419 RTE_FLOW_ERROR_TYPE_ITEM, item, 3420 "Can't match on Geneve option data 0"); 3421 } else { 3422 for (i = 0; i < data_len; i++) 3423 if (spec->data[i] & mask->data[i]) 3424 break; 3425 if (i == data_len) 3426 return rte_flow_error_set(error, ENOTSUP, 3427 RTE_FLOW_ERROR_TYPE_ITEM, item, 3428 "Can't match on Geneve option data and mask 0"); 3429 /* Check data mask supported. */ 3430 for (i = data_max_supported; i < data_len ; i++) 3431 if (mask->data[i]) 3432 return rte_flow_error_set(error, ENOTSUP, 3433 RTE_FLOW_ERROR_TYPE_ITEM, item, 3434 "Data mask is of unsupported size"); 3435 } 3436 /* Check GENEVE option is supported in NIC. */ 3437 if (!hca_attr->geneve_tlv_opt) 3438 return rte_flow_error_set 3439 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3440 "Geneve TLV opt not supported"); 3441 /* Check if we already have geneve option with different type/class. */ 3442 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3443 geneve_opt_resource = sh->geneve_tlv_option_resource; 3444 if (geneve_opt_resource != NULL) 3445 if (geneve_opt_resource->option_class != spec->option_class || 3446 geneve_opt_resource->option_type != spec->option_type || 3447 geneve_opt_resource->length != spec->option_len) { 3448 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3449 return rte_flow_error_set(error, ENOTSUP, 3450 RTE_FLOW_ERROR_TYPE_ITEM, item, 3451 "Only one Geneve TLV option supported"); 3452 } 3453 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3454 return 0; 3455 } 3456 3457 /** 3458 * Validate MPLS item. 3459 * 3460 * @param[in] dev 3461 * Pointer to the rte_eth_dev structure. 3462 * @param[in] item 3463 * Item specification. 3464 * @param[in] item_flags 3465 * Bit-fields that holds the items detected until now. 3466 * @param[in] prev_layer 3467 * The protocol layer indicated in previous item. 3468 * @param[out] error 3469 * Pointer to error structure. 3470 * 3471 * @return 3472 * 0 on success, a negative errno value otherwise and rte_errno is set. 3473 */ 3474 int 3475 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3476 const struct rte_flow_item *item __rte_unused, 3477 uint64_t item_flags __rte_unused, 3478 uint64_t prev_layer __rte_unused, 3479 struct rte_flow_error *error) 3480 { 3481 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3482 const struct rte_flow_item_mpls *mask = item->mask; 3483 struct mlx5_priv *priv = dev->data->dev_private; 3484 int ret; 3485 3486 if (!priv->sh->dev_cap.mpls_en) 3487 return rte_flow_error_set(error, ENOTSUP, 3488 RTE_FLOW_ERROR_TYPE_ITEM, item, 3489 "MPLS not supported or" 3490 " disabled in firmware" 3491 " configuration."); 3492 /* MPLS over UDP, GRE is allowed */ 3493 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3494 MLX5_FLOW_LAYER_GRE | 3495 MLX5_FLOW_LAYER_GRE_KEY))) 3496 return rte_flow_error_set(error, EINVAL, 3497 RTE_FLOW_ERROR_TYPE_ITEM, item, 3498 "protocol filtering not compatible" 3499 " with MPLS layer"); 3500 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3501 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3502 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3503 return rte_flow_error_set(error, ENOTSUP, 3504 RTE_FLOW_ERROR_TYPE_ITEM, item, 3505 "multiple tunnel layers not" 3506 " supported"); 3507 if (!mask) 3508 mask = &rte_flow_item_mpls_mask; 3509 ret = mlx5_flow_item_acceptable 3510 (item, (const uint8_t *)mask, 3511 (const uint8_t *)&rte_flow_item_mpls_mask, 3512 sizeof(struct rte_flow_item_mpls), 3513 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3514 if (ret < 0) 3515 return ret; 3516 return 0; 3517 #else 3518 return rte_flow_error_set(error, ENOTSUP, 3519 RTE_FLOW_ERROR_TYPE_ITEM, item, 3520 "MPLS is not supported by Verbs, please" 3521 " update."); 3522 #endif 3523 } 3524 3525 /** 3526 * Validate NVGRE item. 3527 * 3528 * @param[in] item 3529 * Item specification. 3530 * @param[in] item_flags 3531 * Bit flags to mark detected items. 3532 * @param[in] target_protocol 3533 * The next protocol in the previous item. 3534 * @param[out] error 3535 * Pointer to error structure. 3536 * 3537 * @return 3538 * 0 on success, a negative errno value otherwise and rte_errno is set. 3539 */ 3540 int 3541 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3542 uint64_t item_flags, 3543 uint8_t target_protocol, 3544 struct rte_flow_error *error) 3545 { 3546 const struct rte_flow_item_nvgre *mask = item->mask; 3547 int ret; 3548 3549 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3550 return rte_flow_error_set(error, EINVAL, 3551 RTE_FLOW_ERROR_TYPE_ITEM, item, 3552 "protocol filtering not compatible" 3553 " with this GRE layer"); 3554 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3555 return rte_flow_error_set(error, ENOTSUP, 3556 RTE_FLOW_ERROR_TYPE_ITEM, item, 3557 "multiple tunnel layers not" 3558 " supported"); 3559 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3560 return rte_flow_error_set(error, ENOTSUP, 3561 RTE_FLOW_ERROR_TYPE_ITEM, item, 3562 "L3 Layer is missing"); 3563 if (!mask) 3564 mask = &rte_flow_item_nvgre_mask; 3565 ret = mlx5_flow_item_acceptable 3566 (item, (const uint8_t *)mask, 3567 (const uint8_t *)&rte_flow_item_nvgre_mask, 3568 sizeof(struct rte_flow_item_nvgre), 3569 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3570 if (ret < 0) 3571 return ret; 3572 return 0; 3573 } 3574 3575 /** 3576 * Validate eCPRI item. 3577 * 3578 * @param[in] item 3579 * Item specification. 3580 * @param[in] item_flags 3581 * Bit-fields that holds the items detected until now. 3582 * @param[in] last_item 3583 * Previous validated item in the pattern items. 3584 * @param[in] ether_type 3585 * Type in the ethernet layer header (including dot1q). 3586 * @param[in] acc_mask 3587 * Acceptable mask, if NULL default internal default mask 3588 * will be used to check whether item fields are supported. 3589 * @param[out] error 3590 * Pointer to error structure. 3591 * 3592 * @return 3593 * 0 on success, a negative errno value otherwise and rte_errno is set. 3594 */ 3595 int 3596 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3597 uint64_t item_flags, 3598 uint64_t last_item, 3599 uint16_t ether_type, 3600 const struct rte_flow_item_ecpri *acc_mask, 3601 struct rte_flow_error *error) 3602 { 3603 const struct rte_flow_item_ecpri *mask = item->mask; 3604 const struct rte_flow_item_ecpri nic_mask = { 3605 .hdr = { 3606 .common = { 3607 .u32 = 3608 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3609 .type = 0xFF, 3610 }).u32), 3611 }, 3612 .dummy[0] = 0xFFFFFFFF, 3613 }, 3614 }; 3615 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3616 MLX5_FLOW_LAYER_OUTER_VLAN); 3617 struct rte_flow_item_ecpri mask_lo; 3618 3619 if (!(last_item & outer_l2_vlan) && 3620 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3621 return rte_flow_error_set(error, EINVAL, 3622 RTE_FLOW_ERROR_TYPE_ITEM, item, 3623 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3624 if ((last_item & outer_l2_vlan) && ether_type && 3625 ether_type != RTE_ETHER_TYPE_ECPRI) 3626 return rte_flow_error_set(error, EINVAL, 3627 RTE_FLOW_ERROR_TYPE_ITEM, item, 3628 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3629 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3630 return rte_flow_error_set(error, EINVAL, 3631 RTE_FLOW_ERROR_TYPE_ITEM, item, 3632 "eCPRI with tunnel is not supported right now"); 3633 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3634 return rte_flow_error_set(error, ENOTSUP, 3635 RTE_FLOW_ERROR_TYPE_ITEM, item, 3636 "multiple L3 layers not supported"); 3637 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3638 return rte_flow_error_set(error, EINVAL, 3639 RTE_FLOW_ERROR_TYPE_ITEM, item, 3640 "eCPRI cannot coexist with a TCP layer"); 3641 /* In specification, eCPRI could be over UDP layer. */ 3642 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3643 return rte_flow_error_set(error, EINVAL, 3644 RTE_FLOW_ERROR_TYPE_ITEM, item, 3645 "eCPRI over UDP layer is not yet supported right now"); 3646 /* Mask for type field in common header could be zero. */ 3647 if (!mask) 3648 mask = &rte_flow_item_ecpri_mask; 3649 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3650 /* Input mask is in big-endian format. */ 3651 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3652 return rte_flow_error_set(error, EINVAL, 3653 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3654 "partial mask is not supported for protocol"); 3655 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3656 return rte_flow_error_set(error, EINVAL, 3657 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3658 "message header mask must be after a type mask"); 3659 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3660 acc_mask ? (const uint8_t *)acc_mask 3661 : (const uint8_t *)&nic_mask, 3662 sizeof(struct rte_flow_item_ecpri), 3663 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3664 } 3665 3666 static int 3667 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3668 const struct rte_flow_attr *attr __rte_unused, 3669 const struct rte_flow_item items[] __rte_unused, 3670 const struct rte_flow_action actions[] __rte_unused, 3671 bool external __rte_unused, 3672 int hairpin __rte_unused, 3673 struct rte_flow_error *error) 3674 { 3675 return rte_flow_error_set(error, ENOTSUP, 3676 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3677 } 3678 3679 static struct mlx5_flow * 3680 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3681 const struct rte_flow_attr *attr __rte_unused, 3682 const struct rte_flow_item items[] __rte_unused, 3683 const struct rte_flow_action actions[] __rte_unused, 3684 struct rte_flow_error *error) 3685 { 3686 rte_flow_error_set(error, ENOTSUP, 3687 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3688 return NULL; 3689 } 3690 3691 static int 3692 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3693 struct mlx5_flow *dev_flow __rte_unused, 3694 const struct rte_flow_attr *attr __rte_unused, 3695 const struct rte_flow_item items[] __rte_unused, 3696 const struct rte_flow_action actions[] __rte_unused, 3697 struct rte_flow_error *error) 3698 { 3699 return rte_flow_error_set(error, ENOTSUP, 3700 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3701 } 3702 3703 static int 3704 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3705 struct rte_flow *flow __rte_unused, 3706 struct rte_flow_error *error) 3707 { 3708 return rte_flow_error_set(error, ENOTSUP, 3709 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3710 } 3711 3712 static void 3713 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3714 struct rte_flow *flow __rte_unused) 3715 { 3716 } 3717 3718 static void 3719 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3720 struct rte_flow *flow __rte_unused) 3721 { 3722 } 3723 3724 static int 3725 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3726 struct rte_flow *flow __rte_unused, 3727 const struct rte_flow_action *actions __rte_unused, 3728 void *data __rte_unused, 3729 struct rte_flow_error *error) 3730 { 3731 return rte_flow_error_set(error, ENOTSUP, 3732 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3733 } 3734 3735 static int 3736 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3737 uint32_t domains __rte_unused, 3738 uint32_t flags __rte_unused) 3739 { 3740 return 0; 3741 } 3742 3743 /* Void driver to protect from null pointer reference. */ 3744 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3745 .validate = flow_null_validate, 3746 .prepare = flow_null_prepare, 3747 .translate = flow_null_translate, 3748 .apply = flow_null_apply, 3749 .remove = flow_null_remove, 3750 .destroy = flow_null_destroy, 3751 .query = flow_null_query, 3752 .sync_domain = flow_null_sync_domain, 3753 }; 3754 3755 /** 3756 * Select flow driver type according to flow attributes and device 3757 * configuration. 3758 * 3759 * @param[in] dev 3760 * Pointer to the dev structure. 3761 * @param[in] attr 3762 * Pointer to the flow attributes. 3763 * 3764 * @return 3765 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3766 */ 3767 static enum mlx5_flow_drv_type 3768 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3769 { 3770 struct mlx5_priv *priv = dev->data->dev_private; 3771 /* The OS can determine first a specific flow type (DV, VERBS) */ 3772 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3773 3774 if (type != MLX5_FLOW_TYPE_MAX) 3775 return type; 3776 /* 3777 * Currently when dv_flow_en == 2, only HW steering engine is 3778 * supported. New engines can also be chosen here if ready. 3779 */ 3780 if (priv->sh->config.dv_flow_en == 2) 3781 return MLX5_FLOW_TYPE_HW; 3782 if (!attr) 3783 return MLX5_FLOW_TYPE_MIN; 3784 /* If no OS specific type - continue with DV/VERBS selection */ 3785 if (attr->transfer && priv->sh->config.dv_esw_en) 3786 type = MLX5_FLOW_TYPE_DV; 3787 if (!attr->transfer) 3788 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3789 MLX5_FLOW_TYPE_VERBS; 3790 return type; 3791 } 3792 3793 #define flow_get_drv_ops(type) flow_drv_ops[type] 3794 3795 /** 3796 * Flow driver validation API. This abstracts calling driver specific functions. 3797 * The type of flow driver is determined according to flow attributes. 3798 * 3799 * @param[in] dev 3800 * Pointer to the dev structure. 3801 * @param[in] attr 3802 * Pointer to the flow attributes. 3803 * @param[in] items 3804 * Pointer to the list of items. 3805 * @param[in] actions 3806 * Pointer to the list of actions. 3807 * @param[in] external 3808 * This flow rule is created by request external to PMD. 3809 * @param[in] hairpin 3810 * Number of hairpin TX actions, 0 means classic flow. 3811 * @param[out] error 3812 * Pointer to the error structure. 3813 * 3814 * @return 3815 * 0 on success, a negative errno value otherwise and rte_errno is set. 3816 */ 3817 static inline int 3818 flow_drv_validate(struct rte_eth_dev *dev, 3819 const struct rte_flow_attr *attr, 3820 const struct rte_flow_item items[], 3821 const struct rte_flow_action actions[], 3822 bool external, int hairpin, struct rte_flow_error *error) 3823 { 3824 const struct mlx5_flow_driver_ops *fops; 3825 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3826 3827 fops = flow_get_drv_ops(type); 3828 return fops->validate(dev, attr, items, actions, external, 3829 hairpin, error); 3830 } 3831 3832 /** 3833 * Flow driver preparation API. This abstracts calling driver specific 3834 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3835 * calculates the size of memory required for device flow, allocates the memory, 3836 * initializes the device flow and returns the pointer. 3837 * 3838 * @note 3839 * This function initializes device flow structure such as dv or verbs in 3840 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3841 * rest. For example, adding returning device flow to flow->dev_flow list and 3842 * setting backward reference to the flow should be done out of this function. 3843 * layers field is not filled either. 3844 * 3845 * @param[in] dev 3846 * Pointer to the dev structure. 3847 * @param[in] attr 3848 * Pointer to the flow attributes. 3849 * @param[in] items 3850 * Pointer to the list of items. 3851 * @param[in] actions 3852 * Pointer to the list of actions. 3853 * @param[in] flow_idx 3854 * This memory pool index to the flow. 3855 * @param[out] error 3856 * Pointer to the error structure. 3857 * 3858 * @return 3859 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3860 */ 3861 static inline struct mlx5_flow * 3862 flow_drv_prepare(struct rte_eth_dev *dev, 3863 const struct rte_flow *flow, 3864 const struct rte_flow_attr *attr, 3865 const struct rte_flow_item items[], 3866 const struct rte_flow_action actions[], 3867 uint32_t flow_idx, 3868 struct rte_flow_error *error) 3869 { 3870 const struct mlx5_flow_driver_ops *fops; 3871 enum mlx5_flow_drv_type type = flow->drv_type; 3872 struct mlx5_flow *mlx5_flow = NULL; 3873 3874 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3875 fops = flow_get_drv_ops(type); 3876 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3877 if (mlx5_flow) 3878 mlx5_flow->flow_idx = flow_idx; 3879 return mlx5_flow; 3880 } 3881 3882 /** 3883 * Flow driver translation API. This abstracts calling driver specific 3884 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3885 * translates a generic flow into a driver flow. flow_drv_prepare() must 3886 * precede. 3887 * 3888 * @note 3889 * dev_flow->layers could be filled as a result of parsing during translation 3890 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3891 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3892 * flow->actions could be overwritten even though all the expanded dev_flows 3893 * have the same actions. 3894 * 3895 * @param[in] dev 3896 * Pointer to the rte dev structure. 3897 * @param[in, out] dev_flow 3898 * Pointer to the mlx5 flow. 3899 * @param[in] attr 3900 * Pointer to the flow attributes. 3901 * @param[in] items 3902 * Pointer to the list of items. 3903 * @param[in] actions 3904 * Pointer to the list of actions. 3905 * @param[out] error 3906 * Pointer to the error structure. 3907 * 3908 * @return 3909 * 0 on success, a negative errno value otherwise and rte_errno is set. 3910 */ 3911 static inline int 3912 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3913 const struct rte_flow_attr *attr, 3914 const struct rte_flow_item items[], 3915 const struct rte_flow_action actions[], 3916 struct rte_flow_error *error) 3917 { 3918 const struct mlx5_flow_driver_ops *fops; 3919 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3920 3921 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3922 fops = flow_get_drv_ops(type); 3923 return fops->translate(dev, dev_flow, attr, items, actions, error); 3924 } 3925 3926 /** 3927 * Flow driver apply API. This abstracts calling driver specific functions. 3928 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3929 * translated driver flows on to device. flow_drv_translate() must precede. 3930 * 3931 * @param[in] dev 3932 * Pointer to Ethernet device structure. 3933 * @param[in, out] flow 3934 * Pointer to flow structure. 3935 * @param[out] error 3936 * Pointer to error structure. 3937 * 3938 * @return 3939 * 0 on success, a negative errno value otherwise and rte_errno is set. 3940 */ 3941 static inline int 3942 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3943 struct rte_flow_error *error) 3944 { 3945 const struct mlx5_flow_driver_ops *fops; 3946 enum mlx5_flow_drv_type type = flow->drv_type; 3947 3948 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3949 fops = flow_get_drv_ops(type); 3950 return fops->apply(dev, flow, error); 3951 } 3952 3953 /** 3954 * Flow driver destroy API. This abstracts calling driver specific functions. 3955 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 3956 * on device and releases resources of the flow. 3957 * 3958 * @param[in] dev 3959 * Pointer to Ethernet device. 3960 * @param[in, out] flow 3961 * Pointer to flow structure. 3962 */ 3963 static inline void 3964 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 3965 { 3966 const struct mlx5_flow_driver_ops *fops; 3967 enum mlx5_flow_drv_type type = flow->drv_type; 3968 3969 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3970 fops = flow_get_drv_ops(type); 3971 fops->destroy(dev, flow); 3972 } 3973 3974 /** 3975 * Flow driver find RSS policy tbl API. This abstracts calling driver 3976 * specific functions. Parent flow (rte_flow) should have driver 3977 * type (drv_type). It will find the RSS policy table that has the rss_desc. 3978 * 3979 * @param[in] dev 3980 * Pointer to Ethernet device. 3981 * @param[in, out] flow 3982 * Pointer to flow structure. 3983 * @param[in] policy 3984 * Pointer to meter policy table. 3985 * @param[in] rss_desc 3986 * Pointer to rss_desc 3987 */ 3988 static struct mlx5_flow_meter_sub_policy * 3989 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 3990 struct rte_flow *flow, 3991 struct mlx5_flow_meter_policy *policy, 3992 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 3993 { 3994 const struct mlx5_flow_driver_ops *fops; 3995 enum mlx5_flow_drv_type type = flow->drv_type; 3996 3997 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3998 fops = flow_get_drv_ops(type); 3999 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4000 } 4001 4002 /** 4003 * Flow driver color tag rule API. This abstracts calling driver 4004 * specific functions. Parent flow (rte_flow) should have driver 4005 * type (drv_type). It will create the color tag rules in hierarchy meter. 4006 * 4007 * @param[in] dev 4008 * Pointer to Ethernet device. 4009 * @param[in, out] flow 4010 * Pointer to flow structure. 4011 * @param[in] fm 4012 * Pointer to flow meter structure. 4013 * @param[in] src_port 4014 * The src port this extra rule should use. 4015 * @param[in] item 4016 * The src port id match item. 4017 * @param[out] error 4018 * Pointer to error structure. 4019 */ 4020 static int 4021 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4022 struct rte_flow *flow, 4023 struct mlx5_flow_meter_info *fm, 4024 int32_t src_port, 4025 const struct rte_flow_item *item, 4026 struct rte_flow_error *error) 4027 { 4028 const struct mlx5_flow_driver_ops *fops; 4029 enum mlx5_flow_drv_type type = flow->drv_type; 4030 4031 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4032 fops = flow_get_drv_ops(type); 4033 return fops->meter_hierarchy_rule_create(dev, fm, 4034 src_port, item, error); 4035 } 4036 4037 /** 4038 * Get RSS action from the action list. 4039 * 4040 * @param[in] dev 4041 * Pointer to Ethernet device. 4042 * @param[in] actions 4043 * Pointer to the list of actions. 4044 * @param[in] flow 4045 * Parent flow structure pointer. 4046 * 4047 * @return 4048 * Pointer to the RSS action if exist, else return NULL. 4049 */ 4050 static const struct rte_flow_action_rss* 4051 flow_get_rss_action(struct rte_eth_dev *dev, 4052 const struct rte_flow_action actions[]) 4053 { 4054 struct mlx5_priv *priv = dev->data->dev_private; 4055 const struct rte_flow_action_rss *rss = NULL; 4056 struct mlx5_meter_policy_action_container *acg; 4057 struct mlx5_meter_policy_action_container *acy; 4058 4059 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4060 switch (actions->type) { 4061 case RTE_FLOW_ACTION_TYPE_RSS: 4062 rss = actions->conf; 4063 break; 4064 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4065 { 4066 const struct rte_flow_action_sample *sample = 4067 actions->conf; 4068 const struct rte_flow_action *act = sample->actions; 4069 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4070 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4071 rss = act->conf; 4072 break; 4073 } 4074 case RTE_FLOW_ACTION_TYPE_METER: 4075 { 4076 uint32_t mtr_idx; 4077 struct mlx5_flow_meter_info *fm; 4078 struct mlx5_flow_meter_policy *policy; 4079 const struct rte_flow_action_meter *mtr = actions->conf; 4080 4081 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4082 if (fm && !fm->def_policy) { 4083 policy = mlx5_flow_meter_policy_find(dev, 4084 fm->policy_id, NULL); 4085 MLX5_ASSERT(policy); 4086 if (policy->is_hierarchy) { 4087 policy = 4088 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4089 policy); 4090 if (!policy) 4091 return NULL; 4092 } 4093 if (policy->is_rss) { 4094 acg = 4095 &policy->act_cnt[RTE_COLOR_GREEN]; 4096 acy = 4097 &policy->act_cnt[RTE_COLOR_YELLOW]; 4098 if (acg->fate_action == 4099 MLX5_FLOW_FATE_SHARED_RSS) 4100 rss = acg->rss->conf; 4101 else if (acy->fate_action == 4102 MLX5_FLOW_FATE_SHARED_RSS) 4103 rss = acy->rss->conf; 4104 } 4105 } 4106 break; 4107 } 4108 default: 4109 break; 4110 } 4111 } 4112 return rss; 4113 } 4114 4115 /** 4116 * Get ASO age action by index. 4117 * 4118 * @param[in] dev 4119 * Pointer to the Ethernet device structure. 4120 * @param[in] age_idx 4121 * Index to the ASO age action. 4122 * 4123 * @return 4124 * The specified ASO age action. 4125 */ 4126 struct mlx5_aso_age_action* 4127 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4128 { 4129 uint16_t pool_idx = age_idx & UINT16_MAX; 4130 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4131 struct mlx5_priv *priv = dev->data->dev_private; 4132 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4133 struct mlx5_aso_age_pool *pool; 4134 4135 rte_rwlock_read_lock(&mng->resize_rwl); 4136 pool = mng->pools[pool_idx]; 4137 rte_rwlock_read_unlock(&mng->resize_rwl); 4138 return &pool->actions[offset - 1]; 4139 } 4140 4141 /* maps indirect action to translated direct in some actions array */ 4142 struct mlx5_translated_action_handle { 4143 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4144 int index; /**< Index in related array of rte_flow_action. */ 4145 }; 4146 4147 /** 4148 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4149 * direct action if translation possible. 4150 * This functionality used to run same execution path for both direct and 4151 * indirect actions on flow create. All necessary preparations for indirect 4152 * action handling should be performed on *handle* actions list returned 4153 * from this call. 4154 * 4155 * @param[in] dev 4156 * Pointer to Ethernet device. 4157 * @param[in] actions 4158 * List of actions to translate. 4159 * @param[out] handle 4160 * List to store translated indirect action object handles. 4161 * @param[in, out] indir_n 4162 * Size of *handle* array. On return should be updated with number of 4163 * indirect actions retrieved from the *actions* list. 4164 * @param[out] translated_actions 4165 * List of actions where all indirect actions were translated to direct 4166 * if possible. NULL if no translation took place. 4167 * @param[out] error 4168 * Pointer to the error structure. 4169 * 4170 * @return 4171 * 0 on success, a negative errno value otherwise and rte_errno is set. 4172 */ 4173 static int 4174 flow_action_handles_translate(struct rte_eth_dev *dev, 4175 const struct rte_flow_action actions[], 4176 struct mlx5_translated_action_handle *handle, 4177 int *indir_n, 4178 struct rte_flow_action **translated_actions, 4179 struct rte_flow_error *error) 4180 { 4181 struct mlx5_priv *priv = dev->data->dev_private; 4182 struct rte_flow_action *translated = NULL; 4183 size_t actions_size; 4184 int n; 4185 int copied_n = 0; 4186 struct mlx5_translated_action_handle *handle_end = NULL; 4187 4188 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4189 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4190 continue; 4191 if (copied_n == *indir_n) { 4192 return rte_flow_error_set 4193 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4194 NULL, "too many shared actions"); 4195 } 4196 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4197 sizeof(actions[n].conf)); 4198 handle[copied_n].index = n; 4199 copied_n++; 4200 } 4201 n++; 4202 *indir_n = copied_n; 4203 if (!copied_n) 4204 return 0; 4205 actions_size = sizeof(struct rte_flow_action) * n; 4206 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4207 if (!translated) { 4208 rte_errno = ENOMEM; 4209 return -ENOMEM; 4210 } 4211 memcpy(translated, actions, actions_size); 4212 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4213 struct mlx5_shared_action_rss *shared_rss; 4214 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4215 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4216 uint32_t idx = act_idx & 4217 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4218 4219 switch (type) { 4220 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4221 shared_rss = mlx5_ipool_get 4222 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4223 translated[handle->index].type = 4224 RTE_FLOW_ACTION_TYPE_RSS; 4225 translated[handle->index].conf = 4226 &shared_rss->origin; 4227 break; 4228 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4229 translated[handle->index].type = 4230 (enum rte_flow_action_type) 4231 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4232 translated[handle->index].conf = (void *)(uintptr_t)idx; 4233 break; 4234 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4235 translated[handle->index].type = 4236 (enum rte_flow_action_type) 4237 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4238 translated[handle->index].conf = (void *)(uintptr_t)idx; 4239 break; 4240 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4241 if (priv->sh->flow_hit_aso_en) { 4242 translated[handle->index].type = 4243 (enum rte_flow_action_type) 4244 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4245 translated[handle->index].conf = 4246 (void *)(uintptr_t)idx; 4247 break; 4248 } 4249 /* Fall-through */ 4250 case MLX5_INDIRECT_ACTION_TYPE_CT: 4251 if (priv->sh->ct_aso_en) { 4252 translated[handle->index].type = 4253 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4254 translated[handle->index].conf = 4255 (void *)(uintptr_t)idx; 4256 break; 4257 } 4258 /* Fall-through */ 4259 default: 4260 mlx5_free(translated); 4261 return rte_flow_error_set 4262 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4263 NULL, "invalid indirect action type"); 4264 } 4265 } 4266 *translated_actions = translated; 4267 return 0; 4268 } 4269 4270 /** 4271 * Get Shared RSS action from the action list. 4272 * 4273 * @param[in] dev 4274 * Pointer to Ethernet device. 4275 * @param[in] shared 4276 * Pointer to the list of actions. 4277 * @param[in] shared_n 4278 * Actions list length. 4279 * 4280 * @return 4281 * The MLX5 RSS action ID if exists, otherwise return 0. 4282 */ 4283 static uint32_t 4284 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4285 struct mlx5_translated_action_handle *handle, 4286 int shared_n) 4287 { 4288 struct mlx5_translated_action_handle *handle_end; 4289 struct mlx5_priv *priv = dev->data->dev_private; 4290 struct mlx5_shared_action_rss *shared_rss; 4291 4292 4293 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4294 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4295 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4296 uint32_t idx = act_idx & 4297 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4298 switch (type) { 4299 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4300 shared_rss = mlx5_ipool_get 4301 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4302 idx); 4303 __atomic_add_fetch(&shared_rss->refcnt, 1, 4304 __ATOMIC_RELAXED); 4305 return idx; 4306 default: 4307 break; 4308 } 4309 } 4310 return 0; 4311 } 4312 4313 static unsigned int 4314 find_graph_root(uint32_t rss_level) 4315 { 4316 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4317 MLX5_EXPANSION_ROOT_OUTER; 4318 } 4319 4320 /** 4321 * Get layer flags from the prefix flow. 4322 * 4323 * Some flows may be split to several subflows, the prefix subflow gets the 4324 * match items and the suffix sub flow gets the actions. 4325 * Some actions need the user defined match item flags to get the detail for 4326 * the action. 4327 * This function helps the suffix flow to get the item layer flags from prefix 4328 * subflow. 4329 * 4330 * @param[in] dev_flow 4331 * Pointer the created prefix subflow. 4332 * 4333 * @return 4334 * The layers get from prefix subflow. 4335 */ 4336 static inline uint64_t 4337 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4338 { 4339 uint64_t layers = 0; 4340 4341 /* 4342 * Layers bits could be localization, but usually the compiler will 4343 * help to do the optimization work for source code. 4344 * If no decap actions, use the layers directly. 4345 */ 4346 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4347 return dev_flow->handle->layers; 4348 /* Convert L3 layers with decap action. */ 4349 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4350 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4351 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4352 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4353 /* Convert L4 layers with decap action. */ 4354 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4355 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4356 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4357 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4358 return layers; 4359 } 4360 4361 /** 4362 * Get metadata split action information. 4363 * 4364 * @param[in] actions 4365 * Pointer to the list of actions. 4366 * @param[out] qrss 4367 * Pointer to the return pointer. 4368 * @param[out] qrss_type 4369 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4370 * if no QUEUE/RSS is found. 4371 * @param[out] encap_idx 4372 * Pointer to the index of the encap action if exists, otherwise the last 4373 * action index. 4374 * 4375 * @return 4376 * Total number of actions. 4377 */ 4378 static int 4379 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4380 const struct rte_flow_action **qrss, 4381 int *encap_idx) 4382 { 4383 const struct rte_flow_action_raw_encap *raw_encap; 4384 int actions_n = 0; 4385 int raw_decap_idx = -1; 4386 4387 *encap_idx = -1; 4388 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4389 switch (actions->type) { 4390 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4391 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4392 *encap_idx = actions_n; 4393 break; 4394 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4395 raw_decap_idx = actions_n; 4396 break; 4397 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4398 raw_encap = actions->conf; 4399 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4400 *encap_idx = raw_decap_idx != -1 ? 4401 raw_decap_idx : actions_n; 4402 break; 4403 case RTE_FLOW_ACTION_TYPE_QUEUE: 4404 case RTE_FLOW_ACTION_TYPE_RSS: 4405 *qrss = actions; 4406 break; 4407 default: 4408 break; 4409 } 4410 actions_n++; 4411 } 4412 if (*encap_idx == -1) 4413 *encap_idx = actions_n; 4414 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4415 return actions_n + 1; 4416 } 4417 4418 /** 4419 * Check if the action will change packet. 4420 * 4421 * @param dev 4422 * Pointer to Ethernet device. 4423 * @param[in] type 4424 * action type. 4425 * 4426 * @return 4427 * true if action will change packet, false otherwise. 4428 */ 4429 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4430 enum rte_flow_action_type type) 4431 { 4432 struct mlx5_priv *priv = dev->data->dev_private; 4433 4434 switch (type) { 4435 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4436 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4437 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4438 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4439 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4440 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4441 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4442 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4443 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4444 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4445 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4446 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4447 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4448 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4449 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4450 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4451 case RTE_FLOW_ACTION_TYPE_SET_META: 4452 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4453 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4454 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4455 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4456 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4457 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4458 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4459 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4460 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4461 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4462 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4463 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4464 return true; 4465 case RTE_FLOW_ACTION_TYPE_FLAG: 4466 case RTE_FLOW_ACTION_TYPE_MARK: 4467 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4468 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4469 return true; 4470 else 4471 return false; 4472 default: 4473 return false; 4474 } 4475 } 4476 4477 /** 4478 * Check meter action from the action list. 4479 * 4480 * @param dev 4481 * Pointer to Ethernet device. 4482 * @param[in] actions 4483 * Pointer to the list of actions. 4484 * @param[out] has_mtr 4485 * Pointer to the meter exist flag. 4486 * @param[out] has_modify 4487 * Pointer to the flag showing there's packet change action. 4488 * @param[out] meter_id 4489 * Pointer to the meter id. 4490 * 4491 * @return 4492 * Total number of actions. 4493 */ 4494 static int 4495 flow_check_meter_action(struct rte_eth_dev *dev, 4496 const struct rte_flow_action actions[], 4497 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4498 { 4499 const struct rte_flow_action_meter *mtr = NULL; 4500 int actions_n = 0; 4501 4502 MLX5_ASSERT(has_mtr); 4503 *has_mtr = false; 4504 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4505 switch (actions->type) { 4506 case RTE_FLOW_ACTION_TYPE_METER: 4507 mtr = actions->conf; 4508 *meter_id = mtr->mtr_id; 4509 *has_mtr = true; 4510 break; 4511 default: 4512 break; 4513 } 4514 if (!*has_mtr) 4515 *has_modify |= flow_check_modify_action_type(dev, 4516 actions->type); 4517 actions_n++; 4518 } 4519 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4520 return actions_n + 1; 4521 } 4522 4523 /** 4524 * Check if the flow should be split due to hairpin. 4525 * The reason for the split is that in current HW we can't 4526 * support encap and push-vlan on Rx, so if a flow contains 4527 * these actions we move it to Tx. 4528 * 4529 * @param dev 4530 * Pointer to Ethernet device. 4531 * @param[in] attr 4532 * Flow rule attributes. 4533 * @param[in] actions 4534 * Associated actions (list terminated by the END action). 4535 * 4536 * @return 4537 * > 0 the number of actions and the flow should be split, 4538 * 0 when no split required. 4539 */ 4540 static int 4541 flow_check_hairpin_split(struct rte_eth_dev *dev, 4542 const struct rte_flow_attr *attr, 4543 const struct rte_flow_action actions[]) 4544 { 4545 int queue_action = 0; 4546 int action_n = 0; 4547 int split = 0; 4548 const struct rte_flow_action_queue *queue; 4549 const struct rte_flow_action_rss *rss; 4550 const struct rte_flow_action_raw_encap *raw_encap; 4551 const struct rte_eth_hairpin_conf *conf; 4552 4553 if (!attr->ingress) 4554 return 0; 4555 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4556 switch (actions->type) { 4557 case RTE_FLOW_ACTION_TYPE_QUEUE: 4558 queue = actions->conf; 4559 if (queue == NULL) 4560 return 0; 4561 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4562 if (conf == NULL || conf->tx_explicit != 0) 4563 return 0; 4564 queue_action = 1; 4565 action_n++; 4566 break; 4567 case RTE_FLOW_ACTION_TYPE_RSS: 4568 rss = actions->conf; 4569 if (rss == NULL || rss->queue_num == 0) 4570 return 0; 4571 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4572 if (conf == NULL || conf->tx_explicit != 0) 4573 return 0; 4574 queue_action = 1; 4575 action_n++; 4576 break; 4577 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4578 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4579 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4580 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4581 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4582 split++; 4583 action_n++; 4584 break; 4585 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4586 raw_encap = actions->conf; 4587 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4588 split++; 4589 action_n++; 4590 break; 4591 default: 4592 action_n++; 4593 break; 4594 } 4595 } 4596 if (split && queue_action) 4597 return action_n; 4598 return 0; 4599 } 4600 4601 /* Declare flow create/destroy prototype in advance. */ 4602 static uint32_t 4603 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4604 const struct rte_flow_attr *attr, 4605 const struct rte_flow_item items[], 4606 const struct rte_flow_action actions[], 4607 bool external, struct rte_flow_error *error); 4608 4609 static void 4610 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4611 uint32_t flow_idx); 4612 4613 int 4614 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4615 struct mlx5_list_entry *entry, void *cb_ctx) 4616 { 4617 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4618 struct mlx5_flow_mreg_copy_resource *mcp_res = 4619 container_of(entry, typeof(*mcp_res), hlist_ent); 4620 4621 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4622 } 4623 4624 struct mlx5_list_entry * 4625 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4626 { 4627 struct rte_eth_dev *dev = tool_ctx; 4628 struct mlx5_priv *priv = dev->data->dev_private; 4629 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4630 struct mlx5_flow_mreg_copy_resource *mcp_res; 4631 struct rte_flow_error *error = ctx->error; 4632 uint32_t idx = 0; 4633 int ret; 4634 uint32_t mark_id = *(uint32_t *)(ctx->data); 4635 struct rte_flow_attr attr = { 4636 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4637 .ingress = 1, 4638 }; 4639 struct mlx5_rte_flow_item_tag tag_spec = { 4640 .data = mark_id, 4641 }; 4642 struct rte_flow_item items[] = { 4643 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4644 }; 4645 struct rte_flow_action_mark ftag = { 4646 .id = mark_id, 4647 }; 4648 struct mlx5_flow_action_copy_mreg cp_mreg = { 4649 .dst = REG_B, 4650 .src = REG_NON, 4651 }; 4652 struct rte_flow_action_jump jump = { 4653 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4654 }; 4655 struct rte_flow_action actions[] = { 4656 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4657 }; 4658 4659 /* Fill the register fields in the flow. */ 4660 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4661 if (ret < 0) 4662 return NULL; 4663 tag_spec.id = ret; 4664 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4665 if (ret < 0) 4666 return NULL; 4667 cp_mreg.src = ret; 4668 /* Provide the full width of FLAG specific value. */ 4669 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4670 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4671 /* Build a new flow. */ 4672 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4673 items[0] = (struct rte_flow_item){ 4674 .type = (enum rte_flow_item_type) 4675 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4676 .spec = &tag_spec, 4677 }; 4678 items[1] = (struct rte_flow_item){ 4679 .type = RTE_FLOW_ITEM_TYPE_END, 4680 }; 4681 actions[0] = (struct rte_flow_action){ 4682 .type = (enum rte_flow_action_type) 4683 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4684 .conf = &ftag, 4685 }; 4686 actions[1] = (struct rte_flow_action){ 4687 .type = (enum rte_flow_action_type) 4688 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4689 .conf = &cp_mreg, 4690 }; 4691 actions[2] = (struct rte_flow_action){ 4692 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4693 .conf = &jump, 4694 }; 4695 actions[3] = (struct rte_flow_action){ 4696 .type = RTE_FLOW_ACTION_TYPE_END, 4697 }; 4698 } else { 4699 /* Default rule, wildcard match. */ 4700 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4701 items[0] = (struct rte_flow_item){ 4702 .type = RTE_FLOW_ITEM_TYPE_END, 4703 }; 4704 actions[0] = (struct rte_flow_action){ 4705 .type = (enum rte_flow_action_type) 4706 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4707 .conf = &cp_mreg, 4708 }; 4709 actions[1] = (struct rte_flow_action){ 4710 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4711 .conf = &jump, 4712 }; 4713 actions[2] = (struct rte_flow_action){ 4714 .type = RTE_FLOW_ACTION_TYPE_END, 4715 }; 4716 } 4717 /* Build a new entry. */ 4718 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4719 if (!mcp_res) { 4720 rte_errno = ENOMEM; 4721 return NULL; 4722 } 4723 mcp_res->idx = idx; 4724 mcp_res->mark_id = mark_id; 4725 /* 4726 * The copy Flows are not included in any list. There 4727 * ones are referenced from other Flows and can not 4728 * be applied, removed, deleted in arbitrary order 4729 * by list traversing. 4730 */ 4731 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4732 &attr, items, actions, false, error); 4733 if (!mcp_res->rix_flow) { 4734 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4735 return NULL; 4736 } 4737 return &mcp_res->hlist_ent; 4738 } 4739 4740 struct mlx5_list_entry * 4741 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4742 void *cb_ctx __rte_unused) 4743 { 4744 struct rte_eth_dev *dev = tool_ctx; 4745 struct mlx5_priv *priv = dev->data->dev_private; 4746 struct mlx5_flow_mreg_copy_resource *mcp_res; 4747 uint32_t idx = 0; 4748 4749 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4750 if (!mcp_res) { 4751 rte_errno = ENOMEM; 4752 return NULL; 4753 } 4754 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4755 mcp_res->idx = idx; 4756 return &mcp_res->hlist_ent; 4757 } 4758 4759 void 4760 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4761 { 4762 struct mlx5_flow_mreg_copy_resource *mcp_res = 4763 container_of(entry, typeof(*mcp_res), hlist_ent); 4764 struct rte_eth_dev *dev = tool_ctx; 4765 struct mlx5_priv *priv = dev->data->dev_private; 4766 4767 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4768 } 4769 4770 /** 4771 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4772 * 4773 * As mark_id is unique, if there's already a registered flow for the mark_id, 4774 * return by increasing the reference counter of the resource. Otherwise, create 4775 * the resource (mcp_res) and flow. 4776 * 4777 * Flow looks like, 4778 * - If ingress port is ANY and reg_c[1] is mark_id, 4779 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4780 * 4781 * For default flow (zero mark_id), flow is like, 4782 * - If ingress port is ANY, 4783 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4784 * 4785 * @param dev 4786 * Pointer to Ethernet device. 4787 * @param mark_id 4788 * ID of MARK action, zero means default flow for META. 4789 * @param[out] error 4790 * Perform verbose error reporting if not NULL. 4791 * 4792 * @return 4793 * Associated resource on success, NULL otherwise and rte_errno is set. 4794 */ 4795 static struct mlx5_flow_mreg_copy_resource * 4796 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4797 struct rte_flow_error *error) 4798 { 4799 struct mlx5_priv *priv = dev->data->dev_private; 4800 struct mlx5_list_entry *entry; 4801 struct mlx5_flow_cb_ctx ctx = { 4802 .dev = dev, 4803 .error = error, 4804 .data = &mark_id, 4805 }; 4806 4807 /* Check if already registered. */ 4808 MLX5_ASSERT(priv->mreg_cp_tbl); 4809 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4810 if (!entry) 4811 return NULL; 4812 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4813 hlist_ent); 4814 } 4815 4816 void 4817 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4818 { 4819 struct mlx5_flow_mreg_copy_resource *mcp_res = 4820 container_of(entry, typeof(*mcp_res), hlist_ent); 4821 struct rte_eth_dev *dev = tool_ctx; 4822 struct mlx5_priv *priv = dev->data->dev_private; 4823 4824 MLX5_ASSERT(mcp_res->rix_flow); 4825 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4826 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4827 } 4828 4829 /** 4830 * Release flow in RX_CP_TBL. 4831 * 4832 * @param dev 4833 * Pointer to Ethernet device. 4834 * @flow 4835 * Parent flow for wich copying is provided. 4836 */ 4837 static void 4838 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4839 struct rte_flow *flow) 4840 { 4841 struct mlx5_flow_mreg_copy_resource *mcp_res; 4842 struct mlx5_priv *priv = dev->data->dev_private; 4843 4844 if (!flow->rix_mreg_copy) 4845 return; 4846 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4847 flow->rix_mreg_copy); 4848 if (!mcp_res || !priv->mreg_cp_tbl) 4849 return; 4850 MLX5_ASSERT(mcp_res->rix_flow); 4851 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4852 flow->rix_mreg_copy = 0; 4853 } 4854 4855 /** 4856 * Remove the default copy action from RX_CP_TBL. 4857 * 4858 * This functions is called in the mlx5_dev_start(). No thread safe 4859 * is guaranteed. 4860 * 4861 * @param dev 4862 * Pointer to Ethernet device. 4863 */ 4864 static void 4865 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4866 { 4867 struct mlx5_list_entry *entry; 4868 struct mlx5_priv *priv = dev->data->dev_private; 4869 struct mlx5_flow_cb_ctx ctx; 4870 uint32_t mark_id; 4871 4872 /* Check if default flow is registered. */ 4873 if (!priv->mreg_cp_tbl) 4874 return; 4875 mark_id = MLX5_DEFAULT_COPY_ID; 4876 ctx.data = &mark_id; 4877 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4878 if (!entry) 4879 return; 4880 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4881 } 4882 4883 /** 4884 * Add the default copy action in in RX_CP_TBL. 4885 * 4886 * This functions is called in the mlx5_dev_start(). No thread safe 4887 * is guaranteed. 4888 * 4889 * @param dev 4890 * Pointer to Ethernet device. 4891 * @param[out] error 4892 * Perform verbose error reporting if not NULL. 4893 * 4894 * @return 4895 * 0 for success, negative value otherwise and rte_errno is set. 4896 */ 4897 static int 4898 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4899 struct rte_flow_error *error) 4900 { 4901 struct mlx5_priv *priv = dev->data->dev_private; 4902 struct mlx5_flow_mreg_copy_resource *mcp_res; 4903 struct mlx5_flow_cb_ctx ctx; 4904 uint32_t mark_id; 4905 4906 /* Check whether extensive metadata feature is engaged. */ 4907 if (!priv->sh->config.dv_flow_en || 4908 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4909 !mlx5_flow_ext_mreg_supported(dev) || 4910 !priv->sh->dv_regc0_mask) 4911 return 0; 4912 /* 4913 * Add default mreg copy flow may be called multiple time, but 4914 * only be called once in stop. Avoid register it twice. 4915 */ 4916 mark_id = MLX5_DEFAULT_COPY_ID; 4917 ctx.data = &mark_id; 4918 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4919 return 0; 4920 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4921 if (!mcp_res) 4922 return -rte_errno; 4923 return 0; 4924 } 4925 4926 /** 4927 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4928 * 4929 * All the flow having Q/RSS action should be split by 4930 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4931 * performs the following, 4932 * - CQE->flow_tag := reg_c[1] (MARK) 4933 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4934 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4935 * but there should be a flow per each MARK ID set by MARK action. 4936 * 4937 * For the aforementioned reason, if there's a MARK action in flow's action 4938 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4939 * the MARK ID to CQE's flow_tag like, 4940 * - If reg_c[1] is mark_id, 4941 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4942 * 4943 * For SET_META action which stores value in reg_c[0], as the destination is 4944 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4945 * MARK ID means the default flow. The default flow looks like, 4946 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4947 * 4948 * @param dev 4949 * Pointer to Ethernet device. 4950 * @param flow 4951 * Pointer to flow structure. 4952 * @param[in] actions 4953 * Pointer to the list of actions. 4954 * @param[out] error 4955 * Perform verbose error reporting if not NULL. 4956 * 4957 * @return 4958 * 0 on success, negative value otherwise and rte_errno is set. 4959 */ 4960 static int 4961 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 4962 struct rte_flow *flow, 4963 const struct rte_flow_action *actions, 4964 struct rte_flow_error *error) 4965 { 4966 struct mlx5_priv *priv = dev->data->dev_private; 4967 struct mlx5_sh_config *config = &priv->sh->config; 4968 struct mlx5_flow_mreg_copy_resource *mcp_res; 4969 const struct rte_flow_action_mark *mark; 4970 4971 /* Check whether extensive metadata feature is engaged. */ 4972 if (!config->dv_flow_en || 4973 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4974 !mlx5_flow_ext_mreg_supported(dev) || 4975 !priv->sh->dv_regc0_mask) 4976 return 0; 4977 /* Find MARK action. */ 4978 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4979 switch (actions->type) { 4980 case RTE_FLOW_ACTION_TYPE_FLAG: 4981 mcp_res = flow_mreg_add_copy_action 4982 (dev, MLX5_FLOW_MARK_DEFAULT, error); 4983 if (!mcp_res) 4984 return -rte_errno; 4985 flow->rix_mreg_copy = mcp_res->idx; 4986 return 0; 4987 case RTE_FLOW_ACTION_TYPE_MARK: 4988 mark = (const struct rte_flow_action_mark *) 4989 actions->conf; 4990 mcp_res = 4991 flow_mreg_add_copy_action(dev, mark->id, error); 4992 if (!mcp_res) 4993 return -rte_errno; 4994 flow->rix_mreg_copy = mcp_res->idx; 4995 return 0; 4996 default: 4997 break; 4998 } 4999 } 5000 return 0; 5001 } 5002 5003 #define MLX5_MAX_SPLIT_ACTIONS 24 5004 #define MLX5_MAX_SPLIT_ITEMS 24 5005 5006 /** 5007 * Split the hairpin flow. 5008 * Since HW can't support encap and push-vlan on Rx, we move these 5009 * actions to Tx. 5010 * If the count action is after the encap then we also 5011 * move the count action. in this case the count will also measure 5012 * the outer bytes. 5013 * 5014 * @param dev 5015 * Pointer to Ethernet device. 5016 * @param[in] actions 5017 * Associated actions (list terminated by the END action). 5018 * @param[out] actions_rx 5019 * Rx flow actions. 5020 * @param[out] actions_tx 5021 * Tx flow actions.. 5022 * @param[out] pattern_tx 5023 * The pattern items for the Tx flow. 5024 * @param[out] flow_id 5025 * The flow ID connected to this flow. 5026 * 5027 * @return 5028 * 0 on success. 5029 */ 5030 static int 5031 flow_hairpin_split(struct rte_eth_dev *dev, 5032 const struct rte_flow_action actions[], 5033 struct rte_flow_action actions_rx[], 5034 struct rte_flow_action actions_tx[], 5035 struct rte_flow_item pattern_tx[], 5036 uint32_t flow_id) 5037 { 5038 const struct rte_flow_action_raw_encap *raw_encap; 5039 const struct rte_flow_action_raw_decap *raw_decap; 5040 struct mlx5_rte_flow_action_set_tag *set_tag; 5041 struct rte_flow_action *tag_action; 5042 struct mlx5_rte_flow_item_tag *tag_item; 5043 struct rte_flow_item *item; 5044 char *addr; 5045 int encap = 0; 5046 5047 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5048 switch (actions->type) { 5049 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5050 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5051 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5052 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5053 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5054 rte_memcpy(actions_tx, actions, 5055 sizeof(struct rte_flow_action)); 5056 actions_tx++; 5057 break; 5058 case RTE_FLOW_ACTION_TYPE_COUNT: 5059 if (encap) { 5060 rte_memcpy(actions_tx, actions, 5061 sizeof(struct rte_flow_action)); 5062 actions_tx++; 5063 } else { 5064 rte_memcpy(actions_rx, actions, 5065 sizeof(struct rte_flow_action)); 5066 actions_rx++; 5067 } 5068 break; 5069 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5070 raw_encap = actions->conf; 5071 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5072 memcpy(actions_tx, actions, 5073 sizeof(struct rte_flow_action)); 5074 actions_tx++; 5075 encap = 1; 5076 } else { 5077 rte_memcpy(actions_rx, actions, 5078 sizeof(struct rte_flow_action)); 5079 actions_rx++; 5080 } 5081 break; 5082 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5083 raw_decap = actions->conf; 5084 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5085 memcpy(actions_tx, actions, 5086 sizeof(struct rte_flow_action)); 5087 actions_tx++; 5088 } else { 5089 rte_memcpy(actions_rx, actions, 5090 sizeof(struct rte_flow_action)); 5091 actions_rx++; 5092 } 5093 break; 5094 default: 5095 rte_memcpy(actions_rx, actions, 5096 sizeof(struct rte_flow_action)); 5097 actions_rx++; 5098 break; 5099 } 5100 } 5101 /* Add set meta action and end action for the Rx flow. */ 5102 tag_action = actions_rx; 5103 tag_action->type = (enum rte_flow_action_type) 5104 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5105 actions_rx++; 5106 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5107 actions_rx++; 5108 set_tag = (void *)actions_rx; 5109 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5110 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5111 .data = flow_id, 5112 }; 5113 MLX5_ASSERT(set_tag->id > REG_NON); 5114 tag_action->conf = set_tag; 5115 /* Create Tx item list. */ 5116 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5117 addr = (void *)&pattern_tx[2]; 5118 item = pattern_tx; 5119 item->type = (enum rte_flow_item_type) 5120 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5121 tag_item = (void *)addr; 5122 tag_item->data = flow_id; 5123 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5124 MLX5_ASSERT(set_tag->id > REG_NON); 5125 item->spec = tag_item; 5126 addr += sizeof(struct mlx5_rte_flow_item_tag); 5127 tag_item = (void *)addr; 5128 tag_item->data = UINT32_MAX; 5129 tag_item->id = UINT16_MAX; 5130 item->mask = tag_item; 5131 item->last = NULL; 5132 item++; 5133 item->type = RTE_FLOW_ITEM_TYPE_END; 5134 return 0; 5135 } 5136 5137 /** 5138 * The last stage of splitting chain, just creates the subflow 5139 * without any modification. 5140 * 5141 * @param[in] dev 5142 * Pointer to Ethernet device. 5143 * @param[in] flow 5144 * Parent flow structure pointer. 5145 * @param[in, out] sub_flow 5146 * Pointer to return the created subflow, may be NULL. 5147 * @param[in] attr 5148 * Flow rule attributes. 5149 * @param[in] items 5150 * Pattern specification (list terminated by the END pattern item). 5151 * @param[in] actions 5152 * Associated actions (list terminated by the END action). 5153 * @param[in] flow_split_info 5154 * Pointer to flow split info structure. 5155 * @param[out] error 5156 * Perform verbose error reporting if not NULL. 5157 * @return 5158 * 0 on success, negative value otherwise 5159 */ 5160 static int 5161 flow_create_split_inner(struct rte_eth_dev *dev, 5162 struct rte_flow *flow, 5163 struct mlx5_flow **sub_flow, 5164 const struct rte_flow_attr *attr, 5165 const struct rte_flow_item items[], 5166 const struct rte_flow_action actions[], 5167 struct mlx5_flow_split_info *flow_split_info, 5168 struct rte_flow_error *error) 5169 { 5170 struct mlx5_flow *dev_flow; 5171 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5172 5173 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5174 flow_split_info->flow_idx, error); 5175 if (!dev_flow) 5176 return -rte_errno; 5177 dev_flow->flow = flow; 5178 dev_flow->external = flow_split_info->external; 5179 dev_flow->skip_scale = flow_split_info->skip_scale; 5180 /* Subflow object was created, we must include one in the list. */ 5181 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5182 dev_flow->handle, next); 5183 /* 5184 * If dev_flow is as one of the suffix flow, some actions in suffix 5185 * flow may need some user defined item layer flags, and pass the 5186 * Metadata rxq mark flag to suffix flow as well. 5187 */ 5188 if (flow_split_info->prefix_layers) 5189 dev_flow->handle->layers = flow_split_info->prefix_layers; 5190 if (flow_split_info->prefix_mark) { 5191 MLX5_ASSERT(wks); 5192 wks->mark = 1; 5193 } 5194 if (sub_flow) 5195 *sub_flow = dev_flow; 5196 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5197 dev_flow->dv.table_id = flow_split_info->table_id; 5198 #endif 5199 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5200 } 5201 5202 /** 5203 * Get the sub policy of a meter. 5204 * 5205 * @param[in] dev 5206 * Pointer to Ethernet device. 5207 * @param[in] flow 5208 * Parent flow structure pointer. 5209 * @param wks 5210 * Pointer to thread flow work space. 5211 * @param[in] attr 5212 * Flow rule attributes. 5213 * @param[in] items 5214 * Pattern specification (list terminated by the END pattern item). 5215 * @param[out] error 5216 * Perform verbose error reporting if not NULL. 5217 * 5218 * @return 5219 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5220 */ 5221 static struct mlx5_flow_meter_sub_policy * 5222 get_meter_sub_policy(struct rte_eth_dev *dev, 5223 struct rte_flow *flow, 5224 struct mlx5_flow_workspace *wks, 5225 const struct rte_flow_attr *attr, 5226 const struct rte_flow_item items[], 5227 struct rte_flow_error *error) 5228 { 5229 struct mlx5_flow_meter_policy *policy; 5230 struct mlx5_flow_meter_policy *final_policy; 5231 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5232 5233 policy = wks->policy; 5234 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5235 if (final_policy->is_rss || final_policy->is_queue) { 5236 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5237 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5238 uint32_t i; 5239 5240 /* 5241 * This is a tmp dev_flow, 5242 * no need to register any matcher for it in translate. 5243 */ 5244 wks->skip_matcher_reg = 1; 5245 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5246 struct mlx5_flow dev_flow = {0}; 5247 struct mlx5_flow_handle dev_handle = { {0} }; 5248 uint8_t fate = final_policy->act_cnt[i].fate_action; 5249 5250 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5251 const struct rte_flow_action_rss *rss_act = 5252 final_policy->act_cnt[i].rss->conf; 5253 struct rte_flow_action rss_actions[2] = { 5254 [0] = { 5255 .type = RTE_FLOW_ACTION_TYPE_RSS, 5256 .conf = rss_act, 5257 }, 5258 [1] = { 5259 .type = RTE_FLOW_ACTION_TYPE_END, 5260 .conf = NULL, 5261 } 5262 }; 5263 5264 dev_flow.handle = &dev_handle; 5265 dev_flow.ingress = attr->ingress; 5266 dev_flow.flow = flow; 5267 dev_flow.external = 0; 5268 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5269 dev_flow.dv.transfer = attr->transfer; 5270 #endif 5271 /** 5272 * Translate RSS action to get rss hash fields. 5273 */ 5274 if (flow_drv_translate(dev, &dev_flow, attr, 5275 items, rss_actions, error)) 5276 goto exit; 5277 rss_desc_v[i] = wks->rss_desc; 5278 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5279 rss_desc_v[i].hash_fields = 5280 dev_flow.hash_fields; 5281 rss_desc_v[i].queue_num = 5282 rss_desc_v[i].hash_fields ? 5283 rss_desc_v[i].queue_num : 1; 5284 rss_desc_v[i].tunnel = 5285 !!(dev_flow.handle->layers & 5286 MLX5_FLOW_LAYER_TUNNEL); 5287 /* Use the RSS queues in the containers. */ 5288 rss_desc_v[i].queue = 5289 (uint16_t *)(uintptr_t)rss_act->queue; 5290 rss_desc[i] = &rss_desc_v[i]; 5291 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5292 /* This is queue action. */ 5293 rss_desc_v[i] = wks->rss_desc; 5294 rss_desc_v[i].key_len = 0; 5295 rss_desc_v[i].hash_fields = 0; 5296 rss_desc_v[i].queue = 5297 &final_policy->act_cnt[i].queue; 5298 rss_desc_v[i].queue_num = 1; 5299 rss_desc[i] = &rss_desc_v[i]; 5300 } else { 5301 rss_desc[i] = NULL; 5302 } 5303 } 5304 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5305 flow, policy, rss_desc); 5306 } else { 5307 enum mlx5_meter_domain mtr_domain = 5308 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5309 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5310 MLX5_MTR_DOMAIN_INGRESS); 5311 sub_policy = policy->sub_policys[mtr_domain][0]; 5312 } 5313 if (!sub_policy) 5314 rte_flow_error_set(error, EINVAL, 5315 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5316 "Failed to get meter sub-policy."); 5317 exit: 5318 return sub_policy; 5319 } 5320 5321 /** 5322 * Split the meter flow. 5323 * 5324 * As meter flow will split to three sub flow, other than meter 5325 * action, the other actions make sense to only meter accepts 5326 * the packet. If it need to be dropped, no other additional 5327 * actions should be take. 5328 * 5329 * One kind of special action which decapsulates the L3 tunnel 5330 * header will be in the prefix sub flow, as not to take the 5331 * L3 tunnel header into account. 5332 * 5333 * @param[in] dev 5334 * Pointer to Ethernet device. 5335 * @param[in] flow 5336 * Parent flow structure pointer. 5337 * @param wks 5338 * Pointer to thread flow work space. 5339 * @param[in] attr 5340 * Flow rule attributes. 5341 * @param[in] items 5342 * Pattern specification (list terminated by the END pattern item). 5343 * @param[out] sfx_items 5344 * Suffix flow match items (list terminated by the END pattern item). 5345 * @param[in] actions 5346 * Associated actions (list terminated by the END action). 5347 * @param[out] actions_sfx 5348 * Suffix flow actions. 5349 * @param[out] actions_pre 5350 * Prefix flow actions. 5351 * @param[out] mtr_flow_id 5352 * Pointer to meter flow id. 5353 * @param[out] error 5354 * Perform verbose error reporting if not NULL. 5355 * 5356 * @return 5357 * 0 on success, a negative errno value otherwise and rte_errno is set. 5358 */ 5359 static int 5360 flow_meter_split_prep(struct rte_eth_dev *dev, 5361 struct rte_flow *flow, 5362 struct mlx5_flow_workspace *wks, 5363 const struct rte_flow_attr *attr, 5364 const struct rte_flow_item items[], 5365 struct rte_flow_item sfx_items[], 5366 const struct rte_flow_action actions[], 5367 struct rte_flow_action actions_sfx[], 5368 struct rte_flow_action actions_pre[], 5369 uint32_t *mtr_flow_id, 5370 struct rte_flow_error *error) 5371 { 5372 struct mlx5_priv *priv = dev->data->dev_private; 5373 struct mlx5_flow_meter_info *fm = wks->fm; 5374 struct rte_flow_action *tag_action = NULL; 5375 struct rte_flow_item *tag_item; 5376 struct mlx5_rte_flow_action_set_tag *set_tag; 5377 const struct rte_flow_action_raw_encap *raw_encap; 5378 const struct rte_flow_action_raw_decap *raw_decap; 5379 struct mlx5_rte_flow_item_tag *tag_item_spec; 5380 struct mlx5_rte_flow_item_tag *tag_item_mask; 5381 uint32_t tag_id = 0; 5382 struct rte_flow_item *vlan_item_dst = NULL; 5383 const struct rte_flow_item *vlan_item_src = NULL; 5384 const struct rte_flow_item *orig_items = items; 5385 struct rte_flow_action *hw_mtr_action; 5386 struct rte_flow_action *action_pre_head = NULL; 5387 uint16_t flow_src_port = priv->representor_id; 5388 bool mtr_first; 5389 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5390 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5391 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5392 uint32_t flow_id = 0; 5393 uint32_t flow_id_reversed = 0; 5394 uint8_t flow_id_bits = 0; 5395 bool after_meter = false; 5396 int shift; 5397 5398 /* Prepare the suffix subflow items. */ 5399 tag_item = sfx_items++; 5400 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5401 int item_type = items->type; 5402 5403 switch (item_type) { 5404 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5405 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5406 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5407 return -rte_errno; 5408 if (!fm->def_policy && wks->policy->is_hierarchy && 5409 flow_src_port != priv->representor_id) { 5410 if (flow_drv_mtr_hierarchy_rule_create(dev, 5411 flow, fm, 5412 flow_src_port, 5413 items, 5414 error)) 5415 return -rte_errno; 5416 } 5417 memcpy(sfx_items, items, sizeof(*sfx_items)); 5418 sfx_items++; 5419 break; 5420 case RTE_FLOW_ITEM_TYPE_VLAN: 5421 /* Determine if copy vlan item below. */ 5422 vlan_item_src = items; 5423 vlan_item_dst = sfx_items++; 5424 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5425 break; 5426 default: 5427 break; 5428 } 5429 } 5430 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5431 sfx_items++; 5432 mtr_first = priv->sh->meter_aso_en && 5433 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5434 /* For ASO meter, meter must be before tag in TX direction. */ 5435 if (mtr_first) { 5436 action_pre_head = actions_pre++; 5437 /* Leave space for tag action. */ 5438 tag_action = actions_pre++; 5439 } 5440 /* Prepare the actions for prefix and suffix flow. */ 5441 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5442 struct rte_flow_action *action_cur = NULL; 5443 5444 switch (actions->type) { 5445 case RTE_FLOW_ACTION_TYPE_METER: 5446 if (mtr_first) { 5447 action_cur = action_pre_head; 5448 } else { 5449 /* Leave space for tag action. */ 5450 tag_action = actions_pre++; 5451 action_cur = actions_pre++; 5452 } 5453 after_meter = true; 5454 break; 5455 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5456 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5457 action_cur = actions_pre++; 5458 break; 5459 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5460 raw_encap = actions->conf; 5461 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5462 action_cur = actions_pre++; 5463 break; 5464 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5465 raw_decap = actions->conf; 5466 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5467 action_cur = actions_pre++; 5468 break; 5469 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5470 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5471 if (vlan_item_dst && vlan_item_src) { 5472 memcpy(vlan_item_dst, vlan_item_src, 5473 sizeof(*vlan_item_dst)); 5474 /* 5475 * Convert to internal match item, it is used 5476 * for vlan push and set vid. 5477 */ 5478 vlan_item_dst->type = (enum rte_flow_item_type) 5479 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5480 } 5481 break; 5482 case RTE_FLOW_ACTION_TYPE_COUNT: 5483 if (fm->def_policy) 5484 action_cur = after_meter ? 5485 actions_sfx++ : actions_pre++; 5486 break; 5487 default: 5488 break; 5489 } 5490 if (!action_cur) 5491 action_cur = (fm->def_policy) ? 5492 actions_sfx++ : actions_pre++; 5493 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5494 } 5495 /* Add end action to the actions. */ 5496 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5497 if (priv->sh->meter_aso_en) { 5498 /** 5499 * For ASO meter, need to add an extra jump action explicitly, 5500 * to jump from meter to policer table. 5501 */ 5502 struct mlx5_flow_meter_sub_policy *sub_policy; 5503 struct mlx5_flow_tbl_data_entry *tbl_data; 5504 5505 if (!fm->def_policy) { 5506 sub_policy = get_meter_sub_policy(dev, flow, wks, 5507 attr, orig_items, 5508 error); 5509 if (!sub_policy) 5510 return -rte_errno; 5511 } else { 5512 enum mlx5_meter_domain mtr_domain = 5513 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5514 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5515 MLX5_MTR_DOMAIN_INGRESS); 5516 5517 sub_policy = 5518 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5519 } 5520 tbl_data = container_of(sub_policy->tbl_rsc, 5521 struct mlx5_flow_tbl_data_entry, tbl); 5522 hw_mtr_action = actions_pre++; 5523 hw_mtr_action->type = (enum rte_flow_action_type) 5524 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5525 hw_mtr_action->conf = tbl_data->jump.action; 5526 } 5527 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5528 actions_pre++; 5529 if (!tag_action) 5530 return rte_flow_error_set(error, ENOMEM, 5531 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5532 NULL, "No tag action space."); 5533 if (!mtr_flow_id) { 5534 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5535 goto exit; 5536 } 5537 /* Only default-policy Meter creates mtr flow id. */ 5538 if (fm->def_policy) { 5539 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5540 if (!tag_id) 5541 return rte_flow_error_set(error, ENOMEM, 5542 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5543 "Failed to allocate meter flow id."); 5544 flow_id = tag_id - 1; 5545 flow_id_bits = (!flow_id) ? 1 : 5546 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5547 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5548 mtr_reg_bits) { 5549 mlx5_ipool_free(fm->flow_ipool, tag_id); 5550 return rte_flow_error_set(error, EINVAL, 5551 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5552 "Meter flow id exceeds max limit."); 5553 } 5554 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5555 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5556 } 5557 /* Build tag actions and items for meter_id/meter flow_id. */ 5558 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5559 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5560 tag_item_mask = tag_item_spec + 1; 5561 /* Both flow_id and meter_id share the same register. */ 5562 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5563 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5564 0, error), 5565 .offset = mtr_id_offset, 5566 .length = mtr_reg_bits, 5567 .data = flow->meter, 5568 }; 5569 /* 5570 * The color Reg bits used by flow_id are growing from 5571 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5572 */ 5573 for (shift = 0; shift < flow_id_bits; shift++) 5574 flow_id_reversed = (flow_id_reversed << 1) | 5575 ((flow_id >> shift) & 0x1); 5576 set_tag->data |= 5577 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5578 tag_item_spec->id = set_tag->id; 5579 tag_item_spec->data = set_tag->data << mtr_id_offset; 5580 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5581 tag_action->type = (enum rte_flow_action_type) 5582 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5583 tag_action->conf = set_tag; 5584 tag_item->type = (enum rte_flow_item_type) 5585 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5586 tag_item->spec = tag_item_spec; 5587 tag_item->last = NULL; 5588 tag_item->mask = tag_item_mask; 5589 exit: 5590 if (mtr_flow_id) 5591 *mtr_flow_id = tag_id; 5592 return 0; 5593 } 5594 5595 /** 5596 * Split action list having QUEUE/RSS for metadata register copy. 5597 * 5598 * Once Q/RSS action is detected in user's action list, the flow action 5599 * should be split in order to copy metadata registers, which will happen in 5600 * RX_CP_TBL like, 5601 * - CQE->flow_tag := reg_c[1] (MARK) 5602 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5603 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5604 * This is because the last action of each flow must be a terminal action 5605 * (QUEUE, RSS or DROP). 5606 * 5607 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5608 * stored and kept in the mlx5_flow structure per each sub_flow. 5609 * 5610 * The Q/RSS action is replaced with, 5611 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5612 * And the following JUMP action is added at the end, 5613 * - JUMP, to RX_CP_TBL. 5614 * 5615 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5616 * flow_create_split_metadata() routine. The flow will look like, 5617 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5618 * 5619 * @param dev 5620 * Pointer to Ethernet device. 5621 * @param[out] split_actions 5622 * Pointer to store split actions to jump to CP_TBL. 5623 * @param[in] actions 5624 * Pointer to the list of original flow actions. 5625 * @param[in] qrss 5626 * Pointer to the Q/RSS action. 5627 * @param[in] actions_n 5628 * Number of original actions. 5629 * @param[in] mtr_sfx 5630 * Check if it is in meter suffix table. 5631 * @param[out] error 5632 * Perform verbose error reporting if not NULL. 5633 * 5634 * @return 5635 * non-zero unique flow_id on success, otherwise 0 and 5636 * error/rte_error are set. 5637 */ 5638 static uint32_t 5639 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5640 struct rte_flow_action *split_actions, 5641 const struct rte_flow_action *actions, 5642 const struct rte_flow_action *qrss, 5643 int actions_n, int mtr_sfx, 5644 struct rte_flow_error *error) 5645 { 5646 struct mlx5_priv *priv = dev->data->dev_private; 5647 struct mlx5_rte_flow_action_set_tag *set_tag; 5648 struct rte_flow_action_jump *jump; 5649 const int qrss_idx = qrss - actions; 5650 uint32_t flow_id = 0; 5651 int ret = 0; 5652 5653 /* 5654 * Given actions will be split 5655 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5656 * - Add jump to mreg CP_TBL. 5657 * As a result, there will be one more action. 5658 */ 5659 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5660 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5661 ++actions_n; 5662 set_tag = (void *)(split_actions + actions_n); 5663 /* 5664 * If we are not the meter suffix flow, add the tag action. 5665 * Since meter suffix flow already has the tag added. 5666 */ 5667 if (!mtr_sfx) { 5668 /* 5669 * Allocate the new subflow ID. This one is unique within 5670 * device and not shared with representors. Otherwise, 5671 * we would have to resolve multi-thread access synch 5672 * issue. Each flow on the shared device is appended 5673 * with source vport identifier, so the resulting 5674 * flows will be unique in the shared (by master and 5675 * representors) domain even if they have coinciding 5676 * IDs. 5677 */ 5678 mlx5_ipool_malloc(priv->sh->ipool 5679 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5680 if (!flow_id) 5681 return rte_flow_error_set(error, ENOMEM, 5682 RTE_FLOW_ERROR_TYPE_ACTION, 5683 NULL, "can't allocate id " 5684 "for split Q/RSS subflow"); 5685 /* Internal SET_TAG action to set flow ID. */ 5686 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5687 .data = flow_id, 5688 }; 5689 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5690 if (ret < 0) 5691 return ret; 5692 set_tag->id = ret; 5693 /* Construct new actions array. */ 5694 /* Replace QUEUE/RSS action. */ 5695 split_actions[qrss_idx] = (struct rte_flow_action){ 5696 .type = (enum rte_flow_action_type) 5697 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5698 .conf = set_tag, 5699 }; 5700 } else { 5701 /* 5702 * If we are the suffix flow of meter, tag already exist. 5703 * Set the QUEUE/RSS action to void. 5704 */ 5705 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5706 } 5707 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5708 jump = (void *)(set_tag + 1); 5709 *jump = (struct rte_flow_action_jump){ 5710 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5711 }; 5712 split_actions[actions_n - 2] = (struct rte_flow_action){ 5713 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5714 .conf = jump, 5715 }; 5716 split_actions[actions_n - 1] = (struct rte_flow_action){ 5717 .type = RTE_FLOW_ACTION_TYPE_END, 5718 }; 5719 return flow_id; 5720 } 5721 5722 /** 5723 * Extend the given action list for Tx metadata copy. 5724 * 5725 * Copy the given action list to the ext_actions and add flow metadata register 5726 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5727 * 5728 * @param[out] ext_actions 5729 * Pointer to the extended action list. 5730 * @param[in] actions 5731 * Pointer to the list of actions. 5732 * @param[in] actions_n 5733 * Number of actions in the list. 5734 * @param[out] error 5735 * Perform verbose error reporting if not NULL. 5736 * @param[in] encap_idx 5737 * The encap action index. 5738 * 5739 * @return 5740 * 0 on success, negative value otherwise 5741 */ 5742 static int 5743 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5744 struct rte_flow_action *ext_actions, 5745 const struct rte_flow_action *actions, 5746 int actions_n, struct rte_flow_error *error, 5747 int encap_idx) 5748 { 5749 struct mlx5_flow_action_copy_mreg *cp_mreg = 5750 (struct mlx5_flow_action_copy_mreg *) 5751 (ext_actions + actions_n + 1); 5752 int ret; 5753 5754 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5755 if (ret < 0) 5756 return ret; 5757 cp_mreg->dst = ret; 5758 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5759 if (ret < 0) 5760 return ret; 5761 cp_mreg->src = ret; 5762 if (encap_idx != 0) 5763 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5764 if (encap_idx == actions_n - 1) { 5765 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5766 .type = (enum rte_flow_action_type) 5767 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5768 .conf = cp_mreg, 5769 }; 5770 ext_actions[actions_n] = (struct rte_flow_action){ 5771 .type = RTE_FLOW_ACTION_TYPE_END, 5772 }; 5773 } else { 5774 ext_actions[encap_idx] = (struct rte_flow_action){ 5775 .type = (enum rte_flow_action_type) 5776 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5777 .conf = cp_mreg, 5778 }; 5779 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5780 sizeof(*ext_actions) * (actions_n - encap_idx)); 5781 } 5782 return 0; 5783 } 5784 5785 /** 5786 * Check the match action from the action list. 5787 * 5788 * @param[in] actions 5789 * Pointer to the list of actions. 5790 * @param[in] attr 5791 * Flow rule attributes. 5792 * @param[in] action 5793 * The action to be check if exist. 5794 * @param[out] match_action_pos 5795 * Pointer to the position of the matched action if exists, otherwise is -1. 5796 * @param[out] qrss_action_pos 5797 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5798 * @param[out] modify_after_mirror 5799 * Pointer to the flag of modify action after FDB mirroring. 5800 * 5801 * @return 5802 * > 0 the total number of actions. 5803 * 0 if not found match action in action list. 5804 */ 5805 static int 5806 flow_check_match_action(const struct rte_flow_action actions[], 5807 const struct rte_flow_attr *attr, 5808 enum rte_flow_action_type action, 5809 int *match_action_pos, int *qrss_action_pos, 5810 int *modify_after_mirror) 5811 { 5812 const struct rte_flow_action_sample *sample; 5813 const struct rte_flow_action_raw_decap *decap; 5814 int actions_n = 0; 5815 uint32_t ratio = 0; 5816 int sub_type = 0; 5817 int flag = 0; 5818 int fdb_mirror = 0; 5819 5820 *match_action_pos = -1; 5821 *qrss_action_pos = -1; 5822 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5823 if (actions->type == action) { 5824 flag = 1; 5825 *match_action_pos = actions_n; 5826 } 5827 switch (actions->type) { 5828 case RTE_FLOW_ACTION_TYPE_QUEUE: 5829 case RTE_FLOW_ACTION_TYPE_RSS: 5830 *qrss_action_pos = actions_n; 5831 break; 5832 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5833 sample = actions->conf; 5834 ratio = sample->ratio; 5835 sub_type = ((const struct rte_flow_action *) 5836 (sample->actions))->type; 5837 if (ratio == 1 && attr->transfer) 5838 fdb_mirror = 1; 5839 break; 5840 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5841 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5842 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5843 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5844 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5845 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5846 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5847 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5848 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5849 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5850 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5851 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5852 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5853 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5854 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5855 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5856 case RTE_FLOW_ACTION_TYPE_FLAG: 5857 case RTE_FLOW_ACTION_TYPE_MARK: 5858 case RTE_FLOW_ACTION_TYPE_SET_META: 5859 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5860 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5861 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5862 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5863 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5864 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5865 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5866 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5867 case RTE_FLOW_ACTION_TYPE_METER: 5868 if (fdb_mirror) 5869 *modify_after_mirror = 1; 5870 break; 5871 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5872 decap = actions->conf; 5873 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5874 ; 5875 actions_n++; 5876 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5877 const struct rte_flow_action_raw_encap *encap = 5878 actions->conf; 5879 if (decap->size <= 5880 MLX5_ENCAPSULATION_DECISION_SIZE && 5881 encap->size > 5882 MLX5_ENCAPSULATION_DECISION_SIZE) 5883 /* L3 encap. */ 5884 break; 5885 } 5886 if (fdb_mirror) 5887 *modify_after_mirror = 1; 5888 break; 5889 default: 5890 break; 5891 } 5892 actions_n++; 5893 } 5894 if (flag && fdb_mirror && !*modify_after_mirror) { 5895 /* FDB mirroring uses the destination array to implement 5896 * instead of FLOW_SAMPLER object. 5897 */ 5898 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5899 flag = 0; 5900 } 5901 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5902 return flag ? actions_n + 1 : 0; 5903 } 5904 5905 #define SAMPLE_SUFFIX_ITEM 3 5906 5907 /** 5908 * Split the sample flow. 5909 * 5910 * As sample flow will split to two sub flow, sample flow with 5911 * sample action, the other actions will move to new suffix flow. 5912 * 5913 * Also add unique tag id with tag action in the sample flow, 5914 * the same tag id will be as match in the suffix flow. 5915 * 5916 * @param dev 5917 * Pointer to Ethernet device. 5918 * @param[in] add_tag 5919 * Add extra tag action flag. 5920 * @param[out] sfx_items 5921 * Suffix flow match items (list terminated by the END pattern item). 5922 * @param[in] actions 5923 * Associated actions (list terminated by the END action). 5924 * @param[out] actions_sfx 5925 * Suffix flow actions. 5926 * @param[out] actions_pre 5927 * Prefix flow actions. 5928 * @param[in] actions_n 5929 * The total number of actions. 5930 * @param[in] sample_action_pos 5931 * The sample action position. 5932 * @param[in] qrss_action_pos 5933 * The Queue/RSS action position. 5934 * @param[in] jump_table 5935 * Add extra jump action flag. 5936 * @param[out] error 5937 * Perform verbose error reporting if not NULL. 5938 * 5939 * @return 5940 * 0 on success, or unique flow_id, a negative errno value 5941 * otherwise and rte_errno is set. 5942 */ 5943 static int 5944 flow_sample_split_prep(struct rte_eth_dev *dev, 5945 int add_tag, 5946 const struct rte_flow_item items[], 5947 struct rte_flow_item sfx_items[], 5948 const struct rte_flow_action actions[], 5949 struct rte_flow_action actions_sfx[], 5950 struct rte_flow_action actions_pre[], 5951 int actions_n, 5952 int sample_action_pos, 5953 int qrss_action_pos, 5954 int jump_table, 5955 struct rte_flow_error *error) 5956 { 5957 struct mlx5_priv *priv = dev->data->dev_private; 5958 struct mlx5_rte_flow_action_set_tag *set_tag; 5959 struct mlx5_rte_flow_item_tag *tag_spec; 5960 struct mlx5_rte_flow_item_tag *tag_mask; 5961 struct rte_flow_action_jump *jump_action; 5962 uint32_t tag_id = 0; 5963 int append_index = 0; 5964 int set_tag_idx = -1; 5965 int index; 5966 int ret; 5967 5968 if (sample_action_pos < 0) 5969 return rte_flow_error_set(error, EINVAL, 5970 RTE_FLOW_ERROR_TYPE_ACTION, 5971 NULL, "invalid position of sample " 5972 "action in list"); 5973 /* Prepare the actions for prefix and suffix flow. */ 5974 if (add_tag) { 5975 /* Update the new added tag action index preceding 5976 * the PUSH_VLAN or ENCAP action. 5977 */ 5978 const struct rte_flow_action_raw_encap *raw_encap; 5979 const struct rte_flow_action *action = actions; 5980 int encap_idx; 5981 int action_idx = 0; 5982 int raw_decap_idx = -1; 5983 int push_vlan_idx = -1; 5984 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 5985 switch (action->type) { 5986 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5987 raw_decap_idx = action_idx; 5988 break; 5989 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5990 raw_encap = action->conf; 5991 if (raw_encap->size > 5992 MLX5_ENCAPSULATION_DECISION_SIZE) { 5993 encap_idx = raw_decap_idx != -1 ? 5994 raw_decap_idx : action_idx; 5995 if (encap_idx < sample_action_pos && 5996 push_vlan_idx == -1) 5997 set_tag_idx = encap_idx; 5998 } 5999 break; 6000 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6001 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6002 encap_idx = action_idx; 6003 if (encap_idx < sample_action_pos && 6004 push_vlan_idx == -1) 6005 set_tag_idx = encap_idx; 6006 break; 6007 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6008 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6009 push_vlan_idx = action_idx; 6010 if (push_vlan_idx < sample_action_pos) 6011 set_tag_idx = action_idx; 6012 break; 6013 default: 6014 break; 6015 } 6016 action_idx++; 6017 } 6018 } 6019 /* Prepare the actions for prefix and suffix flow. */ 6020 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6021 index = qrss_action_pos; 6022 /* Put the preceding the Queue/RSS action into prefix flow. */ 6023 if (index != 0) 6024 memcpy(actions_pre, actions, 6025 sizeof(struct rte_flow_action) * index); 6026 /* Put others preceding the sample action into prefix flow. */ 6027 if (sample_action_pos > index + 1) 6028 memcpy(actions_pre + index, actions + index + 1, 6029 sizeof(struct rte_flow_action) * 6030 (sample_action_pos - index - 1)); 6031 index = sample_action_pos - 1; 6032 /* Put Queue/RSS action into Suffix flow. */ 6033 memcpy(actions_sfx, actions + qrss_action_pos, 6034 sizeof(struct rte_flow_action)); 6035 actions_sfx++; 6036 } else if (add_tag && set_tag_idx >= 0) { 6037 if (set_tag_idx > 0) 6038 memcpy(actions_pre, actions, 6039 sizeof(struct rte_flow_action) * set_tag_idx); 6040 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6041 sizeof(struct rte_flow_action) * 6042 (sample_action_pos - set_tag_idx)); 6043 index = sample_action_pos; 6044 } else { 6045 index = sample_action_pos; 6046 if (index != 0) 6047 memcpy(actions_pre, actions, 6048 sizeof(struct rte_flow_action) * index); 6049 } 6050 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6051 * For CX6DX and above, metadata registers Cx preserve their value, 6052 * add an extra tag action for NIC-RX and E-Switch Domain. 6053 */ 6054 if (add_tag) { 6055 /* Prepare the prefix tag action. */ 6056 append_index++; 6057 set_tag = (void *)(actions_pre + actions_n + append_index); 6058 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6059 /* Trust VF/SF on CX5 not supported meter so that the reserved 6060 * metadata regC is REG_NON, back to use application tag 6061 * index 0. 6062 */ 6063 if (unlikely(ret == REG_NON)) 6064 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6065 if (ret < 0) 6066 return ret; 6067 mlx5_ipool_malloc(priv->sh->ipool 6068 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6069 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6070 .id = ret, 6071 .data = tag_id, 6072 }; 6073 /* Prepare the suffix subflow items. */ 6074 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6075 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6076 memcpy(sfx_items, items, sizeof(*sfx_items)); 6077 sfx_items++; 6078 } 6079 } 6080 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6081 tag_spec->data = tag_id; 6082 tag_spec->id = set_tag->id; 6083 tag_mask = tag_spec + 1; 6084 tag_mask->data = UINT32_MAX; 6085 sfx_items[0] = (struct rte_flow_item){ 6086 .type = (enum rte_flow_item_type) 6087 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6088 .spec = tag_spec, 6089 .last = NULL, 6090 .mask = tag_mask, 6091 }; 6092 sfx_items[1] = (struct rte_flow_item){ 6093 .type = (enum rte_flow_item_type) 6094 RTE_FLOW_ITEM_TYPE_END, 6095 }; 6096 /* Prepare the tag action in prefix subflow. */ 6097 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6098 actions_pre[set_tag_idx] = 6099 (struct rte_flow_action){ 6100 .type = (enum rte_flow_action_type) 6101 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6102 .conf = set_tag, 6103 }; 6104 /* Update next sample position due to add one tag action */ 6105 index += 1; 6106 } 6107 /* Copy the sample action into prefix flow. */ 6108 memcpy(actions_pre + index, actions + sample_action_pos, 6109 sizeof(struct rte_flow_action)); 6110 index += 1; 6111 /* For the modify action after the sample action in E-Switch mirroring, 6112 * Add the extra jump action in prefix subflow and jump into the next 6113 * table, then do the modify action in the new table. 6114 */ 6115 if (jump_table) { 6116 /* Prepare the prefix jump action. */ 6117 append_index++; 6118 jump_action = (void *)(actions_pre + actions_n + append_index); 6119 jump_action->group = jump_table; 6120 actions_pre[index++] = 6121 (struct rte_flow_action){ 6122 .type = (enum rte_flow_action_type) 6123 RTE_FLOW_ACTION_TYPE_JUMP, 6124 .conf = jump_action, 6125 }; 6126 } 6127 actions_pre[index] = (struct rte_flow_action){ 6128 .type = (enum rte_flow_action_type) 6129 RTE_FLOW_ACTION_TYPE_END, 6130 }; 6131 /* Put the actions after sample into Suffix flow. */ 6132 memcpy(actions_sfx, actions + sample_action_pos + 1, 6133 sizeof(struct rte_flow_action) * 6134 (actions_n - sample_action_pos - 1)); 6135 return tag_id; 6136 } 6137 6138 /** 6139 * The splitting for metadata feature. 6140 * 6141 * - Q/RSS action on NIC Rx should be split in order to pass by 6142 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6143 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6144 * 6145 * - All the actions on NIC Tx should have a mreg copy action to 6146 * copy reg_a from WQE to reg_c[0]. 6147 * 6148 * @param dev 6149 * Pointer to Ethernet device. 6150 * @param[in] flow 6151 * Parent flow structure pointer. 6152 * @param[in] attr 6153 * Flow rule attributes. 6154 * @param[in] items 6155 * Pattern specification (list terminated by the END pattern item). 6156 * @param[in] actions 6157 * Associated actions (list terminated by the END action). 6158 * @param[in] flow_split_info 6159 * Pointer to flow split info structure. 6160 * @param[out] error 6161 * Perform verbose error reporting if not NULL. 6162 * @return 6163 * 0 on success, negative value otherwise 6164 */ 6165 static int 6166 flow_create_split_metadata(struct rte_eth_dev *dev, 6167 struct rte_flow *flow, 6168 const struct rte_flow_attr *attr, 6169 const struct rte_flow_item items[], 6170 const struct rte_flow_action actions[], 6171 struct mlx5_flow_split_info *flow_split_info, 6172 struct rte_flow_error *error) 6173 { 6174 struct mlx5_priv *priv = dev->data->dev_private; 6175 struct mlx5_sh_config *config = &priv->sh->config; 6176 const struct rte_flow_action *qrss = NULL; 6177 struct rte_flow_action *ext_actions = NULL; 6178 struct mlx5_flow *dev_flow = NULL; 6179 uint32_t qrss_id = 0; 6180 int mtr_sfx = 0; 6181 size_t act_size; 6182 int actions_n; 6183 int encap_idx; 6184 int ret; 6185 6186 /* Check whether extensive metadata feature is engaged. */ 6187 if (!config->dv_flow_en || 6188 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6189 !mlx5_flow_ext_mreg_supported(dev)) 6190 return flow_create_split_inner(dev, flow, NULL, attr, items, 6191 actions, flow_split_info, error); 6192 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6193 &encap_idx); 6194 if (qrss) { 6195 /* Exclude hairpin flows from splitting. */ 6196 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6197 const struct rte_flow_action_queue *queue; 6198 6199 queue = qrss->conf; 6200 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6201 qrss = NULL; 6202 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6203 const struct rte_flow_action_rss *rss; 6204 6205 rss = qrss->conf; 6206 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6207 qrss = NULL; 6208 } 6209 } 6210 if (qrss) { 6211 /* Check if it is in meter suffix table. */ 6212 mtr_sfx = attr->group == 6213 ((attr->transfer && priv->fdb_def_rule) ? 6214 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6215 MLX5_FLOW_TABLE_LEVEL_METER); 6216 /* 6217 * Q/RSS action on NIC Rx should be split in order to pass by 6218 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6219 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6220 */ 6221 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6222 sizeof(struct rte_flow_action_set_tag) + 6223 sizeof(struct rte_flow_action_jump); 6224 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6225 SOCKET_ID_ANY); 6226 if (!ext_actions) 6227 return rte_flow_error_set(error, ENOMEM, 6228 RTE_FLOW_ERROR_TYPE_ACTION, 6229 NULL, "no memory to split " 6230 "metadata flow"); 6231 /* 6232 * Create the new actions list with removed Q/RSS action 6233 * and appended set tag and jump to register copy table 6234 * (RX_CP_TBL). We should preallocate unique tag ID here 6235 * in advance, because it is needed for set tag action. 6236 */ 6237 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6238 qrss, actions_n, 6239 mtr_sfx, error); 6240 if (!mtr_sfx && !qrss_id) { 6241 ret = -rte_errno; 6242 goto exit; 6243 } 6244 } else if (attr->egress && !attr->transfer) { 6245 /* 6246 * All the actions on NIC Tx should have a metadata register 6247 * copy action to copy reg_a from WQE to reg_c[meta] 6248 */ 6249 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6250 sizeof(struct mlx5_flow_action_copy_mreg); 6251 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6252 SOCKET_ID_ANY); 6253 if (!ext_actions) 6254 return rte_flow_error_set(error, ENOMEM, 6255 RTE_FLOW_ERROR_TYPE_ACTION, 6256 NULL, "no memory to split " 6257 "metadata flow"); 6258 /* Create the action list appended with copy register. */ 6259 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6260 actions_n, error, encap_idx); 6261 if (ret < 0) 6262 goto exit; 6263 } 6264 /* Add the unmodified original or prefix subflow. */ 6265 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6266 items, ext_actions ? ext_actions : 6267 actions, flow_split_info, error); 6268 if (ret < 0) 6269 goto exit; 6270 MLX5_ASSERT(dev_flow); 6271 if (qrss) { 6272 const struct rte_flow_attr q_attr = { 6273 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6274 .ingress = 1, 6275 }; 6276 /* Internal PMD action to set register. */ 6277 struct mlx5_rte_flow_item_tag q_tag_spec = { 6278 .data = qrss_id, 6279 .id = REG_NON, 6280 }; 6281 struct rte_flow_item q_items[] = { 6282 { 6283 .type = (enum rte_flow_item_type) 6284 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6285 .spec = &q_tag_spec, 6286 .last = NULL, 6287 .mask = NULL, 6288 }, 6289 { 6290 .type = RTE_FLOW_ITEM_TYPE_END, 6291 }, 6292 }; 6293 struct rte_flow_action q_actions[] = { 6294 { 6295 .type = qrss->type, 6296 .conf = qrss->conf, 6297 }, 6298 { 6299 .type = RTE_FLOW_ACTION_TYPE_END, 6300 }, 6301 }; 6302 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6303 6304 /* 6305 * Configure the tag item only if there is no meter subflow. 6306 * Since tag is already marked in the meter suffix subflow 6307 * we can just use the meter suffix items as is. 6308 */ 6309 if (qrss_id) { 6310 /* Not meter subflow. */ 6311 MLX5_ASSERT(!mtr_sfx); 6312 /* 6313 * Put unique id in prefix flow due to it is destroyed 6314 * after suffix flow and id will be freed after there 6315 * is no actual flows with this id and identifier 6316 * reallocation becomes possible (for example, for 6317 * other flows in other threads). 6318 */ 6319 dev_flow->handle->split_flow_id = qrss_id; 6320 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6321 error); 6322 if (ret < 0) 6323 goto exit; 6324 q_tag_spec.id = ret; 6325 } 6326 dev_flow = NULL; 6327 /* Add suffix subflow to execute Q/RSS. */ 6328 flow_split_info->prefix_layers = layers; 6329 flow_split_info->prefix_mark = 0; 6330 flow_split_info->table_id = 0; 6331 ret = flow_create_split_inner(dev, flow, &dev_flow, 6332 &q_attr, mtr_sfx ? items : 6333 q_items, q_actions, 6334 flow_split_info, error); 6335 if (ret < 0) 6336 goto exit; 6337 /* qrss ID should be freed if failed. */ 6338 qrss_id = 0; 6339 MLX5_ASSERT(dev_flow); 6340 } 6341 6342 exit: 6343 /* 6344 * We do not destroy the partially created sub_flows in case of error. 6345 * These ones are included into parent flow list and will be destroyed 6346 * by flow_drv_destroy. 6347 */ 6348 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6349 qrss_id); 6350 mlx5_free(ext_actions); 6351 return ret; 6352 } 6353 6354 /** 6355 * Create meter internal drop flow with the original pattern. 6356 * 6357 * @param dev 6358 * Pointer to Ethernet device. 6359 * @param[in] flow 6360 * Parent flow structure pointer. 6361 * @param[in] attr 6362 * Flow rule attributes. 6363 * @param[in] items 6364 * Pattern specification (list terminated by the END pattern item). 6365 * @param[in] flow_split_info 6366 * Pointer to flow split info structure. 6367 * @param[in] fm 6368 * Pointer to flow meter structure. 6369 * @param[out] error 6370 * Perform verbose error reporting if not NULL. 6371 * @return 6372 * 0 on success, negative value otherwise 6373 */ 6374 static uint32_t 6375 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6376 struct rte_flow *flow, 6377 const struct rte_flow_attr *attr, 6378 const struct rte_flow_item items[], 6379 struct mlx5_flow_split_info *flow_split_info, 6380 struct mlx5_flow_meter_info *fm, 6381 struct rte_flow_error *error) 6382 { 6383 struct mlx5_flow *dev_flow = NULL; 6384 struct rte_flow_attr drop_attr = *attr; 6385 struct rte_flow_action drop_actions[3]; 6386 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6387 6388 MLX5_ASSERT(fm->drop_cnt); 6389 drop_actions[0].type = 6390 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6391 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6392 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6393 drop_actions[1].conf = NULL; 6394 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6395 drop_actions[2].conf = NULL; 6396 drop_split_info.external = false; 6397 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6398 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6399 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6400 return flow_create_split_inner(dev, flow, &dev_flow, 6401 &drop_attr, items, drop_actions, 6402 &drop_split_info, error); 6403 } 6404 6405 /** 6406 * The splitting for meter feature. 6407 * 6408 * - The meter flow will be split to two flows as prefix and 6409 * suffix flow. The packets make sense only it pass the prefix 6410 * meter action. 6411 * 6412 * - Reg_C_5 is used for the packet to match betweend prefix and 6413 * suffix flow. 6414 * 6415 * @param dev 6416 * Pointer to Ethernet device. 6417 * @param[in] flow 6418 * Parent flow structure pointer. 6419 * @param[in] attr 6420 * Flow rule attributes. 6421 * @param[in] items 6422 * Pattern specification (list terminated by the END pattern item). 6423 * @param[in] actions 6424 * Associated actions (list terminated by the END action). 6425 * @param[in] flow_split_info 6426 * Pointer to flow split info structure. 6427 * @param[out] error 6428 * Perform verbose error reporting if not NULL. 6429 * @return 6430 * 0 on success, negative value otherwise 6431 */ 6432 static int 6433 flow_create_split_meter(struct rte_eth_dev *dev, 6434 struct rte_flow *flow, 6435 const struct rte_flow_attr *attr, 6436 const struct rte_flow_item items[], 6437 const struct rte_flow_action actions[], 6438 struct mlx5_flow_split_info *flow_split_info, 6439 struct rte_flow_error *error) 6440 { 6441 struct mlx5_priv *priv = dev->data->dev_private; 6442 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6443 struct rte_flow_action *sfx_actions = NULL; 6444 struct rte_flow_action *pre_actions = NULL; 6445 struct rte_flow_item *sfx_items = NULL; 6446 struct mlx5_flow *dev_flow = NULL; 6447 struct rte_flow_attr sfx_attr = *attr; 6448 struct mlx5_flow_meter_info *fm = NULL; 6449 uint8_t skip_scale_restore; 6450 bool has_mtr = false; 6451 bool has_modify = false; 6452 bool set_mtr_reg = true; 6453 bool is_mtr_hierarchy = false; 6454 uint32_t meter_id = 0; 6455 uint32_t mtr_idx = 0; 6456 uint32_t mtr_flow_id = 0; 6457 size_t act_size; 6458 size_t item_size; 6459 int actions_n = 0; 6460 int ret = 0; 6461 6462 if (priv->mtr_en) 6463 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6464 &has_modify, &meter_id); 6465 if (has_mtr) { 6466 if (flow->meter) { 6467 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6468 if (!fm) 6469 return rte_flow_error_set(error, EINVAL, 6470 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6471 NULL, "Meter not found."); 6472 } else { 6473 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6474 if (!fm) 6475 return rte_flow_error_set(error, EINVAL, 6476 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6477 NULL, "Meter not found."); 6478 ret = mlx5_flow_meter_attach(priv, fm, 6479 &sfx_attr, error); 6480 if (ret) 6481 return -rte_errno; 6482 flow->meter = mtr_idx; 6483 } 6484 MLX5_ASSERT(wks); 6485 wks->fm = fm; 6486 if (!fm->def_policy) { 6487 wks->policy = mlx5_flow_meter_policy_find(dev, 6488 fm->policy_id, 6489 NULL); 6490 MLX5_ASSERT(wks->policy); 6491 if (wks->policy->mark) 6492 wks->mark = 1; 6493 if (wks->policy->is_hierarchy) { 6494 wks->final_policy = 6495 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6496 wks->policy); 6497 if (!wks->final_policy) 6498 return rte_flow_error_set(error, 6499 EINVAL, 6500 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6501 "Failed to find terminal policy of hierarchy."); 6502 is_mtr_hierarchy = true; 6503 } 6504 } 6505 /* 6506 * If it isn't default-policy Meter, and 6507 * 1. Not meter hierarchy and there's no action in flow to change 6508 * packet (modify/encap/decap etc.), OR 6509 * 2. No drop count needed for this meter. 6510 * Then no need to use regC to save meter id anymore. 6511 */ 6512 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6513 set_mtr_reg = false; 6514 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6515 #define METER_PREFIX_ACTION 7 6516 act_size = (sizeof(struct rte_flow_action) * 6517 (actions_n + METER_PREFIX_ACTION)) + 6518 sizeof(struct mlx5_rte_flow_action_set_tag); 6519 /* Suffix items: tag, vlan, port id, end. */ 6520 #define METER_SUFFIX_ITEM 4 6521 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6522 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6523 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6524 0, SOCKET_ID_ANY); 6525 if (!sfx_actions) 6526 return rte_flow_error_set(error, ENOMEM, 6527 RTE_FLOW_ERROR_TYPE_ACTION, 6528 NULL, "no memory to split " 6529 "meter flow"); 6530 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6531 act_size); 6532 /* There's no suffix flow for meter of non-default policy. */ 6533 if (!fm->def_policy) 6534 pre_actions = sfx_actions + 1; 6535 else 6536 pre_actions = sfx_actions + actions_n; 6537 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6538 items, sfx_items, actions, 6539 sfx_actions, pre_actions, 6540 (set_mtr_reg ? &mtr_flow_id : NULL), 6541 error); 6542 if (ret) { 6543 ret = -rte_errno; 6544 goto exit; 6545 } 6546 /* Add the prefix subflow. */ 6547 skip_scale_restore = flow_split_info->skip_scale; 6548 flow_split_info->skip_scale |= 6549 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6550 ret = flow_create_split_inner(dev, flow, &dev_flow, 6551 attr, items, pre_actions, 6552 flow_split_info, error); 6553 flow_split_info->skip_scale = skip_scale_restore; 6554 if (ret) { 6555 if (mtr_flow_id) 6556 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6557 ret = -rte_errno; 6558 goto exit; 6559 } 6560 if (mtr_flow_id) { 6561 dev_flow->handle->split_flow_id = mtr_flow_id; 6562 dev_flow->handle->is_meter_flow_id = 1; 6563 } 6564 if (!fm->def_policy) { 6565 if (!set_mtr_reg && fm->drop_cnt) 6566 ret = 6567 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6568 &sfx_attr, items, 6569 flow_split_info, 6570 fm, error); 6571 goto exit; 6572 } 6573 /* Setting the sfx group atrr. */ 6574 sfx_attr.group = sfx_attr.transfer ? 6575 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6576 MLX5_FLOW_TABLE_LEVEL_METER; 6577 flow_split_info->prefix_layers = 6578 flow_get_prefix_layer_flags(dev_flow); 6579 flow_split_info->prefix_mark |= wks->mark; 6580 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6581 } 6582 /* Add the prefix subflow. */ 6583 ret = flow_create_split_metadata(dev, flow, 6584 &sfx_attr, sfx_items ? 6585 sfx_items : items, 6586 sfx_actions ? sfx_actions : actions, 6587 flow_split_info, error); 6588 exit: 6589 if (sfx_actions) 6590 mlx5_free(sfx_actions); 6591 return ret; 6592 } 6593 6594 /** 6595 * The splitting for sample feature. 6596 * 6597 * Once Sample action is detected in the action list, the flow actions should 6598 * be split into prefix sub flow and suffix sub flow. 6599 * 6600 * The original items remain in the prefix sub flow, all actions preceding the 6601 * sample action and the sample action itself will be copied to the prefix 6602 * sub flow, the actions following the sample action will be copied to the 6603 * suffix sub flow, Queue action always be located in the suffix sub flow. 6604 * 6605 * In order to make the packet from prefix sub flow matches with suffix sub 6606 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6607 * flow uses tag item with the unique flow id. 6608 * 6609 * @param dev 6610 * Pointer to Ethernet device. 6611 * @param[in] flow 6612 * Parent flow structure pointer. 6613 * @param[in] attr 6614 * Flow rule attributes. 6615 * @param[in] items 6616 * Pattern specification (list terminated by the END pattern item). 6617 * @param[in] actions 6618 * Associated actions (list terminated by the END action). 6619 * @param[in] flow_split_info 6620 * Pointer to flow split info structure. 6621 * @param[out] error 6622 * Perform verbose error reporting if not NULL. 6623 * @return 6624 * 0 on success, negative value otherwise 6625 */ 6626 static int 6627 flow_create_split_sample(struct rte_eth_dev *dev, 6628 struct rte_flow *flow, 6629 const struct rte_flow_attr *attr, 6630 const struct rte_flow_item items[], 6631 const struct rte_flow_action actions[], 6632 struct mlx5_flow_split_info *flow_split_info, 6633 struct rte_flow_error *error) 6634 { 6635 struct mlx5_priv *priv = dev->data->dev_private; 6636 struct rte_flow_action *sfx_actions = NULL; 6637 struct rte_flow_action *pre_actions = NULL; 6638 struct rte_flow_item *sfx_items = NULL; 6639 struct mlx5_flow *dev_flow = NULL; 6640 struct rte_flow_attr sfx_attr = *attr; 6641 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6642 struct mlx5_flow_dv_sample_resource *sample_res; 6643 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6644 struct mlx5_flow_tbl_resource *sfx_tbl; 6645 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6646 #endif 6647 size_t act_size; 6648 size_t item_size; 6649 uint32_t fdb_tx = 0; 6650 int32_t tag_id = 0; 6651 int actions_n = 0; 6652 int sample_action_pos; 6653 int qrss_action_pos; 6654 int add_tag = 0; 6655 int modify_after_mirror = 0; 6656 uint16_t jump_table = 0; 6657 const uint32_t next_ft_step = 1; 6658 int ret = 0; 6659 6660 if (priv->sampler_en) 6661 actions_n = flow_check_match_action(actions, attr, 6662 RTE_FLOW_ACTION_TYPE_SAMPLE, 6663 &sample_action_pos, &qrss_action_pos, 6664 &modify_after_mirror); 6665 if (actions_n) { 6666 /* The prefix actions must includes sample, tag, end. */ 6667 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6668 + sizeof(struct mlx5_rte_flow_action_set_tag); 6669 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6670 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6671 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6672 item_size), 0, SOCKET_ID_ANY); 6673 if (!sfx_actions) 6674 return rte_flow_error_set(error, ENOMEM, 6675 RTE_FLOW_ERROR_TYPE_ACTION, 6676 NULL, "no memory to split " 6677 "sample flow"); 6678 /* The representor_id is UINT16_MAX for uplink. */ 6679 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6680 /* 6681 * When reg_c_preserve is set, metadata registers Cx preserve 6682 * their value even through packet duplication. 6683 */ 6684 add_tag = (!fdb_tx || 6685 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6686 if (add_tag) 6687 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6688 + act_size); 6689 if (modify_after_mirror) 6690 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6691 next_ft_step; 6692 pre_actions = sfx_actions + actions_n; 6693 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6694 actions, sfx_actions, 6695 pre_actions, actions_n, 6696 sample_action_pos, 6697 qrss_action_pos, jump_table, 6698 error); 6699 if (tag_id < 0 || (add_tag && !tag_id)) { 6700 ret = -rte_errno; 6701 goto exit; 6702 } 6703 if (modify_after_mirror) 6704 flow_split_info->skip_scale = 6705 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6706 /* Add the prefix subflow. */ 6707 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6708 items, pre_actions, 6709 flow_split_info, error); 6710 if (ret) { 6711 ret = -rte_errno; 6712 goto exit; 6713 } 6714 dev_flow->handle->split_flow_id = tag_id; 6715 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6716 if (!modify_after_mirror) { 6717 /* Set the sfx group attr. */ 6718 sample_res = (struct mlx5_flow_dv_sample_resource *) 6719 dev_flow->dv.sample_res; 6720 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6721 sample_res->normal_path_tbl; 6722 sfx_tbl_data = container_of(sfx_tbl, 6723 struct mlx5_flow_tbl_data_entry, 6724 tbl); 6725 sfx_attr.group = sfx_attr.transfer ? 6726 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6727 } else { 6728 MLX5_ASSERT(attr->transfer); 6729 sfx_attr.group = jump_table; 6730 } 6731 flow_split_info->prefix_layers = 6732 flow_get_prefix_layer_flags(dev_flow); 6733 MLX5_ASSERT(wks); 6734 flow_split_info->prefix_mark |= wks->mark; 6735 /* Suffix group level already be scaled with factor, set 6736 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6737 * again in translation. 6738 */ 6739 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6740 #endif 6741 } 6742 /* Add the suffix subflow. */ 6743 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6744 sfx_items ? sfx_items : items, 6745 sfx_actions ? sfx_actions : actions, 6746 flow_split_info, error); 6747 exit: 6748 if (sfx_actions) 6749 mlx5_free(sfx_actions); 6750 return ret; 6751 } 6752 6753 /** 6754 * Split the flow to subflow set. The splitters might be linked 6755 * in the chain, like this: 6756 * flow_create_split_outer() calls: 6757 * flow_create_split_meter() calls: 6758 * flow_create_split_metadata(meter_subflow_0) calls: 6759 * flow_create_split_inner(metadata_subflow_0) 6760 * flow_create_split_inner(metadata_subflow_1) 6761 * flow_create_split_inner(metadata_subflow_2) 6762 * flow_create_split_metadata(meter_subflow_1) calls: 6763 * flow_create_split_inner(metadata_subflow_0) 6764 * flow_create_split_inner(metadata_subflow_1) 6765 * flow_create_split_inner(metadata_subflow_2) 6766 * 6767 * This provide flexible way to add new levels of flow splitting. 6768 * The all of successfully created subflows are included to the 6769 * parent flow dev_flow list. 6770 * 6771 * @param dev 6772 * Pointer to Ethernet device. 6773 * @param[in] flow 6774 * Parent flow structure pointer. 6775 * @param[in] attr 6776 * Flow rule attributes. 6777 * @param[in] items 6778 * Pattern specification (list terminated by the END pattern item). 6779 * @param[in] actions 6780 * Associated actions (list terminated by the END action). 6781 * @param[in] flow_split_info 6782 * Pointer to flow split info structure. 6783 * @param[out] error 6784 * Perform verbose error reporting if not NULL. 6785 * @return 6786 * 0 on success, negative value otherwise 6787 */ 6788 static int 6789 flow_create_split_outer(struct rte_eth_dev *dev, 6790 struct rte_flow *flow, 6791 const struct rte_flow_attr *attr, 6792 const struct rte_flow_item items[], 6793 const struct rte_flow_action actions[], 6794 struct mlx5_flow_split_info *flow_split_info, 6795 struct rte_flow_error *error) 6796 { 6797 int ret; 6798 6799 ret = flow_create_split_sample(dev, flow, attr, items, 6800 actions, flow_split_info, error); 6801 MLX5_ASSERT(ret <= 0); 6802 return ret; 6803 } 6804 6805 static inline struct mlx5_flow_tunnel * 6806 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6807 { 6808 struct mlx5_flow_tunnel *tunnel; 6809 6810 #pragma GCC diagnostic push 6811 #pragma GCC diagnostic ignored "-Wcast-qual" 6812 tunnel = (typeof(tunnel))flow->tunnel; 6813 #pragma GCC diagnostic pop 6814 6815 return tunnel; 6816 } 6817 6818 /** 6819 * Adjust flow RSS workspace if needed. 6820 * 6821 * @param wks 6822 * Pointer to thread flow work space. 6823 * @param rss_desc 6824 * Pointer to RSS descriptor. 6825 * @param[in] nrssq_num 6826 * New RSS queue number. 6827 * 6828 * @return 6829 * 0 on success, -1 otherwise and rte_errno is set. 6830 */ 6831 static int 6832 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6833 struct mlx5_flow_rss_desc *rss_desc, 6834 uint32_t nrssq_num) 6835 { 6836 if (likely(nrssq_num <= wks->rssq_num)) 6837 return 0; 6838 rss_desc->queue = realloc(rss_desc->queue, 6839 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6840 if (!rss_desc->queue) { 6841 rte_errno = ENOMEM; 6842 return -1; 6843 } 6844 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6845 return 0; 6846 } 6847 6848 /** 6849 * Create a flow and add it to @p list. 6850 * 6851 * @param dev 6852 * Pointer to Ethernet device. 6853 * @param list 6854 * Pointer to a TAILQ flow list. If this parameter NULL, 6855 * no list insertion occurred, flow is just created, 6856 * this is caller's responsibility to track the 6857 * created flow. 6858 * @param[in] attr 6859 * Flow rule attributes. 6860 * @param[in] items 6861 * Pattern specification (list terminated by the END pattern item). 6862 * @param[in] actions 6863 * Associated actions (list terminated by the END action). 6864 * @param[in] external 6865 * This flow rule is created by request external to PMD. 6866 * @param[out] error 6867 * Perform verbose error reporting if not NULL. 6868 * 6869 * @return 6870 * A flow index on success, 0 otherwise and rte_errno is set. 6871 */ 6872 static uint32_t 6873 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6874 const struct rte_flow_attr *attr, 6875 const struct rte_flow_item items[], 6876 const struct rte_flow_action original_actions[], 6877 bool external, struct rte_flow_error *error) 6878 { 6879 struct mlx5_priv *priv = dev->data->dev_private; 6880 struct rte_flow *flow = NULL; 6881 struct mlx5_flow *dev_flow; 6882 const struct rte_flow_action_rss *rss = NULL; 6883 struct mlx5_translated_action_handle 6884 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6885 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6886 union { 6887 struct mlx5_flow_expand_rss buf; 6888 uint8_t buffer[4096]; 6889 } expand_buffer; 6890 union { 6891 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6892 uint8_t buffer[2048]; 6893 } actions_rx; 6894 union { 6895 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6896 uint8_t buffer[2048]; 6897 } actions_hairpin_tx; 6898 union { 6899 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 6900 uint8_t buffer[2048]; 6901 } items_tx; 6902 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 6903 struct mlx5_flow_rss_desc *rss_desc; 6904 const struct rte_flow_action *p_actions_rx; 6905 uint32_t i; 6906 uint32_t idx = 0; 6907 int hairpin_flow; 6908 struct rte_flow_attr attr_tx = { .priority = 0 }; 6909 const struct rte_flow_action *actions; 6910 struct rte_flow_action *translated_actions = NULL; 6911 struct mlx5_flow_tunnel *tunnel; 6912 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 6913 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 6914 struct mlx5_flow_split_info flow_split_info = { 6915 .external = !!external, 6916 .skip_scale = 0, 6917 .flow_idx = 0, 6918 .prefix_mark = 0, 6919 .prefix_layers = 0, 6920 .table_id = 0 6921 }; 6922 int ret; 6923 6924 MLX5_ASSERT(wks); 6925 rss_desc = &wks->rss_desc; 6926 ret = flow_action_handles_translate(dev, original_actions, 6927 indir_actions, 6928 &indir_actions_n, 6929 &translated_actions, error); 6930 if (ret < 0) { 6931 MLX5_ASSERT(translated_actions == NULL); 6932 return 0; 6933 } 6934 actions = translated_actions ? translated_actions : original_actions; 6935 p_actions_rx = actions; 6936 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 6937 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 6938 external, hairpin_flow, error); 6939 if (ret < 0) 6940 goto error_before_hairpin_split; 6941 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 6942 if (!flow) { 6943 rte_errno = ENOMEM; 6944 goto error_before_hairpin_split; 6945 } 6946 if (hairpin_flow > 0) { 6947 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 6948 rte_errno = EINVAL; 6949 goto error_before_hairpin_split; 6950 } 6951 flow_hairpin_split(dev, actions, actions_rx.actions, 6952 actions_hairpin_tx.actions, items_tx.items, 6953 idx); 6954 p_actions_rx = actions_rx.actions; 6955 } 6956 flow_split_info.flow_idx = idx; 6957 flow->drv_type = flow_get_drv_type(dev, attr); 6958 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 6959 flow->drv_type < MLX5_FLOW_TYPE_MAX); 6960 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 6961 /* RSS Action only works on NIC RX domain */ 6962 if (attr->ingress && !attr->transfer) 6963 rss = flow_get_rss_action(dev, p_actions_rx); 6964 if (rss) { 6965 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 6966 return 0; 6967 /* 6968 * The following information is required by 6969 * mlx5_flow_hashfields_adjust() in advance. 6970 */ 6971 rss_desc->level = rss->level; 6972 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 6973 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 6974 } 6975 flow->dev_handles = 0; 6976 if (rss && rss->types) { 6977 unsigned int graph_root; 6978 6979 graph_root = find_graph_root(rss->level); 6980 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 6981 items, rss->types, 6982 mlx5_support_expansion, graph_root); 6983 MLX5_ASSERT(ret > 0 && 6984 (unsigned int)ret < sizeof(expand_buffer.buffer)); 6985 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 6986 for (i = 0; i < buf->entries; ++i) 6987 mlx5_dbg__print_pattern(buf->entry[i].pattern); 6988 } 6989 } else { 6990 buf->entries = 1; 6991 buf->entry[0].pattern = (void *)(uintptr_t)items; 6992 } 6993 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 6994 indir_actions_n); 6995 for (i = 0; i < buf->entries; ++i) { 6996 /* Initialize flow split data. */ 6997 flow_split_info.prefix_layers = 0; 6998 flow_split_info.prefix_mark = 0; 6999 flow_split_info.skip_scale = 0; 7000 /* 7001 * The splitter may create multiple dev_flows, 7002 * depending on configuration. In the simplest 7003 * case it just creates unmodified original flow. 7004 */ 7005 ret = flow_create_split_outer(dev, flow, attr, 7006 buf->entry[i].pattern, 7007 p_actions_rx, &flow_split_info, 7008 error); 7009 if (ret < 0) 7010 goto error; 7011 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7012 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7013 p_actions_rx, 7014 idx, 7015 wks->flows[0].tunnel, 7016 &default_miss_ctx, 7017 error); 7018 if (ret < 0) { 7019 mlx5_free(default_miss_ctx.queue); 7020 goto error; 7021 } 7022 } 7023 } 7024 /* Create the tx flow. */ 7025 if (hairpin_flow) { 7026 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7027 attr_tx.ingress = 0; 7028 attr_tx.egress = 1; 7029 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7030 actions_hairpin_tx.actions, 7031 idx, error); 7032 if (!dev_flow) 7033 goto error; 7034 dev_flow->flow = flow; 7035 dev_flow->external = 0; 7036 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7037 dev_flow->handle, next); 7038 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7039 items_tx.items, 7040 actions_hairpin_tx.actions, error); 7041 if (ret < 0) 7042 goto error; 7043 } 7044 /* 7045 * Update the metadata register copy table. If extensive 7046 * metadata feature is enabled and registers are supported 7047 * we might create the extra rte_flow for each unique 7048 * MARK/FLAG action ID. 7049 * 7050 * The table is updated for ingress Flows only, because 7051 * the egress Flows belong to the different device and 7052 * copy table should be updated in peer NIC Rx domain. 7053 */ 7054 if (attr->ingress && 7055 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7056 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7057 if (ret) 7058 goto error; 7059 } 7060 /* 7061 * If the flow is external (from application) OR device is started, 7062 * OR mreg discover, then apply immediately. 7063 */ 7064 if (external || dev->data->dev_started || 7065 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7066 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7067 ret = flow_drv_apply(dev, flow, error); 7068 if (ret < 0) 7069 goto error; 7070 } 7071 flow->type = type; 7072 flow_rxq_flags_set(dev, flow); 7073 rte_free(translated_actions); 7074 tunnel = flow_tunnel_from_rule(wks->flows); 7075 if (tunnel) { 7076 flow->tunnel = 1; 7077 flow->tunnel_id = tunnel->tunnel_id; 7078 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7079 mlx5_free(default_miss_ctx.queue); 7080 } 7081 mlx5_flow_pop_thread_workspace(); 7082 return idx; 7083 error: 7084 MLX5_ASSERT(flow); 7085 ret = rte_errno; /* Save rte_errno before cleanup. */ 7086 flow_mreg_del_copy_action(dev, flow); 7087 flow_drv_destroy(dev, flow); 7088 if (rss_desc->shared_rss) 7089 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7090 mlx5_ipool_get 7091 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7092 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7093 mlx5_ipool_free(priv->flows[type], idx); 7094 rte_errno = ret; /* Restore rte_errno. */ 7095 ret = rte_errno; 7096 rte_errno = ret; 7097 error_before_hairpin_split: 7098 mlx5_flow_pop_thread_workspace(); 7099 rte_free(translated_actions); 7100 return 0; 7101 } 7102 7103 /** 7104 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7105 * incoming packets to table 1. 7106 * 7107 * Other flow rules, requested for group n, will be created in 7108 * e-switch table n+1. 7109 * Jump action to e-switch group n will be created to group n+1. 7110 * 7111 * Used when working in switchdev mode, to utilise advantages of table 1 7112 * and above. 7113 * 7114 * @param dev 7115 * Pointer to Ethernet device. 7116 * 7117 * @return 7118 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7119 */ 7120 struct rte_flow * 7121 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7122 { 7123 const struct rte_flow_attr attr = { 7124 .group = 0, 7125 .priority = 0, 7126 .ingress = 1, 7127 .egress = 0, 7128 .transfer = 1, 7129 }; 7130 const struct rte_flow_item pattern = { 7131 .type = RTE_FLOW_ITEM_TYPE_END, 7132 }; 7133 struct rte_flow_action_jump jump = { 7134 .group = 1, 7135 }; 7136 const struct rte_flow_action actions[] = { 7137 { 7138 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7139 .conf = &jump, 7140 }, 7141 { 7142 .type = RTE_FLOW_ACTION_TYPE_END, 7143 }, 7144 }; 7145 struct rte_flow_error error; 7146 7147 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7148 &attr, &pattern, 7149 actions, false, &error); 7150 } 7151 7152 /** 7153 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7154 * and sq number, directs all packets to peer vport. 7155 * 7156 * @param dev 7157 * Pointer to Ethernet device. 7158 * @param sq_num 7159 * SQ number. 7160 * 7161 * @return 7162 * Flow ID on success, 0 otherwise and rte_errno is set. 7163 */ 7164 uint32_t 7165 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7166 { 7167 struct rte_flow_attr attr = { 7168 .group = 0, 7169 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7170 .ingress = 1, 7171 .egress = 0, 7172 .transfer = 1, 7173 }; 7174 struct rte_flow_item_port_id port_spec = { 7175 .id = MLX5_PORT_ESW_MGR, 7176 }; 7177 struct mlx5_rte_flow_item_sq sq_spec = { 7178 .queue = sq_num, 7179 }; 7180 struct rte_flow_item pattern[] = { 7181 { 7182 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7183 .spec = &port_spec, 7184 }, 7185 { 7186 .type = (enum rte_flow_item_type) 7187 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7188 .spec = &sq_spec, 7189 }, 7190 { 7191 .type = RTE_FLOW_ITEM_TYPE_END, 7192 }, 7193 }; 7194 struct rte_flow_action_jump jump = { 7195 .group = 1, 7196 }; 7197 struct rte_flow_action_port_id port = { 7198 .id = dev->data->port_id, 7199 }; 7200 struct rte_flow_action actions[] = { 7201 { 7202 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7203 .conf = &jump, 7204 }, 7205 { 7206 .type = RTE_FLOW_ACTION_TYPE_END, 7207 }, 7208 }; 7209 struct rte_flow_error error; 7210 7211 /* 7212 * Creates group 0, highest priority jump flow. 7213 * Matches txq to bypass kernel packets. 7214 */ 7215 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7216 false, &error) == 0) 7217 return 0; 7218 /* Create group 1, lowest priority redirect flow for txq. */ 7219 attr.group = 1; 7220 actions[0].conf = &port; 7221 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7222 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7223 actions, false, &error); 7224 } 7225 7226 /** 7227 * Validate a flow supported by the NIC. 7228 * 7229 * @see rte_flow_validate() 7230 * @see rte_flow_ops 7231 */ 7232 int 7233 mlx5_flow_validate(struct rte_eth_dev *dev, 7234 const struct rte_flow_attr *attr, 7235 const struct rte_flow_item items[], 7236 const struct rte_flow_action original_actions[], 7237 struct rte_flow_error *error) 7238 { 7239 int hairpin_flow; 7240 struct mlx5_translated_action_handle 7241 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7242 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7243 const struct rte_flow_action *actions; 7244 struct rte_flow_action *translated_actions = NULL; 7245 int ret = flow_action_handles_translate(dev, original_actions, 7246 indir_actions, 7247 &indir_actions_n, 7248 &translated_actions, error); 7249 7250 if (ret) 7251 return ret; 7252 actions = translated_actions ? translated_actions : original_actions; 7253 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7254 ret = flow_drv_validate(dev, attr, items, actions, 7255 true, hairpin_flow, error); 7256 rte_free(translated_actions); 7257 return ret; 7258 } 7259 7260 /** 7261 * Create a flow. 7262 * 7263 * @see rte_flow_create() 7264 * @see rte_flow_ops 7265 */ 7266 struct rte_flow * 7267 mlx5_flow_create(struct rte_eth_dev *dev, 7268 const struct rte_flow_attr *attr, 7269 const struct rte_flow_item items[], 7270 const struct rte_flow_action actions[], 7271 struct rte_flow_error *error) 7272 { 7273 struct mlx5_priv *priv = dev->data->dev_private; 7274 7275 if (priv->sh->config.dv_flow_en == 2) { 7276 rte_flow_error_set(error, ENOTSUP, 7277 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7278 NULL, 7279 "Flow non-Q creation not supported"); 7280 return NULL; 7281 } 7282 /* 7283 * If the device is not started yet, it is not allowed to created a 7284 * flow from application. PMD default flows and traffic control flows 7285 * are not affected. 7286 */ 7287 if (unlikely(!dev->data->dev_started)) { 7288 DRV_LOG(DEBUG, "port %u is not started when " 7289 "inserting a flow", dev->data->port_id); 7290 rte_flow_error_set(error, ENODEV, 7291 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7292 NULL, 7293 "port not started"); 7294 return NULL; 7295 } 7296 7297 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7298 attr, items, actions, 7299 true, error); 7300 } 7301 7302 /** 7303 * Destroy a flow in a list. 7304 * 7305 * @param dev 7306 * Pointer to Ethernet device. 7307 * @param[in] flow_idx 7308 * Index of flow to destroy. 7309 */ 7310 static void 7311 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7312 uint32_t flow_idx) 7313 { 7314 struct mlx5_priv *priv = dev->data->dev_private; 7315 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7316 7317 if (!flow) 7318 return; 7319 MLX5_ASSERT(flow->type == type); 7320 /* 7321 * Update RX queue flags only if port is started, otherwise it is 7322 * already clean. 7323 */ 7324 if (dev->data->dev_started) 7325 flow_rxq_flags_trim(dev, flow); 7326 flow_drv_destroy(dev, flow); 7327 if (flow->tunnel) { 7328 struct mlx5_flow_tunnel *tunnel; 7329 7330 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7331 RTE_VERIFY(tunnel); 7332 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7333 mlx5_flow_tunnel_free(dev, tunnel); 7334 } 7335 flow_mreg_del_copy_action(dev, flow); 7336 mlx5_ipool_free(priv->flows[type], flow_idx); 7337 } 7338 7339 /** 7340 * Destroy all flows. 7341 * 7342 * @param dev 7343 * Pointer to Ethernet device. 7344 * @param type 7345 * Flow type to be flushed. 7346 * @param active 7347 * If flushing is called actively. 7348 */ 7349 void 7350 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7351 bool active) 7352 { 7353 struct mlx5_priv *priv = dev->data->dev_private; 7354 uint32_t num_flushed = 0, fidx = 1; 7355 struct rte_flow *flow; 7356 7357 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7358 if (priv->sh->config.dv_flow_en == 2 && 7359 type == MLX5_FLOW_TYPE_GEN) { 7360 flow_hw_q_flow_flush(dev, NULL); 7361 return; 7362 } 7363 #endif 7364 7365 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7366 flow_list_destroy(dev, type, fidx); 7367 num_flushed++; 7368 } 7369 if (active) { 7370 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7371 dev->data->port_id, num_flushed); 7372 } 7373 } 7374 7375 /** 7376 * Stop all default actions for flows. 7377 * 7378 * @param dev 7379 * Pointer to Ethernet device. 7380 */ 7381 void 7382 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7383 { 7384 flow_mreg_del_default_copy_action(dev); 7385 flow_rxq_flags_clear(dev); 7386 } 7387 7388 /** 7389 * Start all default actions for flows. 7390 * 7391 * @param dev 7392 * Pointer to Ethernet device. 7393 * @return 7394 * 0 on success, a negative errno value otherwise and rte_errno is set. 7395 */ 7396 int 7397 mlx5_flow_start_default(struct rte_eth_dev *dev) 7398 { 7399 struct rte_flow_error error; 7400 7401 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7402 return flow_mreg_add_default_copy_action(dev, &error); 7403 } 7404 7405 /** 7406 * Release key of thread specific flow workspace data. 7407 */ 7408 void 7409 flow_release_workspace(void *data) 7410 { 7411 struct mlx5_flow_workspace *wks = data; 7412 struct mlx5_flow_workspace *next; 7413 7414 while (wks) { 7415 next = wks->next; 7416 free(wks->rss_desc.queue); 7417 free(wks); 7418 wks = next; 7419 } 7420 } 7421 7422 /** 7423 * Get thread specific current flow workspace. 7424 * 7425 * @return pointer to thread specific flow workspace data, NULL on error. 7426 */ 7427 struct mlx5_flow_workspace* 7428 mlx5_flow_get_thread_workspace(void) 7429 { 7430 struct mlx5_flow_workspace *data; 7431 7432 data = mlx5_flow_os_get_specific_workspace(); 7433 MLX5_ASSERT(data && data->inuse); 7434 if (!data || !data->inuse) 7435 DRV_LOG(ERR, "flow workspace not initialized."); 7436 return data; 7437 } 7438 7439 /** 7440 * Allocate and init new flow workspace. 7441 * 7442 * @return pointer to flow workspace data, NULL on error. 7443 */ 7444 static struct mlx5_flow_workspace* 7445 flow_alloc_thread_workspace(void) 7446 { 7447 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7448 7449 if (!data) { 7450 DRV_LOG(ERR, "Failed to allocate flow workspace " 7451 "memory."); 7452 return NULL; 7453 } 7454 data->rss_desc.queue = calloc(1, 7455 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7456 if (!data->rss_desc.queue) 7457 goto err; 7458 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7459 return data; 7460 err: 7461 free(data->rss_desc.queue); 7462 free(data); 7463 return NULL; 7464 } 7465 7466 /** 7467 * Get new thread specific flow workspace. 7468 * 7469 * If current workspace inuse, create new one and set as current. 7470 * 7471 * @return pointer to thread specific flow workspace data, NULL on error. 7472 */ 7473 struct mlx5_flow_workspace* 7474 mlx5_flow_push_thread_workspace(void) 7475 { 7476 struct mlx5_flow_workspace *curr; 7477 struct mlx5_flow_workspace *data; 7478 7479 curr = mlx5_flow_os_get_specific_workspace(); 7480 if (!curr) { 7481 data = flow_alloc_thread_workspace(); 7482 if (!data) 7483 return NULL; 7484 } else if (!curr->inuse) { 7485 data = curr; 7486 } else if (curr->next) { 7487 data = curr->next; 7488 } else { 7489 data = flow_alloc_thread_workspace(); 7490 if (!data) 7491 return NULL; 7492 curr->next = data; 7493 data->prev = curr; 7494 } 7495 data->inuse = 1; 7496 data->flow_idx = 0; 7497 /* Set as current workspace */ 7498 if (mlx5_flow_os_set_specific_workspace(data)) 7499 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7500 return data; 7501 } 7502 7503 /** 7504 * Close current thread specific flow workspace. 7505 * 7506 * If previous workspace available, set it as current. 7507 * 7508 * @return pointer to thread specific flow workspace data, NULL on error. 7509 */ 7510 void 7511 mlx5_flow_pop_thread_workspace(void) 7512 { 7513 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7514 7515 if (!data) 7516 return; 7517 if (!data->inuse) { 7518 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7519 return; 7520 } 7521 data->inuse = 0; 7522 if (!data->prev) 7523 return; 7524 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7525 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7526 } 7527 7528 /** 7529 * Verify the flow list is empty 7530 * 7531 * @param dev 7532 * Pointer to Ethernet device. 7533 * 7534 * @return the number of flows not released. 7535 */ 7536 int 7537 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7538 { 7539 struct mlx5_priv *priv = dev->data->dev_private; 7540 struct rte_flow *flow; 7541 uint32_t idx = 0; 7542 int ret = 0, i; 7543 7544 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7545 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7546 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7547 dev->data->port_id, (void *)flow); 7548 ret++; 7549 } 7550 } 7551 return ret; 7552 } 7553 7554 /** 7555 * Enable default hairpin egress flow. 7556 * 7557 * @param dev 7558 * Pointer to Ethernet device. 7559 * @param sq_num 7560 * The SQ hw number. 7561 * 7562 * @return 7563 * 0 on success, a negative errno value otherwise and rte_errno is set. 7564 */ 7565 int 7566 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7567 uint32_t sq_num) 7568 { 7569 const struct rte_flow_attr attr = { 7570 .egress = 1, 7571 .priority = 0, 7572 }; 7573 struct mlx5_rte_flow_item_sq queue_spec = { 7574 .queue = sq_num, 7575 }; 7576 struct mlx5_rte_flow_item_sq queue_mask = { 7577 .queue = UINT32_MAX, 7578 }; 7579 struct rte_flow_item items[] = { 7580 { 7581 .type = (enum rte_flow_item_type) 7582 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7583 .spec = &queue_spec, 7584 .last = NULL, 7585 .mask = &queue_mask, 7586 }, 7587 { 7588 .type = RTE_FLOW_ITEM_TYPE_END, 7589 }, 7590 }; 7591 struct rte_flow_action_jump jump = { 7592 .group = MLX5_HAIRPIN_TX_TABLE, 7593 }; 7594 struct rte_flow_action actions[2]; 7595 uint32_t flow_idx; 7596 struct rte_flow_error error; 7597 7598 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7599 actions[0].conf = &jump; 7600 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7601 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7602 &attr, items, actions, false, &error); 7603 if (!flow_idx) { 7604 DRV_LOG(DEBUG, 7605 "Failed to create ctrl flow: rte_errno(%d)," 7606 " type(%d), message(%s)", 7607 rte_errno, error.type, 7608 error.message ? error.message : " (no stated reason)"); 7609 return -rte_errno; 7610 } 7611 return 0; 7612 } 7613 7614 /** 7615 * Enable a control flow configured from the control plane. 7616 * 7617 * @param dev 7618 * Pointer to Ethernet device. 7619 * @param eth_spec 7620 * An Ethernet flow spec to apply. 7621 * @param eth_mask 7622 * An Ethernet flow mask to apply. 7623 * @param vlan_spec 7624 * A VLAN flow spec to apply. 7625 * @param vlan_mask 7626 * A VLAN flow mask to apply. 7627 * 7628 * @return 7629 * 0 on success, a negative errno value otherwise and rte_errno is set. 7630 */ 7631 int 7632 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7633 struct rte_flow_item_eth *eth_spec, 7634 struct rte_flow_item_eth *eth_mask, 7635 struct rte_flow_item_vlan *vlan_spec, 7636 struct rte_flow_item_vlan *vlan_mask) 7637 { 7638 struct mlx5_priv *priv = dev->data->dev_private; 7639 const struct rte_flow_attr attr = { 7640 .ingress = 1, 7641 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7642 }; 7643 struct rte_flow_item items[] = { 7644 { 7645 .type = RTE_FLOW_ITEM_TYPE_ETH, 7646 .spec = eth_spec, 7647 .last = NULL, 7648 .mask = eth_mask, 7649 }, 7650 { 7651 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7652 RTE_FLOW_ITEM_TYPE_END, 7653 .spec = vlan_spec, 7654 .last = NULL, 7655 .mask = vlan_mask, 7656 }, 7657 { 7658 .type = RTE_FLOW_ITEM_TYPE_END, 7659 }, 7660 }; 7661 uint16_t queue[priv->reta_idx_n]; 7662 struct rte_flow_action_rss action_rss = { 7663 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7664 .level = 0, 7665 .types = priv->rss_conf.rss_hf, 7666 .key_len = priv->rss_conf.rss_key_len, 7667 .queue_num = priv->reta_idx_n, 7668 .key = priv->rss_conf.rss_key, 7669 .queue = queue, 7670 }; 7671 struct rte_flow_action actions[] = { 7672 { 7673 .type = RTE_FLOW_ACTION_TYPE_RSS, 7674 .conf = &action_rss, 7675 }, 7676 { 7677 .type = RTE_FLOW_ACTION_TYPE_END, 7678 }, 7679 }; 7680 uint32_t flow_idx; 7681 struct rte_flow_error error; 7682 unsigned int i; 7683 7684 if (!priv->reta_idx_n || !priv->rxqs_n) { 7685 return 0; 7686 } 7687 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7688 action_rss.types = 0; 7689 for (i = 0; i != priv->reta_idx_n; ++i) 7690 queue[i] = (*priv->reta_idx)[i]; 7691 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7692 &attr, items, actions, false, &error); 7693 if (!flow_idx) 7694 return -rte_errno; 7695 return 0; 7696 } 7697 7698 /** 7699 * Enable a flow control configured from the control plane. 7700 * 7701 * @param dev 7702 * Pointer to Ethernet device. 7703 * @param eth_spec 7704 * An Ethernet flow spec to apply. 7705 * @param eth_mask 7706 * An Ethernet flow mask to apply. 7707 * 7708 * @return 7709 * 0 on success, a negative errno value otherwise and rte_errno is set. 7710 */ 7711 int 7712 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7713 struct rte_flow_item_eth *eth_spec, 7714 struct rte_flow_item_eth *eth_mask) 7715 { 7716 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7717 } 7718 7719 /** 7720 * Create default miss flow rule matching lacp traffic 7721 * 7722 * @param dev 7723 * Pointer to Ethernet device. 7724 * @param eth_spec 7725 * An Ethernet flow spec to apply. 7726 * 7727 * @return 7728 * 0 on success, a negative errno value otherwise and rte_errno is set. 7729 */ 7730 int 7731 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7732 { 7733 /* 7734 * The LACP matching is done by only using ether type since using 7735 * a multicast dst mac causes kernel to give low priority to this flow. 7736 */ 7737 static const struct rte_flow_item_eth lacp_spec = { 7738 .type = RTE_BE16(0x8809), 7739 }; 7740 static const struct rte_flow_item_eth lacp_mask = { 7741 .type = 0xffff, 7742 }; 7743 const struct rte_flow_attr attr = { 7744 .ingress = 1, 7745 }; 7746 struct rte_flow_item items[] = { 7747 { 7748 .type = RTE_FLOW_ITEM_TYPE_ETH, 7749 .spec = &lacp_spec, 7750 .mask = &lacp_mask, 7751 }, 7752 { 7753 .type = RTE_FLOW_ITEM_TYPE_END, 7754 }, 7755 }; 7756 struct rte_flow_action actions[] = { 7757 { 7758 .type = (enum rte_flow_action_type) 7759 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7760 }, 7761 { 7762 .type = RTE_FLOW_ACTION_TYPE_END, 7763 }, 7764 }; 7765 struct rte_flow_error error; 7766 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7767 &attr, items, actions, 7768 false, &error); 7769 7770 if (!flow_idx) 7771 return -rte_errno; 7772 return 0; 7773 } 7774 7775 /** 7776 * Destroy a flow. 7777 * 7778 * @see rte_flow_destroy() 7779 * @see rte_flow_ops 7780 */ 7781 int 7782 mlx5_flow_destroy(struct rte_eth_dev *dev, 7783 struct rte_flow *flow, 7784 struct rte_flow_error *error __rte_unused) 7785 { 7786 struct mlx5_priv *priv = dev->data->dev_private; 7787 7788 if (priv->sh->config.dv_flow_en == 2) 7789 return rte_flow_error_set(error, ENOTSUP, 7790 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7791 NULL, 7792 "Flow non-Q destruction not supported"); 7793 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7794 (uintptr_t)(void *)flow); 7795 return 0; 7796 } 7797 7798 /** 7799 * Destroy all flows. 7800 * 7801 * @see rte_flow_flush() 7802 * @see rte_flow_ops 7803 */ 7804 int 7805 mlx5_flow_flush(struct rte_eth_dev *dev, 7806 struct rte_flow_error *error __rte_unused) 7807 { 7808 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7809 return 0; 7810 } 7811 7812 /** 7813 * Isolated mode. 7814 * 7815 * @see rte_flow_isolate() 7816 * @see rte_flow_ops 7817 */ 7818 int 7819 mlx5_flow_isolate(struct rte_eth_dev *dev, 7820 int enable, 7821 struct rte_flow_error *error) 7822 { 7823 struct mlx5_priv *priv = dev->data->dev_private; 7824 7825 if (dev->data->dev_started) { 7826 rte_flow_error_set(error, EBUSY, 7827 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7828 NULL, 7829 "port must be stopped first"); 7830 return -rte_errno; 7831 } 7832 priv->isolated = !!enable; 7833 if (enable) 7834 dev->dev_ops = &mlx5_dev_ops_isolate; 7835 else 7836 dev->dev_ops = &mlx5_dev_ops; 7837 7838 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7839 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7840 7841 return 0; 7842 } 7843 7844 /** 7845 * Query a flow. 7846 * 7847 * @see rte_flow_query() 7848 * @see rte_flow_ops 7849 */ 7850 static int 7851 flow_drv_query(struct rte_eth_dev *dev, 7852 struct rte_flow *eflow, 7853 const struct rte_flow_action *actions, 7854 void *data, 7855 struct rte_flow_error *error) 7856 { 7857 struct mlx5_priv *priv = dev->data->dev_private; 7858 const struct mlx5_flow_driver_ops *fops; 7859 struct rte_flow *flow = NULL; 7860 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 7861 7862 if (priv->sh->config.dv_flow_en == 2) { 7863 #ifdef HAVE_MLX5_HWS_SUPPORT 7864 flow = eflow; 7865 ftype = MLX5_FLOW_TYPE_HW; 7866 #endif 7867 } else { 7868 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7869 (uintptr_t)(void *)eflow); 7870 } 7871 if (!flow) { 7872 return rte_flow_error_set(error, ENOENT, 7873 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7874 NULL, 7875 "invalid flow handle"); 7876 } 7877 if (ftype == MLX5_FLOW_TYPE_MIN) 7878 ftype = flow->drv_type; 7879 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7880 fops = flow_get_drv_ops(ftype); 7881 7882 return fops->query(dev, flow, actions, data, error); 7883 } 7884 7885 /** 7886 * Query a flow. 7887 * 7888 * @see rte_flow_query() 7889 * @see rte_flow_ops 7890 */ 7891 int 7892 mlx5_flow_query(struct rte_eth_dev *dev, 7893 struct rte_flow *flow, 7894 const struct rte_flow_action *actions, 7895 void *data, 7896 struct rte_flow_error *error) 7897 { 7898 int ret; 7899 7900 ret = flow_drv_query(dev, flow, actions, data, 7901 error); 7902 if (ret < 0) 7903 return ret; 7904 return 0; 7905 } 7906 7907 /** 7908 * Get rte_flow callbacks. 7909 * 7910 * @param dev 7911 * Pointer to Ethernet device structure. 7912 * @param ops 7913 * Pointer to operation-specific structure. 7914 * 7915 * @return 0 7916 */ 7917 int 7918 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 7919 const struct rte_flow_ops **ops) 7920 { 7921 *ops = &mlx5_flow_ops; 7922 return 0; 7923 } 7924 7925 /** 7926 * Validate meter policy actions. 7927 * Dispatcher for action type specific validation. 7928 * 7929 * @param[in] dev 7930 * Pointer to the Ethernet device structure. 7931 * @param[in] action 7932 * The meter policy action object to validate. 7933 * @param[in] attr 7934 * Attributes of flow to determine steering domain. 7935 * @param[out] is_rss 7936 * Is RSS or not. 7937 * @param[out] domain_bitmap 7938 * Domain bitmap. 7939 * @param[out] is_def_policy 7940 * Is default policy or not. 7941 * @param[out] error 7942 * Perform verbose error reporting if not NULL. Initialized in case of 7943 * error only. 7944 * 7945 * @return 7946 * 0 on success, otherwise negative errno value. 7947 */ 7948 int 7949 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 7950 const struct rte_flow_action *actions[RTE_COLORS], 7951 struct rte_flow_attr *attr, 7952 bool *is_rss, 7953 uint8_t *domain_bitmap, 7954 uint8_t *policy_mode, 7955 struct rte_mtr_error *error) 7956 { 7957 const struct mlx5_flow_driver_ops *fops; 7958 7959 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7960 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 7961 domain_bitmap, policy_mode, error); 7962 } 7963 7964 /** 7965 * Destroy the meter table set. 7966 * 7967 * @param[in] dev 7968 * Pointer to Ethernet device. 7969 * @param[in] mtr_policy 7970 * Meter policy struct. 7971 */ 7972 void 7973 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 7974 struct mlx5_flow_meter_policy *mtr_policy) 7975 { 7976 const struct mlx5_flow_driver_ops *fops; 7977 7978 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 7979 fops->destroy_mtr_acts(dev, mtr_policy); 7980 } 7981 7982 /** 7983 * Create policy action, lock free, 7984 * (mutex should be acquired by caller). 7985 * Dispatcher for action type specific call. 7986 * 7987 * @param[in] dev 7988 * Pointer to the Ethernet device structure. 7989 * @param[in] mtr_policy 7990 * Meter policy struct. 7991 * @param[in] action 7992 * Action specification used to create meter actions. 7993 * @param[in] attr 7994 * Flow rule attributes. 7995 * @param[out] error 7996 * Perform verbose error reporting if not NULL. Initialized in case of 7997 * error only. 7998 * 7999 * @return 8000 * 0 on success, otherwise negative errno value. 8001 */ 8002 int 8003 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8004 struct mlx5_flow_meter_policy *mtr_policy, 8005 const struct rte_flow_action *actions[RTE_COLORS], 8006 struct rte_flow_attr *attr, 8007 struct rte_mtr_error *error) 8008 { 8009 const struct mlx5_flow_driver_ops *fops; 8010 8011 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8012 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8013 } 8014 8015 /** 8016 * Create policy rules, lock free, 8017 * (mutex should be acquired by caller). 8018 * Dispatcher for action type specific call. 8019 * 8020 * @param[in] dev 8021 * Pointer to the Ethernet device structure. 8022 * @param[in] mtr_policy 8023 * Meter policy struct. 8024 * 8025 * @return 8026 * 0 on success, -1 otherwise. 8027 */ 8028 int 8029 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8030 struct mlx5_flow_meter_policy *mtr_policy) 8031 { 8032 const struct mlx5_flow_driver_ops *fops; 8033 8034 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8035 return fops->create_policy_rules(dev, mtr_policy); 8036 } 8037 8038 /** 8039 * Destroy policy rules, lock free, 8040 * (mutex should be acquired by caller). 8041 * Dispatcher for action type specific call. 8042 * 8043 * @param[in] dev 8044 * Pointer to the Ethernet device structure. 8045 * @param[in] mtr_policy 8046 * Meter policy struct. 8047 */ 8048 void 8049 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8050 struct mlx5_flow_meter_policy *mtr_policy) 8051 { 8052 const struct mlx5_flow_driver_ops *fops; 8053 8054 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8055 fops->destroy_policy_rules(dev, mtr_policy); 8056 } 8057 8058 /** 8059 * Destroy the default policy table set. 8060 * 8061 * @param[in] dev 8062 * Pointer to Ethernet device. 8063 */ 8064 void 8065 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8066 { 8067 const struct mlx5_flow_driver_ops *fops; 8068 8069 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8070 fops->destroy_def_policy(dev); 8071 } 8072 8073 /** 8074 * Destroy the default policy table set. 8075 * 8076 * @param[in] dev 8077 * Pointer to Ethernet device. 8078 * 8079 * @return 8080 * 0 on success, -1 otherwise. 8081 */ 8082 int 8083 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8084 { 8085 const struct mlx5_flow_driver_ops *fops; 8086 8087 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8088 return fops->create_def_policy(dev); 8089 } 8090 8091 /** 8092 * Create the needed meter and suffix tables. 8093 * 8094 * @param[in] dev 8095 * Pointer to Ethernet device. 8096 * 8097 * @return 8098 * 0 on success, -1 otherwise. 8099 */ 8100 int 8101 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8102 struct mlx5_flow_meter_info *fm, 8103 uint32_t mtr_idx, 8104 uint8_t domain_bitmap) 8105 { 8106 const struct mlx5_flow_driver_ops *fops; 8107 8108 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8109 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8110 } 8111 8112 /** 8113 * Destroy the meter table set. 8114 * 8115 * @param[in] dev 8116 * Pointer to Ethernet device. 8117 * @param[in] tbl 8118 * Pointer to the meter table set. 8119 */ 8120 void 8121 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8122 struct mlx5_flow_meter_info *fm) 8123 { 8124 const struct mlx5_flow_driver_ops *fops; 8125 8126 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8127 fops->destroy_mtr_tbls(dev, fm); 8128 } 8129 8130 /** 8131 * Destroy the global meter drop table. 8132 * 8133 * @param[in] dev 8134 * Pointer to Ethernet device. 8135 */ 8136 void 8137 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8138 { 8139 const struct mlx5_flow_driver_ops *fops; 8140 8141 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8142 fops->destroy_mtr_drop_tbls(dev); 8143 } 8144 8145 /** 8146 * Destroy the sub policy table with RX queue. 8147 * 8148 * @param[in] dev 8149 * Pointer to Ethernet device. 8150 * @param[in] mtr_policy 8151 * Pointer to meter policy table. 8152 */ 8153 void 8154 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8155 struct mlx5_flow_meter_policy *mtr_policy) 8156 { 8157 const struct mlx5_flow_driver_ops *fops; 8158 8159 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8160 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8161 } 8162 8163 /** 8164 * Allocate the needed aso flow meter id. 8165 * 8166 * @param[in] dev 8167 * Pointer to Ethernet device. 8168 * 8169 * @return 8170 * Index to aso flow meter on success, NULL otherwise. 8171 */ 8172 uint32_t 8173 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8174 { 8175 const struct mlx5_flow_driver_ops *fops; 8176 8177 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8178 return fops->create_meter(dev); 8179 } 8180 8181 /** 8182 * Free the aso flow meter id. 8183 * 8184 * @param[in] dev 8185 * Pointer to Ethernet device. 8186 * @param[in] mtr_idx 8187 * Index to aso flow meter to be free. 8188 * 8189 * @return 8190 * 0 on success. 8191 */ 8192 void 8193 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8194 { 8195 const struct mlx5_flow_driver_ops *fops; 8196 8197 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8198 fops->free_meter(dev, mtr_idx); 8199 } 8200 8201 /** 8202 * Allocate a counter. 8203 * 8204 * @param[in] dev 8205 * Pointer to Ethernet device structure. 8206 * 8207 * @return 8208 * Index to allocated counter on success, 0 otherwise. 8209 */ 8210 uint32_t 8211 mlx5_counter_alloc(struct rte_eth_dev *dev) 8212 { 8213 const struct mlx5_flow_driver_ops *fops; 8214 struct rte_flow_attr attr = { .transfer = 0 }; 8215 8216 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8217 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8218 return fops->counter_alloc(dev); 8219 } 8220 DRV_LOG(ERR, 8221 "port %u counter allocate is not supported.", 8222 dev->data->port_id); 8223 return 0; 8224 } 8225 8226 /** 8227 * Free a counter. 8228 * 8229 * @param[in] dev 8230 * Pointer to Ethernet device structure. 8231 * @param[in] cnt 8232 * Index to counter to be free. 8233 */ 8234 void 8235 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8236 { 8237 const struct mlx5_flow_driver_ops *fops; 8238 struct rte_flow_attr attr = { .transfer = 0 }; 8239 8240 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8241 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8242 fops->counter_free(dev, cnt); 8243 return; 8244 } 8245 DRV_LOG(ERR, 8246 "port %u counter free is not supported.", 8247 dev->data->port_id); 8248 } 8249 8250 /** 8251 * Query counter statistics. 8252 * 8253 * @param[in] dev 8254 * Pointer to Ethernet device structure. 8255 * @param[in] cnt 8256 * Index to counter to query. 8257 * @param[in] clear 8258 * Set to clear counter statistics. 8259 * @param[out] pkts 8260 * The counter hits packets number to save. 8261 * @param[out] bytes 8262 * The counter hits bytes number to save. 8263 * 8264 * @return 8265 * 0 on success, a negative errno value otherwise. 8266 */ 8267 int 8268 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8269 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8270 { 8271 const struct mlx5_flow_driver_ops *fops; 8272 struct rte_flow_attr attr = { .transfer = 0 }; 8273 8274 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8275 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8276 return fops->counter_query(dev, cnt, clear, pkts, 8277 bytes, action); 8278 } 8279 DRV_LOG(ERR, 8280 "port %u counter query is not supported.", 8281 dev->data->port_id); 8282 return -ENOTSUP; 8283 } 8284 8285 /** 8286 * Get information about HWS pre-configurable resources. 8287 * 8288 * @param[in] dev 8289 * Pointer to the rte_eth_dev structure. 8290 * @param[out] port_info 8291 * Pointer to port information. 8292 * @param[out] queue_info 8293 * Pointer to queue information. 8294 * @param[out] error 8295 * Pointer to error structure. 8296 * 8297 * @return 8298 * 0 on success, a negative errno value otherwise and rte_errno is set. 8299 */ 8300 static int 8301 mlx5_flow_info_get(struct rte_eth_dev *dev, 8302 struct rte_flow_port_info *port_info, 8303 struct rte_flow_queue_info *queue_info, 8304 struct rte_flow_error *error) 8305 { 8306 const struct mlx5_flow_driver_ops *fops; 8307 struct rte_flow_attr attr = {0}; 8308 8309 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8310 return rte_flow_error_set(error, ENOTSUP, 8311 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8312 NULL, 8313 "info get with incorrect steering mode"); 8314 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8315 return fops->info_get(dev, port_info, queue_info, error); 8316 } 8317 8318 /** 8319 * Configure port HWS resources. 8320 * 8321 * @param[in] dev 8322 * Pointer to the rte_eth_dev structure. 8323 * @param[in] port_attr 8324 * Port configuration attributes. 8325 * @param[in] nb_queue 8326 * Number of queue. 8327 * @param[in] queue_attr 8328 * Array that holds attributes for each flow queue. 8329 * @param[out] error 8330 * Pointer to error structure. 8331 * 8332 * @return 8333 * 0 on success, a negative errno value otherwise and rte_errno is set. 8334 */ 8335 static int 8336 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8337 const struct rte_flow_port_attr *port_attr, 8338 uint16_t nb_queue, 8339 const struct rte_flow_queue_attr *queue_attr[], 8340 struct rte_flow_error *error) 8341 { 8342 const struct mlx5_flow_driver_ops *fops; 8343 struct rte_flow_attr attr = {0}; 8344 8345 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8346 return rte_flow_error_set(error, ENOTSUP, 8347 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8348 NULL, 8349 "port configure with incorrect steering mode"); 8350 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8351 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8352 } 8353 8354 /** 8355 * Validate item template. 8356 * 8357 * @param[in] dev 8358 * Pointer to the rte_eth_dev structure. 8359 * @param[in] attr 8360 * Pointer to the item template attributes. 8361 * @param[in] items 8362 * The template item pattern. 8363 * @param[out] error 8364 * Pointer to error structure. 8365 * 8366 * @return 8367 * 0 on success, a negative errno value otherwise and rte_errno is set. 8368 */ 8369 int 8370 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8371 const struct rte_flow_pattern_template_attr *attr, 8372 const struct rte_flow_item items[], 8373 struct rte_flow_error *error) 8374 { 8375 const struct mlx5_flow_driver_ops *fops; 8376 struct rte_flow_attr fattr = {0}; 8377 8378 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8379 rte_flow_error_set(error, ENOTSUP, 8380 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8381 "pattern validate with incorrect steering mode"); 8382 return -ENOTSUP; 8383 } 8384 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8385 return fops->pattern_validate(dev, attr, items, error); 8386 } 8387 8388 /** 8389 * Create flow item template. 8390 * 8391 * @param[in] dev 8392 * Pointer to the rte_eth_dev structure. 8393 * @param[in] attr 8394 * Pointer to the item template attributes. 8395 * @param[in] items 8396 * The template item pattern. 8397 * @param[out] error 8398 * Pointer to error structure. 8399 * 8400 * @return 8401 * 0 on success, a negative errno value otherwise and rte_errno is set. 8402 */ 8403 static struct rte_flow_pattern_template * 8404 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8405 const struct rte_flow_pattern_template_attr *attr, 8406 const struct rte_flow_item items[], 8407 struct rte_flow_error *error) 8408 { 8409 const struct mlx5_flow_driver_ops *fops; 8410 struct rte_flow_attr fattr = {0}; 8411 8412 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8413 rte_flow_error_set(error, ENOTSUP, 8414 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8415 NULL, 8416 "pattern create with incorrect steering mode"); 8417 return NULL; 8418 } 8419 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8420 return fops->pattern_template_create(dev, attr, items, error); 8421 } 8422 8423 /** 8424 * Destroy flow item template. 8425 * 8426 * @param[in] dev 8427 * Pointer to the rte_eth_dev structure. 8428 * @param[in] template 8429 * Pointer to the item template to be destroyed. 8430 * @param[out] error 8431 * Pointer to error structure. 8432 * 8433 * @return 8434 * 0 on success, a negative errno value otherwise and rte_errno is set. 8435 */ 8436 static int 8437 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8438 struct rte_flow_pattern_template *template, 8439 struct rte_flow_error *error) 8440 { 8441 const struct mlx5_flow_driver_ops *fops; 8442 struct rte_flow_attr attr = {0}; 8443 8444 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8445 return rte_flow_error_set(error, ENOTSUP, 8446 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8447 NULL, 8448 "pattern destroy with incorrect steering mode"); 8449 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8450 return fops->pattern_template_destroy(dev, template, error); 8451 } 8452 8453 /** 8454 * Validate flow actions template. 8455 * 8456 * @param[in] dev 8457 * Pointer to the rte_eth_dev structure. 8458 * @param[in] attr 8459 * Pointer to the action template attributes. 8460 * @param[in] actions 8461 * Associated actions (list terminated by the END action). 8462 * @param[in] masks 8463 * List of actions that marks which of the action's member is constant. 8464 * @param[out] error 8465 * Pointer to error structure. 8466 * 8467 * @return 8468 * 0 on success, a negative errno value otherwise and rte_errno is set. 8469 */ 8470 int 8471 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8472 const struct rte_flow_actions_template_attr *attr, 8473 const struct rte_flow_action actions[], 8474 const struct rte_flow_action masks[], 8475 struct rte_flow_error *error) 8476 { 8477 const struct mlx5_flow_driver_ops *fops; 8478 struct rte_flow_attr fattr = {0}; 8479 8480 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8481 rte_flow_error_set(error, ENOTSUP, 8482 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8483 "actions validate with incorrect steering mode"); 8484 return -ENOTSUP; 8485 } 8486 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8487 return fops->actions_validate(dev, attr, actions, masks, error); 8488 } 8489 8490 /** 8491 * Create flow item template. 8492 * 8493 * @param[in] dev 8494 * Pointer to the rte_eth_dev structure. 8495 * @param[in] attr 8496 * Pointer to the action template attributes. 8497 * @param[in] actions 8498 * Associated actions (list terminated by the END action). 8499 * @param[in] masks 8500 * List of actions that marks which of the action's member is constant. 8501 * @param[out] error 8502 * Pointer to error structure. 8503 * 8504 * @return 8505 * 0 on success, a negative errno value otherwise and rte_errno is set. 8506 */ 8507 static struct rte_flow_actions_template * 8508 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8509 const struct rte_flow_actions_template_attr *attr, 8510 const struct rte_flow_action actions[], 8511 const struct rte_flow_action masks[], 8512 struct rte_flow_error *error) 8513 { 8514 const struct mlx5_flow_driver_ops *fops; 8515 struct rte_flow_attr fattr = {0}; 8516 8517 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8518 rte_flow_error_set(error, ENOTSUP, 8519 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8520 NULL, 8521 "action create with incorrect steering mode"); 8522 return NULL; 8523 } 8524 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8525 return fops->actions_template_create(dev, attr, actions, masks, error); 8526 } 8527 8528 /** 8529 * Destroy flow action template. 8530 * 8531 * @param[in] dev 8532 * Pointer to the rte_eth_dev structure. 8533 * @param[in] template 8534 * Pointer to the action template to be destroyed. 8535 * @param[out] error 8536 * Pointer to error structure. 8537 * 8538 * @return 8539 * 0 on success, a negative errno value otherwise and rte_errno is set. 8540 */ 8541 static int 8542 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8543 struct rte_flow_actions_template *template, 8544 struct rte_flow_error *error) 8545 { 8546 const struct mlx5_flow_driver_ops *fops; 8547 struct rte_flow_attr attr = {0}; 8548 8549 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8550 return rte_flow_error_set(error, ENOTSUP, 8551 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8552 NULL, 8553 "action destroy with incorrect steering mode"); 8554 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8555 return fops->actions_template_destroy(dev, template, error); 8556 } 8557 8558 /** 8559 * Create flow table. 8560 * 8561 * @param[in] dev 8562 * Pointer to the rte_eth_dev structure. 8563 * @param[in] attr 8564 * Pointer to the table attributes. 8565 * @param[in] item_templates 8566 * Item template array to be binded to the table. 8567 * @param[in] nb_item_templates 8568 * Number of item template. 8569 * @param[in] action_templates 8570 * Action template array to be binded to the table. 8571 * @param[in] nb_action_templates 8572 * Number of action template. 8573 * @param[out] error 8574 * Pointer to error structure. 8575 * 8576 * @return 8577 * Table on success, NULL otherwise and rte_errno is set. 8578 */ 8579 static struct rte_flow_template_table * 8580 mlx5_flow_table_create(struct rte_eth_dev *dev, 8581 const struct rte_flow_template_table_attr *attr, 8582 struct rte_flow_pattern_template *item_templates[], 8583 uint8_t nb_item_templates, 8584 struct rte_flow_actions_template *action_templates[], 8585 uint8_t nb_action_templates, 8586 struct rte_flow_error *error) 8587 { 8588 const struct mlx5_flow_driver_ops *fops; 8589 struct rte_flow_attr fattr = {0}; 8590 8591 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8592 rte_flow_error_set(error, ENOTSUP, 8593 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8594 NULL, 8595 "table create with incorrect steering mode"); 8596 return NULL; 8597 } 8598 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8599 return fops->template_table_create(dev, 8600 attr, 8601 item_templates, 8602 nb_item_templates, 8603 action_templates, 8604 nb_action_templates, 8605 error); 8606 } 8607 8608 /** 8609 * PMD destroy flow table. 8610 * 8611 * @param[in] dev 8612 * Pointer to the rte_eth_dev structure. 8613 * @param[in] table 8614 * Pointer to the table to be destroyed. 8615 * @param[out] error 8616 * Pointer to error structure. 8617 * 8618 * @return 8619 * 0 on success, a negative errno value otherwise and rte_errno is set. 8620 */ 8621 static int 8622 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8623 struct rte_flow_template_table *table, 8624 struct rte_flow_error *error) 8625 { 8626 const struct mlx5_flow_driver_ops *fops; 8627 struct rte_flow_attr attr = {0}; 8628 8629 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8630 return rte_flow_error_set(error, ENOTSUP, 8631 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8632 NULL, 8633 "table destroy with incorrect steering mode"); 8634 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8635 return fops->template_table_destroy(dev, table, error); 8636 } 8637 8638 /** 8639 * Enqueue flow creation. 8640 * 8641 * @param[in] dev 8642 * Pointer to the rte_eth_dev structure. 8643 * @param[in] queue_id 8644 * The queue to create the flow. 8645 * @param[in] attr 8646 * Pointer to the flow operation attributes. 8647 * @param[in] items 8648 * Items with flow spec value. 8649 * @param[in] pattern_template_index 8650 * The item pattern flow follows from the table. 8651 * @param[in] actions 8652 * Action with flow spec value. 8653 * @param[in] action_template_index 8654 * The action pattern flow follows from the table. 8655 * @param[in] user_data 8656 * Pointer to the user_data. 8657 * @param[out] error 8658 * Pointer to error structure. 8659 * 8660 * @return 8661 * Flow pointer on success, NULL otherwise and rte_errno is set. 8662 */ 8663 static struct rte_flow * 8664 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8665 uint32_t queue_id, 8666 const struct rte_flow_op_attr *attr, 8667 struct rte_flow_template_table *table, 8668 const struct rte_flow_item items[], 8669 uint8_t pattern_template_index, 8670 const struct rte_flow_action actions[], 8671 uint8_t action_template_index, 8672 void *user_data, 8673 struct rte_flow_error *error) 8674 { 8675 const struct mlx5_flow_driver_ops *fops; 8676 struct rte_flow_attr fattr = {0}; 8677 8678 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8679 rte_flow_error_set(error, ENOTSUP, 8680 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8681 NULL, 8682 "flow_q create with incorrect steering mode"); 8683 return NULL; 8684 } 8685 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8686 return fops->async_flow_create(dev, queue_id, attr, table, 8687 items, pattern_template_index, 8688 actions, action_template_index, 8689 user_data, error); 8690 } 8691 8692 /** 8693 * Enqueue flow destruction. 8694 * 8695 * @param[in] dev 8696 * Pointer to the rte_eth_dev structure. 8697 * @param[in] queue 8698 * The queue to destroy the flow. 8699 * @param[in] attr 8700 * Pointer to the flow operation attributes. 8701 * @param[in] flow 8702 * Pointer to the flow to be destroyed. 8703 * @param[in] user_data 8704 * Pointer to the user_data. 8705 * @param[out] error 8706 * Pointer to error structure. 8707 * 8708 * @return 8709 * 0 on success, negative value otherwise and rte_errno is set. 8710 */ 8711 static int 8712 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8713 uint32_t queue, 8714 const struct rte_flow_op_attr *attr, 8715 struct rte_flow *flow, 8716 void *user_data, 8717 struct rte_flow_error *error) 8718 { 8719 const struct mlx5_flow_driver_ops *fops; 8720 struct rte_flow_attr fattr = {0}; 8721 8722 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8723 return rte_flow_error_set(error, ENOTSUP, 8724 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8725 NULL, 8726 "flow_q destroy with incorrect steering mode"); 8727 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8728 return fops->async_flow_destroy(dev, queue, attr, flow, 8729 user_data, error); 8730 } 8731 8732 /** 8733 * Pull the enqueued flows. 8734 * 8735 * @param[in] dev 8736 * Pointer to the rte_eth_dev structure. 8737 * @param[in] queue 8738 * The queue to pull the result. 8739 * @param[in/out] res 8740 * Array to save the results. 8741 * @param[in] n_res 8742 * Available result with the array. 8743 * @param[out] error 8744 * Pointer to error structure. 8745 * 8746 * @return 8747 * Result number on success, negative value otherwise and rte_errno is set. 8748 */ 8749 static int 8750 mlx5_flow_pull(struct rte_eth_dev *dev, 8751 uint32_t queue, 8752 struct rte_flow_op_result res[], 8753 uint16_t n_res, 8754 struct rte_flow_error *error) 8755 { 8756 const struct mlx5_flow_driver_ops *fops; 8757 struct rte_flow_attr attr = {0}; 8758 8759 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8760 return rte_flow_error_set(error, ENOTSUP, 8761 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8762 NULL, 8763 "flow_q pull with incorrect steering mode"); 8764 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8765 return fops->pull(dev, queue, res, n_res, error); 8766 } 8767 8768 /** 8769 * Push the enqueued flows. 8770 * 8771 * @param[in] dev 8772 * Pointer to the rte_eth_dev structure. 8773 * @param[in] queue 8774 * The queue to push the flows. 8775 * @param[out] error 8776 * Pointer to error structure. 8777 * 8778 * @return 8779 * 0 on success, negative value otherwise and rte_errno is set. 8780 */ 8781 static int 8782 mlx5_flow_push(struct rte_eth_dev *dev, 8783 uint32_t queue, 8784 struct rte_flow_error *error) 8785 { 8786 const struct mlx5_flow_driver_ops *fops; 8787 struct rte_flow_attr attr = {0}; 8788 8789 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8790 return rte_flow_error_set(error, ENOTSUP, 8791 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8792 NULL, 8793 "flow_q push with incorrect steering mode"); 8794 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8795 return fops->push(dev, queue, error); 8796 } 8797 8798 /** 8799 * Create shared action. 8800 * 8801 * @param[in] dev 8802 * Pointer to the rte_eth_dev structure. 8803 * @param[in] queue 8804 * Which queue to be used.. 8805 * @param[in] attr 8806 * Operation attribute. 8807 * @param[in] conf 8808 * Indirect action configuration. 8809 * @param[in] action 8810 * rte_flow action detail. 8811 * @param[in] user_data 8812 * Pointer to the user_data. 8813 * @param[out] error 8814 * Pointer to error structure. 8815 * 8816 * @return 8817 * Action handle on success, NULL otherwise and rte_errno is set. 8818 */ 8819 static struct rte_flow_action_handle * 8820 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8821 const struct rte_flow_op_attr *attr, 8822 const struct rte_flow_indir_action_conf *conf, 8823 const struct rte_flow_action *action, 8824 void *user_data, 8825 struct rte_flow_error *error) 8826 { 8827 const struct mlx5_flow_driver_ops *fops = 8828 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8829 8830 return fops->async_action_create(dev, queue, attr, conf, action, 8831 user_data, error); 8832 } 8833 8834 /** 8835 * Update shared action. 8836 * 8837 * @param[in] dev 8838 * Pointer to the rte_eth_dev structure. 8839 * @param[in] queue 8840 * Which queue to be used.. 8841 * @param[in] attr 8842 * Operation attribute. 8843 * @param[in] handle 8844 * Action handle to be updated. 8845 * @param[in] update 8846 * Update value. 8847 * @param[in] user_data 8848 * Pointer to the user_data. 8849 * @param[out] error 8850 * Pointer to error structure. 8851 * 8852 * @return 8853 * 0 on success, negative value otherwise and rte_errno is set. 8854 */ 8855 static int 8856 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8857 const struct rte_flow_op_attr *attr, 8858 struct rte_flow_action_handle *handle, 8859 const void *update, 8860 void *user_data, 8861 struct rte_flow_error *error) 8862 { 8863 const struct mlx5_flow_driver_ops *fops = 8864 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8865 8866 return fops->async_action_update(dev, queue, attr, handle, 8867 update, user_data, error); 8868 } 8869 8870 /** 8871 * Query shared action. 8872 * 8873 * @param[in] dev 8874 * Pointer to the rte_eth_dev structure. 8875 * @param[in] queue 8876 * Which queue to be used.. 8877 * @param[in] attr 8878 * Operation attribute. 8879 * @param[in] handle 8880 * Action handle to be updated. 8881 * @param[in] data 8882 * Pointer query result data. 8883 * @param[in] user_data 8884 * Pointer to the user_data. 8885 * @param[out] error 8886 * Pointer to error structure. 8887 * 8888 * @return 8889 * 0 on success, negative value otherwise and rte_errno is set. 8890 */ 8891 static int 8892 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 8893 const struct rte_flow_op_attr *attr, 8894 const struct rte_flow_action_handle *handle, 8895 void *data, 8896 void *user_data, 8897 struct rte_flow_error *error) 8898 { 8899 const struct mlx5_flow_driver_ops *fops = 8900 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8901 8902 return fops->async_action_query(dev, queue, attr, handle, 8903 data, user_data, error); 8904 } 8905 8906 /** 8907 * Destroy shared action. 8908 * 8909 * @param[in] dev 8910 * Pointer to the rte_eth_dev structure. 8911 * @param[in] queue 8912 * Which queue to be used.. 8913 * @param[in] attr 8914 * Operation attribute. 8915 * @param[in] handle 8916 * Action handle to be destroyed. 8917 * @param[in] user_data 8918 * Pointer to the user_data. 8919 * @param[out] error 8920 * Pointer to error structure. 8921 * 8922 * @return 8923 * 0 on success, negative value otherwise and rte_errno is set. 8924 */ 8925 static int 8926 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 8927 const struct rte_flow_op_attr *attr, 8928 struct rte_flow_action_handle *handle, 8929 void *user_data, 8930 struct rte_flow_error *error) 8931 { 8932 const struct mlx5_flow_driver_ops *fops = 8933 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8934 8935 return fops->async_action_destroy(dev, queue, attr, handle, 8936 user_data, error); 8937 } 8938 8939 /** 8940 * Allocate a new memory for the counter values wrapped by all the needed 8941 * management. 8942 * 8943 * @param[in] sh 8944 * Pointer to mlx5_dev_ctx_shared object. 8945 * 8946 * @return 8947 * 0 on success, a negative errno value otherwise. 8948 */ 8949 static int 8950 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 8951 { 8952 struct mlx5_counter_stats_mem_mng *mem_mng; 8953 volatile struct flow_counter_stats *raw_data; 8954 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 8955 int size = (sizeof(struct flow_counter_stats) * 8956 MLX5_COUNTERS_PER_POOL + 8957 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 8958 sizeof(struct mlx5_counter_stats_mem_mng); 8959 size_t pgsize = rte_mem_page_size(); 8960 uint8_t *mem; 8961 int ret; 8962 int i; 8963 8964 if (pgsize == (size_t)-1) { 8965 DRV_LOG(ERR, "Failed to get mem page size"); 8966 rte_errno = ENOMEM; 8967 return -ENOMEM; 8968 } 8969 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 8970 if (!mem) { 8971 rte_errno = ENOMEM; 8972 return -ENOMEM; 8973 } 8974 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 8975 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 8976 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 8977 sh->cdev->pdn, mem, size, 8978 &mem_mng->wm); 8979 if (ret) { 8980 rte_errno = errno; 8981 mlx5_free(mem); 8982 return -rte_errno; 8983 } 8984 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 8985 raw_data = (volatile struct flow_counter_stats *)mem; 8986 for (i = 0; i < raws_n; ++i) { 8987 mem_mng->raws[i].mem_mng = mem_mng; 8988 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 8989 } 8990 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 8991 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 8992 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 8993 next); 8994 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 8995 sh->sws_cmng.mem_mng = mem_mng; 8996 return 0; 8997 } 8998 8999 /** 9000 * Set the statistic memory to the new counter pool. 9001 * 9002 * @param[in] sh 9003 * Pointer to mlx5_dev_ctx_shared object. 9004 * @param[in] pool 9005 * Pointer to the pool to set the statistic memory. 9006 * 9007 * @return 9008 * 0 on success, a negative errno value otherwise. 9009 */ 9010 static int 9011 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9012 struct mlx5_flow_counter_pool *pool) 9013 { 9014 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9015 /* Resize statistic memory once used out. */ 9016 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 9017 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9018 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9019 return -1; 9020 } 9021 rte_spinlock_lock(&pool->sl); 9022 pool->raw = cmng->mem_mng->raws + pool->index % 9023 MLX5_CNT_CONTAINER_RESIZE; 9024 rte_spinlock_unlock(&pool->sl); 9025 pool->raw_hw = NULL; 9026 return 0; 9027 } 9028 9029 #define MLX5_POOL_QUERY_FREQ_US 1000000 9030 9031 /** 9032 * Set the periodic procedure for triggering asynchronous batch queries for all 9033 * the counter pools. 9034 * 9035 * @param[in] sh 9036 * Pointer to mlx5_dev_ctx_shared object. 9037 */ 9038 void 9039 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9040 { 9041 uint32_t pools_n, us; 9042 9043 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9044 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9045 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9046 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9047 sh->sws_cmng.query_thread_on = 0; 9048 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9049 } else { 9050 sh->sws_cmng.query_thread_on = 1; 9051 } 9052 } 9053 9054 /** 9055 * The periodic procedure for triggering asynchronous batch queries for all the 9056 * counter pools. This function is probably called by the host thread. 9057 * 9058 * @param[in] arg 9059 * The parameter for the alarm process. 9060 */ 9061 void 9062 mlx5_flow_query_alarm(void *arg) 9063 { 9064 struct mlx5_dev_ctx_shared *sh = arg; 9065 int ret; 9066 uint16_t pool_index = sh->sws_cmng.pool_index; 9067 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9068 struct mlx5_flow_counter_pool *pool; 9069 uint16_t n_valid; 9070 9071 if (sh->sws_cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 9072 goto set_alarm; 9073 rte_spinlock_lock(&cmng->pool_update_sl); 9074 pool = cmng->pools[pool_index]; 9075 n_valid = cmng->n_valid; 9076 rte_spinlock_unlock(&cmng->pool_update_sl); 9077 /* Set the statistic memory to the new created pool. */ 9078 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9079 goto set_alarm; 9080 if (pool->raw_hw) 9081 /* There is a pool query in progress. */ 9082 goto set_alarm; 9083 pool->raw_hw = 9084 LIST_FIRST(&sh->sws_cmng.free_stat_raws); 9085 if (!pool->raw_hw) 9086 /* No free counter statistics raw memory. */ 9087 goto set_alarm; 9088 /* 9089 * Identify the counters released between query trigger and query 9090 * handle more efficiently. The counter released in this gap period 9091 * should wait for a new round of query as the new arrived packets 9092 * will not be taken into account. 9093 */ 9094 pool->query_gen++; 9095 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9096 MLX5_COUNTERS_PER_POOL, 9097 NULL, NULL, 9098 pool->raw_hw->mem_mng->wm.lkey, 9099 (void *)(uintptr_t) 9100 pool->raw_hw->data, 9101 sh->devx_comp, 9102 (uint64_t)(uintptr_t)pool); 9103 if (ret) { 9104 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9105 " %d", pool->min_dcs->id); 9106 pool->raw_hw = NULL; 9107 goto set_alarm; 9108 } 9109 LIST_REMOVE(pool->raw_hw, next); 9110 sh->sws_cmng.pending_queries++; 9111 pool_index++; 9112 if (pool_index >= n_valid) 9113 pool_index = 0; 9114 set_alarm: 9115 sh->sws_cmng.pool_index = pool_index; 9116 mlx5_set_query_alarm(sh); 9117 } 9118 9119 /** 9120 * Check and callback event for new aged flow in the counter pool 9121 * 9122 * @param[in] sh 9123 * Pointer to mlx5_dev_ctx_shared object. 9124 * @param[in] pool 9125 * Pointer to Current counter pool. 9126 */ 9127 static void 9128 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9129 struct mlx5_flow_counter_pool *pool) 9130 { 9131 struct mlx5_priv *priv; 9132 struct mlx5_flow_counter *cnt; 9133 struct mlx5_age_info *age_info; 9134 struct mlx5_age_param *age_param; 9135 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9136 struct mlx5_counter_stats_raw *prev = pool->raw; 9137 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9138 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9139 uint16_t expected = AGE_CANDIDATE; 9140 uint32_t i; 9141 9142 pool->time_of_last_age_check = curr_time; 9143 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9144 cnt = MLX5_POOL_GET_CNT(pool, i); 9145 age_param = MLX5_CNT_TO_AGE(cnt); 9146 if (__atomic_load_n(&age_param->state, 9147 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9148 continue; 9149 if (cur->data[i].hits != prev->data[i].hits) { 9150 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9151 __ATOMIC_RELAXED); 9152 continue; 9153 } 9154 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9155 time_delta, 9156 __ATOMIC_RELAXED) <= age_param->timeout) 9157 continue; 9158 /** 9159 * Hold the lock first, or if between the 9160 * state AGE_TMOUT and tailq operation the 9161 * release happened, the release procedure 9162 * may delete a non-existent tailq node. 9163 */ 9164 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9165 age_info = GET_PORT_AGE_INFO(priv); 9166 rte_spinlock_lock(&age_info->aged_sl); 9167 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9168 AGE_TMOUT, false, 9169 __ATOMIC_RELAXED, 9170 __ATOMIC_RELAXED)) { 9171 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9172 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9173 } 9174 rte_spinlock_unlock(&age_info->aged_sl); 9175 } 9176 mlx5_age_event_prepare(sh); 9177 } 9178 9179 /** 9180 * Handler for the HW respond about ready values from an asynchronous batch 9181 * query. This function is probably called by the host thread. 9182 * 9183 * @param[in] sh 9184 * The pointer to the shared device context. 9185 * @param[in] async_id 9186 * The Devx async ID. 9187 * @param[in] status 9188 * The status of the completion. 9189 */ 9190 void 9191 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9192 uint64_t async_id, int status) 9193 { 9194 struct mlx5_flow_counter_pool *pool = 9195 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9196 struct mlx5_counter_stats_raw *raw_to_free; 9197 uint8_t query_gen = pool->query_gen ^ 1; 9198 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9199 enum mlx5_counter_type cnt_type = 9200 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9201 MLX5_COUNTER_TYPE_ORIGIN; 9202 9203 if (unlikely(status)) { 9204 raw_to_free = pool->raw_hw; 9205 } else { 9206 raw_to_free = pool->raw; 9207 if (pool->is_aged) 9208 mlx5_flow_aging_check(sh, pool); 9209 rte_spinlock_lock(&pool->sl); 9210 pool->raw = pool->raw_hw; 9211 rte_spinlock_unlock(&pool->sl); 9212 /* Be sure the new raw counters data is updated in memory. */ 9213 rte_io_wmb(); 9214 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9215 rte_spinlock_lock(&cmng->csl[cnt_type]); 9216 TAILQ_CONCAT(&cmng->counters[cnt_type], 9217 &pool->counters[query_gen], next); 9218 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9219 } 9220 } 9221 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9222 pool->raw_hw = NULL; 9223 sh->sws_cmng.pending_queries--; 9224 } 9225 9226 static int 9227 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9228 const struct flow_grp_info *grp_info, 9229 struct rte_flow_error *error) 9230 { 9231 if (grp_info->transfer && grp_info->external && 9232 grp_info->fdb_def_rule) { 9233 if (group == UINT32_MAX) 9234 return rte_flow_error_set 9235 (error, EINVAL, 9236 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9237 NULL, 9238 "group index not supported"); 9239 *table = group + 1; 9240 } else { 9241 *table = group; 9242 } 9243 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9244 return 0; 9245 } 9246 9247 /** 9248 * Translate the rte_flow group index to HW table value. 9249 * 9250 * If tunnel offload is disabled, all group ids converted to flow table 9251 * id using the standard method. 9252 * If tunnel offload is enabled, group id can be converted using the 9253 * standard or tunnel conversion method. Group conversion method 9254 * selection depends on flags in `grp_info` parameter: 9255 * - Internal (grp_info.external == 0) groups conversion uses the 9256 * standard method. 9257 * - Group ids in JUMP action converted with the tunnel conversion. 9258 * - Group id in rule attribute conversion depends on a rule type and 9259 * group id value: 9260 * ** non zero group attributes converted with the tunnel method 9261 * ** zero group attribute in non-tunnel rule is converted using the 9262 * standard method - there's only one root table 9263 * ** zero group attribute in steer tunnel rule is converted with the 9264 * standard method - single root table 9265 * ** zero group attribute in match tunnel rule is a special OvS 9266 * case: that value is used for portability reasons. That group 9267 * id is converted with the tunnel conversion method. 9268 * 9269 * @param[in] dev 9270 * Port device 9271 * @param[in] tunnel 9272 * PMD tunnel offload object 9273 * @param[in] group 9274 * rte_flow group index value. 9275 * @param[out] table 9276 * HW table value. 9277 * @param[in] grp_info 9278 * flags used for conversion 9279 * @param[out] error 9280 * Pointer to error structure. 9281 * 9282 * @return 9283 * 0 on success, a negative errno value otherwise and rte_errno is set. 9284 */ 9285 int 9286 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9287 const struct mlx5_flow_tunnel *tunnel, 9288 uint32_t group, uint32_t *table, 9289 const struct flow_grp_info *grp_info, 9290 struct rte_flow_error *error) 9291 { 9292 int ret; 9293 bool standard_translation; 9294 9295 if (!grp_info->skip_scale && grp_info->external && 9296 group < MLX5_MAX_TABLES_EXTERNAL) 9297 group *= MLX5_FLOW_TABLE_FACTOR; 9298 if (is_tunnel_offload_active(dev)) { 9299 standard_translation = !grp_info->external || 9300 grp_info->std_tbl_fix; 9301 } else { 9302 standard_translation = true; 9303 } 9304 DRV_LOG(DEBUG, 9305 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9306 dev->data->port_id, group, grp_info->transfer, 9307 grp_info->external, grp_info->fdb_def_rule, 9308 standard_translation ? "STANDARD" : "TUNNEL"); 9309 if (standard_translation) 9310 ret = flow_group_to_table(dev->data->port_id, group, table, 9311 grp_info, error); 9312 else 9313 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9314 table, error); 9315 9316 return ret; 9317 } 9318 9319 /** 9320 * Discover availability of metadata reg_c's. 9321 * 9322 * Iteratively use test flows to check availability. 9323 * 9324 * @param[in] dev 9325 * Pointer to the Ethernet device structure. 9326 * 9327 * @return 9328 * 0 on success, a negative errno value otherwise and rte_errno is set. 9329 */ 9330 int 9331 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9332 { 9333 struct mlx5_priv *priv = dev->data->dev_private; 9334 enum modify_reg idx; 9335 int n = 0; 9336 9337 /* reg_c[0] and reg_c[1] are reserved. */ 9338 priv->sh->flow_mreg_c[n++] = REG_C_0; 9339 priv->sh->flow_mreg_c[n++] = REG_C_1; 9340 /* Discover availability of other reg_c's. */ 9341 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9342 struct rte_flow_attr attr = { 9343 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9344 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9345 .ingress = 1, 9346 }; 9347 struct rte_flow_item items[] = { 9348 [0] = { 9349 .type = RTE_FLOW_ITEM_TYPE_END, 9350 }, 9351 }; 9352 struct rte_flow_action actions[] = { 9353 [0] = { 9354 .type = (enum rte_flow_action_type) 9355 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9356 .conf = &(struct mlx5_flow_action_copy_mreg){ 9357 .src = REG_C_1, 9358 .dst = idx, 9359 }, 9360 }, 9361 [1] = { 9362 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9363 .conf = &(struct rte_flow_action_jump){ 9364 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9365 }, 9366 }, 9367 [2] = { 9368 .type = RTE_FLOW_ACTION_TYPE_END, 9369 }, 9370 }; 9371 uint32_t flow_idx; 9372 struct rte_flow *flow; 9373 struct rte_flow_error error; 9374 9375 if (!priv->sh->config.dv_flow_en) 9376 break; 9377 /* Create internal flow, validation skips copy action. */ 9378 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9379 items, actions, false, &error); 9380 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9381 flow_idx); 9382 if (!flow) 9383 continue; 9384 priv->sh->flow_mreg_c[n++] = idx; 9385 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9386 } 9387 for (; n < MLX5_MREG_C_NUM; ++n) 9388 priv->sh->flow_mreg_c[n] = REG_NON; 9389 priv->sh->metadata_regc_check_flag = 1; 9390 return 0; 9391 } 9392 9393 int 9394 save_dump_file(const uint8_t *data, uint32_t size, 9395 uint32_t type, uint64_t id, void *arg, FILE *file) 9396 { 9397 char line[BUF_SIZE]; 9398 uint32_t out = 0; 9399 uint32_t k; 9400 uint32_t actions_num; 9401 struct rte_flow_query_count *count; 9402 9403 memset(line, 0, BUF_SIZE); 9404 switch (type) { 9405 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9406 actions_num = *(uint32_t *)(arg); 9407 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9408 type, id, actions_num); 9409 break; 9410 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9411 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9412 type, id); 9413 break; 9414 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9415 count = (struct rte_flow_query_count *)arg; 9416 fprintf(file, 9417 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9418 type, id, count->hits, count->bytes); 9419 return 0; 9420 default: 9421 return -1; 9422 } 9423 9424 for (k = 0; k < size; k++) { 9425 /* Make sure we do not overrun the line buffer length. */ 9426 if (out >= BUF_SIZE - 4) { 9427 line[out] = '\0'; 9428 break; 9429 } 9430 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9431 (data[k]) & 0xff); 9432 } 9433 fprintf(file, "%s\n", line); 9434 return 0; 9435 } 9436 9437 int 9438 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9439 struct rte_flow_query_count *count, struct rte_flow_error *error) 9440 { 9441 struct rte_flow_action action[2]; 9442 enum mlx5_flow_drv_type ftype; 9443 const struct mlx5_flow_driver_ops *fops; 9444 9445 if (!flow) { 9446 return rte_flow_error_set(error, ENOENT, 9447 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9448 NULL, 9449 "invalid flow handle"); 9450 } 9451 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9452 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9453 if (flow->counter) { 9454 memset(count, 0, sizeof(struct rte_flow_query_count)); 9455 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9456 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9457 ftype < MLX5_FLOW_TYPE_MAX); 9458 fops = flow_get_drv_ops(ftype); 9459 return fops->query(dev, flow, action, count, error); 9460 } 9461 return -1; 9462 } 9463 9464 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9465 /** 9466 * Dump flow ipool data to file 9467 * 9468 * @param[in] dev 9469 * The pointer to Ethernet device. 9470 * @param[in] file 9471 * A pointer to a file for output. 9472 * @param[out] error 9473 * Perform verbose error reporting if not NULL. PMDs initialize this 9474 * structure in case of error only. 9475 * @return 9476 * 0 on success, a negative value otherwise. 9477 */ 9478 int 9479 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9480 struct rte_flow *flow, FILE *file, 9481 struct rte_flow_error *error) 9482 { 9483 struct mlx5_priv *priv = dev->data->dev_private; 9484 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9485 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9486 uint32_t handle_idx; 9487 struct mlx5_flow_handle *dh; 9488 struct rte_flow_query_count count; 9489 uint32_t actions_num; 9490 const uint8_t *data; 9491 size_t size; 9492 uint64_t id; 9493 uint32_t type; 9494 void *action = NULL; 9495 9496 if (!flow) { 9497 return rte_flow_error_set(error, ENOENT, 9498 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9499 NULL, 9500 "invalid flow handle"); 9501 } 9502 handle_idx = flow->dev_handles; 9503 /* query counter */ 9504 if (flow->counter && 9505 (!mlx5_counter_query(dev, flow->counter, false, 9506 &count.hits, &count.bytes, &action)) && action) { 9507 id = (uint64_t)(uintptr_t)action; 9508 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9509 save_dump_file(NULL, 0, type, 9510 id, (void *)&count, file); 9511 } 9512 9513 while (handle_idx) { 9514 dh = mlx5_ipool_get(priv->sh->ipool 9515 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9516 if (!dh) 9517 continue; 9518 handle_idx = dh->next.next; 9519 9520 /* Get modify_hdr and encap_decap buf from ipools. */ 9521 encap_decap = NULL; 9522 modify_hdr = dh->dvh.modify_hdr; 9523 9524 if (dh->dvh.rix_encap_decap) { 9525 encap_decap = mlx5_ipool_get(priv->sh->ipool 9526 [MLX5_IPOOL_DECAP_ENCAP], 9527 dh->dvh.rix_encap_decap); 9528 } 9529 if (modify_hdr) { 9530 data = (const uint8_t *)modify_hdr->actions; 9531 size = (size_t)(modify_hdr->actions_num) * 8; 9532 id = (uint64_t)(uintptr_t)modify_hdr->action; 9533 actions_num = modify_hdr->actions_num; 9534 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9535 save_dump_file(data, size, type, id, 9536 (void *)(&actions_num), file); 9537 } 9538 if (encap_decap) { 9539 data = encap_decap->buf; 9540 size = encap_decap->size; 9541 id = (uint64_t)(uintptr_t)encap_decap->action; 9542 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9543 save_dump_file(data, size, type, 9544 id, NULL, file); 9545 } 9546 } 9547 return 0; 9548 } 9549 9550 /** 9551 * Dump all flow's encap_decap/modify_hdr/counter data to file 9552 * 9553 * @param[in] dev 9554 * The pointer to Ethernet device. 9555 * @param[in] file 9556 * A pointer to a file for output. 9557 * @param[out] error 9558 * Perform verbose error reporting if not NULL. PMDs initialize this 9559 * structure in case of error only. 9560 * @return 9561 * 0 on success, a negative value otherwise. 9562 */ 9563 static int 9564 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9565 FILE *file, struct rte_flow_error *error __rte_unused) 9566 { 9567 struct mlx5_priv *priv = dev->data->dev_private; 9568 struct mlx5_dev_ctx_shared *sh = priv->sh; 9569 struct mlx5_hlist *h; 9570 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9571 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9572 struct rte_flow_query_count count; 9573 uint32_t actions_num; 9574 const uint8_t *data; 9575 size_t size; 9576 uint64_t id; 9577 uint32_t type; 9578 uint32_t i; 9579 uint32_t j; 9580 struct mlx5_list_inconst *l_inconst; 9581 struct mlx5_list_entry *e; 9582 int lcore_index; 9583 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9584 uint32_t max; 9585 void *action; 9586 9587 /* encap_decap hlist is lcore_share, get global core cache. */ 9588 i = MLX5_LIST_GLOBAL; 9589 h = sh->encaps_decaps; 9590 if (h) { 9591 for (j = 0; j <= h->mask; j++) { 9592 l_inconst = &h->buckets[j].l; 9593 if (!l_inconst || !l_inconst->cache[i]) 9594 continue; 9595 9596 e = LIST_FIRST(&l_inconst->cache[i]->h); 9597 while (e) { 9598 encap_decap = 9599 (struct mlx5_flow_dv_encap_decap_resource *)e; 9600 data = encap_decap->buf; 9601 size = encap_decap->size; 9602 id = (uint64_t)(uintptr_t)encap_decap->action; 9603 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9604 save_dump_file(data, size, type, 9605 id, NULL, file); 9606 e = LIST_NEXT(e, next); 9607 } 9608 } 9609 } 9610 9611 /* get modify_hdr */ 9612 h = sh->modify_cmds; 9613 if (h) { 9614 lcore_index = rte_lcore_index(rte_lcore_id()); 9615 if (unlikely(lcore_index == -1)) { 9616 lcore_index = MLX5_LIST_NLCORE; 9617 rte_spinlock_lock(&h->l_const.lcore_lock); 9618 } 9619 i = lcore_index; 9620 9621 for (j = 0; j <= h->mask; j++) { 9622 l_inconst = &h->buckets[j].l; 9623 if (!l_inconst || !l_inconst->cache[i]) 9624 continue; 9625 9626 e = LIST_FIRST(&l_inconst->cache[i]->h); 9627 while (e) { 9628 modify_hdr = 9629 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9630 data = (const uint8_t *)modify_hdr->actions; 9631 size = (size_t)(modify_hdr->actions_num) * 8; 9632 actions_num = modify_hdr->actions_num; 9633 id = (uint64_t)(uintptr_t)modify_hdr->action; 9634 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9635 save_dump_file(data, size, type, id, 9636 (void *)(&actions_num), file); 9637 e = LIST_NEXT(e, next); 9638 } 9639 } 9640 9641 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9642 rte_spinlock_unlock(&h->l_const.lcore_lock); 9643 } 9644 9645 /* get counter */ 9646 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9647 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9648 for (j = 1; j <= max; j++) { 9649 action = NULL; 9650 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9651 &count.bytes, &action)) && action) { 9652 id = (uint64_t)(uintptr_t)action; 9653 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9654 save_dump_file(NULL, 0, type, 9655 id, (void *)&count, file); 9656 } 9657 } 9658 return 0; 9659 } 9660 #endif 9661 9662 /** 9663 * Dump flow raw hw data to file 9664 * 9665 * @param[in] dev 9666 * The pointer to Ethernet device. 9667 * @param[in] file 9668 * A pointer to a file for output. 9669 * @param[out] error 9670 * Perform verbose error reporting if not NULL. PMDs initialize this 9671 * structure in case of error only. 9672 * @return 9673 * 0 on success, a negative value otherwise. 9674 */ 9675 int 9676 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9677 FILE *file, 9678 struct rte_flow_error *error __rte_unused) 9679 { 9680 struct mlx5_priv *priv = dev->data->dev_private; 9681 struct mlx5_dev_ctx_shared *sh = priv->sh; 9682 uint32_t handle_idx; 9683 int ret; 9684 struct mlx5_flow_handle *dh; 9685 struct rte_flow *flow; 9686 9687 if (!sh->config.dv_flow_en) { 9688 if (fputs("device dv flow disabled\n", file) <= 0) 9689 return -errno; 9690 return -ENOTSUP; 9691 } 9692 9693 /* dump all */ 9694 if (!flow_idx) { 9695 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9696 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9697 return -EINVAL; 9698 #endif 9699 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9700 sh->rx_domain, 9701 sh->tx_domain, file); 9702 } 9703 /* dump one */ 9704 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9705 (uintptr_t)(void *)flow_idx); 9706 if (!flow) 9707 return -EINVAL; 9708 9709 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9710 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9711 #endif 9712 handle_idx = flow->dev_handles; 9713 while (handle_idx) { 9714 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9715 handle_idx); 9716 if (!dh) 9717 return -ENOENT; 9718 if (dh->drv_flow) { 9719 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9720 file); 9721 if (ret) 9722 return -ENOENT; 9723 } 9724 handle_idx = dh->next.next; 9725 } 9726 return 0; 9727 } 9728 9729 /** 9730 * Get aged-out flows. 9731 * 9732 * @param[in] dev 9733 * Pointer to the Ethernet device structure. 9734 * @param[in] context 9735 * The address of an array of pointers to the aged-out flows contexts. 9736 * @param[in] nb_countexts 9737 * The length of context array pointers. 9738 * @param[out] error 9739 * Perform verbose error reporting if not NULL. Initialized in case of 9740 * error only. 9741 * 9742 * @return 9743 * how many contexts get in success, otherwise negative errno value. 9744 * if nb_contexts is 0, return the amount of all aged contexts. 9745 * if nb_contexts is not 0 , return the amount of aged flows reported 9746 * in the context array. 9747 */ 9748 int 9749 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9750 uint32_t nb_contexts, struct rte_flow_error *error) 9751 { 9752 const struct mlx5_flow_driver_ops *fops; 9753 struct rte_flow_attr attr = { .transfer = 0 }; 9754 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, &attr); 9755 9756 if (type == MLX5_FLOW_TYPE_DV || type == MLX5_FLOW_TYPE_HW) { 9757 fops = flow_get_drv_ops(type); 9758 return fops->get_aged_flows(dev, contexts, nb_contexts, error); 9759 } 9760 DRV_LOG(ERR, "port %u get aged flows is not supported.", 9761 dev->data->port_id); 9762 return -ENOTSUP; 9763 } 9764 9765 /** 9766 * Get aged-out flows per HWS queue. 9767 * 9768 * @param[in] dev 9769 * Pointer to the Ethernet device structure. 9770 * @param[in] queue_id 9771 * Flow queue to query. 9772 * @param[in] context 9773 * The address of an array of pointers to the aged-out flows contexts. 9774 * @param[in] nb_countexts 9775 * The length of context array pointers. 9776 * @param[out] error 9777 * Perform verbose error reporting if not NULL. Initialized in case of 9778 * error only. 9779 * 9780 * @return 9781 * how many contexts get in success, otherwise negative errno value. 9782 * if nb_contexts is 0, return the amount of all aged contexts. 9783 * if nb_contexts is not 0 , return the amount of aged flows reported 9784 * in the context array. 9785 */ 9786 int 9787 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 9788 void **contexts, uint32_t nb_contexts, 9789 struct rte_flow_error *error) 9790 { 9791 const struct mlx5_flow_driver_ops *fops; 9792 struct rte_flow_attr attr = { 0 }; 9793 9794 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 9795 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9796 return fops->get_q_aged_flows(dev, queue_id, contexts, 9797 nb_contexts, error); 9798 } 9799 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 9800 dev->data->port_id, queue_id); 9801 return rte_flow_error_set(error, ENOTSUP, 9802 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9803 "get Q aged flows with incorrect steering mode"); 9804 } 9805 9806 /* Wrapper for driver action_validate op callback */ 9807 static int 9808 flow_drv_action_validate(struct rte_eth_dev *dev, 9809 const struct rte_flow_indir_action_conf *conf, 9810 const struct rte_flow_action *action, 9811 const struct mlx5_flow_driver_ops *fops, 9812 struct rte_flow_error *error) 9813 { 9814 static const char err_msg[] = "indirect action validation unsupported"; 9815 9816 if (!fops->action_validate) { 9817 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9818 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9819 NULL, err_msg); 9820 return -rte_errno; 9821 } 9822 return fops->action_validate(dev, conf, action, error); 9823 } 9824 9825 /** 9826 * Destroys the shared action by handle. 9827 * 9828 * @param dev 9829 * Pointer to Ethernet device structure. 9830 * @param[in] handle 9831 * Handle for the indirect action object to be destroyed. 9832 * @param[out] error 9833 * Perform verbose error reporting if not NULL. PMDs initialize this 9834 * structure in case of error only. 9835 * 9836 * @return 9837 * 0 on success, a negative errno value otherwise and rte_errno is set. 9838 * 9839 * @note: wrapper for driver action_create op callback. 9840 */ 9841 static int 9842 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9843 struct rte_flow_action_handle *handle, 9844 struct rte_flow_error *error) 9845 { 9846 static const char err_msg[] = "indirect action destruction unsupported"; 9847 struct rte_flow_attr attr = { .transfer = 0 }; 9848 const struct mlx5_flow_driver_ops *fops = 9849 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9850 9851 if (!fops->action_destroy) { 9852 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9853 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9854 NULL, err_msg); 9855 return -rte_errno; 9856 } 9857 return fops->action_destroy(dev, handle, error); 9858 } 9859 9860 /* Wrapper for driver action_destroy op callback */ 9861 static int 9862 flow_drv_action_update(struct rte_eth_dev *dev, 9863 struct rte_flow_action_handle *handle, 9864 const void *update, 9865 const struct mlx5_flow_driver_ops *fops, 9866 struct rte_flow_error *error) 9867 { 9868 static const char err_msg[] = "indirect action update unsupported"; 9869 9870 if (!fops->action_update) { 9871 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9872 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9873 NULL, err_msg); 9874 return -rte_errno; 9875 } 9876 return fops->action_update(dev, handle, update, error); 9877 } 9878 9879 /* Wrapper for driver action_destroy op callback */ 9880 static int 9881 flow_drv_action_query(struct rte_eth_dev *dev, 9882 const struct rte_flow_action_handle *handle, 9883 void *data, 9884 const struct mlx5_flow_driver_ops *fops, 9885 struct rte_flow_error *error) 9886 { 9887 static const char err_msg[] = "indirect action query unsupported"; 9888 9889 if (!fops->action_query) { 9890 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9891 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9892 NULL, err_msg); 9893 return -rte_errno; 9894 } 9895 return fops->action_query(dev, handle, data, error); 9896 } 9897 9898 /** 9899 * Create indirect action for reuse in multiple flow rules. 9900 * 9901 * @param dev 9902 * Pointer to Ethernet device structure. 9903 * @param conf 9904 * Pointer to indirect action object configuration. 9905 * @param[in] action 9906 * Action configuration for indirect action object creation. 9907 * @param[out] error 9908 * Perform verbose error reporting if not NULL. PMDs initialize this 9909 * structure in case of error only. 9910 * @return 9911 * A valid handle in case of success, NULL otherwise and rte_errno is set. 9912 */ 9913 static struct rte_flow_action_handle * 9914 mlx5_action_handle_create(struct rte_eth_dev *dev, 9915 const struct rte_flow_indir_action_conf *conf, 9916 const struct rte_flow_action *action, 9917 struct rte_flow_error *error) 9918 { 9919 static const char err_msg[] = "indirect action creation unsupported"; 9920 struct rte_flow_attr attr = { .transfer = 0 }; 9921 const struct mlx5_flow_driver_ops *fops = 9922 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9923 9924 if (flow_drv_action_validate(dev, conf, action, fops, error)) 9925 return NULL; 9926 if (!fops->action_create) { 9927 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9928 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9929 NULL, err_msg); 9930 return NULL; 9931 } 9932 return fops->action_create(dev, conf, action, error); 9933 } 9934 9935 /** 9936 * Updates inplace the indirect action configuration pointed by *handle* 9937 * with the configuration provided as *update* argument. 9938 * The update of the indirect action configuration effects all flow rules 9939 * reusing the action via handle. 9940 * 9941 * @param dev 9942 * Pointer to Ethernet device structure. 9943 * @param[in] handle 9944 * Handle for the indirect action to be updated. 9945 * @param[in] update 9946 * Action specification used to modify the action pointed by handle. 9947 * *update* could be of same type with the action pointed by the *handle* 9948 * handle argument, or some other structures like a wrapper, depending on 9949 * the indirect action type. 9950 * @param[out] error 9951 * Perform verbose error reporting if not NULL. PMDs initialize this 9952 * structure in case of error only. 9953 * 9954 * @return 9955 * 0 on success, a negative errno value otherwise and rte_errno is set. 9956 */ 9957 static int 9958 mlx5_action_handle_update(struct rte_eth_dev *dev, 9959 struct rte_flow_action_handle *handle, 9960 const void *update, 9961 struct rte_flow_error *error) 9962 { 9963 struct rte_flow_attr attr = { .transfer = 0 }; 9964 const struct mlx5_flow_driver_ops *fops = 9965 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9966 int ret; 9967 9968 ret = flow_drv_action_validate(dev, NULL, 9969 (const struct rte_flow_action *)update, fops, error); 9970 if (ret) 9971 return ret; 9972 return flow_drv_action_update(dev, handle, update, fops, 9973 error); 9974 } 9975 9976 /** 9977 * Query the indirect action by handle. 9978 * 9979 * This function allows retrieving action-specific data such as counters. 9980 * Data is gathered by special action which may be present/referenced in 9981 * more than one flow rule definition. 9982 * 9983 * see @RTE_FLOW_ACTION_TYPE_COUNT 9984 * 9985 * @param dev 9986 * Pointer to Ethernet device structure. 9987 * @param[in] handle 9988 * Handle for the indirect action to query. 9989 * @param[in, out] data 9990 * Pointer to storage for the associated query data type. 9991 * @param[out] error 9992 * Perform verbose error reporting if not NULL. PMDs initialize this 9993 * structure in case of error only. 9994 * 9995 * @return 9996 * 0 on success, a negative errno value otherwise and rte_errno is set. 9997 */ 9998 static int 9999 mlx5_action_handle_query(struct rte_eth_dev *dev, 10000 const struct rte_flow_action_handle *handle, 10001 void *data, 10002 struct rte_flow_error *error) 10003 { 10004 struct rte_flow_attr attr = { .transfer = 0 }; 10005 const struct mlx5_flow_driver_ops *fops = 10006 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10007 10008 return flow_drv_action_query(dev, handle, data, fops, error); 10009 } 10010 10011 /** 10012 * Destroy all indirect actions (shared RSS). 10013 * 10014 * @param dev 10015 * Pointer to Ethernet device. 10016 * 10017 * @return 10018 * 0 on success, a negative errno value otherwise and rte_errno is set. 10019 */ 10020 int 10021 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10022 { 10023 struct rte_flow_error error; 10024 struct mlx5_priv *priv = dev->data->dev_private; 10025 struct mlx5_shared_action_rss *shared_rss; 10026 int ret = 0; 10027 uint32_t idx; 10028 10029 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10030 priv->rss_shared_actions, idx, shared_rss, next) { 10031 ret |= mlx5_action_handle_destroy(dev, 10032 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10033 } 10034 return ret; 10035 } 10036 10037 /** 10038 * Validate existing indirect actions against current device configuration 10039 * and attach them to device resources. 10040 * 10041 * @param dev 10042 * Pointer to Ethernet device. 10043 * 10044 * @return 10045 * 0 on success, a negative errno value otherwise and rte_errno is set. 10046 */ 10047 int 10048 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10049 { 10050 struct mlx5_priv *priv = dev->data->dev_private; 10051 int ret = 0; 10052 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10053 10054 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10055 const char *message; 10056 uint32_t queue_idx; 10057 10058 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10059 ind_tbl->queues_n, 10060 &message, &queue_idx); 10061 if (ret != 0) { 10062 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10063 dev->data->port_id, ind_tbl->queues[queue_idx], 10064 message); 10065 break; 10066 } 10067 } 10068 if (ret != 0) 10069 return ret; 10070 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10071 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10072 if (ret != 0) { 10073 DRV_LOG(ERR, "Port %u could not attach " 10074 "indirection table obj %p", 10075 dev->data->port_id, (void *)ind_tbl); 10076 goto error; 10077 } 10078 } 10079 10080 return 0; 10081 error: 10082 ind_tbl_last = ind_tbl; 10083 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10084 if (ind_tbl == ind_tbl_last) 10085 break; 10086 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10087 DRV_LOG(CRIT, "Port %u could not detach " 10088 "indirection table obj %p on rollback", 10089 dev->data->port_id, (void *)ind_tbl); 10090 } 10091 return ret; 10092 } 10093 10094 /** 10095 * Detach indirect actions of the device from its resources. 10096 * 10097 * @param dev 10098 * Pointer to Ethernet device. 10099 * 10100 * @return 10101 * 0 on success, a negative errno value otherwise and rte_errno is set. 10102 */ 10103 int 10104 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10105 { 10106 struct mlx5_priv *priv = dev->data->dev_private; 10107 int ret = 0; 10108 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10109 10110 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10111 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10112 if (ret != 0) { 10113 DRV_LOG(ERR, "Port %u could not detach " 10114 "indirection table obj %p", 10115 dev->data->port_id, (void *)ind_tbl); 10116 goto error; 10117 } 10118 } 10119 return 0; 10120 error: 10121 ind_tbl_last = ind_tbl; 10122 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10123 if (ind_tbl == ind_tbl_last) 10124 break; 10125 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10126 DRV_LOG(CRIT, "Port %u could not attach " 10127 "indirection table obj %p on rollback", 10128 dev->data->port_id, (void *)ind_tbl); 10129 } 10130 return ret; 10131 } 10132 10133 #ifndef HAVE_MLX5DV_DR 10134 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10135 #else 10136 #define MLX5_DOMAIN_SYNC_FLOW \ 10137 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10138 #endif 10139 10140 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10141 { 10142 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10143 const struct mlx5_flow_driver_ops *fops; 10144 int ret; 10145 struct rte_flow_attr attr = { .transfer = 0 }; 10146 10147 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10148 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10149 if (ret > 0) 10150 ret = -ret; 10151 return ret; 10152 } 10153 10154 const struct mlx5_flow_tunnel * 10155 mlx5_get_tof(const struct rte_flow_item *item, 10156 const struct rte_flow_action *action, 10157 enum mlx5_tof_rule_type *rule_type) 10158 { 10159 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10160 if (item->type == (typeof(item->type)) 10161 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10162 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10163 return flow_items_to_tunnel(item); 10164 } 10165 } 10166 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10167 if (action->type == (typeof(action->type)) 10168 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10169 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10170 return flow_actions_to_tunnel(action); 10171 } 10172 } 10173 return NULL; 10174 } 10175 10176 /** 10177 * tunnel offload functionality is defined for DV environment only 10178 */ 10179 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10180 __extension__ 10181 union tunnel_offload_mark { 10182 uint32_t val; 10183 struct { 10184 uint32_t app_reserve:8; 10185 uint32_t table_id:15; 10186 uint32_t transfer:1; 10187 uint32_t _unused_:8; 10188 }; 10189 }; 10190 10191 static bool 10192 mlx5_access_tunnel_offload_db 10193 (struct rte_eth_dev *dev, 10194 bool (*match)(struct rte_eth_dev *, 10195 struct mlx5_flow_tunnel *, const void *), 10196 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10197 void (*miss)(struct rte_eth_dev *, void *), 10198 void *ctx, bool lock_op); 10199 10200 static int 10201 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10202 struct rte_flow *flow, 10203 const struct rte_flow_attr *attr, 10204 const struct rte_flow_action *app_actions, 10205 uint32_t flow_idx, 10206 const struct mlx5_flow_tunnel *tunnel, 10207 struct tunnel_default_miss_ctx *ctx, 10208 struct rte_flow_error *error) 10209 { 10210 struct mlx5_priv *priv = dev->data->dev_private; 10211 struct mlx5_flow *dev_flow; 10212 struct rte_flow_attr miss_attr = *attr; 10213 const struct rte_flow_item miss_items[2] = { 10214 { 10215 .type = RTE_FLOW_ITEM_TYPE_ETH, 10216 .spec = NULL, 10217 .last = NULL, 10218 .mask = NULL 10219 }, 10220 { 10221 .type = RTE_FLOW_ITEM_TYPE_END, 10222 .spec = NULL, 10223 .last = NULL, 10224 .mask = NULL 10225 } 10226 }; 10227 union tunnel_offload_mark mark_id; 10228 struct rte_flow_action_mark miss_mark; 10229 struct rte_flow_action miss_actions[3] = { 10230 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10231 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10232 }; 10233 const struct rte_flow_action_jump *jump_data; 10234 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10235 struct flow_grp_info grp_info = { 10236 .external = 1, 10237 .transfer = attr->transfer, 10238 .fdb_def_rule = !!priv->fdb_def_rule, 10239 .std_tbl_fix = 0, 10240 }; 10241 int ret; 10242 10243 if (!attr->transfer) { 10244 uint32_t q_size; 10245 10246 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10247 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10248 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10249 0, SOCKET_ID_ANY); 10250 if (!ctx->queue) 10251 return rte_flow_error_set 10252 (error, ENOMEM, 10253 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10254 NULL, "invalid default miss RSS"); 10255 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10256 ctx->action_rss.level = 0, 10257 ctx->action_rss.types = priv->rss_conf.rss_hf, 10258 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10259 ctx->action_rss.queue_num = priv->reta_idx_n, 10260 ctx->action_rss.key = priv->rss_conf.rss_key, 10261 ctx->action_rss.queue = ctx->queue; 10262 if (!priv->reta_idx_n || !priv->rxqs_n) 10263 return rte_flow_error_set 10264 (error, EINVAL, 10265 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10266 NULL, "invalid port configuration"); 10267 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10268 ctx->action_rss.types = 0; 10269 for (i = 0; i != priv->reta_idx_n; ++i) 10270 ctx->queue[i] = (*priv->reta_idx)[i]; 10271 } else { 10272 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10273 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10274 } 10275 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10276 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10277 jump_data = app_actions->conf; 10278 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10279 miss_attr.group = jump_data->group; 10280 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10281 &flow_table, &grp_info, error); 10282 if (ret) 10283 return rte_flow_error_set(error, EINVAL, 10284 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10285 NULL, "invalid tunnel id"); 10286 mark_id.app_reserve = 0; 10287 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10288 mark_id.transfer = !!attr->transfer; 10289 mark_id._unused_ = 0; 10290 miss_mark.id = mark_id.val; 10291 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10292 miss_items, miss_actions, flow_idx, error); 10293 if (!dev_flow) 10294 return -rte_errno; 10295 dev_flow->flow = flow; 10296 dev_flow->external = true; 10297 dev_flow->tunnel = tunnel; 10298 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10299 /* Subflow object was created, we must include one in the list. */ 10300 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10301 dev_flow->handle, next); 10302 DRV_LOG(DEBUG, 10303 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10304 dev->data->port_id, tunnel->app_tunnel.type, 10305 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10306 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10307 miss_actions, error); 10308 if (!ret) 10309 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10310 error); 10311 10312 return ret; 10313 } 10314 10315 static const struct mlx5_flow_tbl_data_entry * 10316 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10317 { 10318 struct mlx5_priv *priv = dev->data->dev_private; 10319 struct mlx5_dev_ctx_shared *sh = priv->sh; 10320 struct mlx5_list_entry *he; 10321 union tunnel_offload_mark mbits = { .val = mark }; 10322 union mlx5_flow_tbl_key table_key = { 10323 { 10324 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10325 .id = 0, 10326 .reserved = 0, 10327 .dummy = 0, 10328 .is_fdb = !!mbits.transfer, 10329 .is_egress = 0, 10330 } 10331 }; 10332 struct mlx5_flow_cb_ctx ctx = { 10333 .data = &table_key.v64, 10334 }; 10335 10336 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10337 return he ? 10338 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10339 } 10340 10341 static void 10342 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10343 struct mlx5_list_entry *entry) 10344 { 10345 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10346 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10347 10348 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10349 tunnel_flow_tbl_to_id(tte->flow_table)); 10350 mlx5_free(tte); 10351 } 10352 10353 static int 10354 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10355 struct mlx5_list_entry *entry, void *cb_ctx) 10356 { 10357 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10358 union tunnel_tbl_key tbl = { 10359 .val = *(uint64_t *)(ctx->data), 10360 }; 10361 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10362 10363 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10364 } 10365 10366 static struct mlx5_list_entry * 10367 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10368 { 10369 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10370 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10371 struct tunnel_tbl_entry *tte; 10372 union tunnel_tbl_key tbl = { 10373 .val = *(uint64_t *)(ctx->data), 10374 }; 10375 10376 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10377 sizeof(*tte), 0, 10378 SOCKET_ID_ANY); 10379 if (!tte) 10380 goto err; 10381 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10382 &tte->flow_table); 10383 if (tte->flow_table >= MLX5_MAX_TABLES) { 10384 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10385 tte->flow_table); 10386 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10387 tte->flow_table); 10388 goto err; 10389 } else if (!tte->flow_table) { 10390 goto err; 10391 } 10392 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10393 tte->tunnel_id = tbl.tunnel_id; 10394 tte->group = tbl.group; 10395 return &tte->hash; 10396 err: 10397 if (tte) 10398 mlx5_free(tte); 10399 return NULL; 10400 } 10401 10402 static struct mlx5_list_entry * 10403 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10404 struct mlx5_list_entry *oentry, 10405 void *cb_ctx __rte_unused) 10406 { 10407 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10408 0, SOCKET_ID_ANY); 10409 10410 if (!tte) 10411 return NULL; 10412 memcpy(tte, oentry, sizeof(*tte)); 10413 return &tte->hash; 10414 } 10415 10416 static void 10417 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10418 struct mlx5_list_entry *entry) 10419 { 10420 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10421 10422 mlx5_free(tte); 10423 } 10424 10425 static uint32_t 10426 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10427 const struct mlx5_flow_tunnel *tunnel, 10428 uint32_t group, uint32_t *table, 10429 struct rte_flow_error *error) 10430 { 10431 struct mlx5_list_entry *he; 10432 struct tunnel_tbl_entry *tte; 10433 union tunnel_tbl_key key = { 10434 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10435 .group = group 10436 }; 10437 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10438 struct mlx5_hlist *group_hash; 10439 struct mlx5_flow_cb_ctx ctx = { 10440 .data = &key.val, 10441 }; 10442 10443 group_hash = tunnel ? tunnel->groups : thub->groups; 10444 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10445 if (!he) 10446 return rte_flow_error_set(error, EINVAL, 10447 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10448 NULL, 10449 "tunnel group index not supported"); 10450 tte = container_of(he, typeof(*tte), hash); 10451 *table = tte->flow_table; 10452 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10453 dev->data->port_id, key.tunnel_id, group, *table); 10454 return 0; 10455 } 10456 10457 static void 10458 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10459 struct mlx5_flow_tunnel *tunnel) 10460 { 10461 struct mlx5_priv *priv = dev->data->dev_private; 10462 struct mlx5_indexed_pool *ipool; 10463 10464 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10465 dev->data->port_id, tunnel->tunnel_id); 10466 LIST_REMOVE(tunnel, chain); 10467 mlx5_hlist_destroy(tunnel->groups); 10468 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10469 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10470 } 10471 10472 static bool 10473 mlx5_access_tunnel_offload_db 10474 (struct rte_eth_dev *dev, 10475 bool (*match)(struct rte_eth_dev *, 10476 struct mlx5_flow_tunnel *, const void *), 10477 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10478 void (*miss)(struct rte_eth_dev *, void *), 10479 void *ctx, bool lock_op) 10480 { 10481 bool verdict = false; 10482 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10483 struct mlx5_flow_tunnel *tunnel; 10484 10485 rte_spinlock_lock(&thub->sl); 10486 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10487 verdict = match(dev, tunnel, (const void *)ctx); 10488 if (verdict) 10489 break; 10490 } 10491 if (!lock_op) 10492 rte_spinlock_unlock(&thub->sl); 10493 if (verdict && hit) 10494 hit(dev, tunnel, ctx); 10495 if (!verdict && miss) 10496 miss(dev, ctx); 10497 if (lock_op) 10498 rte_spinlock_unlock(&thub->sl); 10499 10500 return verdict; 10501 } 10502 10503 struct tunnel_db_find_tunnel_id_ctx { 10504 uint32_t tunnel_id; 10505 struct mlx5_flow_tunnel *tunnel; 10506 }; 10507 10508 static bool 10509 find_tunnel_id_match(struct rte_eth_dev *dev, 10510 struct mlx5_flow_tunnel *tunnel, const void *x) 10511 { 10512 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10513 10514 RTE_SET_USED(dev); 10515 return tunnel->tunnel_id == ctx->tunnel_id; 10516 } 10517 10518 static void 10519 find_tunnel_id_hit(struct rte_eth_dev *dev, 10520 struct mlx5_flow_tunnel *tunnel, void *x) 10521 { 10522 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10523 RTE_SET_USED(dev); 10524 ctx->tunnel = tunnel; 10525 } 10526 10527 static struct mlx5_flow_tunnel * 10528 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10529 { 10530 struct tunnel_db_find_tunnel_id_ctx ctx = { 10531 .tunnel_id = id, 10532 }; 10533 10534 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10535 find_tunnel_id_hit, NULL, &ctx, true); 10536 10537 return ctx.tunnel; 10538 } 10539 10540 static struct mlx5_flow_tunnel * 10541 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10542 const struct rte_flow_tunnel *app_tunnel) 10543 { 10544 struct mlx5_priv *priv = dev->data->dev_private; 10545 struct mlx5_indexed_pool *ipool; 10546 struct mlx5_flow_tunnel *tunnel; 10547 uint32_t id; 10548 10549 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10550 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10551 if (!tunnel) 10552 return NULL; 10553 if (id >= MLX5_MAX_TUNNELS) { 10554 mlx5_ipool_free(ipool, id); 10555 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10556 return NULL; 10557 } 10558 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10559 priv->sh, 10560 mlx5_flow_tunnel_grp2tbl_create_cb, 10561 mlx5_flow_tunnel_grp2tbl_match_cb, 10562 mlx5_flow_tunnel_grp2tbl_remove_cb, 10563 mlx5_flow_tunnel_grp2tbl_clone_cb, 10564 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10565 if (!tunnel->groups) { 10566 mlx5_ipool_free(ipool, id); 10567 return NULL; 10568 } 10569 /* initiate new PMD tunnel */ 10570 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10571 tunnel->tunnel_id = id; 10572 tunnel->action.type = (typeof(tunnel->action.type)) 10573 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10574 tunnel->action.conf = tunnel; 10575 tunnel->item.type = (typeof(tunnel->item.type)) 10576 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10577 tunnel->item.spec = tunnel; 10578 tunnel->item.last = NULL; 10579 tunnel->item.mask = NULL; 10580 10581 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10582 dev->data->port_id, tunnel->tunnel_id); 10583 10584 return tunnel; 10585 } 10586 10587 struct tunnel_db_get_tunnel_ctx { 10588 const struct rte_flow_tunnel *app_tunnel; 10589 struct mlx5_flow_tunnel *tunnel; 10590 }; 10591 10592 static bool get_tunnel_match(struct rte_eth_dev *dev, 10593 struct mlx5_flow_tunnel *tunnel, const void *x) 10594 { 10595 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10596 10597 RTE_SET_USED(dev); 10598 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10599 sizeof(*ctx->app_tunnel)); 10600 } 10601 10602 static void get_tunnel_hit(struct rte_eth_dev *dev, 10603 struct mlx5_flow_tunnel *tunnel, void *x) 10604 { 10605 /* called under tunnel spinlock protection */ 10606 struct tunnel_db_get_tunnel_ctx *ctx = x; 10607 10608 RTE_SET_USED(dev); 10609 tunnel->refctn++; 10610 ctx->tunnel = tunnel; 10611 } 10612 10613 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10614 { 10615 /* called under tunnel spinlock protection */ 10616 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10617 struct tunnel_db_get_tunnel_ctx *ctx = x; 10618 10619 rte_spinlock_unlock(&thub->sl); 10620 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10621 rte_spinlock_lock(&thub->sl); 10622 if (ctx->tunnel) { 10623 ctx->tunnel->refctn = 1; 10624 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10625 } 10626 } 10627 10628 10629 static int 10630 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10631 const struct rte_flow_tunnel *app_tunnel, 10632 struct mlx5_flow_tunnel **tunnel) 10633 { 10634 struct tunnel_db_get_tunnel_ctx ctx = { 10635 .app_tunnel = app_tunnel, 10636 }; 10637 10638 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10639 get_tunnel_miss, &ctx, true); 10640 *tunnel = ctx.tunnel; 10641 return ctx.tunnel ? 0 : -ENOMEM; 10642 } 10643 10644 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10645 { 10646 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10647 10648 if (!thub) 10649 return; 10650 if (!LIST_EMPTY(&thub->tunnels)) 10651 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10652 mlx5_hlist_destroy(thub->groups); 10653 mlx5_free(thub); 10654 } 10655 10656 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10657 { 10658 int err; 10659 struct mlx5_flow_tunnel_hub *thub; 10660 10661 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10662 0, SOCKET_ID_ANY); 10663 if (!thub) 10664 return -ENOMEM; 10665 LIST_INIT(&thub->tunnels); 10666 rte_spinlock_init(&thub->sl); 10667 thub->groups = mlx5_hlist_create("flow groups", 64, 10668 false, true, sh, 10669 mlx5_flow_tunnel_grp2tbl_create_cb, 10670 mlx5_flow_tunnel_grp2tbl_match_cb, 10671 mlx5_flow_tunnel_grp2tbl_remove_cb, 10672 mlx5_flow_tunnel_grp2tbl_clone_cb, 10673 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10674 if (!thub->groups) { 10675 err = -rte_errno; 10676 goto err; 10677 } 10678 sh->tunnel_hub = thub; 10679 10680 return 0; 10681 10682 err: 10683 if (thub->groups) 10684 mlx5_hlist_destroy(thub->groups); 10685 if (thub) 10686 mlx5_free(thub); 10687 return err; 10688 } 10689 10690 static inline int 10691 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10692 struct rte_flow_tunnel *tunnel, 10693 struct rte_flow_error *error) 10694 { 10695 struct mlx5_priv *priv = dev->data->dev_private; 10696 10697 if (!priv->sh->config.dv_flow_en) 10698 return rte_flow_error_set(error, ENOTSUP, 10699 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10700 "flow DV interface is off"); 10701 if (!is_tunnel_offload_active(dev)) 10702 return rte_flow_error_set(error, ENOTSUP, 10703 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10704 "tunnel offload was not activated"); 10705 if (!tunnel) 10706 return rte_flow_error_set(error, EINVAL, 10707 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10708 "no application tunnel"); 10709 switch (tunnel->type) { 10710 default: 10711 return rte_flow_error_set(error, EINVAL, 10712 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10713 "unsupported tunnel type"); 10714 case RTE_FLOW_ITEM_TYPE_VXLAN: 10715 case RTE_FLOW_ITEM_TYPE_GRE: 10716 case RTE_FLOW_ITEM_TYPE_NVGRE: 10717 case RTE_FLOW_ITEM_TYPE_GENEVE: 10718 break; 10719 } 10720 return 0; 10721 } 10722 10723 static int 10724 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10725 struct rte_flow_tunnel *app_tunnel, 10726 struct rte_flow_action **actions, 10727 uint32_t *num_of_actions, 10728 struct rte_flow_error *error) 10729 { 10730 struct mlx5_flow_tunnel *tunnel; 10731 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10732 10733 if (ret) 10734 return ret; 10735 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10736 if (ret < 0) { 10737 return rte_flow_error_set(error, ret, 10738 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10739 "failed to initialize pmd tunnel"); 10740 } 10741 *actions = &tunnel->action; 10742 *num_of_actions = 1; 10743 return 0; 10744 } 10745 10746 static int 10747 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10748 struct rte_flow_tunnel *app_tunnel, 10749 struct rte_flow_item **items, 10750 uint32_t *num_of_items, 10751 struct rte_flow_error *error) 10752 { 10753 struct mlx5_flow_tunnel *tunnel; 10754 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10755 10756 if (ret) 10757 return ret; 10758 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10759 if (ret < 0) { 10760 return rte_flow_error_set(error, ret, 10761 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10762 "failed to initialize pmd tunnel"); 10763 } 10764 *items = &tunnel->item; 10765 *num_of_items = 1; 10766 return 0; 10767 } 10768 10769 struct tunnel_db_element_release_ctx { 10770 struct rte_flow_item *items; 10771 struct rte_flow_action *actions; 10772 uint32_t num_elements; 10773 struct rte_flow_error *error; 10774 int ret; 10775 }; 10776 10777 static bool 10778 tunnel_element_release_match(struct rte_eth_dev *dev, 10779 struct mlx5_flow_tunnel *tunnel, const void *x) 10780 { 10781 const struct tunnel_db_element_release_ctx *ctx = x; 10782 10783 RTE_SET_USED(dev); 10784 if (ctx->num_elements != 1) 10785 return false; 10786 else if (ctx->items) 10787 return ctx->items == &tunnel->item; 10788 else if (ctx->actions) 10789 return ctx->actions == &tunnel->action; 10790 10791 return false; 10792 } 10793 10794 static void 10795 tunnel_element_release_hit(struct rte_eth_dev *dev, 10796 struct mlx5_flow_tunnel *tunnel, void *x) 10797 { 10798 struct tunnel_db_element_release_ctx *ctx = x; 10799 ctx->ret = 0; 10800 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10801 mlx5_flow_tunnel_free(dev, tunnel); 10802 } 10803 10804 static void 10805 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10806 { 10807 struct tunnel_db_element_release_ctx *ctx = x; 10808 RTE_SET_USED(dev); 10809 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10810 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10811 "invalid argument"); 10812 } 10813 10814 static int 10815 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10816 struct rte_flow_item *pmd_items, 10817 uint32_t num_items, struct rte_flow_error *err) 10818 { 10819 struct tunnel_db_element_release_ctx ctx = { 10820 .items = pmd_items, 10821 .actions = NULL, 10822 .num_elements = num_items, 10823 .error = err, 10824 }; 10825 10826 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10827 tunnel_element_release_hit, 10828 tunnel_element_release_miss, &ctx, false); 10829 10830 return ctx.ret; 10831 } 10832 10833 static int 10834 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10835 struct rte_flow_action *pmd_actions, 10836 uint32_t num_actions, struct rte_flow_error *err) 10837 { 10838 struct tunnel_db_element_release_ctx ctx = { 10839 .items = NULL, 10840 .actions = pmd_actions, 10841 .num_elements = num_actions, 10842 .error = err, 10843 }; 10844 10845 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10846 tunnel_element_release_hit, 10847 tunnel_element_release_miss, &ctx, false); 10848 10849 return ctx.ret; 10850 } 10851 10852 static int 10853 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10854 struct rte_mbuf *m, 10855 struct rte_flow_restore_info *info, 10856 struct rte_flow_error *err) 10857 { 10858 uint64_t ol_flags = m->ol_flags; 10859 const struct mlx5_flow_tbl_data_entry *tble; 10860 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10861 10862 if (!is_tunnel_offload_active(dev)) { 10863 info->flags = 0; 10864 return 0; 10865 } 10866 10867 if ((ol_flags & mask) != mask) 10868 goto err; 10869 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10870 if (!tble) { 10871 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10872 dev->data->port_id, m->hash.fdir.hi); 10873 goto err; 10874 } 10875 MLX5_ASSERT(tble->tunnel); 10876 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10877 info->group_id = tble->group_id; 10878 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10879 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10880 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10881 10882 return 0; 10883 10884 err: 10885 return rte_flow_error_set(err, EINVAL, 10886 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10887 "failed to get restore info"); 10888 } 10889 10890 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 10891 static int 10892 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 10893 __rte_unused struct rte_flow_tunnel *app_tunnel, 10894 __rte_unused struct rte_flow_action **actions, 10895 __rte_unused uint32_t *num_of_actions, 10896 __rte_unused struct rte_flow_error *error) 10897 { 10898 return -ENOTSUP; 10899 } 10900 10901 static int 10902 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 10903 __rte_unused struct rte_flow_tunnel *app_tunnel, 10904 __rte_unused struct rte_flow_item **items, 10905 __rte_unused uint32_t *num_of_items, 10906 __rte_unused struct rte_flow_error *error) 10907 { 10908 return -ENOTSUP; 10909 } 10910 10911 static int 10912 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 10913 __rte_unused struct rte_flow_item *pmd_items, 10914 __rte_unused uint32_t num_items, 10915 __rte_unused struct rte_flow_error *err) 10916 { 10917 return -ENOTSUP; 10918 } 10919 10920 static int 10921 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 10922 __rte_unused struct rte_flow_action *pmd_action, 10923 __rte_unused uint32_t num_actions, 10924 __rte_unused struct rte_flow_error *err) 10925 { 10926 return -ENOTSUP; 10927 } 10928 10929 static int 10930 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 10931 __rte_unused struct rte_mbuf *m, 10932 __rte_unused struct rte_flow_restore_info *i, 10933 __rte_unused struct rte_flow_error *err) 10934 { 10935 return -ENOTSUP; 10936 } 10937 10938 static int 10939 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 10940 __rte_unused struct rte_flow *flow, 10941 __rte_unused const struct rte_flow_attr *attr, 10942 __rte_unused const struct rte_flow_action *actions, 10943 __rte_unused uint32_t flow_idx, 10944 __rte_unused const struct mlx5_flow_tunnel *tunnel, 10945 __rte_unused struct tunnel_default_miss_ctx *ctx, 10946 __rte_unused struct rte_flow_error *error) 10947 { 10948 return -ENOTSUP; 10949 } 10950 10951 static struct mlx5_flow_tunnel * 10952 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 10953 __rte_unused uint32_t id) 10954 { 10955 return NULL; 10956 } 10957 10958 static void 10959 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 10960 __rte_unused struct mlx5_flow_tunnel *tunnel) 10961 { 10962 } 10963 10964 static uint32_t 10965 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 10966 __rte_unused const struct mlx5_flow_tunnel *t, 10967 __rte_unused uint32_t group, 10968 __rte_unused uint32_t *table, 10969 struct rte_flow_error *error) 10970 { 10971 return rte_flow_error_set(error, ENOTSUP, 10972 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10973 "tunnel offload requires DV support"); 10974 } 10975 10976 void 10977 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 10978 __rte_unused uint16_t port_id) 10979 { 10980 } 10981 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 10982 10983 /* Flex flow item API */ 10984 static struct rte_flow_item_flex_handle * 10985 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 10986 const struct rte_flow_item_flex_conf *conf, 10987 struct rte_flow_error *error) 10988 { 10989 static const char err_msg[] = "flex item creation unsupported"; 10990 struct mlx5_priv *priv = dev->data->dev_private; 10991 struct rte_flow_attr attr = { .transfer = 0 }; 10992 const struct mlx5_flow_driver_ops *fops = 10993 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10994 10995 if (!priv->pci_dev) { 10996 rte_flow_error_set(error, ENOTSUP, 10997 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10998 "create flex item on PF only"); 10999 return NULL; 11000 } 11001 switch (priv->pci_dev->id.device_id) { 11002 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11003 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11004 break; 11005 default: 11006 rte_flow_error_set(error, ENOTSUP, 11007 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11008 "flex item available on BlueField ports only"); 11009 return NULL; 11010 } 11011 if (!fops->item_create) { 11012 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11013 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11014 NULL, err_msg); 11015 return NULL; 11016 } 11017 return fops->item_create(dev, conf, error); 11018 } 11019 11020 static int 11021 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11022 const struct rte_flow_item_flex_handle *handle, 11023 struct rte_flow_error *error) 11024 { 11025 static const char err_msg[] = "flex item release unsupported"; 11026 struct rte_flow_attr attr = { .transfer = 0 }; 11027 const struct mlx5_flow_driver_ops *fops = 11028 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11029 11030 if (!fops->item_release) { 11031 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11032 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11033 NULL, err_msg); 11034 return -rte_errno; 11035 } 11036 return fops->item_release(dev, handle, error); 11037 } 11038 11039 static void 11040 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11041 { 11042 int ret; 11043 struct rte_flow_error error; 11044 11045 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11046 char *item_name; 11047 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11048 sizeof(item_name), 11049 (void *)(uintptr_t)item->type, &error); 11050 if (ret > 0) 11051 printf("%s ", item_name); 11052 else 11053 printf("%d\n", (int)item->type); 11054 } 11055 printf("END\n"); 11056 } 11057 11058 static int 11059 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11060 { 11061 const struct rte_flow_item_udp *spec = udp_item->spec; 11062 const struct rte_flow_item_udp *mask = udp_item->mask; 11063 uint16_t udp_dport = 0; 11064 11065 if (spec != NULL) { 11066 if (!mask) 11067 mask = &rte_flow_item_udp_mask; 11068 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11069 mask->hdr.dst_port); 11070 } 11071 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11072 } 11073 11074 static const struct mlx5_flow_expand_node * 11075 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11076 unsigned int item_idx, 11077 const struct mlx5_flow_expand_node graph[], 11078 const struct mlx5_flow_expand_node *node) 11079 { 11080 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11081 11082 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11083 node != NULL && 11084 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11085 /* 11086 * The expansion node is VXLAN and it is also the last 11087 * expandable item in the pattern, so need to continue 11088 * expansion of the inner tunnel. 11089 */ 11090 MLX5_ASSERT(item_idx > 0); 11091 prev_item = pattern + item_idx - 1; 11092 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11093 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11094 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11095 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11096 } 11097 return node; 11098 } 11099 11100 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11101 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11102 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11103 }; 11104 11105 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11106 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11107 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11108 { 9, 10, 11 }, { 12, 13, 14 }, 11109 }; 11110 11111 /** 11112 * Discover the number of available flow priorities. 11113 * 11114 * @param dev 11115 * Ethernet device. 11116 * 11117 * @return 11118 * On success, number of available flow priorities. 11119 * On failure, a negative errno-style code and rte_errno is set. 11120 */ 11121 int 11122 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11123 { 11124 static const uint16_t vprio[] = {8, 16}; 11125 const struct mlx5_priv *priv = dev->data->dev_private; 11126 const struct mlx5_flow_driver_ops *fops; 11127 enum mlx5_flow_drv_type type; 11128 int ret; 11129 11130 type = mlx5_flow_os_get_type(); 11131 if (type == MLX5_FLOW_TYPE_MAX) { 11132 type = MLX5_FLOW_TYPE_VERBS; 11133 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11134 type = MLX5_FLOW_TYPE_DV; 11135 } 11136 fops = flow_get_drv_ops(type); 11137 if (fops->discover_priorities == NULL) { 11138 DRV_LOG(ERR, "Priority discovery not supported"); 11139 rte_errno = ENOTSUP; 11140 return -rte_errno; 11141 } 11142 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11143 if (ret < 0) 11144 return ret; 11145 switch (ret) { 11146 case 8: 11147 ret = RTE_DIM(priority_map_3); 11148 break; 11149 case 16: 11150 ret = RTE_DIM(priority_map_5); 11151 break; 11152 default: 11153 rte_errno = ENOTSUP; 11154 DRV_LOG(ERR, 11155 "port %u maximum priority: %d expected 8/16", 11156 dev->data->port_id, ret); 11157 return -rte_errno; 11158 } 11159 DRV_LOG(INFO, "port %u supported flow priorities:" 11160 " 0-%d for ingress or egress root table," 11161 " 0-%d for non-root table or transfer root table.", 11162 dev->data->port_id, ret - 2, 11163 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11164 return ret; 11165 } 11166 11167 /** 11168 * Adjust flow priority based on the highest layer and the request priority. 11169 * 11170 * @param[in] dev 11171 * Pointer to the Ethernet device structure. 11172 * @param[in] priority 11173 * The rule base priority. 11174 * @param[in] subpriority 11175 * The priority based on the items. 11176 * 11177 * @return 11178 * The new priority. 11179 */ 11180 uint32_t 11181 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11182 uint32_t subpriority) 11183 { 11184 uint32_t res = 0; 11185 struct mlx5_priv *priv = dev->data->dev_private; 11186 11187 switch (priv->sh->flow_max_priority) { 11188 case RTE_DIM(priority_map_3): 11189 res = priority_map_3[priority][subpriority]; 11190 break; 11191 case RTE_DIM(priority_map_5): 11192 res = priority_map_5[priority][subpriority]; 11193 break; 11194 } 11195 return res; 11196 } 11197 11198 /** 11199 * Get the priority for sending traffic to kernel table. 11200 * 11201 * @param[in] dev 11202 * Pointer to the Ethernet device structure. 11203 * 11204 * @return 11205 * On success: the value of priority for sending traffic to kernel table 11206 * On failure: -1 11207 */ 11208 uint32_t 11209 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11210 { 11211 struct mlx5_priv *priv = dev->data->dev_private; 11212 uint32_t res; 11213 11214 switch (priv->sh->flow_max_priority) { 11215 case RTE_DIM(priority_map_5): 11216 res = 15; 11217 break; 11218 case RTE_DIM(priority_map_3): 11219 res = 7; 11220 break; 11221 default: 11222 DRV_LOG(ERR, 11223 "port %u maximum priority: %d expected 8/16", 11224 dev->data->port_id, priv->sh->flow_max_priority); 11225 res = (uint32_t)-1; 11226 } 11227 return res; 11228 } 11229 11230 /** 11231 * Get the E-Switch Manager vport id. 11232 * 11233 * @param[in] dev 11234 * Pointer to the Ethernet device structure. 11235 * 11236 * @return 11237 * The vport id. 11238 */ 11239 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11240 { 11241 struct mlx5_priv *priv = dev->data->dev_private; 11242 struct mlx5_common_device *cdev = priv->sh->cdev; 11243 11244 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11245 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11246 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11247 11248 if (priv->pci_dev == NULL) 11249 return 0; 11250 switch (priv->pci_dev->id.device_id) { 11251 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 11252 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11253 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11254 /* 11255 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11256 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11257 * ECPF vport ID is selected and not the host port (0) in any BF case. 11258 */ 11259 return (int16_t)MLX5_ECPF_VPORT_ID; 11260 default: 11261 return MLX5_PF_VPORT_ID; 11262 } 11263 } 11264 11265 /** 11266 * Parse item to get the vport id. 11267 * 11268 * @param[in] dev 11269 * Pointer to the Ethernet device structure. 11270 * @param[in] item 11271 * The src port id match item. 11272 * @param[out] vport_id 11273 * Pointer to put the vport id. 11274 * @param[out] all_ports 11275 * Indicate if the item matches all ports. 11276 * @param[out] error 11277 * Pointer to error structure. 11278 * 11279 * @return 11280 * 0 on success, a negative errno value otherwise and rte_errno is set. 11281 */ 11282 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11283 const struct rte_flow_item *item, 11284 uint16_t *vport_id, 11285 bool *all_ports, 11286 struct rte_flow_error *error) 11287 { 11288 struct mlx5_priv *port_priv; 11289 const struct rte_flow_item_port_id *pid_v; 11290 uint32_t esw_mgr_port; 11291 11292 if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID && 11293 item->type != RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) 11294 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11295 NULL, "Incorrect item type."); 11296 pid_v = item->spec; 11297 if (!pid_v) { 11298 if (all_ports) 11299 *all_ports = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT); 11300 return 0; 11301 } 11302 if (all_ports) 11303 *all_ports = false; 11304 esw_mgr_port = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) ? 11305 MLX5_REPRESENTED_PORT_ESW_MGR : MLX5_PORT_ESW_MGR; 11306 if (pid_v->id == esw_mgr_port) { 11307 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11308 } else { 11309 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 11310 if (!port_priv) 11311 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11312 NULL, "Failed to get port info."); 11313 *vport_id = port_priv->representor_id; 11314 } 11315 11316 return 0; 11317 } 11318 11319 int 11320 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11321 uint16_t *proxy_port_id, 11322 struct rte_flow_error *error) 11323 { 11324 const struct mlx5_priv *priv = dev->data->dev_private; 11325 uint16_t port_id; 11326 11327 if (!priv->sh->config.dv_esw_en) 11328 return rte_flow_error_set(error, EINVAL, 11329 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11330 NULL, 11331 "unable to provide a proxy port" 11332 " without E-Switch configured"); 11333 if (!priv->master && !priv->representor) 11334 return rte_flow_error_set(error, EINVAL, 11335 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11336 NULL, 11337 "unable to provide a proxy port" 11338 " for port which is not a master" 11339 " or a representor port"); 11340 if (priv->master) { 11341 *proxy_port_id = dev->data->port_id; 11342 return 0; 11343 } 11344 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11345 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11346 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11347 11348 if (port_priv->master && 11349 port_priv->domain_id == priv->domain_id) { 11350 *proxy_port_id = port_id; 11351 return 0; 11352 } 11353 } 11354 return rte_flow_error_set(error, EINVAL, 11355 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11356 NULL, "unable to find a proxy port"); 11357 } 11358