1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static bool 168 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 169 { 170 switch (item->type) { 171 case RTE_FLOW_ITEM_TYPE_ETH: 172 case RTE_FLOW_ITEM_TYPE_VLAN: 173 case RTE_FLOW_ITEM_TYPE_IPV4: 174 case RTE_FLOW_ITEM_TYPE_IPV6: 175 case RTE_FLOW_ITEM_TYPE_UDP: 176 case RTE_FLOW_ITEM_TYPE_TCP: 177 case RTE_FLOW_ITEM_TYPE_ESP: 178 case RTE_FLOW_ITEM_TYPE_ICMP: 179 case RTE_FLOW_ITEM_TYPE_ICMP6: 180 case RTE_FLOW_ITEM_TYPE_VXLAN: 181 case RTE_FLOW_ITEM_TYPE_NVGRE: 182 case RTE_FLOW_ITEM_TYPE_GRE: 183 case RTE_FLOW_ITEM_TYPE_GENEVE: 184 case RTE_FLOW_ITEM_TYPE_MPLS: 185 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 186 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 187 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 188 case RTE_FLOW_ITEM_TYPE_GTP: 189 return true; 190 default: 191 break; 192 } 193 return false; 194 } 195 196 /** 197 * Network Service Header (NSH) and its next protocol values 198 * are described in RFC-8393. 199 */ 200 static enum rte_flow_item_type 201 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 202 { 203 enum rte_flow_item_type type; 204 205 switch (proto_mask & proto_spec) { 206 case 0: 207 type = RTE_FLOW_ITEM_TYPE_VOID; 208 break; 209 case RTE_VXLAN_GPE_TYPE_IPV4: 210 type = RTE_FLOW_ITEM_TYPE_IPV4; 211 break; 212 case RTE_VXLAN_GPE_TYPE_IPV6: 213 type = RTE_VXLAN_GPE_TYPE_IPV6; 214 break; 215 case RTE_VXLAN_GPE_TYPE_ETH: 216 type = RTE_FLOW_ITEM_TYPE_ETH; 217 break; 218 default: 219 type = RTE_FLOW_ITEM_TYPE_END; 220 } 221 return type; 222 } 223 224 static enum rte_flow_item_type 225 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 226 { 227 enum rte_flow_item_type type; 228 229 switch (proto_mask & proto_spec) { 230 case 0: 231 type = RTE_FLOW_ITEM_TYPE_VOID; 232 break; 233 case IPPROTO_UDP: 234 type = RTE_FLOW_ITEM_TYPE_UDP; 235 break; 236 case IPPROTO_TCP: 237 type = RTE_FLOW_ITEM_TYPE_TCP; 238 break; 239 case IPPROTO_IPIP: 240 type = RTE_FLOW_ITEM_TYPE_IPV4; 241 break; 242 case IPPROTO_IPV6: 243 type = RTE_FLOW_ITEM_TYPE_IPV6; 244 break; 245 case IPPROTO_ESP: 246 type = RTE_FLOW_ITEM_TYPE_ESP; 247 break; 248 default: 249 type = RTE_FLOW_ITEM_TYPE_END; 250 } 251 return type; 252 } 253 254 static enum rte_flow_item_type 255 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 256 rte_be16_t type_mask, bool is_tunnel) 257 { 258 enum rte_flow_item_type type; 259 260 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 261 case 0: 262 type = RTE_FLOW_ITEM_TYPE_VOID; 263 break; 264 case RTE_ETHER_TYPE_TEB: 265 type = is_tunnel ? 266 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 267 break; 268 case RTE_ETHER_TYPE_VLAN: 269 type = !is_tunnel ? 270 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 271 break; 272 case RTE_ETHER_TYPE_IPV4: 273 type = RTE_FLOW_ITEM_TYPE_IPV4; 274 break; 275 case RTE_ETHER_TYPE_IPV6: 276 type = RTE_FLOW_ITEM_TYPE_IPV6; 277 break; 278 default: 279 type = RTE_FLOW_ITEM_TYPE_END; 280 } 281 return type; 282 } 283 284 static enum rte_flow_item_type 285 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 286 { 287 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 288 do { \ 289 const void *m = item->mask; \ 290 const void *s = item->spec; \ 291 mask = m ? \ 292 ((const struct rte_flow_item_##type *)m)->fld : \ 293 rte_flow_item_##type##_mask.fld; \ 294 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 295 } while (0) 296 297 enum rte_flow_item_type ret; 298 uint16_t spec, mask; 299 300 if (item == NULL || item->spec == NULL) 301 return RTE_FLOW_ITEM_TYPE_VOID; 302 switch (item->type) { 303 case RTE_FLOW_ITEM_TYPE_ETH: 304 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 305 if (!mask) 306 return RTE_FLOW_ITEM_TYPE_VOID; 307 ret = mlx5_ethertype_to_item_type(spec, mask, false); 308 break; 309 case RTE_FLOW_ITEM_TYPE_VLAN: 310 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 311 if (!mask) 312 return RTE_FLOW_ITEM_TYPE_VOID; 313 ret = mlx5_ethertype_to_item_type(spec, mask, false); 314 break; 315 case RTE_FLOW_ITEM_TYPE_IPV4: 316 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 317 if (!mask) 318 return RTE_FLOW_ITEM_TYPE_VOID; 319 ret = mlx5_inet_proto_to_item_type(spec, mask); 320 break; 321 case RTE_FLOW_ITEM_TYPE_IPV6: 322 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 323 if (!mask) 324 return RTE_FLOW_ITEM_TYPE_VOID; 325 ret = mlx5_inet_proto_to_item_type(spec, mask); 326 break; 327 case RTE_FLOW_ITEM_TYPE_GENEVE: 328 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 329 ret = mlx5_ethertype_to_item_type(spec, mask, true); 330 break; 331 case RTE_FLOW_ITEM_TYPE_GRE: 332 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 333 ret = mlx5_ethertype_to_item_type(spec, mask, true); 334 break; 335 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 336 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 337 ret = mlx5_nsh_proto_to_item_type(spec, mask); 338 break; 339 default: 340 ret = RTE_FLOW_ITEM_TYPE_VOID; 341 break; 342 } 343 return ret; 344 #undef MLX5_XSET_ITEM_MASK_SPEC 345 } 346 347 static const int * 348 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 349 const int *next_node) 350 { 351 const struct mlx5_flow_expand_node *node = NULL; 352 const int *next = next_node; 353 354 while (next && *next) { 355 /* 356 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 357 * flag set, because they were not found in the flow pattern. 358 */ 359 node = &graph[*next]; 360 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 361 break; 362 next = node->next; 363 } 364 return next; 365 } 366 367 #define MLX5_RSS_EXP_ELT_N 16 368 369 /** 370 * Expand RSS flows into several possible flows according to the RSS hash 371 * fields requested and the driver capabilities. 372 * 373 * @param[out] buf 374 * Buffer to store the result expansion. 375 * @param[in] size 376 * Buffer size in bytes. If 0, @p buf can be NULL. 377 * @param[in] pattern 378 * User flow pattern. 379 * @param[in] types 380 * RSS types to expand (see RTE_ETH_RSS_* definitions). 381 * @param[in] graph 382 * Input graph to expand @p pattern according to @p types. 383 * @param[in] graph_root_index 384 * Index of root node in @p graph, typically 0. 385 * 386 * @return 387 * A positive value representing the size of @p buf in bytes regardless of 388 * @p size on success, a negative errno value otherwise and rte_errno is 389 * set, the following errors are defined: 390 * 391 * -E2BIG: graph-depth @p graph is too deep. 392 * -EINVAL: @p size has not enough space for expanded pattern. 393 */ 394 static int 395 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 396 const struct rte_flow_item *pattern, uint64_t types, 397 const struct mlx5_flow_expand_node graph[], 398 int graph_root_index) 399 { 400 const struct rte_flow_item *item; 401 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 402 const int *next_node; 403 const int *stack[MLX5_RSS_EXP_ELT_N]; 404 int stack_pos = 0; 405 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 406 unsigned int i, item_idx, last_expand_item_idx = 0; 407 size_t lsize; 408 size_t user_pattern_size = 0; 409 void *addr = NULL; 410 const struct mlx5_flow_expand_node *next = NULL; 411 struct rte_flow_item missed_item; 412 int missed = 0; 413 int elt = 0; 414 const struct rte_flow_item *last_expand_item = NULL; 415 416 memset(&missed_item, 0, sizeof(missed_item)); 417 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 418 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 419 if (lsize > size) 420 return -EINVAL; 421 buf->entry[0].priority = 0; 422 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 423 buf->entries = 0; 424 addr = buf->entry[0].pattern; 425 for (item = pattern, item_idx = 0; 426 item->type != RTE_FLOW_ITEM_TYPE_END; 427 item++, item_idx++) { 428 if (!mlx5_flow_is_rss_expandable_item(item)) { 429 user_pattern_size += sizeof(*item); 430 continue; 431 } 432 last_expand_item = item; 433 last_expand_item_idx = item_idx; 434 i = 0; 435 while (node->next && node->next[i]) { 436 next = &graph[node->next[i]]; 437 if (next->type == item->type) 438 break; 439 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 440 node = next; 441 i = 0; 442 } else { 443 ++i; 444 } 445 } 446 if (next) 447 node = next; 448 user_pattern_size += sizeof(*item); 449 } 450 user_pattern_size += sizeof(*item); /* Handle END item. */ 451 lsize += user_pattern_size; 452 if (lsize > size) 453 return -EINVAL; 454 /* Copy the user pattern in the first entry of the buffer. */ 455 rte_memcpy(addr, pattern, user_pattern_size); 456 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 457 buf->entries = 1; 458 /* Start expanding. */ 459 memset(flow_items, 0, sizeof(flow_items)); 460 user_pattern_size -= sizeof(*item); 461 /* 462 * Check if the last valid item has spec set, need complete pattern, 463 * and the pattern can be used for expansion. 464 */ 465 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 466 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 467 /* Item type END indicates expansion is not required. */ 468 return lsize; 469 } 470 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 471 next = NULL; 472 missed = 1; 473 i = 0; 474 while (node->next && node->next[i]) { 475 next = &graph[node->next[i]]; 476 if (next->type == missed_item.type) { 477 flow_items[0].type = missed_item.type; 478 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 479 break; 480 } 481 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 482 node = next; 483 i = 0; 484 } else { 485 ++i; 486 } 487 next = NULL; 488 } 489 } 490 if (next && missed) { 491 elt = 2; /* missed item + item end. */ 492 node = next; 493 lsize += elt * sizeof(*item) + user_pattern_size; 494 if (lsize > size) 495 return -EINVAL; 496 if (node->rss_types & types) { 497 buf->entry[buf->entries].priority = 1; 498 buf->entry[buf->entries].pattern = addr; 499 buf->entries++; 500 rte_memcpy(addr, buf->entry[0].pattern, 501 user_pattern_size); 502 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 503 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 504 addr = (void *)(((uintptr_t)addr) + 505 elt * sizeof(*item)); 506 } 507 } else if (last_expand_item != NULL) { 508 node = mlx5_flow_expand_rss_adjust_node(pattern, 509 last_expand_item_idx, graph, node); 510 } 511 memset(flow_items, 0, sizeof(flow_items)); 512 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 513 node->next); 514 stack[stack_pos] = next_node; 515 node = next_node ? &graph[*next_node] : NULL; 516 while (node) { 517 flow_items[stack_pos].type = node->type; 518 if (node->rss_types & types) { 519 size_t n; 520 /* 521 * compute the number of items to copy from the 522 * expansion and copy it. 523 * When the stack_pos is 0, there are 1 element in it, 524 * plus the addition END item. 525 */ 526 elt = stack_pos + 2; 527 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 528 lsize += elt * sizeof(*item) + user_pattern_size; 529 if (lsize > size) 530 return -EINVAL; 531 n = elt * sizeof(*item); 532 buf->entry[buf->entries].priority = 533 stack_pos + 1 + missed; 534 buf->entry[buf->entries].pattern = addr; 535 buf->entries++; 536 rte_memcpy(addr, buf->entry[0].pattern, 537 user_pattern_size); 538 addr = (void *)(((uintptr_t)addr) + 539 user_pattern_size); 540 rte_memcpy(addr, &missed_item, 541 missed * sizeof(*item)); 542 addr = (void *)(((uintptr_t)addr) + 543 missed * sizeof(*item)); 544 rte_memcpy(addr, flow_items, n); 545 addr = (void *)(((uintptr_t)addr) + n); 546 } 547 /* Go deeper. */ 548 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 549 node->next) { 550 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 551 node->next); 552 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 553 rte_errno = E2BIG; 554 return -rte_errno; 555 } 556 stack[stack_pos] = next_node; 557 } else if (*(next_node + 1)) { 558 /* Follow up with the next possibility. */ 559 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 560 ++next_node); 561 } else if (!stack_pos) { 562 /* 563 * Completing the traverse over the different paths. 564 * The next_node is advanced to the terminator. 565 */ 566 ++next_node; 567 } else { 568 /* Move to the next path. */ 569 while (stack_pos) { 570 next_node = stack[--stack_pos]; 571 next_node++; 572 if (*next_node) 573 break; 574 } 575 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 576 next_node); 577 stack[stack_pos] = next_node; 578 } 579 node = next_node && *next_node ? &graph[*next_node] : NULL; 580 }; 581 return lsize; 582 } 583 584 /** 585 * Expand SQN flows into several possible flows according to the Tx queue 586 * number 587 * 588 * @param[in] buf 589 * Buffer to store the result expansion. 590 * @param[in] size 591 * Buffer size in bytes. If 0, @p buf can be NULL. 592 * @param[in] pattern 593 * User flow pattern. 594 * @param[in] sq_specs 595 * Buffer to store sq spec. 596 * 597 * @return 598 * 0 for success and negative value for failure 599 * 600 */ 601 static int 602 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 603 const struct rte_flow_item *pattern, 604 struct mlx5_rte_flow_item_sq *sq_specs) 605 { 606 const struct rte_flow_item *item; 607 bool port_representor = false; 608 size_t user_pattern_size = 0; 609 struct rte_eth_dev *dev; 610 struct mlx5_priv *priv; 611 void *addr = NULL; 612 uint16_t port_id; 613 size_t lsize; 614 int elt = 2; 615 uint16_t i; 616 617 buf->entries = 0; 618 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 619 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 620 const struct rte_flow_item_ethdev *pid_v = item->spec; 621 622 if (!pid_v) 623 return 0; 624 port_id = pid_v->port_id; 625 port_representor = true; 626 } 627 user_pattern_size += sizeof(*item); 628 } 629 if (!port_representor) 630 return 0; 631 dev = &rte_eth_devices[port_id]; 632 priv = dev->data->dev_private; 633 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 634 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 635 sizeof(buf->entry[0]) * priv->txqs_n; 636 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 637 return -EINVAL; 638 addr = buf->entry[0].pattern; 639 for (i = 0; i != priv->txqs_n; ++i) { 640 struct rte_flow_item pattern_add[] = { 641 { 642 .type = (enum rte_flow_item_type) 643 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 644 .spec = &sq_specs[i], 645 }, 646 { 647 .type = RTE_FLOW_ITEM_TYPE_END, 648 }, 649 }; 650 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 651 652 if (txq == NULL) 653 return -EINVAL; 654 buf->entry[i].pattern = addr; 655 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 656 mlx5_txq_release(dev, i); 657 rte_memcpy(addr, pattern, user_pattern_size); 658 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 659 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 660 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 661 buf->entries++; 662 } 663 return 0; 664 } 665 666 enum mlx5_expansion { 667 MLX5_EXPANSION_ROOT, 668 MLX5_EXPANSION_ROOT_OUTER, 669 MLX5_EXPANSION_OUTER_ETH, 670 MLX5_EXPANSION_OUTER_VLAN, 671 MLX5_EXPANSION_OUTER_IPV4, 672 MLX5_EXPANSION_OUTER_IPV4_UDP, 673 MLX5_EXPANSION_OUTER_IPV4_TCP, 674 MLX5_EXPANSION_OUTER_IPV4_ESP, 675 MLX5_EXPANSION_OUTER_IPV4_ICMP, 676 MLX5_EXPANSION_OUTER_IPV6, 677 MLX5_EXPANSION_OUTER_IPV6_UDP, 678 MLX5_EXPANSION_OUTER_IPV6_TCP, 679 MLX5_EXPANSION_OUTER_IPV6_ESP, 680 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 681 MLX5_EXPANSION_VXLAN, 682 MLX5_EXPANSION_STD_VXLAN, 683 MLX5_EXPANSION_L3_VXLAN, 684 MLX5_EXPANSION_VXLAN_GPE, 685 MLX5_EXPANSION_GRE, 686 MLX5_EXPANSION_NVGRE, 687 MLX5_EXPANSION_GRE_KEY, 688 MLX5_EXPANSION_MPLS, 689 MLX5_EXPANSION_ETH, 690 MLX5_EXPANSION_VLAN, 691 MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV4_UDP, 693 MLX5_EXPANSION_IPV4_TCP, 694 MLX5_EXPANSION_IPV4_ESP, 695 MLX5_EXPANSION_IPV4_ICMP, 696 MLX5_EXPANSION_IPV6, 697 MLX5_EXPANSION_IPV6_UDP, 698 MLX5_EXPANSION_IPV6_TCP, 699 MLX5_EXPANSION_IPV6_ESP, 700 MLX5_EXPANSION_IPV6_ICMP6, 701 MLX5_EXPANSION_IPV6_FRAG_EXT, 702 MLX5_EXPANSION_GTP, 703 MLX5_EXPANSION_GENEVE, 704 }; 705 706 /** Supported expansion of items. */ 707 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 708 [MLX5_EXPANSION_ROOT] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 710 MLX5_EXPANSION_IPV4, 711 MLX5_EXPANSION_IPV6), 712 .type = RTE_FLOW_ITEM_TYPE_END, 713 }, 714 [MLX5_EXPANSION_ROOT_OUTER] = { 715 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 716 MLX5_EXPANSION_OUTER_IPV4, 717 MLX5_EXPANSION_OUTER_IPV6), 718 .type = RTE_FLOW_ITEM_TYPE_END, 719 }, 720 [MLX5_EXPANSION_OUTER_ETH] = { 721 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 722 .type = RTE_FLOW_ITEM_TYPE_ETH, 723 .rss_types = 0, 724 }, 725 [MLX5_EXPANSION_OUTER_VLAN] = { 726 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 727 MLX5_EXPANSION_OUTER_IPV6), 728 .type = RTE_FLOW_ITEM_TYPE_VLAN, 729 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 730 }, 731 [MLX5_EXPANSION_OUTER_IPV4] = { 732 .next = MLX5_FLOW_EXPAND_RSS_NEXT 733 (MLX5_EXPANSION_OUTER_IPV4_UDP, 734 MLX5_EXPANSION_OUTER_IPV4_TCP, 735 MLX5_EXPANSION_OUTER_IPV4_ESP, 736 MLX5_EXPANSION_OUTER_IPV4_ICMP, 737 MLX5_EXPANSION_GRE, 738 MLX5_EXPANSION_NVGRE, 739 MLX5_EXPANSION_IPV4, 740 MLX5_EXPANSION_IPV6), 741 .type = RTE_FLOW_ITEM_TYPE_IPV4, 742 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 743 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 744 }, 745 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 746 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 747 MLX5_EXPANSION_VXLAN_GPE, 748 MLX5_EXPANSION_MPLS, 749 MLX5_EXPANSION_GENEVE, 750 MLX5_EXPANSION_GTP), 751 .type = RTE_FLOW_ITEM_TYPE_UDP, 752 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 753 }, 754 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 755 .type = RTE_FLOW_ITEM_TYPE_TCP, 756 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 757 }, 758 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 759 .type = RTE_FLOW_ITEM_TYPE_ESP, 760 .rss_types = RTE_ETH_RSS_ESP, 761 }, 762 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 763 .type = RTE_FLOW_ITEM_TYPE_ICMP, 764 }, 765 [MLX5_EXPANSION_OUTER_IPV6] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT 767 (MLX5_EXPANSION_OUTER_IPV6_UDP, 768 MLX5_EXPANSION_OUTER_IPV6_TCP, 769 MLX5_EXPANSION_OUTER_IPV6_ESP, 770 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 771 MLX5_EXPANSION_IPV4, 772 MLX5_EXPANSION_IPV6, 773 MLX5_EXPANSION_GRE, 774 MLX5_EXPANSION_NVGRE), 775 .type = RTE_FLOW_ITEM_TYPE_IPV6, 776 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 777 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 778 }, 779 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 780 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 781 MLX5_EXPANSION_VXLAN_GPE, 782 MLX5_EXPANSION_MPLS, 783 MLX5_EXPANSION_GENEVE, 784 MLX5_EXPANSION_GTP), 785 .type = RTE_FLOW_ITEM_TYPE_UDP, 786 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 787 }, 788 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 789 .type = RTE_FLOW_ITEM_TYPE_TCP, 790 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 791 }, 792 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 793 .type = RTE_FLOW_ITEM_TYPE_ESP, 794 .rss_types = RTE_ETH_RSS_ESP, 795 }, 796 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 797 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 798 }, 799 [MLX5_EXPANSION_VXLAN] = { 800 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 801 MLX5_EXPANSION_IPV4, 802 MLX5_EXPANSION_IPV6), 803 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 804 }, 805 [MLX5_EXPANSION_STD_VXLAN] = { 806 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 807 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 808 }, 809 [MLX5_EXPANSION_L3_VXLAN] = { 810 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 811 MLX5_EXPANSION_IPV6), 812 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 813 }, 814 [MLX5_EXPANSION_VXLAN_GPE] = { 815 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 816 MLX5_EXPANSION_IPV4, 817 MLX5_EXPANSION_IPV6), 818 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 819 }, 820 [MLX5_EXPANSION_GRE] = { 821 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 822 MLX5_EXPANSION_IPV4, 823 MLX5_EXPANSION_IPV6, 824 MLX5_EXPANSION_GRE_KEY, 825 MLX5_EXPANSION_MPLS), 826 .type = RTE_FLOW_ITEM_TYPE_GRE, 827 }, 828 [MLX5_EXPANSION_GRE_KEY] = { 829 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 830 MLX5_EXPANSION_IPV6, 831 MLX5_EXPANSION_MPLS), 832 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 833 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 834 }, 835 [MLX5_EXPANSION_NVGRE] = { 836 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 837 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 838 }, 839 [MLX5_EXPANSION_MPLS] = { 840 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 841 MLX5_EXPANSION_IPV6, 842 MLX5_EXPANSION_ETH), 843 .type = RTE_FLOW_ITEM_TYPE_MPLS, 844 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 845 }, 846 [MLX5_EXPANSION_ETH] = { 847 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 848 .type = RTE_FLOW_ITEM_TYPE_ETH, 849 }, 850 [MLX5_EXPANSION_VLAN] = { 851 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 852 MLX5_EXPANSION_IPV6), 853 .type = RTE_FLOW_ITEM_TYPE_VLAN, 854 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 855 }, 856 [MLX5_EXPANSION_IPV4] = { 857 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 858 MLX5_EXPANSION_IPV4_TCP, 859 MLX5_EXPANSION_IPV4_ESP, 860 MLX5_EXPANSION_IPV4_ICMP), 861 .type = RTE_FLOW_ITEM_TYPE_IPV4, 862 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 863 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 864 }, 865 [MLX5_EXPANSION_IPV4_UDP] = { 866 .type = RTE_FLOW_ITEM_TYPE_UDP, 867 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 868 }, 869 [MLX5_EXPANSION_IPV4_TCP] = { 870 .type = RTE_FLOW_ITEM_TYPE_TCP, 871 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 872 }, 873 [MLX5_EXPANSION_IPV4_ESP] = { 874 .type = RTE_FLOW_ITEM_TYPE_ESP, 875 .rss_types = RTE_ETH_RSS_ESP, 876 }, 877 [MLX5_EXPANSION_IPV4_ICMP] = { 878 .type = RTE_FLOW_ITEM_TYPE_ICMP, 879 }, 880 [MLX5_EXPANSION_IPV6] = { 881 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 882 MLX5_EXPANSION_IPV6_TCP, 883 MLX5_EXPANSION_IPV6_ESP, 884 MLX5_EXPANSION_IPV6_ICMP6, 885 MLX5_EXPANSION_IPV6_FRAG_EXT), 886 .type = RTE_FLOW_ITEM_TYPE_IPV6, 887 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 888 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 889 }, 890 [MLX5_EXPANSION_IPV6_UDP] = { 891 .type = RTE_FLOW_ITEM_TYPE_UDP, 892 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 893 }, 894 [MLX5_EXPANSION_IPV6_TCP] = { 895 .type = RTE_FLOW_ITEM_TYPE_TCP, 896 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 897 }, 898 [MLX5_EXPANSION_IPV6_ESP] = { 899 .type = RTE_FLOW_ITEM_TYPE_ESP, 900 .rss_types = RTE_ETH_RSS_ESP, 901 }, 902 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 903 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 904 }, 905 [MLX5_EXPANSION_IPV6_ICMP6] = { 906 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 907 }, 908 [MLX5_EXPANSION_GTP] = { 909 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 910 MLX5_EXPANSION_IPV6), 911 .type = RTE_FLOW_ITEM_TYPE_GTP, 912 }, 913 [MLX5_EXPANSION_GENEVE] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 915 MLX5_EXPANSION_IPV4, 916 MLX5_EXPANSION_IPV6), 917 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 918 }, 919 }; 920 921 static struct rte_flow_action_handle * 922 mlx5_action_handle_create(struct rte_eth_dev *dev, 923 const struct rte_flow_indir_action_conf *conf, 924 const struct rte_flow_action *action, 925 struct rte_flow_error *error); 926 static int mlx5_action_handle_destroy 927 (struct rte_eth_dev *dev, 928 struct rte_flow_action_handle *handle, 929 struct rte_flow_error *error); 930 static int mlx5_action_handle_update 931 (struct rte_eth_dev *dev, 932 struct rte_flow_action_handle *handle, 933 const void *update, 934 struct rte_flow_error *error); 935 static int mlx5_action_handle_query 936 (struct rte_eth_dev *dev, 937 const struct rte_flow_action_handle *handle, 938 void *data, 939 struct rte_flow_error *error); 940 static int 941 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 942 struct rte_flow_tunnel *app_tunnel, 943 struct rte_flow_action **actions, 944 uint32_t *num_of_actions, 945 struct rte_flow_error *error); 946 static int 947 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 948 struct rte_flow_tunnel *app_tunnel, 949 struct rte_flow_item **items, 950 uint32_t *num_of_items, 951 struct rte_flow_error *error); 952 static int 953 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 954 struct rte_flow_item *pmd_items, 955 uint32_t num_items, struct rte_flow_error *err); 956 static int 957 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 958 struct rte_flow_action *pmd_actions, 959 uint32_t num_actions, 960 struct rte_flow_error *err); 961 static int 962 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 963 struct rte_mbuf *m, 964 struct rte_flow_restore_info *info, 965 struct rte_flow_error *err); 966 static struct rte_flow_item_flex_handle * 967 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 968 const struct rte_flow_item_flex_conf *conf, 969 struct rte_flow_error *error); 970 static int 971 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 972 const struct rte_flow_item_flex_handle *handle, 973 struct rte_flow_error *error); 974 static int 975 mlx5_flow_info_get(struct rte_eth_dev *dev, 976 struct rte_flow_port_info *port_info, 977 struct rte_flow_queue_info *queue_info, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_port_configure(struct rte_eth_dev *dev, 981 const struct rte_flow_port_attr *port_attr, 982 uint16_t nb_queue, 983 const struct rte_flow_queue_attr *queue_attr[], 984 struct rte_flow_error *err); 985 986 static struct rte_flow_pattern_template * 987 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 988 const struct rte_flow_pattern_template_attr *attr, 989 const struct rte_flow_item items[], 990 struct rte_flow_error *error); 991 992 static int 993 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 994 struct rte_flow_pattern_template *template, 995 struct rte_flow_error *error); 996 static struct rte_flow_actions_template * 997 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 998 const struct rte_flow_actions_template_attr *attr, 999 const struct rte_flow_action actions[], 1000 const struct rte_flow_action masks[], 1001 struct rte_flow_error *error); 1002 static int 1003 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1004 struct rte_flow_actions_template *template, 1005 struct rte_flow_error *error); 1006 1007 static struct rte_flow_template_table * 1008 mlx5_flow_table_create(struct rte_eth_dev *dev, 1009 const struct rte_flow_template_table_attr *attr, 1010 struct rte_flow_pattern_template *item_templates[], 1011 uint8_t nb_item_templates, 1012 struct rte_flow_actions_template *action_templates[], 1013 uint8_t nb_action_templates, 1014 struct rte_flow_error *error); 1015 static int 1016 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1017 struct rte_flow_template_table *table, 1018 struct rte_flow_error *error); 1019 static struct rte_flow * 1020 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1021 uint32_t queue, 1022 const struct rte_flow_op_attr *attr, 1023 struct rte_flow_template_table *table, 1024 const struct rte_flow_item items[], 1025 uint8_t pattern_template_index, 1026 const struct rte_flow_action actions[], 1027 uint8_t action_template_index, 1028 void *user_data, 1029 struct rte_flow_error *error); 1030 static int 1031 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1032 uint32_t queue, 1033 const struct rte_flow_op_attr *attr, 1034 struct rte_flow *flow, 1035 void *user_data, 1036 struct rte_flow_error *error); 1037 static int 1038 mlx5_flow_pull(struct rte_eth_dev *dev, 1039 uint32_t queue, 1040 struct rte_flow_op_result res[], 1041 uint16_t n_res, 1042 struct rte_flow_error *error); 1043 static int 1044 mlx5_flow_push(struct rte_eth_dev *dev, 1045 uint32_t queue, 1046 struct rte_flow_error *error); 1047 1048 static struct rte_flow_action_handle * 1049 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1050 const struct rte_flow_op_attr *attr, 1051 const struct rte_flow_indir_action_conf *conf, 1052 const struct rte_flow_action *action, 1053 void *user_data, 1054 struct rte_flow_error *error); 1055 1056 static int 1057 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1058 const struct rte_flow_op_attr *attr, 1059 struct rte_flow_action_handle *handle, 1060 const void *update, 1061 void *user_data, 1062 struct rte_flow_error *error); 1063 1064 static int 1065 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1066 const struct rte_flow_op_attr *attr, 1067 struct rte_flow_action_handle *handle, 1068 void *user_data, 1069 struct rte_flow_error *error); 1070 1071 static int 1072 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1073 const struct rte_flow_op_attr *attr, 1074 const struct rte_flow_action_handle *handle, 1075 void *data, 1076 void *user_data, 1077 struct rte_flow_error *error); 1078 1079 static const struct rte_flow_ops mlx5_flow_ops = { 1080 .validate = mlx5_flow_validate, 1081 .create = mlx5_flow_create, 1082 .destroy = mlx5_flow_destroy, 1083 .flush = mlx5_flow_flush, 1084 .isolate = mlx5_flow_isolate, 1085 .query = mlx5_flow_query, 1086 .dev_dump = mlx5_flow_dev_dump, 1087 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1088 .get_aged_flows = mlx5_flow_get_aged_flows, 1089 .action_handle_create = mlx5_action_handle_create, 1090 .action_handle_destroy = mlx5_action_handle_destroy, 1091 .action_handle_update = mlx5_action_handle_update, 1092 .action_handle_query = mlx5_action_handle_query, 1093 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1094 .tunnel_match = mlx5_flow_tunnel_match, 1095 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1096 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1097 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1098 .flex_item_create = mlx5_flow_flex_item_create, 1099 .flex_item_release = mlx5_flow_flex_item_release, 1100 .info_get = mlx5_flow_info_get, 1101 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1102 .configure = mlx5_flow_port_configure, 1103 .pattern_template_create = mlx5_flow_pattern_template_create, 1104 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1105 .actions_template_create = mlx5_flow_actions_template_create, 1106 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1107 .template_table_create = mlx5_flow_table_create, 1108 .template_table_destroy = mlx5_flow_table_destroy, 1109 .async_create = mlx5_flow_async_flow_create, 1110 .async_destroy = mlx5_flow_async_flow_destroy, 1111 .pull = mlx5_flow_pull, 1112 .push = mlx5_flow_push, 1113 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1114 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1115 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1116 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1117 }; 1118 1119 /* Tunnel information. */ 1120 struct mlx5_flow_tunnel_info { 1121 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1122 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1123 }; 1124 1125 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1126 { 1127 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1128 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1129 }, 1130 { 1131 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1132 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1133 }, 1134 { 1135 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1136 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1137 }, 1138 { 1139 .tunnel = MLX5_FLOW_LAYER_GRE, 1140 .ptype = RTE_PTYPE_TUNNEL_GRE, 1141 }, 1142 { 1143 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1144 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1145 }, 1146 { 1147 .tunnel = MLX5_FLOW_LAYER_MPLS, 1148 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1149 }, 1150 { 1151 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1152 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1153 }, 1154 { 1155 .tunnel = MLX5_FLOW_LAYER_IPIP, 1156 .ptype = RTE_PTYPE_TUNNEL_IP, 1157 }, 1158 { 1159 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1160 .ptype = RTE_PTYPE_TUNNEL_IP, 1161 }, 1162 { 1163 .tunnel = MLX5_FLOW_LAYER_GTP, 1164 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1165 }, 1166 }; 1167 1168 1169 1170 /** 1171 * Translate tag ID to register. 1172 * 1173 * @param[in] dev 1174 * Pointer to the Ethernet device structure. 1175 * @param[in] feature 1176 * The feature that request the register. 1177 * @param[in] id 1178 * The request register ID. 1179 * @param[out] error 1180 * Error description in case of any. 1181 * 1182 * @return 1183 * The request register on success, a negative errno 1184 * value otherwise and rte_errno is set. 1185 */ 1186 int 1187 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1188 enum mlx5_feature_name feature, 1189 uint32_t id, 1190 struct rte_flow_error *error) 1191 { 1192 struct mlx5_priv *priv = dev->data->dev_private; 1193 struct mlx5_sh_config *config = &priv->sh->config; 1194 enum modify_reg start_reg; 1195 bool skip_mtr_reg = false; 1196 1197 switch (feature) { 1198 case MLX5_HAIRPIN_RX: 1199 return REG_B; 1200 case MLX5_HAIRPIN_TX: 1201 return REG_A; 1202 case MLX5_METADATA_RX: 1203 switch (config->dv_xmeta_en) { 1204 case MLX5_XMETA_MODE_LEGACY: 1205 return REG_B; 1206 case MLX5_XMETA_MODE_META16: 1207 return REG_C_0; 1208 case MLX5_XMETA_MODE_META32: 1209 return REG_C_1; 1210 case MLX5_XMETA_MODE_META32_HWS: 1211 return REG_C_1; 1212 } 1213 break; 1214 case MLX5_METADATA_TX: 1215 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1216 return REG_C_1; 1217 } else { 1218 return REG_A; 1219 } 1220 case MLX5_METADATA_FDB: 1221 switch (config->dv_xmeta_en) { 1222 case MLX5_XMETA_MODE_LEGACY: 1223 return REG_NON; 1224 case MLX5_XMETA_MODE_META16: 1225 return REG_C_0; 1226 case MLX5_XMETA_MODE_META32: 1227 return REG_C_1; 1228 case MLX5_XMETA_MODE_META32_HWS: 1229 return REG_C_1; 1230 } 1231 break; 1232 case MLX5_FLOW_MARK: 1233 switch (config->dv_xmeta_en) { 1234 case MLX5_XMETA_MODE_LEGACY: 1235 case MLX5_XMETA_MODE_META32_HWS: 1236 return REG_NON; 1237 case MLX5_XMETA_MODE_META16: 1238 return REG_C_1; 1239 case MLX5_XMETA_MODE_META32: 1240 return REG_C_0; 1241 } 1242 break; 1243 case MLX5_MTR_ID: 1244 /* 1245 * If meter color and meter id share one register, flow match 1246 * should use the meter color register for match. 1247 */ 1248 if (priv->mtr_reg_share) 1249 return priv->mtr_color_reg; 1250 else 1251 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1252 REG_C_3; 1253 case MLX5_MTR_COLOR: 1254 case MLX5_ASO_FLOW_HIT: 1255 case MLX5_ASO_CONNTRACK: 1256 case MLX5_SAMPLE_ID: 1257 /* All features use the same REG_C. */ 1258 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1259 return priv->mtr_color_reg; 1260 case MLX5_COPY_MARK: 1261 /* 1262 * Metadata COPY_MARK register using is in meter suffix sub 1263 * flow while with meter. It's safe to share the same register. 1264 */ 1265 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1266 case MLX5_APP_TAG: 1267 /* 1268 * If meter is enable, it will engage the register for color 1269 * match and flow match. If meter color match is not using the 1270 * REG_C_2, need to skip the REG_C_x be used by meter color 1271 * match. 1272 * If meter is disable, free to use all available registers. 1273 */ 1274 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1275 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1276 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1277 if (id > (uint32_t)(REG_C_7 - start_reg)) 1278 return rte_flow_error_set(error, EINVAL, 1279 RTE_FLOW_ERROR_TYPE_ITEM, 1280 NULL, "invalid tag id"); 1281 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1282 return rte_flow_error_set(error, ENOTSUP, 1283 RTE_FLOW_ERROR_TYPE_ITEM, 1284 NULL, "unsupported tag id"); 1285 /* 1286 * This case means meter is using the REG_C_x great than 2. 1287 * Take care not to conflict with meter color REG_C_x. 1288 * If the available index REG_C_y >= REG_C_x, skip the 1289 * color register. 1290 */ 1291 if (skip_mtr_reg && priv->sh->flow_mreg_c 1292 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1293 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1294 return rte_flow_error_set(error, EINVAL, 1295 RTE_FLOW_ERROR_TYPE_ITEM, 1296 NULL, "invalid tag id"); 1297 if (priv->sh->flow_mreg_c 1298 [id + 1 + start_reg - REG_C_0] != REG_NON) 1299 return priv->sh->flow_mreg_c 1300 [id + 1 + start_reg - REG_C_0]; 1301 return rte_flow_error_set(error, ENOTSUP, 1302 RTE_FLOW_ERROR_TYPE_ITEM, 1303 NULL, "unsupported tag id"); 1304 } 1305 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1306 } 1307 MLX5_ASSERT(false); 1308 return rte_flow_error_set(error, EINVAL, 1309 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1310 NULL, "invalid feature name"); 1311 } 1312 1313 /** 1314 * Check extensive flow metadata register support. 1315 * 1316 * @param dev 1317 * Pointer to rte_eth_dev structure. 1318 * 1319 * @return 1320 * True if device supports extensive flow metadata register, otherwise false. 1321 */ 1322 bool 1323 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1324 { 1325 struct mlx5_priv *priv = dev->data->dev_private; 1326 1327 /* 1328 * Having available reg_c can be regarded inclusively as supporting 1329 * extensive flow metadata register, which could mean, 1330 * - metadata register copy action by modify header. 1331 * - 16 modify header actions is supported. 1332 * - reg_c's are preserved across different domain (FDB and NIC) on 1333 * packet loopback by flow lookup miss. 1334 */ 1335 return priv->sh->flow_mreg_c[2] != REG_NON; 1336 } 1337 1338 /** 1339 * Get the lowest priority. 1340 * 1341 * @param[in] dev 1342 * Pointer to the Ethernet device structure. 1343 * @param[in] attributes 1344 * Pointer to device flow rule attributes. 1345 * 1346 * @return 1347 * The value of lowest priority of flow. 1348 */ 1349 uint32_t 1350 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1351 const struct rte_flow_attr *attr) 1352 { 1353 struct mlx5_priv *priv = dev->data->dev_private; 1354 1355 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1356 return priv->sh->flow_max_priority - 2; 1357 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1358 } 1359 1360 /** 1361 * Calculate matcher priority of the flow. 1362 * 1363 * @param[in] dev 1364 * Pointer to the Ethernet device structure. 1365 * @param[in] attr 1366 * Pointer to device flow rule attributes. 1367 * @param[in] subpriority 1368 * The priority based on the items. 1369 * @param[in] external 1370 * Flow is user flow. 1371 * @return 1372 * The matcher priority of the flow. 1373 */ 1374 uint16_t 1375 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1376 const struct rte_flow_attr *attr, 1377 uint32_t subpriority, bool external) 1378 { 1379 uint16_t priority = (uint16_t)attr->priority; 1380 struct mlx5_priv *priv = dev->data->dev_private; 1381 1382 /* NIC root rules */ 1383 if (!attr->group && !attr->transfer) { 1384 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1385 priority = priv->sh->flow_max_priority - 1; 1386 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1387 /* FDB root rules */ 1388 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1389 attr->group == 0 && 1390 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1391 return (priv->sh->flow_max_priority - 1) * 3; 1392 } 1393 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1394 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1395 return priority * 3 + subpriority; 1396 } 1397 1398 /** 1399 * Verify the @p item specifications (spec, last, mask) are compatible with the 1400 * NIC capabilities. 1401 * 1402 * @param[in] item 1403 * Item specification. 1404 * @param[in] mask 1405 * @p item->mask or flow default bit-masks. 1406 * @param[in] nic_mask 1407 * Bit-masks covering supported fields by the NIC to compare with user mask. 1408 * @param[in] size 1409 * Bit-masks size in bytes. 1410 * @param[in] range_accepted 1411 * True if range of values is accepted for specific fields, false otherwise. 1412 * @param[out] error 1413 * Pointer to error structure. 1414 * 1415 * @return 1416 * 0 on success, a negative errno value otherwise and rte_errno is set. 1417 */ 1418 int 1419 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1420 const uint8_t *mask, 1421 const uint8_t *nic_mask, 1422 unsigned int size, 1423 bool range_accepted, 1424 struct rte_flow_error *error) 1425 { 1426 unsigned int i; 1427 1428 MLX5_ASSERT(nic_mask); 1429 for (i = 0; i < size; ++i) 1430 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1431 return rte_flow_error_set(error, ENOTSUP, 1432 RTE_FLOW_ERROR_TYPE_ITEM, 1433 item, 1434 "mask enables non supported" 1435 " bits"); 1436 if (!item->spec && (item->mask || item->last)) 1437 return rte_flow_error_set(error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ITEM, item, 1439 "mask/last without a spec is not" 1440 " supported"); 1441 if (item->spec && item->last && !range_accepted) { 1442 uint8_t spec[size]; 1443 uint8_t last[size]; 1444 unsigned int i; 1445 int ret; 1446 1447 for (i = 0; i < size; ++i) { 1448 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1449 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1450 } 1451 ret = memcmp(spec, last, size); 1452 if (ret != 0) 1453 return rte_flow_error_set(error, EINVAL, 1454 RTE_FLOW_ERROR_TYPE_ITEM, 1455 item, 1456 "range is not valid"); 1457 } 1458 return 0; 1459 } 1460 1461 /** 1462 * Adjust the hash fields according to the @p flow information. 1463 * 1464 * @param[in] dev_flow. 1465 * Pointer to the mlx5_flow. 1466 * @param[in] tunnel 1467 * 1 when the hash field is for a tunnel item. 1468 * @param[in] layer_types 1469 * RTE_ETH_RSS_* types. 1470 * @param[in] hash_fields 1471 * Item hash fields. 1472 * 1473 * @return 1474 * The hash fields that should be used. 1475 */ 1476 uint64_t 1477 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1478 int tunnel __rte_unused, uint64_t layer_types, 1479 uint64_t hash_fields) 1480 { 1481 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1482 int rss_request_inner = rss_desc->level >= 2; 1483 1484 /* Check RSS hash level for tunnel. */ 1485 if (tunnel && rss_request_inner) 1486 hash_fields |= IBV_RX_HASH_INNER; 1487 else if (tunnel || rss_request_inner) 1488 return 0; 1489 #endif 1490 /* Check if requested layer matches RSS hash fields. */ 1491 if (!(rss_desc->types & layer_types)) 1492 return 0; 1493 return hash_fields; 1494 } 1495 1496 /** 1497 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1498 * if several tunnel rules are used on this queue, the tunnel ptype will be 1499 * cleared. 1500 * 1501 * @param rxq_ctrl 1502 * Rx queue to update. 1503 */ 1504 static void 1505 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1506 { 1507 unsigned int i; 1508 uint32_t tunnel_ptype = 0; 1509 1510 /* Look up for the ptype to use. */ 1511 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1512 if (!rxq_ctrl->flow_tunnels_n[i]) 1513 continue; 1514 if (!tunnel_ptype) { 1515 tunnel_ptype = tunnels_info[i].ptype; 1516 } else { 1517 tunnel_ptype = 0; 1518 break; 1519 } 1520 } 1521 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1522 } 1523 1524 /** 1525 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1526 * flow. 1527 * 1528 * @param[in] dev 1529 * Pointer to the Ethernet device structure. 1530 * @param[in] dev_handle 1531 * Pointer to device flow handle structure. 1532 */ 1533 void 1534 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1535 struct mlx5_flow_handle *dev_handle) 1536 { 1537 struct mlx5_priv *priv = dev->data->dev_private; 1538 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1539 struct mlx5_ind_table_obj *ind_tbl = NULL; 1540 unsigned int i; 1541 1542 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1543 struct mlx5_hrxq *hrxq; 1544 1545 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1546 dev_handle->rix_hrxq); 1547 if (hrxq) 1548 ind_tbl = hrxq->ind_table; 1549 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1550 struct mlx5_shared_action_rss *shared_rss; 1551 1552 shared_rss = mlx5_ipool_get 1553 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1554 dev_handle->rix_srss); 1555 if (shared_rss) 1556 ind_tbl = shared_rss->ind_tbl; 1557 } 1558 if (!ind_tbl) 1559 return; 1560 for (i = 0; i != ind_tbl->queues_n; ++i) { 1561 int idx = ind_tbl->queues[i]; 1562 struct mlx5_rxq_ctrl *rxq_ctrl; 1563 1564 if (mlx5_is_external_rxq(dev, idx)) 1565 continue; 1566 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1567 MLX5_ASSERT(rxq_ctrl != NULL); 1568 if (rxq_ctrl == NULL) 1569 continue; 1570 /* 1571 * To support metadata register copy on Tx loopback, 1572 * this must be always enabled (metadata may arive 1573 * from other port - not from local flows only. 1574 */ 1575 if (tunnel) { 1576 unsigned int j; 1577 1578 /* Increase the counter matching the flow. */ 1579 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1580 if ((tunnels_info[j].tunnel & 1581 dev_handle->layers) == 1582 tunnels_info[j].tunnel) { 1583 rxq_ctrl->flow_tunnels_n[j]++; 1584 break; 1585 } 1586 } 1587 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1588 } 1589 } 1590 } 1591 1592 static void 1593 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1594 { 1595 struct mlx5_priv *priv = dev->data->dev_private; 1596 struct mlx5_rxq_ctrl *rxq_ctrl; 1597 uint16_t port_id; 1598 1599 if (priv->sh->shared_mark_enabled) 1600 return; 1601 if (priv->master || priv->representor) { 1602 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1603 struct mlx5_priv *opriv = 1604 rte_eth_devices[port_id].data->dev_private; 1605 1606 if (!opriv || 1607 opriv->sh != priv->sh || 1608 opriv->domain_id != priv->domain_id || 1609 opriv->mark_enabled) 1610 continue; 1611 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1612 rxq_ctrl->rxq.mark = 1; 1613 } 1614 opriv->mark_enabled = 1; 1615 } 1616 } else { 1617 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1618 rxq_ctrl->rxq.mark = 1; 1619 } 1620 priv->mark_enabled = 1; 1621 } 1622 priv->sh->shared_mark_enabled = 1; 1623 } 1624 1625 /** 1626 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1627 * 1628 * @param[in] dev 1629 * Pointer to the Ethernet device structure. 1630 * @param[in] flow 1631 * Pointer to flow structure. 1632 */ 1633 static void 1634 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1635 { 1636 struct mlx5_priv *priv = dev->data->dev_private; 1637 uint32_t handle_idx; 1638 struct mlx5_flow_handle *dev_handle; 1639 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1640 1641 MLX5_ASSERT(wks); 1642 if (wks->mark) 1643 flow_rxq_mark_flag_set(dev); 1644 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1645 handle_idx, dev_handle, next) 1646 flow_drv_rxq_flags_set(dev, dev_handle); 1647 } 1648 1649 /** 1650 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1651 * device flow if no other flow uses it with the same kind of request. 1652 * 1653 * @param dev 1654 * Pointer to Ethernet device. 1655 * @param[in] dev_handle 1656 * Pointer to the device flow handle structure. 1657 */ 1658 static void 1659 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1660 struct mlx5_flow_handle *dev_handle) 1661 { 1662 struct mlx5_priv *priv = dev->data->dev_private; 1663 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1664 struct mlx5_ind_table_obj *ind_tbl = NULL; 1665 unsigned int i; 1666 1667 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1668 struct mlx5_hrxq *hrxq; 1669 1670 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1671 dev_handle->rix_hrxq); 1672 if (hrxq) 1673 ind_tbl = hrxq->ind_table; 1674 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1675 struct mlx5_shared_action_rss *shared_rss; 1676 1677 shared_rss = mlx5_ipool_get 1678 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1679 dev_handle->rix_srss); 1680 if (shared_rss) 1681 ind_tbl = shared_rss->ind_tbl; 1682 } 1683 if (!ind_tbl) 1684 return; 1685 MLX5_ASSERT(dev->data->dev_started); 1686 for (i = 0; i != ind_tbl->queues_n; ++i) { 1687 int idx = ind_tbl->queues[i]; 1688 struct mlx5_rxq_ctrl *rxq_ctrl; 1689 1690 if (mlx5_is_external_rxq(dev, idx)) 1691 continue; 1692 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1693 MLX5_ASSERT(rxq_ctrl != NULL); 1694 if (rxq_ctrl == NULL) 1695 continue; 1696 if (tunnel) { 1697 unsigned int j; 1698 1699 /* Decrease the counter matching the flow. */ 1700 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1701 if ((tunnels_info[j].tunnel & 1702 dev_handle->layers) == 1703 tunnels_info[j].tunnel) { 1704 rxq_ctrl->flow_tunnels_n[j]--; 1705 break; 1706 } 1707 } 1708 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1709 } 1710 } 1711 } 1712 1713 /** 1714 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1715 * @p flow if no other flow uses it with the same kind of request. 1716 * 1717 * @param dev 1718 * Pointer to Ethernet device. 1719 * @param[in] flow 1720 * Pointer to the flow. 1721 */ 1722 static void 1723 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1724 { 1725 struct mlx5_priv *priv = dev->data->dev_private; 1726 uint32_t handle_idx; 1727 struct mlx5_flow_handle *dev_handle; 1728 1729 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1730 handle_idx, dev_handle, next) 1731 flow_drv_rxq_flags_trim(dev, dev_handle); 1732 } 1733 1734 /** 1735 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1736 * 1737 * @param dev 1738 * Pointer to Ethernet device. 1739 */ 1740 static void 1741 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1742 { 1743 struct mlx5_priv *priv = dev->data->dev_private; 1744 unsigned int i; 1745 1746 for (i = 0; i != priv->rxqs_n; ++i) { 1747 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1748 unsigned int j; 1749 1750 if (rxq == NULL || rxq->ctrl == NULL) 1751 continue; 1752 rxq->ctrl->rxq.mark = 0; 1753 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1754 rxq->ctrl->flow_tunnels_n[j] = 0; 1755 rxq->ctrl->rxq.tunnel = 0; 1756 } 1757 priv->mark_enabled = 0; 1758 priv->sh->shared_mark_enabled = 0; 1759 } 1760 1761 /** 1762 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1763 * 1764 * @param[in] dev 1765 * Pointer to the Ethernet device structure. 1766 */ 1767 void 1768 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1769 { 1770 struct mlx5_priv *priv = dev->data->dev_private; 1771 unsigned int i; 1772 1773 for (i = 0; i != priv->rxqs_n; ++i) { 1774 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1775 struct mlx5_rxq_data *data; 1776 1777 if (rxq == NULL || rxq->ctrl == NULL) 1778 continue; 1779 data = &rxq->ctrl->rxq; 1780 if (!rte_flow_dynf_metadata_avail()) { 1781 data->dynf_meta = 0; 1782 data->flow_meta_mask = 0; 1783 data->flow_meta_offset = -1; 1784 data->flow_meta_port_mask = 0; 1785 } else { 1786 data->dynf_meta = 1; 1787 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1788 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1789 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1790 } 1791 } 1792 } 1793 1794 /* 1795 * return a pointer to the desired action in the list of actions. 1796 * 1797 * @param[in] actions 1798 * The list of actions to search the action in. 1799 * @param[in] action 1800 * The action to find. 1801 * 1802 * @return 1803 * Pointer to the action in the list, if found. NULL otherwise. 1804 */ 1805 const struct rte_flow_action * 1806 mlx5_flow_find_action(const struct rte_flow_action *actions, 1807 enum rte_flow_action_type action) 1808 { 1809 if (actions == NULL) 1810 return NULL; 1811 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1812 if (actions->type == action) 1813 return actions; 1814 return NULL; 1815 } 1816 1817 /* 1818 * Validate the flag action. 1819 * 1820 * @param[in] action_flags 1821 * Bit-fields that holds the actions detected until now. 1822 * @param[in] attr 1823 * Attributes of flow that includes this action. 1824 * @param[out] error 1825 * Pointer to error structure. 1826 * 1827 * @return 1828 * 0 on success, a negative errno value otherwise and rte_errno is set. 1829 */ 1830 int 1831 mlx5_flow_validate_action_flag(uint64_t action_flags, 1832 const struct rte_flow_attr *attr, 1833 struct rte_flow_error *error) 1834 { 1835 if (action_flags & MLX5_FLOW_ACTION_MARK) 1836 return rte_flow_error_set(error, EINVAL, 1837 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1838 "can't mark and flag in same flow"); 1839 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1840 return rte_flow_error_set(error, EINVAL, 1841 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1842 "can't have 2 flag" 1843 " actions in same flow"); 1844 if (attr->egress) 1845 return rte_flow_error_set(error, ENOTSUP, 1846 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1847 "flag action not supported for " 1848 "egress"); 1849 return 0; 1850 } 1851 1852 /* 1853 * Validate the mark action. 1854 * 1855 * @param[in] action 1856 * Pointer to the queue action. 1857 * @param[in] action_flags 1858 * Bit-fields that holds the actions detected until now. 1859 * @param[in] attr 1860 * Attributes of flow that includes this action. 1861 * @param[out] error 1862 * Pointer to error structure. 1863 * 1864 * @return 1865 * 0 on success, a negative errno value otherwise and rte_errno is set. 1866 */ 1867 int 1868 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1869 uint64_t action_flags, 1870 const struct rte_flow_attr *attr, 1871 struct rte_flow_error *error) 1872 { 1873 const struct rte_flow_action_mark *mark = action->conf; 1874 1875 if (!mark) 1876 return rte_flow_error_set(error, EINVAL, 1877 RTE_FLOW_ERROR_TYPE_ACTION, 1878 action, 1879 "configuration cannot be null"); 1880 if (mark->id >= MLX5_FLOW_MARK_MAX) 1881 return rte_flow_error_set(error, EINVAL, 1882 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1883 &mark->id, 1884 "mark id must in 0 <= id < " 1885 RTE_STR(MLX5_FLOW_MARK_MAX)); 1886 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1887 return rte_flow_error_set(error, EINVAL, 1888 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1889 "can't flag and mark in same flow"); 1890 if (action_flags & MLX5_FLOW_ACTION_MARK) 1891 return rte_flow_error_set(error, EINVAL, 1892 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1893 "can't have 2 mark actions in same" 1894 " flow"); 1895 if (attr->egress) 1896 return rte_flow_error_set(error, ENOTSUP, 1897 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1898 "mark action not supported for " 1899 "egress"); 1900 return 0; 1901 } 1902 1903 /* 1904 * Validate the drop action. 1905 * 1906 * @param[in] action_flags 1907 * Bit-fields that holds the actions detected until now. 1908 * @param[in] attr 1909 * Attributes of flow that includes this action. 1910 * @param[out] error 1911 * Pointer to error structure. 1912 * 1913 * @return 1914 * 0 on success, a negative errno value otherwise and rte_errno is set. 1915 */ 1916 int 1917 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1918 const struct rte_flow_attr *attr, 1919 struct rte_flow_error *error) 1920 { 1921 if (attr->egress) 1922 return rte_flow_error_set(error, ENOTSUP, 1923 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1924 "drop action not supported for " 1925 "egress"); 1926 return 0; 1927 } 1928 1929 /* 1930 * Validate the queue action. 1931 * 1932 * @param[in] action 1933 * Pointer to the queue action. 1934 * @param[in] action_flags 1935 * Bit-fields that holds the actions detected until now. 1936 * @param[in] dev 1937 * Pointer to the Ethernet device structure. 1938 * @param[in] attr 1939 * Attributes of flow that includes this action. 1940 * @param[out] error 1941 * Pointer to error structure. 1942 * 1943 * @return 1944 * 0 on success, a negative errno value otherwise and rte_errno is set. 1945 */ 1946 int 1947 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1948 uint64_t action_flags, 1949 struct rte_eth_dev *dev, 1950 const struct rte_flow_attr *attr, 1951 struct rte_flow_error *error) 1952 { 1953 struct mlx5_priv *priv = dev->data->dev_private; 1954 const struct rte_flow_action_queue *queue = action->conf; 1955 1956 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1957 return rte_flow_error_set(error, EINVAL, 1958 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1959 "can't have 2 fate actions in" 1960 " same flow"); 1961 if (attr->egress) 1962 return rte_flow_error_set(error, ENOTSUP, 1963 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1964 "queue action not supported for egress."); 1965 if (mlx5_is_external_rxq(dev, queue->index)) 1966 return 0; 1967 if (!priv->rxqs_n) 1968 return rte_flow_error_set(error, EINVAL, 1969 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1970 NULL, "No Rx queues configured"); 1971 if (queue->index >= priv->rxqs_n) 1972 return rte_flow_error_set(error, EINVAL, 1973 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1974 &queue->index, 1975 "queue index out of range"); 1976 if (mlx5_rxq_get(dev, queue->index) == NULL) 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1979 &queue->index, 1980 "queue is not configured"); 1981 return 0; 1982 } 1983 1984 /** 1985 * Validate queue numbers for device RSS. 1986 * 1987 * @param[in] dev 1988 * Configured device. 1989 * @param[in] queues 1990 * Array of queue numbers. 1991 * @param[in] queues_n 1992 * Size of the @p queues array. 1993 * @param[out] error 1994 * On error, filled with a textual error description. 1995 * @param[out] queue_idx 1996 * On error, filled with an offending queue index in @p queues array. 1997 * 1998 * @return 1999 * 0 on success, a negative errno code on error. 2000 */ 2001 static int 2002 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2003 const uint16_t *queues, uint32_t queues_n, 2004 const char **error, uint32_t *queue_idx) 2005 { 2006 const struct mlx5_priv *priv = dev->data->dev_private; 2007 bool is_hairpin = false; 2008 bool is_ext_rss = false; 2009 uint32_t i; 2010 2011 for (i = 0; i != queues_n; ++i) { 2012 struct mlx5_rxq_ctrl *rxq_ctrl; 2013 2014 if (mlx5_is_external_rxq(dev, queues[0])) { 2015 is_ext_rss = true; 2016 continue; 2017 } 2018 if (is_ext_rss) { 2019 *error = "Combining external and regular RSS queues is not supported"; 2020 *queue_idx = i; 2021 return -ENOTSUP; 2022 } 2023 if (queues[i] >= priv->rxqs_n) { 2024 *error = "queue index out of range"; 2025 *queue_idx = i; 2026 return -EINVAL; 2027 } 2028 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2029 if (rxq_ctrl == NULL) { 2030 *error = "queue is not configured"; 2031 *queue_idx = i; 2032 return -EINVAL; 2033 } 2034 if (i == 0 && rxq_ctrl->is_hairpin) 2035 is_hairpin = true; 2036 if (is_hairpin != rxq_ctrl->is_hairpin) { 2037 *error = "combining hairpin and regular RSS queues is not supported"; 2038 *queue_idx = i; 2039 return -ENOTSUP; 2040 } 2041 } 2042 return 0; 2043 } 2044 2045 /* 2046 * Validate the rss action. 2047 * 2048 * @param[in] dev 2049 * Pointer to the Ethernet device structure. 2050 * @param[in] action 2051 * Pointer to the queue action. 2052 * @param[out] error 2053 * Pointer to error structure. 2054 * 2055 * @return 2056 * 0 on success, a negative errno value otherwise and rte_errno is set. 2057 */ 2058 int 2059 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2060 const struct rte_flow_action *action, 2061 struct rte_flow_error *error) 2062 { 2063 struct mlx5_priv *priv = dev->data->dev_private; 2064 const struct rte_flow_action_rss *rss = action->conf; 2065 int ret; 2066 const char *message; 2067 uint32_t queue_idx; 2068 2069 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2070 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2071 return rte_flow_error_set(error, ENOTSUP, 2072 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2073 &rss->func, 2074 "RSS hash function not supported"); 2075 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2076 if (rss->level > 2) 2077 #else 2078 if (rss->level > 1) 2079 #endif 2080 return rte_flow_error_set(error, ENOTSUP, 2081 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2082 &rss->level, 2083 "tunnel RSS is not supported"); 2084 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2085 if (rss->key_len == 0 && rss->key != NULL) 2086 return rte_flow_error_set(error, ENOTSUP, 2087 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2088 &rss->key_len, 2089 "RSS hash key length 0"); 2090 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2091 return rte_flow_error_set(error, ENOTSUP, 2092 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2093 &rss->key_len, 2094 "RSS hash key too small"); 2095 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2096 return rte_flow_error_set(error, ENOTSUP, 2097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2098 &rss->key_len, 2099 "RSS hash key too large"); 2100 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2101 return rte_flow_error_set(error, ENOTSUP, 2102 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2103 &rss->queue_num, 2104 "number of queues too large"); 2105 if (rss->types & MLX5_RSS_HF_MASK) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2108 &rss->types, 2109 "some RSS protocols are not" 2110 " supported"); 2111 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2112 !(rss->types & RTE_ETH_RSS_IP)) 2113 return rte_flow_error_set(error, EINVAL, 2114 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2115 "L3 partial RSS requested but L3 RSS" 2116 " type not specified"); 2117 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2118 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2119 return rte_flow_error_set(error, EINVAL, 2120 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2121 "L4 partial RSS requested but L4 RSS" 2122 " type not specified"); 2123 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2124 return rte_flow_error_set(error, EINVAL, 2125 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2126 NULL, "No Rx queues configured"); 2127 if (!rss->queue_num) 2128 return rte_flow_error_set(error, EINVAL, 2129 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2130 NULL, "No queues configured"); 2131 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2132 &message, &queue_idx); 2133 if (ret != 0) { 2134 return rte_flow_error_set(error, -ret, 2135 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2136 &rss->queue[queue_idx], message); 2137 } 2138 return 0; 2139 } 2140 2141 /* 2142 * Validate the rss action. 2143 * 2144 * @param[in] action 2145 * Pointer to the queue action. 2146 * @param[in] action_flags 2147 * Bit-fields that holds the actions detected until now. 2148 * @param[in] dev 2149 * Pointer to the Ethernet device structure. 2150 * @param[in] attr 2151 * Attributes of flow that includes this action. 2152 * @param[in] item_flags 2153 * Items that were detected. 2154 * @param[out] error 2155 * Pointer to error structure. 2156 * 2157 * @return 2158 * 0 on success, a negative errno value otherwise and rte_errno is set. 2159 */ 2160 int 2161 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2162 uint64_t action_flags, 2163 struct rte_eth_dev *dev, 2164 const struct rte_flow_attr *attr, 2165 uint64_t item_flags, 2166 struct rte_flow_error *error) 2167 { 2168 const struct rte_flow_action_rss *rss = action->conf; 2169 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2170 int ret; 2171 2172 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2175 "can't have 2 fate actions" 2176 " in same flow"); 2177 ret = mlx5_validate_action_rss(dev, action, error); 2178 if (ret) 2179 return ret; 2180 if (attr->egress) 2181 return rte_flow_error_set(error, ENOTSUP, 2182 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2183 "rss action not supported for " 2184 "egress"); 2185 if (rss->level > 1 && !tunnel) 2186 return rte_flow_error_set(error, EINVAL, 2187 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2188 "inner RSS is not supported for " 2189 "non-tunnel flows"); 2190 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2191 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2192 return rte_flow_error_set(error, EINVAL, 2193 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2194 "RSS on eCPRI is not supported now"); 2195 } 2196 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2197 !(item_flags & 2198 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2199 rss->level > 1) 2200 return rte_flow_error_set(error, EINVAL, 2201 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2202 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2203 return 0; 2204 } 2205 2206 /* 2207 * Validate the default miss action. 2208 * 2209 * @param[in] action_flags 2210 * Bit-fields that holds the actions detected until now. 2211 * @param[out] error 2212 * Pointer to error structure. 2213 * 2214 * @return 2215 * 0 on success, a negative errno value otherwise and rte_errno is set. 2216 */ 2217 int 2218 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2219 const struct rte_flow_attr *attr, 2220 struct rte_flow_error *error) 2221 { 2222 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2225 "can't have 2 fate actions in" 2226 " same flow"); 2227 if (attr->egress) 2228 return rte_flow_error_set(error, ENOTSUP, 2229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2230 "default miss action not supported " 2231 "for egress"); 2232 if (attr->group) 2233 return rte_flow_error_set(error, ENOTSUP, 2234 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2235 "only group 0 is supported"); 2236 if (attr->transfer) 2237 return rte_flow_error_set(error, ENOTSUP, 2238 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2239 NULL, "transfer is not supported"); 2240 return 0; 2241 } 2242 2243 /* 2244 * Validate the count action. 2245 * 2246 * @param[in] dev 2247 * Pointer to the Ethernet device structure. 2248 * @param[in] attr 2249 * Attributes of flow that includes this action. 2250 * @param[out] error 2251 * Pointer to error structure. 2252 * 2253 * @return 2254 * 0 on success, a negative errno value otherwise and rte_errno is set. 2255 */ 2256 int 2257 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2258 const struct rte_flow_attr *attr, 2259 struct rte_flow_error *error) 2260 { 2261 if (attr->egress) 2262 return rte_flow_error_set(error, ENOTSUP, 2263 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2264 "count action not supported for " 2265 "egress"); 2266 return 0; 2267 } 2268 2269 /* 2270 * Validate the ASO CT action. 2271 * 2272 * @param[in] dev 2273 * Pointer to the Ethernet device structure. 2274 * @param[in] conntrack 2275 * Pointer to the CT action profile. 2276 * @param[out] error 2277 * Pointer to error structure. 2278 * 2279 * @return 2280 * 0 on success, a negative errno value otherwise and rte_errno is set. 2281 */ 2282 int 2283 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2284 const struct rte_flow_action_conntrack *conntrack, 2285 struct rte_flow_error *error) 2286 { 2287 RTE_SET_USED(dev); 2288 2289 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2290 return rte_flow_error_set(error, EINVAL, 2291 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2292 "Invalid CT state"); 2293 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2296 "Invalid last TCP packet flag"); 2297 return 0; 2298 } 2299 2300 /** 2301 * Validate ICMP6 item. 2302 * 2303 * @param[in] item 2304 * Item specification. 2305 * @param[in] item_flags 2306 * Bit-fields that holds the items detected until now. 2307 * @param[in] ext_vlan_sup 2308 * Whether extended VLAN features are supported or not. 2309 * @param[out] error 2310 * Pointer to error structure. 2311 * 2312 * @return 2313 * 0 on success, a negative errno value otherwise and rte_errno is set. 2314 */ 2315 int 2316 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2317 uint64_t item_flags, 2318 uint8_t target_protocol, 2319 struct rte_flow_error *error) 2320 { 2321 const struct rte_flow_item_icmp6 *mask = item->mask; 2322 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2323 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2324 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2325 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2326 MLX5_FLOW_LAYER_OUTER_L4; 2327 int ret; 2328 2329 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2330 return rte_flow_error_set(error, EINVAL, 2331 RTE_FLOW_ERROR_TYPE_ITEM, item, 2332 "protocol filtering not compatible" 2333 " with ICMP6 layer"); 2334 if (!(item_flags & l3m)) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "IPv6 is mandatory to filter on" 2338 " ICMP6"); 2339 if (item_flags & l4m) 2340 return rte_flow_error_set(error, EINVAL, 2341 RTE_FLOW_ERROR_TYPE_ITEM, item, 2342 "multiple L4 layers not supported"); 2343 if (!mask) 2344 mask = &rte_flow_item_icmp6_mask; 2345 ret = mlx5_flow_item_acceptable 2346 (item, (const uint8_t *)mask, 2347 (const uint8_t *)&rte_flow_item_icmp6_mask, 2348 sizeof(struct rte_flow_item_icmp6), 2349 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2350 if (ret < 0) 2351 return ret; 2352 return 0; 2353 } 2354 2355 /** 2356 * Validate ICMP item. 2357 * 2358 * @param[in] item 2359 * Item specification. 2360 * @param[in] item_flags 2361 * Bit-fields that holds the items detected until now. 2362 * @param[out] error 2363 * Pointer to error structure. 2364 * 2365 * @return 2366 * 0 on success, a negative errno value otherwise and rte_errno is set. 2367 */ 2368 int 2369 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2370 uint64_t item_flags, 2371 uint8_t target_protocol, 2372 struct rte_flow_error *error) 2373 { 2374 const struct rte_flow_item_icmp *mask = item->mask; 2375 const struct rte_flow_item_icmp nic_mask = { 2376 .hdr.icmp_type = 0xff, 2377 .hdr.icmp_code = 0xff, 2378 .hdr.icmp_ident = RTE_BE16(0xffff), 2379 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2380 }; 2381 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2382 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2383 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2384 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2385 MLX5_FLOW_LAYER_OUTER_L4; 2386 int ret; 2387 2388 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2389 return rte_flow_error_set(error, EINVAL, 2390 RTE_FLOW_ERROR_TYPE_ITEM, item, 2391 "protocol filtering not compatible" 2392 " with ICMP layer"); 2393 if (!(item_flags & l3m)) 2394 return rte_flow_error_set(error, EINVAL, 2395 RTE_FLOW_ERROR_TYPE_ITEM, item, 2396 "IPv4 is mandatory to filter" 2397 " on ICMP"); 2398 if (item_flags & l4m) 2399 return rte_flow_error_set(error, EINVAL, 2400 RTE_FLOW_ERROR_TYPE_ITEM, item, 2401 "multiple L4 layers not supported"); 2402 if (!mask) 2403 mask = &nic_mask; 2404 ret = mlx5_flow_item_acceptable 2405 (item, (const uint8_t *)mask, 2406 (const uint8_t *)&nic_mask, 2407 sizeof(struct rte_flow_item_icmp), 2408 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2409 if (ret < 0) 2410 return ret; 2411 return 0; 2412 } 2413 2414 /** 2415 * Validate Ethernet item. 2416 * 2417 * @param[in] item 2418 * Item specification. 2419 * @param[in] item_flags 2420 * Bit-fields that holds the items detected until now. 2421 * @param[out] error 2422 * Pointer to error structure. 2423 * 2424 * @return 2425 * 0 on success, a negative errno value otherwise and rte_errno is set. 2426 */ 2427 int 2428 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2429 uint64_t item_flags, bool ext_vlan_sup, 2430 struct rte_flow_error *error) 2431 { 2432 const struct rte_flow_item_eth *mask = item->mask; 2433 const struct rte_flow_item_eth nic_mask = { 2434 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2435 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2436 .type = RTE_BE16(0xffff), 2437 .has_vlan = ext_vlan_sup ? 1 : 0, 2438 }; 2439 int ret; 2440 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2441 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2442 MLX5_FLOW_LAYER_OUTER_L2; 2443 2444 if (item_flags & ethm) 2445 return rte_flow_error_set(error, ENOTSUP, 2446 RTE_FLOW_ERROR_TYPE_ITEM, item, 2447 "multiple L2 layers not supported"); 2448 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2449 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2450 return rte_flow_error_set(error, EINVAL, 2451 RTE_FLOW_ERROR_TYPE_ITEM, item, 2452 "L2 layer should not follow " 2453 "L3 layers"); 2454 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2455 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2456 return rte_flow_error_set(error, EINVAL, 2457 RTE_FLOW_ERROR_TYPE_ITEM, item, 2458 "L2 layer should not follow VLAN"); 2459 if (item_flags & MLX5_FLOW_LAYER_GTP) 2460 return rte_flow_error_set(error, EINVAL, 2461 RTE_FLOW_ERROR_TYPE_ITEM, item, 2462 "L2 layer should not follow GTP"); 2463 if (!mask) 2464 mask = &rte_flow_item_eth_mask; 2465 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2466 (const uint8_t *)&nic_mask, 2467 sizeof(struct rte_flow_item_eth), 2468 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2469 return ret; 2470 } 2471 2472 /** 2473 * Validate VLAN item. 2474 * 2475 * @param[in] item 2476 * Item specification. 2477 * @param[in] item_flags 2478 * Bit-fields that holds the items detected until now. 2479 * @param[in] dev 2480 * Ethernet device flow is being created on. 2481 * @param[out] error 2482 * Pointer to error structure. 2483 * 2484 * @return 2485 * 0 on success, a negative errno value otherwise and rte_errno is set. 2486 */ 2487 int 2488 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2489 uint64_t item_flags, 2490 struct rte_eth_dev *dev, 2491 struct rte_flow_error *error) 2492 { 2493 const struct rte_flow_item_vlan *spec = item->spec; 2494 const struct rte_flow_item_vlan *mask = item->mask; 2495 const struct rte_flow_item_vlan nic_mask = { 2496 .tci = RTE_BE16(UINT16_MAX), 2497 .inner_type = RTE_BE16(UINT16_MAX), 2498 }; 2499 uint16_t vlan_tag = 0; 2500 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2501 int ret; 2502 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2503 MLX5_FLOW_LAYER_INNER_L4) : 2504 (MLX5_FLOW_LAYER_OUTER_L3 | 2505 MLX5_FLOW_LAYER_OUTER_L4); 2506 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2507 MLX5_FLOW_LAYER_OUTER_VLAN; 2508 2509 if (item_flags & vlanm) 2510 return rte_flow_error_set(error, EINVAL, 2511 RTE_FLOW_ERROR_TYPE_ITEM, item, 2512 "multiple VLAN layers not supported"); 2513 else if ((item_flags & l34m) != 0) 2514 return rte_flow_error_set(error, EINVAL, 2515 RTE_FLOW_ERROR_TYPE_ITEM, item, 2516 "VLAN cannot follow L3/L4 layer"); 2517 if (!mask) 2518 mask = &rte_flow_item_vlan_mask; 2519 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2520 (const uint8_t *)&nic_mask, 2521 sizeof(struct rte_flow_item_vlan), 2522 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2523 if (ret) 2524 return ret; 2525 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2526 struct mlx5_priv *priv = dev->data->dev_private; 2527 2528 if (priv->vmwa_context) { 2529 /* 2530 * Non-NULL context means we have a virtual machine 2531 * and SR-IOV enabled, we have to create VLAN interface 2532 * to make hypervisor to setup E-Switch vport 2533 * context correctly. We avoid creating the multiple 2534 * VLAN interfaces, so we cannot support VLAN tag mask. 2535 */ 2536 return rte_flow_error_set(error, EINVAL, 2537 RTE_FLOW_ERROR_TYPE_ITEM, 2538 item, 2539 "VLAN tag mask is not" 2540 " supported in virtual" 2541 " environment"); 2542 } 2543 } 2544 if (spec) { 2545 vlan_tag = spec->tci; 2546 vlan_tag &= mask->tci; 2547 } 2548 /* 2549 * From verbs perspective an empty VLAN is equivalent 2550 * to a packet without VLAN layer. 2551 */ 2552 if (!vlan_tag) 2553 return rte_flow_error_set(error, EINVAL, 2554 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2555 item->spec, 2556 "VLAN cannot be empty"); 2557 return 0; 2558 } 2559 2560 /** 2561 * Validate IPV4 item. 2562 * 2563 * @param[in] item 2564 * Item specification. 2565 * @param[in] item_flags 2566 * Bit-fields that holds the items detected until now. 2567 * @param[in] last_item 2568 * Previous validated item in the pattern items. 2569 * @param[in] ether_type 2570 * Type in the ethernet layer header (including dot1q). 2571 * @param[in] acc_mask 2572 * Acceptable mask, if NULL default internal default mask 2573 * will be used to check whether item fields are supported. 2574 * @param[in] range_accepted 2575 * True if range of values is accepted for specific fields, false otherwise. 2576 * @param[out] error 2577 * Pointer to error structure. 2578 * 2579 * @return 2580 * 0 on success, a negative errno value otherwise and rte_errno is set. 2581 */ 2582 int 2583 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2584 uint64_t item_flags, 2585 uint64_t last_item, 2586 uint16_t ether_type, 2587 const struct rte_flow_item_ipv4 *acc_mask, 2588 bool range_accepted, 2589 struct rte_flow_error *error) 2590 { 2591 const struct rte_flow_item_ipv4 *mask = item->mask; 2592 const struct rte_flow_item_ipv4 *spec = item->spec; 2593 const struct rte_flow_item_ipv4 nic_mask = { 2594 .hdr = { 2595 .src_addr = RTE_BE32(0xffffffff), 2596 .dst_addr = RTE_BE32(0xffffffff), 2597 .type_of_service = 0xff, 2598 .next_proto_id = 0xff, 2599 }, 2600 }; 2601 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2602 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2603 MLX5_FLOW_LAYER_OUTER_L3; 2604 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2605 MLX5_FLOW_LAYER_OUTER_L4; 2606 int ret; 2607 uint8_t next_proto = 0xFF; 2608 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2609 MLX5_FLOW_LAYER_OUTER_VLAN | 2610 MLX5_FLOW_LAYER_INNER_VLAN); 2611 2612 if ((last_item & l2_vlan) && ether_type && 2613 ether_type != RTE_ETHER_TYPE_IPV4) 2614 return rte_flow_error_set(error, EINVAL, 2615 RTE_FLOW_ERROR_TYPE_ITEM, item, 2616 "IPv4 cannot follow L2/VLAN layer " 2617 "which ether type is not IPv4"); 2618 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2619 if (mask && spec) 2620 next_proto = mask->hdr.next_proto_id & 2621 spec->hdr.next_proto_id; 2622 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2623 return rte_flow_error_set(error, EINVAL, 2624 RTE_FLOW_ERROR_TYPE_ITEM, 2625 item, 2626 "multiple tunnel " 2627 "not supported"); 2628 } 2629 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2630 return rte_flow_error_set(error, EINVAL, 2631 RTE_FLOW_ERROR_TYPE_ITEM, item, 2632 "wrong tunnel type - IPv6 specified " 2633 "but IPv4 item provided"); 2634 if (item_flags & l3m) 2635 return rte_flow_error_set(error, ENOTSUP, 2636 RTE_FLOW_ERROR_TYPE_ITEM, item, 2637 "multiple L3 layers not supported"); 2638 else if (item_flags & l4m) 2639 return rte_flow_error_set(error, EINVAL, 2640 RTE_FLOW_ERROR_TYPE_ITEM, item, 2641 "L3 cannot follow an L4 layer."); 2642 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2643 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2644 return rte_flow_error_set(error, EINVAL, 2645 RTE_FLOW_ERROR_TYPE_ITEM, item, 2646 "L3 cannot follow an NVGRE layer."); 2647 if (!mask) 2648 mask = &rte_flow_item_ipv4_mask; 2649 else if (mask->hdr.next_proto_id != 0 && 2650 mask->hdr.next_proto_id != 0xff) 2651 return rte_flow_error_set(error, EINVAL, 2652 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2653 "partial mask is not supported" 2654 " for protocol"); 2655 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2656 acc_mask ? (const uint8_t *)acc_mask 2657 : (const uint8_t *)&nic_mask, 2658 sizeof(struct rte_flow_item_ipv4), 2659 range_accepted, error); 2660 if (ret < 0) 2661 return ret; 2662 return 0; 2663 } 2664 2665 /** 2666 * Validate IPV6 item. 2667 * 2668 * @param[in] item 2669 * Item specification. 2670 * @param[in] item_flags 2671 * Bit-fields that holds the items detected until now. 2672 * @param[in] last_item 2673 * Previous validated item in the pattern items. 2674 * @param[in] ether_type 2675 * Type in the ethernet layer header (including dot1q). 2676 * @param[in] acc_mask 2677 * Acceptable mask, if NULL default internal default mask 2678 * will be used to check whether item fields are supported. 2679 * @param[out] error 2680 * Pointer to error structure. 2681 * 2682 * @return 2683 * 0 on success, a negative errno value otherwise and rte_errno is set. 2684 */ 2685 int 2686 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2687 uint64_t item_flags, 2688 uint64_t last_item, 2689 uint16_t ether_type, 2690 const struct rte_flow_item_ipv6 *acc_mask, 2691 struct rte_flow_error *error) 2692 { 2693 const struct rte_flow_item_ipv6 *mask = item->mask; 2694 const struct rte_flow_item_ipv6 *spec = item->spec; 2695 const struct rte_flow_item_ipv6 nic_mask = { 2696 .hdr = { 2697 .src_addr = 2698 "\xff\xff\xff\xff\xff\xff\xff\xff" 2699 "\xff\xff\xff\xff\xff\xff\xff\xff", 2700 .dst_addr = 2701 "\xff\xff\xff\xff\xff\xff\xff\xff" 2702 "\xff\xff\xff\xff\xff\xff\xff\xff", 2703 .vtc_flow = RTE_BE32(0xffffffff), 2704 .proto = 0xff, 2705 }, 2706 }; 2707 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2708 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2709 MLX5_FLOW_LAYER_OUTER_L3; 2710 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2711 MLX5_FLOW_LAYER_OUTER_L4; 2712 int ret; 2713 uint8_t next_proto = 0xFF; 2714 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2715 MLX5_FLOW_LAYER_OUTER_VLAN | 2716 MLX5_FLOW_LAYER_INNER_VLAN); 2717 2718 if ((last_item & l2_vlan) && ether_type && 2719 ether_type != RTE_ETHER_TYPE_IPV6) 2720 return rte_flow_error_set(error, EINVAL, 2721 RTE_FLOW_ERROR_TYPE_ITEM, item, 2722 "IPv6 cannot follow L2/VLAN layer " 2723 "which ether type is not IPv6"); 2724 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2725 next_proto = spec->hdr.proto; 2726 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2727 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2728 return rte_flow_error_set(error, EINVAL, 2729 RTE_FLOW_ERROR_TYPE_ITEM, 2730 item, 2731 "multiple tunnel " 2732 "not supported"); 2733 } 2734 if (next_proto == IPPROTO_HOPOPTS || 2735 next_proto == IPPROTO_ROUTING || 2736 next_proto == IPPROTO_FRAGMENT || 2737 next_proto == IPPROTO_ESP || 2738 next_proto == IPPROTO_AH || 2739 next_proto == IPPROTO_DSTOPTS) 2740 return rte_flow_error_set(error, EINVAL, 2741 RTE_FLOW_ERROR_TYPE_ITEM, item, 2742 "IPv6 proto (next header) should " 2743 "not be set as extension header"); 2744 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2745 return rte_flow_error_set(error, EINVAL, 2746 RTE_FLOW_ERROR_TYPE_ITEM, item, 2747 "wrong tunnel type - IPv4 specified " 2748 "but IPv6 item provided"); 2749 if (item_flags & l3m) 2750 return rte_flow_error_set(error, ENOTSUP, 2751 RTE_FLOW_ERROR_TYPE_ITEM, item, 2752 "multiple L3 layers not supported"); 2753 else if (item_flags & l4m) 2754 return rte_flow_error_set(error, EINVAL, 2755 RTE_FLOW_ERROR_TYPE_ITEM, item, 2756 "L3 cannot follow an L4 layer."); 2757 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2758 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2759 return rte_flow_error_set(error, EINVAL, 2760 RTE_FLOW_ERROR_TYPE_ITEM, item, 2761 "L3 cannot follow an NVGRE layer."); 2762 if (!mask) 2763 mask = &rte_flow_item_ipv6_mask; 2764 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2765 acc_mask ? (const uint8_t *)acc_mask 2766 : (const uint8_t *)&nic_mask, 2767 sizeof(struct rte_flow_item_ipv6), 2768 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2769 if (ret < 0) 2770 return ret; 2771 return 0; 2772 } 2773 2774 /** 2775 * Validate UDP item. 2776 * 2777 * @param[in] item 2778 * Item specification. 2779 * @param[in] item_flags 2780 * Bit-fields that holds the items detected until now. 2781 * @param[in] target_protocol 2782 * The next protocol in the previous item. 2783 * @param[in] flow_mask 2784 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2785 * @param[out] error 2786 * Pointer to error structure. 2787 * 2788 * @return 2789 * 0 on success, a negative errno value otherwise and rte_errno is set. 2790 */ 2791 int 2792 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2793 uint64_t item_flags, 2794 uint8_t target_protocol, 2795 struct rte_flow_error *error) 2796 { 2797 const struct rte_flow_item_udp *mask = item->mask; 2798 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2799 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2800 MLX5_FLOW_LAYER_OUTER_L3; 2801 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2802 MLX5_FLOW_LAYER_OUTER_L4; 2803 int ret; 2804 2805 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2806 return rte_flow_error_set(error, EINVAL, 2807 RTE_FLOW_ERROR_TYPE_ITEM, item, 2808 "protocol filtering not compatible" 2809 " with UDP layer"); 2810 if (!(item_flags & l3m)) 2811 return rte_flow_error_set(error, EINVAL, 2812 RTE_FLOW_ERROR_TYPE_ITEM, item, 2813 "L3 is mandatory to filter on L4"); 2814 if (item_flags & l4m) 2815 return rte_flow_error_set(error, EINVAL, 2816 RTE_FLOW_ERROR_TYPE_ITEM, item, 2817 "multiple L4 layers not supported"); 2818 if (!mask) 2819 mask = &rte_flow_item_udp_mask; 2820 ret = mlx5_flow_item_acceptable 2821 (item, (const uint8_t *)mask, 2822 (const uint8_t *)&rte_flow_item_udp_mask, 2823 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2824 error); 2825 if (ret < 0) 2826 return ret; 2827 return 0; 2828 } 2829 2830 /** 2831 * Validate TCP item. 2832 * 2833 * @param[in] item 2834 * Item specification. 2835 * @param[in] item_flags 2836 * Bit-fields that holds the items detected until now. 2837 * @param[in] target_protocol 2838 * The next protocol in the previous item. 2839 * @param[out] error 2840 * Pointer to error structure. 2841 * 2842 * @return 2843 * 0 on success, a negative errno value otherwise and rte_errno is set. 2844 */ 2845 int 2846 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2847 uint64_t item_flags, 2848 uint8_t target_protocol, 2849 const struct rte_flow_item_tcp *flow_mask, 2850 struct rte_flow_error *error) 2851 { 2852 const struct rte_flow_item_tcp *mask = item->mask; 2853 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2854 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2855 MLX5_FLOW_LAYER_OUTER_L3; 2856 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2857 MLX5_FLOW_LAYER_OUTER_L4; 2858 int ret; 2859 2860 MLX5_ASSERT(flow_mask); 2861 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2862 return rte_flow_error_set(error, EINVAL, 2863 RTE_FLOW_ERROR_TYPE_ITEM, item, 2864 "protocol filtering not compatible" 2865 " with TCP layer"); 2866 if (!(item_flags & l3m)) 2867 return rte_flow_error_set(error, EINVAL, 2868 RTE_FLOW_ERROR_TYPE_ITEM, item, 2869 "L3 is mandatory to filter on L4"); 2870 if (item_flags & l4m) 2871 return rte_flow_error_set(error, EINVAL, 2872 RTE_FLOW_ERROR_TYPE_ITEM, item, 2873 "multiple L4 layers not supported"); 2874 if (!mask) 2875 mask = &rte_flow_item_tcp_mask; 2876 ret = mlx5_flow_item_acceptable 2877 (item, (const uint8_t *)mask, 2878 (const uint8_t *)flow_mask, 2879 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2880 error); 2881 if (ret < 0) 2882 return ret; 2883 return 0; 2884 } 2885 2886 /** 2887 * Validate VXLAN item. 2888 * 2889 * @param[in] dev 2890 * Pointer to the Ethernet device structure. 2891 * @param[in] udp_dport 2892 * UDP destination port 2893 * @param[in] item 2894 * Item specification. 2895 * @param[in] item_flags 2896 * Bit-fields that holds the items detected until now. 2897 * @param root 2898 * Whether action is on root table. 2899 * @param[out] error 2900 * Pointer to error structure. 2901 * 2902 * @return 2903 * 0 on success, a negative errno value otherwise and rte_errno is set. 2904 */ 2905 int 2906 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2907 uint16_t udp_dport, 2908 const struct rte_flow_item *item, 2909 uint64_t item_flags, 2910 bool root, 2911 struct rte_flow_error *error) 2912 { 2913 const struct rte_flow_item_vxlan *spec = item->spec; 2914 const struct rte_flow_item_vxlan *mask = item->mask; 2915 int ret; 2916 struct mlx5_priv *priv = dev->data->dev_private; 2917 union vni { 2918 uint32_t vlan_id; 2919 uint8_t vni[4]; 2920 } id = { .vlan_id = 0, }; 2921 const struct rte_flow_item_vxlan nic_mask = { 2922 .vni = "\xff\xff\xff", 2923 .rsvd1 = 0xff, 2924 }; 2925 const struct rte_flow_item_vxlan *valid_mask; 2926 2927 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2928 return rte_flow_error_set(error, ENOTSUP, 2929 RTE_FLOW_ERROR_TYPE_ITEM, item, 2930 "multiple tunnel layers not" 2931 " supported"); 2932 valid_mask = &rte_flow_item_vxlan_mask; 2933 /* 2934 * Verify only UDPv4 is present as defined in 2935 * https://tools.ietf.org/html/rfc7348 2936 */ 2937 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2938 return rte_flow_error_set(error, EINVAL, 2939 RTE_FLOW_ERROR_TYPE_ITEM, item, 2940 "no outer UDP layer found"); 2941 if (!mask) 2942 mask = &rte_flow_item_vxlan_mask; 2943 2944 if (priv->sh->steering_format_version != 2945 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2946 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2947 /* non-root table */ 2948 if (!root && priv->sh->misc5_cap) 2949 valid_mask = &nic_mask; 2950 /* Group zero in NIC domain */ 2951 if (!root && priv->sh->tunnel_header_0_1) 2952 valid_mask = &nic_mask; 2953 } 2954 ret = mlx5_flow_item_acceptable 2955 (item, (const uint8_t *)mask, 2956 (const uint8_t *)valid_mask, 2957 sizeof(struct rte_flow_item_vxlan), 2958 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2959 if (ret < 0) 2960 return ret; 2961 if (spec) { 2962 memcpy(&id.vni[1], spec->vni, 3); 2963 memcpy(&id.vni[1], mask->vni, 3); 2964 } 2965 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2966 return rte_flow_error_set(error, ENOTSUP, 2967 RTE_FLOW_ERROR_TYPE_ITEM, item, 2968 "VXLAN tunnel must be fully defined"); 2969 return 0; 2970 } 2971 2972 /** 2973 * Validate VXLAN_GPE item. 2974 * 2975 * @param[in] item 2976 * Item specification. 2977 * @param[in] item_flags 2978 * Bit-fields that holds the items detected until now. 2979 * @param[in] priv 2980 * Pointer to the private data structure. 2981 * @param[in] target_protocol 2982 * The next protocol in the previous item. 2983 * @param[out] error 2984 * Pointer to error structure. 2985 * 2986 * @return 2987 * 0 on success, a negative errno value otherwise and rte_errno is set. 2988 */ 2989 int 2990 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2991 uint64_t item_flags, 2992 struct rte_eth_dev *dev, 2993 struct rte_flow_error *error) 2994 { 2995 struct mlx5_priv *priv = dev->data->dev_private; 2996 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2997 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2998 int ret; 2999 union vni { 3000 uint32_t vlan_id; 3001 uint8_t vni[4]; 3002 } id = { .vlan_id = 0, }; 3003 3004 if (!priv->sh->config.l3_vxlan_en) 3005 return rte_flow_error_set(error, ENOTSUP, 3006 RTE_FLOW_ERROR_TYPE_ITEM, item, 3007 "L3 VXLAN is not enabled by device" 3008 " parameter and/or not configured in" 3009 " firmware"); 3010 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3011 return rte_flow_error_set(error, ENOTSUP, 3012 RTE_FLOW_ERROR_TYPE_ITEM, item, 3013 "multiple tunnel layers not" 3014 " supported"); 3015 /* 3016 * Verify only UDPv4 is present as defined in 3017 * https://tools.ietf.org/html/rfc7348 3018 */ 3019 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3020 return rte_flow_error_set(error, EINVAL, 3021 RTE_FLOW_ERROR_TYPE_ITEM, item, 3022 "no outer UDP layer found"); 3023 if (!mask) 3024 mask = &rte_flow_item_vxlan_gpe_mask; 3025 ret = mlx5_flow_item_acceptable 3026 (item, (const uint8_t *)mask, 3027 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 3028 sizeof(struct rte_flow_item_vxlan_gpe), 3029 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3030 if (ret < 0) 3031 return ret; 3032 if (spec) { 3033 if (spec->protocol) 3034 return rte_flow_error_set(error, ENOTSUP, 3035 RTE_FLOW_ERROR_TYPE_ITEM, 3036 item, 3037 "VxLAN-GPE protocol" 3038 " not supported"); 3039 memcpy(&id.vni[1], spec->vni, 3); 3040 memcpy(&id.vni[1], mask->vni, 3); 3041 } 3042 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3043 return rte_flow_error_set(error, ENOTSUP, 3044 RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "VXLAN-GPE tunnel must be fully" 3046 " defined"); 3047 return 0; 3048 } 3049 /** 3050 * Validate GRE Key item. 3051 * 3052 * @param[in] item 3053 * Item specification. 3054 * @param[in] item_flags 3055 * Bit flags to mark detected items. 3056 * @param[in] gre_item 3057 * Pointer to gre_item 3058 * @param[out] error 3059 * Pointer to error structure. 3060 * 3061 * @return 3062 * 0 on success, a negative errno value otherwise and rte_errno is set. 3063 */ 3064 int 3065 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3066 uint64_t item_flags, 3067 const struct rte_flow_item *gre_item, 3068 struct rte_flow_error *error) 3069 { 3070 const rte_be32_t *mask = item->mask; 3071 int ret = 0; 3072 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3073 const struct rte_flow_item_gre *gre_spec; 3074 const struct rte_flow_item_gre *gre_mask; 3075 3076 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3077 return rte_flow_error_set(error, ENOTSUP, 3078 RTE_FLOW_ERROR_TYPE_ITEM, item, 3079 "Multiple GRE key not support"); 3080 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3081 return rte_flow_error_set(error, ENOTSUP, 3082 RTE_FLOW_ERROR_TYPE_ITEM, item, 3083 "No preceding GRE header"); 3084 if (item_flags & MLX5_FLOW_LAYER_INNER) 3085 return rte_flow_error_set(error, ENOTSUP, 3086 RTE_FLOW_ERROR_TYPE_ITEM, item, 3087 "GRE key following a wrong item"); 3088 gre_mask = gre_item->mask; 3089 if (!gre_mask) 3090 gre_mask = &rte_flow_item_gre_mask; 3091 gre_spec = gre_item->spec; 3092 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3093 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3094 return rte_flow_error_set(error, EINVAL, 3095 RTE_FLOW_ERROR_TYPE_ITEM, item, 3096 "Key bit must be on"); 3097 3098 if (!mask) 3099 mask = &gre_key_default_mask; 3100 ret = mlx5_flow_item_acceptable 3101 (item, (const uint8_t *)mask, 3102 (const uint8_t *)&gre_key_default_mask, 3103 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3104 return ret; 3105 } 3106 3107 /** 3108 * Validate GRE optional item. 3109 * 3110 * @param[in] dev 3111 * Pointer to the Ethernet device structure. 3112 * @param[in] item 3113 * Item specification. 3114 * @param[in] item_flags 3115 * Bit flags to mark detected items. 3116 * @param[in] attr 3117 * Flow rule attributes. 3118 * @param[in] gre_item 3119 * Pointer to gre_item 3120 * @param[out] error 3121 * Pointer to error structure. 3122 * 3123 * @return 3124 * 0 on success, a negative errno value otherwise and rte_errno is set. 3125 */ 3126 int 3127 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3128 const struct rte_flow_item *item, 3129 uint64_t item_flags, 3130 const struct rte_flow_attr *attr, 3131 const struct rte_flow_item *gre_item, 3132 struct rte_flow_error *error) 3133 { 3134 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3135 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3136 const struct rte_flow_item_gre_opt *spec = item->spec; 3137 const struct rte_flow_item_gre_opt *mask = item->mask; 3138 struct mlx5_priv *priv = dev->data->dev_private; 3139 int ret = 0; 3140 struct rte_flow_item_gre_opt nic_mask = { 3141 .checksum_rsvd = { 3142 .checksum = RTE_BE16(UINT16_MAX), 3143 .reserved1 = 0x0, 3144 }, 3145 .key = { 3146 .key = RTE_BE32(UINT32_MAX), 3147 }, 3148 .sequence = { 3149 .sequence = RTE_BE32(UINT32_MAX), 3150 }, 3151 }; 3152 3153 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3154 return rte_flow_error_set(error, ENOTSUP, 3155 RTE_FLOW_ERROR_TYPE_ITEM, item, 3156 "No preceding GRE header"); 3157 if (item_flags & MLX5_FLOW_LAYER_INNER) 3158 return rte_flow_error_set(error, ENOTSUP, 3159 RTE_FLOW_ERROR_TYPE_ITEM, item, 3160 "GRE option following a wrong item"); 3161 if (!spec || !mask) 3162 return rte_flow_error_set(error, EINVAL, 3163 RTE_FLOW_ERROR_TYPE_ITEM, item, 3164 "At least one field gre_option(checksum/key/sequence) must be specified"); 3165 if (!gre_mask) 3166 gre_mask = &rte_flow_item_gre_mask; 3167 if (mask->checksum_rsvd.checksum) 3168 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3169 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3170 return rte_flow_error_set(error, EINVAL, 3171 RTE_FLOW_ERROR_TYPE_ITEM, 3172 item, 3173 "Checksum bit must be on"); 3174 if (mask->key.key) 3175 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3176 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3177 return rte_flow_error_set(error, EINVAL, 3178 RTE_FLOW_ERROR_TYPE_ITEM, 3179 item, "Key bit must be on"); 3180 if (mask->sequence.sequence) 3181 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3182 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3183 return rte_flow_error_set(error, EINVAL, 3184 RTE_FLOW_ERROR_TYPE_ITEM, 3185 item, 3186 "Sequence bit must be on"); 3187 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3188 if (priv->sh->steering_format_version == 3189 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3190 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3191 !priv->sh->misc5_cap) || 3192 (!(priv->sh->tunnel_header_0_1 && 3193 priv->sh->tunnel_header_2_3) && 3194 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3195 return rte_flow_error_set(error, EINVAL, 3196 RTE_FLOW_ERROR_TYPE_ITEM, 3197 item, 3198 "Checksum/Sequence not supported"); 3199 } 3200 ret = mlx5_flow_item_acceptable 3201 (item, (const uint8_t *)mask, 3202 (const uint8_t *)&nic_mask, 3203 sizeof(struct rte_flow_item_gre_opt), 3204 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3205 return ret; 3206 } 3207 3208 /** 3209 * Validate GRE item. 3210 * 3211 * @param[in] item 3212 * Item specification. 3213 * @param[in] item_flags 3214 * Bit flags to mark detected items. 3215 * @param[in] target_protocol 3216 * The next protocol in the previous item. 3217 * @param[out] error 3218 * Pointer to error structure. 3219 * 3220 * @return 3221 * 0 on success, a negative errno value otherwise and rte_errno is set. 3222 */ 3223 int 3224 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3225 uint64_t item_flags, 3226 uint8_t target_protocol, 3227 struct rte_flow_error *error) 3228 { 3229 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3230 const struct rte_flow_item_gre *mask = item->mask; 3231 int ret; 3232 const struct rte_flow_item_gre nic_mask = { 3233 .c_rsvd0_ver = RTE_BE16(0xB000), 3234 .protocol = RTE_BE16(UINT16_MAX), 3235 }; 3236 3237 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3238 return rte_flow_error_set(error, EINVAL, 3239 RTE_FLOW_ERROR_TYPE_ITEM, item, 3240 "protocol filtering not compatible" 3241 " with this GRE layer"); 3242 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3243 return rte_flow_error_set(error, ENOTSUP, 3244 RTE_FLOW_ERROR_TYPE_ITEM, item, 3245 "multiple tunnel layers not" 3246 " supported"); 3247 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3248 return rte_flow_error_set(error, ENOTSUP, 3249 RTE_FLOW_ERROR_TYPE_ITEM, item, 3250 "L3 Layer is missing"); 3251 if (!mask) 3252 mask = &rte_flow_item_gre_mask; 3253 ret = mlx5_flow_item_acceptable 3254 (item, (const uint8_t *)mask, 3255 (const uint8_t *)&nic_mask, 3256 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3257 error); 3258 if (ret < 0) 3259 return ret; 3260 #ifndef HAVE_MLX5DV_DR 3261 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3262 if (spec && (spec->protocol & mask->protocol)) 3263 return rte_flow_error_set(error, ENOTSUP, 3264 RTE_FLOW_ERROR_TYPE_ITEM, item, 3265 "without MPLS support the" 3266 " specification cannot be used for" 3267 " filtering"); 3268 #endif 3269 #endif 3270 return 0; 3271 } 3272 3273 /** 3274 * Validate Geneve item. 3275 * 3276 * @param[in] item 3277 * Item specification. 3278 * @param[in] itemFlags 3279 * Bit-fields that holds the items detected until now. 3280 * @param[in] enPriv 3281 * Pointer to the private data structure. 3282 * @param[out] error 3283 * Pointer to error structure. 3284 * 3285 * @return 3286 * 0 on success, a negative errno value otherwise and rte_errno is set. 3287 */ 3288 3289 int 3290 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3291 uint64_t item_flags, 3292 struct rte_eth_dev *dev, 3293 struct rte_flow_error *error) 3294 { 3295 struct mlx5_priv *priv = dev->data->dev_private; 3296 const struct rte_flow_item_geneve *spec = item->spec; 3297 const struct rte_flow_item_geneve *mask = item->mask; 3298 int ret; 3299 uint16_t gbhdr; 3300 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3301 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3302 const struct rte_flow_item_geneve nic_mask = { 3303 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3304 .vni = "\xff\xff\xff", 3305 .protocol = RTE_BE16(UINT16_MAX), 3306 }; 3307 3308 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3309 return rte_flow_error_set(error, ENOTSUP, 3310 RTE_FLOW_ERROR_TYPE_ITEM, item, 3311 "L3 Geneve is not enabled by device" 3312 " parameter and/or not configured in" 3313 " firmware"); 3314 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3315 return rte_flow_error_set(error, ENOTSUP, 3316 RTE_FLOW_ERROR_TYPE_ITEM, item, 3317 "multiple tunnel layers not" 3318 " supported"); 3319 /* 3320 * Verify only UDPv4 is present as defined in 3321 * https://tools.ietf.org/html/rfc7348 3322 */ 3323 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3324 return rte_flow_error_set(error, EINVAL, 3325 RTE_FLOW_ERROR_TYPE_ITEM, item, 3326 "no outer UDP layer found"); 3327 if (!mask) 3328 mask = &rte_flow_item_geneve_mask; 3329 ret = mlx5_flow_item_acceptable 3330 (item, (const uint8_t *)mask, 3331 (const uint8_t *)&nic_mask, 3332 sizeof(struct rte_flow_item_geneve), 3333 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3334 if (ret) 3335 return ret; 3336 if (spec) { 3337 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3338 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3339 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3340 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3341 return rte_flow_error_set(error, ENOTSUP, 3342 RTE_FLOW_ERROR_TYPE_ITEM, 3343 item, 3344 "Geneve protocol unsupported" 3345 " fields are being used"); 3346 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3347 return rte_flow_error_set 3348 (error, ENOTSUP, 3349 RTE_FLOW_ERROR_TYPE_ITEM, 3350 item, 3351 "Unsupported Geneve options length"); 3352 } 3353 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3354 return rte_flow_error_set 3355 (error, ENOTSUP, 3356 RTE_FLOW_ERROR_TYPE_ITEM, item, 3357 "Geneve tunnel must be fully defined"); 3358 return 0; 3359 } 3360 3361 /** 3362 * Validate Geneve TLV option item. 3363 * 3364 * @param[in] item 3365 * Item specification. 3366 * @param[in] last_item 3367 * Previous validated item in the pattern items. 3368 * @param[in] geneve_item 3369 * Previous GENEVE item specification. 3370 * @param[in] dev 3371 * Pointer to the rte_eth_dev structure. 3372 * @param[out] error 3373 * Pointer to error structure. 3374 * 3375 * @return 3376 * 0 on success, a negative errno value otherwise and rte_errno is set. 3377 */ 3378 int 3379 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3380 uint64_t last_item, 3381 const struct rte_flow_item *geneve_item, 3382 struct rte_eth_dev *dev, 3383 struct rte_flow_error *error) 3384 { 3385 struct mlx5_priv *priv = dev->data->dev_private; 3386 struct mlx5_dev_ctx_shared *sh = priv->sh; 3387 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3388 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3389 uint8_t data_max_supported = 3390 hca_attr->max_geneve_tlv_option_data_len * 4; 3391 const struct rte_flow_item_geneve *geneve_spec; 3392 const struct rte_flow_item_geneve *geneve_mask; 3393 const struct rte_flow_item_geneve_opt *spec = item->spec; 3394 const struct rte_flow_item_geneve_opt *mask = item->mask; 3395 unsigned int i; 3396 unsigned int data_len; 3397 uint8_t tlv_option_len; 3398 uint16_t optlen_m, optlen_v; 3399 const struct rte_flow_item_geneve_opt full_mask = { 3400 .option_class = RTE_BE16(0xffff), 3401 .option_type = 0xff, 3402 .option_len = 0x1f, 3403 }; 3404 3405 if (!mask) 3406 mask = &rte_flow_item_geneve_opt_mask; 3407 if (!spec) 3408 return rte_flow_error_set 3409 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3410 "Geneve TLV opt class/type/length must be specified"); 3411 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3412 return rte_flow_error_set 3413 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3414 "Geneve TLV opt length exceeds the limit (31)"); 3415 /* Check if class type and length masks are full. */ 3416 if (full_mask.option_class != mask->option_class || 3417 full_mask.option_type != mask->option_type || 3418 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3419 return rte_flow_error_set 3420 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3421 "Geneve TLV opt class/type/length masks must be full"); 3422 /* Check if length is supported */ 3423 if ((uint32_t)spec->option_len > 3424 hca_attr->max_geneve_tlv_option_data_len) 3425 return rte_flow_error_set 3426 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3427 "Geneve TLV opt length not supported"); 3428 if (hca_attr->max_geneve_tlv_options > 1) 3429 DRV_LOG(DEBUG, 3430 "max_geneve_tlv_options supports more than 1 option"); 3431 /* Check GENEVE item preceding. */ 3432 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3433 return rte_flow_error_set 3434 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3435 "Geneve opt item must be preceded with Geneve item"); 3436 geneve_spec = geneve_item->spec; 3437 geneve_mask = geneve_item->mask ? geneve_item->mask : 3438 &rte_flow_item_geneve_mask; 3439 /* Check if GENEVE TLV option size doesn't exceed option length */ 3440 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3441 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3442 tlv_option_len = spec->option_len & mask->option_len; 3443 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3444 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3445 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3446 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3447 if ((optlen_v & optlen_m) <= tlv_option_len) 3448 return rte_flow_error_set 3449 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3450 "GENEVE TLV option length exceeds optlen"); 3451 } 3452 /* Check if length is 0 or data is 0. */ 3453 if (spec->data == NULL || spec->option_len == 0) 3454 return rte_flow_error_set 3455 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3456 "Geneve TLV opt with zero data/length not supported"); 3457 /* Check not all data & mask are 0. */ 3458 data_len = spec->option_len * 4; 3459 if (mask->data == NULL) { 3460 for (i = 0; i < data_len; i++) 3461 if (spec->data[i]) 3462 break; 3463 if (i == data_len) 3464 return rte_flow_error_set(error, ENOTSUP, 3465 RTE_FLOW_ERROR_TYPE_ITEM, item, 3466 "Can't match on Geneve option data 0"); 3467 } else { 3468 for (i = 0; i < data_len; i++) 3469 if (spec->data[i] & mask->data[i]) 3470 break; 3471 if (i == data_len) 3472 return rte_flow_error_set(error, ENOTSUP, 3473 RTE_FLOW_ERROR_TYPE_ITEM, item, 3474 "Can't match on Geneve option data and mask 0"); 3475 /* Check data mask supported. */ 3476 for (i = data_max_supported; i < data_len ; i++) 3477 if (mask->data[i]) 3478 return rte_flow_error_set(error, ENOTSUP, 3479 RTE_FLOW_ERROR_TYPE_ITEM, item, 3480 "Data mask is of unsupported size"); 3481 } 3482 /* Check GENEVE option is supported in NIC. */ 3483 if (!hca_attr->geneve_tlv_opt) 3484 return rte_flow_error_set 3485 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3486 "Geneve TLV opt not supported"); 3487 /* Check if we already have geneve option with different type/class. */ 3488 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3489 geneve_opt_resource = sh->geneve_tlv_option_resource; 3490 if (geneve_opt_resource != NULL) 3491 if (geneve_opt_resource->option_class != spec->option_class || 3492 geneve_opt_resource->option_type != spec->option_type || 3493 geneve_opt_resource->length != spec->option_len) { 3494 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3495 return rte_flow_error_set(error, ENOTSUP, 3496 RTE_FLOW_ERROR_TYPE_ITEM, item, 3497 "Only one Geneve TLV option supported"); 3498 } 3499 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3500 return 0; 3501 } 3502 3503 /** 3504 * Validate MPLS item. 3505 * 3506 * @param[in] dev 3507 * Pointer to the rte_eth_dev structure. 3508 * @param[in] item 3509 * Item specification. 3510 * @param[in] item_flags 3511 * Bit-fields that holds the items detected until now. 3512 * @param[in] prev_layer 3513 * The protocol layer indicated in previous item. 3514 * @param[out] error 3515 * Pointer to error structure. 3516 * 3517 * @return 3518 * 0 on success, a negative errno value otherwise and rte_errno is set. 3519 */ 3520 int 3521 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3522 const struct rte_flow_item *item __rte_unused, 3523 uint64_t item_flags __rte_unused, 3524 uint64_t prev_layer __rte_unused, 3525 struct rte_flow_error *error) 3526 { 3527 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3528 const struct rte_flow_item_mpls *mask = item->mask; 3529 struct mlx5_priv *priv = dev->data->dev_private; 3530 int ret; 3531 3532 if (!priv->sh->dev_cap.mpls_en) 3533 return rte_flow_error_set(error, ENOTSUP, 3534 RTE_FLOW_ERROR_TYPE_ITEM, item, 3535 "MPLS not supported or" 3536 " disabled in firmware" 3537 " configuration."); 3538 /* MPLS over UDP, GRE is allowed */ 3539 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3540 MLX5_FLOW_LAYER_GRE | 3541 MLX5_FLOW_LAYER_GRE_KEY))) 3542 return rte_flow_error_set(error, EINVAL, 3543 RTE_FLOW_ERROR_TYPE_ITEM, item, 3544 "protocol filtering not compatible" 3545 " with MPLS layer"); 3546 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3547 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3548 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3549 return rte_flow_error_set(error, ENOTSUP, 3550 RTE_FLOW_ERROR_TYPE_ITEM, item, 3551 "multiple tunnel layers not" 3552 " supported"); 3553 if (!mask) 3554 mask = &rte_flow_item_mpls_mask; 3555 ret = mlx5_flow_item_acceptable 3556 (item, (const uint8_t *)mask, 3557 (const uint8_t *)&rte_flow_item_mpls_mask, 3558 sizeof(struct rte_flow_item_mpls), 3559 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3560 if (ret < 0) 3561 return ret; 3562 return 0; 3563 #else 3564 return rte_flow_error_set(error, ENOTSUP, 3565 RTE_FLOW_ERROR_TYPE_ITEM, item, 3566 "MPLS is not supported by Verbs, please" 3567 " update."); 3568 #endif 3569 } 3570 3571 /** 3572 * Validate NVGRE item. 3573 * 3574 * @param[in] item 3575 * Item specification. 3576 * @param[in] item_flags 3577 * Bit flags to mark detected items. 3578 * @param[in] target_protocol 3579 * The next protocol in the previous item. 3580 * @param[out] error 3581 * Pointer to error structure. 3582 * 3583 * @return 3584 * 0 on success, a negative errno value otherwise and rte_errno is set. 3585 */ 3586 int 3587 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3588 uint64_t item_flags, 3589 uint8_t target_protocol, 3590 struct rte_flow_error *error) 3591 { 3592 const struct rte_flow_item_nvgre *mask = item->mask; 3593 int ret; 3594 3595 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3596 return rte_flow_error_set(error, EINVAL, 3597 RTE_FLOW_ERROR_TYPE_ITEM, item, 3598 "protocol filtering not compatible" 3599 " with this GRE layer"); 3600 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3601 return rte_flow_error_set(error, ENOTSUP, 3602 RTE_FLOW_ERROR_TYPE_ITEM, item, 3603 "multiple tunnel layers not" 3604 " supported"); 3605 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3606 return rte_flow_error_set(error, ENOTSUP, 3607 RTE_FLOW_ERROR_TYPE_ITEM, item, 3608 "L3 Layer is missing"); 3609 if (!mask) 3610 mask = &rte_flow_item_nvgre_mask; 3611 ret = mlx5_flow_item_acceptable 3612 (item, (const uint8_t *)mask, 3613 (const uint8_t *)&rte_flow_item_nvgre_mask, 3614 sizeof(struct rte_flow_item_nvgre), 3615 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3616 if (ret < 0) 3617 return ret; 3618 return 0; 3619 } 3620 3621 /** 3622 * Validate eCPRI item. 3623 * 3624 * @param[in] item 3625 * Item specification. 3626 * @param[in] item_flags 3627 * Bit-fields that holds the items detected until now. 3628 * @param[in] last_item 3629 * Previous validated item in the pattern items. 3630 * @param[in] ether_type 3631 * Type in the ethernet layer header (including dot1q). 3632 * @param[in] acc_mask 3633 * Acceptable mask, if NULL default internal default mask 3634 * will be used to check whether item fields are supported. 3635 * @param[out] error 3636 * Pointer to error structure. 3637 * 3638 * @return 3639 * 0 on success, a negative errno value otherwise and rte_errno is set. 3640 */ 3641 int 3642 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3643 uint64_t item_flags, 3644 uint64_t last_item, 3645 uint16_t ether_type, 3646 const struct rte_flow_item_ecpri *acc_mask, 3647 struct rte_flow_error *error) 3648 { 3649 const struct rte_flow_item_ecpri *mask = item->mask; 3650 const struct rte_flow_item_ecpri nic_mask = { 3651 .hdr = { 3652 .common = { 3653 .u32 = 3654 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3655 .type = 0xFF, 3656 }).u32), 3657 }, 3658 .dummy[0] = 0xFFFFFFFF, 3659 }, 3660 }; 3661 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3662 MLX5_FLOW_LAYER_OUTER_VLAN); 3663 struct rte_flow_item_ecpri mask_lo; 3664 3665 if (!(last_item & outer_l2_vlan) && 3666 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3667 return rte_flow_error_set(error, EINVAL, 3668 RTE_FLOW_ERROR_TYPE_ITEM, item, 3669 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3670 if ((last_item & outer_l2_vlan) && ether_type && 3671 ether_type != RTE_ETHER_TYPE_ECPRI) 3672 return rte_flow_error_set(error, EINVAL, 3673 RTE_FLOW_ERROR_TYPE_ITEM, item, 3674 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3675 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3676 return rte_flow_error_set(error, EINVAL, 3677 RTE_FLOW_ERROR_TYPE_ITEM, item, 3678 "eCPRI with tunnel is not supported right now"); 3679 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3680 return rte_flow_error_set(error, ENOTSUP, 3681 RTE_FLOW_ERROR_TYPE_ITEM, item, 3682 "multiple L3 layers not supported"); 3683 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3684 return rte_flow_error_set(error, EINVAL, 3685 RTE_FLOW_ERROR_TYPE_ITEM, item, 3686 "eCPRI cannot coexist with a TCP layer"); 3687 /* In specification, eCPRI could be over UDP layer. */ 3688 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3689 return rte_flow_error_set(error, EINVAL, 3690 RTE_FLOW_ERROR_TYPE_ITEM, item, 3691 "eCPRI over UDP layer is not yet supported right now"); 3692 /* Mask for type field in common header could be zero. */ 3693 if (!mask) 3694 mask = &rte_flow_item_ecpri_mask; 3695 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3696 /* Input mask is in big-endian format. */ 3697 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3698 return rte_flow_error_set(error, EINVAL, 3699 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3700 "partial mask is not supported for protocol"); 3701 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3702 return rte_flow_error_set(error, EINVAL, 3703 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3704 "message header mask must be after a type mask"); 3705 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3706 acc_mask ? (const uint8_t *)acc_mask 3707 : (const uint8_t *)&nic_mask, 3708 sizeof(struct rte_flow_item_ecpri), 3709 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3710 } 3711 3712 static int 3713 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3714 const struct rte_flow_attr *attr __rte_unused, 3715 const struct rte_flow_item items[] __rte_unused, 3716 const struct rte_flow_action actions[] __rte_unused, 3717 bool external __rte_unused, 3718 int hairpin __rte_unused, 3719 struct rte_flow_error *error) 3720 { 3721 return rte_flow_error_set(error, ENOTSUP, 3722 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3723 } 3724 3725 static struct mlx5_flow * 3726 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3727 const struct rte_flow_attr *attr __rte_unused, 3728 const struct rte_flow_item items[] __rte_unused, 3729 const struct rte_flow_action actions[] __rte_unused, 3730 struct rte_flow_error *error) 3731 { 3732 rte_flow_error_set(error, ENOTSUP, 3733 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3734 return NULL; 3735 } 3736 3737 static int 3738 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3739 struct mlx5_flow *dev_flow __rte_unused, 3740 const struct rte_flow_attr *attr __rte_unused, 3741 const struct rte_flow_item items[] __rte_unused, 3742 const struct rte_flow_action actions[] __rte_unused, 3743 struct rte_flow_error *error) 3744 { 3745 return rte_flow_error_set(error, ENOTSUP, 3746 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3747 } 3748 3749 static int 3750 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3751 struct rte_flow *flow __rte_unused, 3752 struct rte_flow_error *error) 3753 { 3754 return rte_flow_error_set(error, ENOTSUP, 3755 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3756 } 3757 3758 static void 3759 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3760 struct rte_flow *flow __rte_unused) 3761 { 3762 } 3763 3764 static void 3765 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3766 struct rte_flow *flow __rte_unused) 3767 { 3768 } 3769 3770 static int 3771 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3772 struct rte_flow *flow __rte_unused, 3773 const struct rte_flow_action *actions __rte_unused, 3774 void *data __rte_unused, 3775 struct rte_flow_error *error) 3776 { 3777 return rte_flow_error_set(error, ENOTSUP, 3778 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3779 } 3780 3781 static int 3782 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3783 uint32_t domains __rte_unused, 3784 uint32_t flags __rte_unused) 3785 { 3786 return 0; 3787 } 3788 3789 /* Void driver to protect from null pointer reference. */ 3790 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3791 .validate = flow_null_validate, 3792 .prepare = flow_null_prepare, 3793 .translate = flow_null_translate, 3794 .apply = flow_null_apply, 3795 .remove = flow_null_remove, 3796 .destroy = flow_null_destroy, 3797 .query = flow_null_query, 3798 .sync_domain = flow_null_sync_domain, 3799 }; 3800 3801 /** 3802 * Select flow driver type according to flow attributes and device 3803 * configuration. 3804 * 3805 * @param[in] dev 3806 * Pointer to the dev structure. 3807 * @param[in] attr 3808 * Pointer to the flow attributes. 3809 * 3810 * @return 3811 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3812 */ 3813 static enum mlx5_flow_drv_type 3814 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3815 { 3816 struct mlx5_priv *priv = dev->data->dev_private; 3817 /* The OS can determine first a specific flow type (DV, VERBS) */ 3818 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3819 3820 if (type != MLX5_FLOW_TYPE_MAX) 3821 return type; 3822 /* 3823 * Currently when dv_flow_en == 2, only HW steering engine is 3824 * supported. New engines can also be chosen here if ready. 3825 */ 3826 if (priv->sh->config.dv_flow_en == 2) 3827 return MLX5_FLOW_TYPE_HW; 3828 if (!attr) 3829 return MLX5_FLOW_TYPE_MIN; 3830 /* If no OS specific type - continue with DV/VERBS selection */ 3831 if (attr->transfer && priv->sh->config.dv_esw_en) 3832 type = MLX5_FLOW_TYPE_DV; 3833 if (!attr->transfer) 3834 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3835 MLX5_FLOW_TYPE_VERBS; 3836 return type; 3837 } 3838 3839 #define flow_get_drv_ops(type) flow_drv_ops[type] 3840 3841 /** 3842 * Flow driver validation API. This abstracts calling driver specific functions. 3843 * The type of flow driver is determined according to flow attributes. 3844 * 3845 * @param[in] dev 3846 * Pointer to the dev structure. 3847 * @param[in] attr 3848 * Pointer to the flow attributes. 3849 * @param[in] items 3850 * Pointer to the list of items. 3851 * @param[in] actions 3852 * Pointer to the list of actions. 3853 * @param[in] external 3854 * This flow rule is created by request external to PMD. 3855 * @param[in] hairpin 3856 * Number of hairpin TX actions, 0 means classic flow. 3857 * @param[out] error 3858 * Pointer to the error structure. 3859 * 3860 * @return 3861 * 0 on success, a negative errno value otherwise and rte_errno is set. 3862 */ 3863 static inline int 3864 flow_drv_validate(struct rte_eth_dev *dev, 3865 const struct rte_flow_attr *attr, 3866 const struct rte_flow_item items[], 3867 const struct rte_flow_action actions[], 3868 bool external, int hairpin, struct rte_flow_error *error) 3869 { 3870 const struct mlx5_flow_driver_ops *fops; 3871 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3872 3873 fops = flow_get_drv_ops(type); 3874 return fops->validate(dev, attr, items, actions, external, 3875 hairpin, error); 3876 } 3877 3878 /** 3879 * Flow driver preparation API. This abstracts calling driver specific 3880 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3881 * calculates the size of memory required for device flow, allocates the memory, 3882 * initializes the device flow and returns the pointer. 3883 * 3884 * @note 3885 * This function initializes device flow structure such as dv or verbs in 3886 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3887 * rest. For example, adding returning device flow to flow->dev_flow list and 3888 * setting backward reference to the flow should be done out of this function. 3889 * layers field is not filled either. 3890 * 3891 * @param[in] dev 3892 * Pointer to the dev structure. 3893 * @param[in] attr 3894 * Pointer to the flow attributes. 3895 * @param[in] items 3896 * Pointer to the list of items. 3897 * @param[in] actions 3898 * Pointer to the list of actions. 3899 * @param[in] flow_idx 3900 * This memory pool index to the flow. 3901 * @param[out] error 3902 * Pointer to the error structure. 3903 * 3904 * @return 3905 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3906 */ 3907 static inline struct mlx5_flow * 3908 flow_drv_prepare(struct rte_eth_dev *dev, 3909 const struct rte_flow *flow, 3910 const struct rte_flow_attr *attr, 3911 const struct rte_flow_item items[], 3912 const struct rte_flow_action actions[], 3913 uint32_t flow_idx, 3914 struct rte_flow_error *error) 3915 { 3916 const struct mlx5_flow_driver_ops *fops; 3917 enum mlx5_flow_drv_type type = flow->drv_type; 3918 struct mlx5_flow *mlx5_flow = NULL; 3919 3920 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3921 fops = flow_get_drv_ops(type); 3922 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3923 if (mlx5_flow) 3924 mlx5_flow->flow_idx = flow_idx; 3925 return mlx5_flow; 3926 } 3927 3928 /** 3929 * Flow driver translation API. This abstracts calling driver specific 3930 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3931 * translates a generic flow into a driver flow. flow_drv_prepare() must 3932 * precede. 3933 * 3934 * @note 3935 * dev_flow->layers could be filled as a result of parsing during translation 3936 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3937 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3938 * flow->actions could be overwritten even though all the expanded dev_flows 3939 * have the same actions. 3940 * 3941 * @param[in] dev 3942 * Pointer to the rte dev structure. 3943 * @param[in, out] dev_flow 3944 * Pointer to the mlx5 flow. 3945 * @param[in] attr 3946 * Pointer to the flow attributes. 3947 * @param[in] items 3948 * Pointer to the list of items. 3949 * @param[in] actions 3950 * Pointer to the list of actions. 3951 * @param[out] error 3952 * Pointer to the error structure. 3953 * 3954 * @return 3955 * 0 on success, a negative errno value otherwise and rte_errno is set. 3956 */ 3957 static inline int 3958 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3959 const struct rte_flow_attr *attr, 3960 const struct rte_flow_item items[], 3961 const struct rte_flow_action actions[], 3962 struct rte_flow_error *error) 3963 { 3964 const struct mlx5_flow_driver_ops *fops; 3965 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3966 3967 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3968 fops = flow_get_drv_ops(type); 3969 return fops->translate(dev, dev_flow, attr, items, actions, error); 3970 } 3971 3972 /** 3973 * Flow driver apply API. This abstracts calling driver specific functions. 3974 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3975 * translated driver flows on to device. flow_drv_translate() must precede. 3976 * 3977 * @param[in] dev 3978 * Pointer to Ethernet device structure. 3979 * @param[in, out] flow 3980 * Pointer to flow structure. 3981 * @param[out] error 3982 * Pointer to error structure. 3983 * 3984 * @return 3985 * 0 on success, a negative errno value otherwise and rte_errno is set. 3986 */ 3987 static inline int 3988 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3989 struct rte_flow_error *error) 3990 { 3991 const struct mlx5_flow_driver_ops *fops; 3992 enum mlx5_flow_drv_type type = flow->drv_type; 3993 3994 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3995 fops = flow_get_drv_ops(type); 3996 return fops->apply(dev, flow, error); 3997 } 3998 3999 /** 4000 * Flow driver destroy API. This abstracts calling driver specific functions. 4001 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4002 * on device and releases resources of the flow. 4003 * 4004 * @param[in] dev 4005 * Pointer to Ethernet device. 4006 * @param[in, out] flow 4007 * Pointer to flow structure. 4008 */ 4009 static inline void 4010 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4011 { 4012 const struct mlx5_flow_driver_ops *fops; 4013 enum mlx5_flow_drv_type type = flow->drv_type; 4014 4015 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4016 fops = flow_get_drv_ops(type); 4017 fops->destroy(dev, flow); 4018 } 4019 4020 /** 4021 * Flow driver find RSS policy tbl API. This abstracts calling driver 4022 * specific functions. Parent flow (rte_flow) should have driver 4023 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4024 * 4025 * @param[in] dev 4026 * Pointer to Ethernet device. 4027 * @param[in, out] flow 4028 * Pointer to flow structure. 4029 * @param[in] policy 4030 * Pointer to meter policy table. 4031 * @param[in] rss_desc 4032 * Pointer to rss_desc 4033 */ 4034 static struct mlx5_flow_meter_sub_policy * 4035 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4036 struct rte_flow *flow, 4037 struct mlx5_flow_meter_policy *policy, 4038 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4039 { 4040 const struct mlx5_flow_driver_ops *fops; 4041 enum mlx5_flow_drv_type type = flow->drv_type; 4042 4043 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4044 fops = flow_get_drv_ops(type); 4045 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4046 } 4047 4048 /** 4049 * Flow driver color tag rule API. This abstracts calling driver 4050 * specific functions. Parent flow (rte_flow) should have driver 4051 * type (drv_type). It will create the color tag rules in hierarchy meter. 4052 * 4053 * @param[in] dev 4054 * Pointer to Ethernet device. 4055 * @param[in, out] flow 4056 * Pointer to flow structure. 4057 * @param[in] fm 4058 * Pointer to flow meter structure. 4059 * @param[in] src_port 4060 * The src port this extra rule should use. 4061 * @param[in] item 4062 * The src port id match item. 4063 * @param[out] error 4064 * Pointer to error structure. 4065 */ 4066 static int 4067 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4068 struct rte_flow *flow, 4069 struct mlx5_flow_meter_info *fm, 4070 int32_t src_port, 4071 const struct rte_flow_item *item, 4072 struct rte_flow_error *error) 4073 { 4074 const struct mlx5_flow_driver_ops *fops; 4075 enum mlx5_flow_drv_type type = flow->drv_type; 4076 4077 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4078 fops = flow_get_drv_ops(type); 4079 return fops->meter_hierarchy_rule_create(dev, fm, 4080 src_port, item, error); 4081 } 4082 4083 /** 4084 * Get RSS action from the action list. 4085 * 4086 * @param[in] dev 4087 * Pointer to Ethernet device. 4088 * @param[in] actions 4089 * Pointer to the list of actions. 4090 * @param[in] flow 4091 * Parent flow structure pointer. 4092 * 4093 * @return 4094 * Pointer to the RSS action if exist, else return NULL. 4095 */ 4096 static const struct rte_flow_action_rss* 4097 flow_get_rss_action(struct rte_eth_dev *dev, 4098 const struct rte_flow_action actions[]) 4099 { 4100 struct mlx5_priv *priv = dev->data->dev_private; 4101 const struct rte_flow_action_rss *rss = NULL; 4102 struct mlx5_meter_policy_action_container *acg; 4103 struct mlx5_meter_policy_action_container *acy; 4104 4105 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4106 switch (actions->type) { 4107 case RTE_FLOW_ACTION_TYPE_RSS: 4108 rss = actions->conf; 4109 break; 4110 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4111 { 4112 const struct rte_flow_action_sample *sample = 4113 actions->conf; 4114 const struct rte_flow_action *act = sample->actions; 4115 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4116 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4117 rss = act->conf; 4118 break; 4119 } 4120 case RTE_FLOW_ACTION_TYPE_METER: 4121 { 4122 uint32_t mtr_idx; 4123 struct mlx5_flow_meter_info *fm; 4124 struct mlx5_flow_meter_policy *policy; 4125 const struct rte_flow_action_meter *mtr = actions->conf; 4126 4127 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4128 if (fm && !fm->def_policy) { 4129 policy = mlx5_flow_meter_policy_find(dev, 4130 fm->policy_id, NULL); 4131 MLX5_ASSERT(policy); 4132 if (policy->is_hierarchy) { 4133 policy = 4134 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4135 policy); 4136 if (!policy) 4137 return NULL; 4138 } 4139 if (policy->is_rss) { 4140 acg = 4141 &policy->act_cnt[RTE_COLOR_GREEN]; 4142 acy = 4143 &policy->act_cnt[RTE_COLOR_YELLOW]; 4144 if (acg->fate_action == 4145 MLX5_FLOW_FATE_SHARED_RSS) 4146 rss = acg->rss->conf; 4147 else if (acy->fate_action == 4148 MLX5_FLOW_FATE_SHARED_RSS) 4149 rss = acy->rss->conf; 4150 } 4151 } 4152 break; 4153 } 4154 default: 4155 break; 4156 } 4157 } 4158 return rss; 4159 } 4160 4161 /** 4162 * Get ASO age action by index. 4163 * 4164 * @param[in] dev 4165 * Pointer to the Ethernet device structure. 4166 * @param[in] age_idx 4167 * Index to the ASO age action. 4168 * 4169 * @return 4170 * The specified ASO age action. 4171 */ 4172 struct mlx5_aso_age_action* 4173 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4174 { 4175 uint16_t pool_idx = age_idx & UINT16_MAX; 4176 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4177 struct mlx5_priv *priv = dev->data->dev_private; 4178 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4179 struct mlx5_aso_age_pool *pool; 4180 4181 rte_rwlock_read_lock(&mng->resize_rwl); 4182 pool = mng->pools[pool_idx]; 4183 rte_rwlock_read_unlock(&mng->resize_rwl); 4184 return &pool->actions[offset - 1]; 4185 } 4186 4187 /* maps indirect action to translated direct in some actions array */ 4188 struct mlx5_translated_action_handle { 4189 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4190 int index; /**< Index in related array of rte_flow_action. */ 4191 }; 4192 4193 /** 4194 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4195 * direct action if translation possible. 4196 * This functionality used to run same execution path for both direct and 4197 * indirect actions on flow create. All necessary preparations for indirect 4198 * action handling should be performed on *handle* actions list returned 4199 * from this call. 4200 * 4201 * @param[in] dev 4202 * Pointer to Ethernet device. 4203 * @param[in] actions 4204 * List of actions to translate. 4205 * @param[out] handle 4206 * List to store translated indirect action object handles. 4207 * @param[in, out] indir_n 4208 * Size of *handle* array. On return should be updated with number of 4209 * indirect actions retrieved from the *actions* list. 4210 * @param[out] translated_actions 4211 * List of actions where all indirect actions were translated to direct 4212 * if possible. NULL if no translation took place. 4213 * @param[out] error 4214 * Pointer to the error structure. 4215 * 4216 * @return 4217 * 0 on success, a negative errno value otherwise and rte_errno is set. 4218 */ 4219 static int 4220 flow_action_handles_translate(struct rte_eth_dev *dev, 4221 const struct rte_flow_action actions[], 4222 struct mlx5_translated_action_handle *handle, 4223 int *indir_n, 4224 struct rte_flow_action **translated_actions, 4225 struct rte_flow_error *error) 4226 { 4227 struct mlx5_priv *priv = dev->data->dev_private; 4228 struct rte_flow_action *translated = NULL; 4229 size_t actions_size; 4230 int n; 4231 int copied_n = 0; 4232 struct mlx5_translated_action_handle *handle_end = NULL; 4233 4234 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4235 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4236 continue; 4237 if (copied_n == *indir_n) { 4238 return rte_flow_error_set 4239 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4240 NULL, "too many shared actions"); 4241 } 4242 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4243 sizeof(actions[n].conf)); 4244 handle[copied_n].index = n; 4245 copied_n++; 4246 } 4247 n++; 4248 *indir_n = copied_n; 4249 if (!copied_n) 4250 return 0; 4251 actions_size = sizeof(struct rte_flow_action) * n; 4252 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4253 if (!translated) { 4254 rte_errno = ENOMEM; 4255 return -ENOMEM; 4256 } 4257 memcpy(translated, actions, actions_size); 4258 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4259 struct mlx5_shared_action_rss *shared_rss; 4260 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4261 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4262 uint32_t idx = act_idx & 4263 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4264 4265 switch (type) { 4266 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4267 shared_rss = mlx5_ipool_get 4268 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4269 translated[handle->index].type = 4270 RTE_FLOW_ACTION_TYPE_RSS; 4271 translated[handle->index].conf = 4272 &shared_rss->origin; 4273 break; 4274 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4275 translated[handle->index].type = 4276 (enum rte_flow_action_type) 4277 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4278 translated[handle->index].conf = (void *)(uintptr_t)idx; 4279 break; 4280 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4281 translated[handle->index].type = 4282 (enum rte_flow_action_type) 4283 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4284 translated[handle->index].conf = (void *)(uintptr_t)idx; 4285 break; 4286 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4287 if (priv->sh->flow_hit_aso_en) { 4288 translated[handle->index].type = 4289 (enum rte_flow_action_type) 4290 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4291 translated[handle->index].conf = 4292 (void *)(uintptr_t)idx; 4293 break; 4294 } 4295 /* Fall-through */ 4296 case MLX5_INDIRECT_ACTION_TYPE_CT: 4297 if (priv->sh->ct_aso_en) { 4298 translated[handle->index].type = 4299 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4300 translated[handle->index].conf = 4301 (void *)(uintptr_t)idx; 4302 break; 4303 } 4304 /* Fall-through */ 4305 default: 4306 mlx5_free(translated); 4307 return rte_flow_error_set 4308 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4309 NULL, "invalid indirect action type"); 4310 } 4311 } 4312 *translated_actions = translated; 4313 return 0; 4314 } 4315 4316 /** 4317 * Get Shared RSS action from the action list. 4318 * 4319 * @param[in] dev 4320 * Pointer to Ethernet device. 4321 * @param[in] shared 4322 * Pointer to the list of actions. 4323 * @param[in] shared_n 4324 * Actions list length. 4325 * 4326 * @return 4327 * The MLX5 RSS action ID if exists, otherwise return 0. 4328 */ 4329 static uint32_t 4330 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4331 struct mlx5_translated_action_handle *handle, 4332 int shared_n) 4333 { 4334 struct mlx5_translated_action_handle *handle_end; 4335 struct mlx5_priv *priv = dev->data->dev_private; 4336 struct mlx5_shared_action_rss *shared_rss; 4337 4338 4339 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4340 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4341 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4342 uint32_t idx = act_idx & 4343 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4344 switch (type) { 4345 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4346 shared_rss = mlx5_ipool_get 4347 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4348 idx); 4349 __atomic_add_fetch(&shared_rss->refcnt, 1, 4350 __ATOMIC_RELAXED); 4351 return idx; 4352 default: 4353 break; 4354 } 4355 } 4356 return 0; 4357 } 4358 4359 static unsigned int 4360 find_graph_root(uint32_t rss_level) 4361 { 4362 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4363 MLX5_EXPANSION_ROOT_OUTER; 4364 } 4365 4366 /** 4367 * Get layer flags from the prefix flow. 4368 * 4369 * Some flows may be split to several subflows, the prefix subflow gets the 4370 * match items and the suffix sub flow gets the actions. 4371 * Some actions need the user defined match item flags to get the detail for 4372 * the action. 4373 * This function helps the suffix flow to get the item layer flags from prefix 4374 * subflow. 4375 * 4376 * @param[in] dev_flow 4377 * Pointer the created prefix subflow. 4378 * 4379 * @return 4380 * The layers get from prefix subflow. 4381 */ 4382 static inline uint64_t 4383 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4384 { 4385 uint64_t layers = 0; 4386 4387 /* 4388 * Layers bits could be localization, but usually the compiler will 4389 * help to do the optimization work for source code. 4390 * If no decap actions, use the layers directly. 4391 */ 4392 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4393 return dev_flow->handle->layers; 4394 /* Convert L3 layers with decap action. */ 4395 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4396 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4397 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4398 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4399 /* Convert L4 layers with decap action. */ 4400 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4401 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4402 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4403 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4404 return layers; 4405 } 4406 4407 /** 4408 * Get metadata split action information. 4409 * 4410 * @param[in] actions 4411 * Pointer to the list of actions. 4412 * @param[out] qrss 4413 * Pointer to the return pointer. 4414 * @param[out] qrss_type 4415 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4416 * if no QUEUE/RSS is found. 4417 * @param[out] encap_idx 4418 * Pointer to the index of the encap action if exists, otherwise the last 4419 * action index. 4420 * 4421 * @return 4422 * Total number of actions. 4423 */ 4424 static int 4425 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4426 const struct rte_flow_action **qrss, 4427 int *encap_idx) 4428 { 4429 const struct rte_flow_action_raw_encap *raw_encap; 4430 int actions_n = 0; 4431 int raw_decap_idx = -1; 4432 4433 *encap_idx = -1; 4434 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4435 switch (actions->type) { 4436 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4437 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4438 *encap_idx = actions_n; 4439 break; 4440 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4441 raw_decap_idx = actions_n; 4442 break; 4443 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4444 raw_encap = actions->conf; 4445 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4446 *encap_idx = raw_decap_idx != -1 ? 4447 raw_decap_idx : actions_n; 4448 break; 4449 case RTE_FLOW_ACTION_TYPE_QUEUE: 4450 case RTE_FLOW_ACTION_TYPE_RSS: 4451 *qrss = actions; 4452 break; 4453 default: 4454 break; 4455 } 4456 actions_n++; 4457 } 4458 if (*encap_idx == -1) 4459 *encap_idx = actions_n; 4460 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4461 return actions_n + 1; 4462 } 4463 4464 /** 4465 * Check if the action will change packet. 4466 * 4467 * @param dev 4468 * Pointer to Ethernet device. 4469 * @param[in] type 4470 * action type. 4471 * 4472 * @return 4473 * true if action will change packet, false otherwise. 4474 */ 4475 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4476 enum rte_flow_action_type type) 4477 { 4478 struct mlx5_priv *priv = dev->data->dev_private; 4479 4480 switch (type) { 4481 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4482 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4483 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4484 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4485 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4486 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4487 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4488 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4489 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4490 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4491 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4492 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4493 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4494 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4495 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4496 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4497 case RTE_FLOW_ACTION_TYPE_SET_META: 4498 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4499 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4500 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4501 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4502 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4503 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4504 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4505 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4506 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4507 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4508 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4509 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4510 return true; 4511 case RTE_FLOW_ACTION_TYPE_FLAG: 4512 case RTE_FLOW_ACTION_TYPE_MARK: 4513 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4514 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4515 return true; 4516 else 4517 return false; 4518 default: 4519 return false; 4520 } 4521 } 4522 4523 /** 4524 * Check meter action from the action list. 4525 * 4526 * @param dev 4527 * Pointer to Ethernet device. 4528 * @param[in] actions 4529 * Pointer to the list of actions. 4530 * @param[out] has_mtr 4531 * Pointer to the meter exist flag. 4532 * @param[out] has_modify 4533 * Pointer to the flag showing there's packet change action. 4534 * @param[out] meter_id 4535 * Pointer to the meter id. 4536 * 4537 * @return 4538 * Total number of actions. 4539 */ 4540 static int 4541 flow_check_meter_action(struct rte_eth_dev *dev, 4542 const struct rte_flow_action actions[], 4543 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4544 { 4545 const struct rte_flow_action_meter *mtr = NULL; 4546 int actions_n = 0; 4547 4548 MLX5_ASSERT(has_mtr); 4549 *has_mtr = false; 4550 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4551 switch (actions->type) { 4552 case RTE_FLOW_ACTION_TYPE_METER: 4553 mtr = actions->conf; 4554 *meter_id = mtr->mtr_id; 4555 *has_mtr = true; 4556 break; 4557 default: 4558 break; 4559 } 4560 if (!*has_mtr) 4561 *has_modify |= flow_check_modify_action_type(dev, 4562 actions->type); 4563 actions_n++; 4564 } 4565 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4566 return actions_n + 1; 4567 } 4568 4569 /** 4570 * Check if the flow should be split due to hairpin. 4571 * The reason for the split is that in current HW we can't 4572 * support encap and push-vlan on Rx, so if a flow contains 4573 * these actions we move it to Tx. 4574 * 4575 * @param dev 4576 * Pointer to Ethernet device. 4577 * @param[in] attr 4578 * Flow rule attributes. 4579 * @param[in] actions 4580 * Associated actions (list terminated by the END action). 4581 * 4582 * @return 4583 * > 0 the number of actions and the flow should be split, 4584 * 0 when no split required. 4585 */ 4586 static int 4587 flow_check_hairpin_split(struct rte_eth_dev *dev, 4588 const struct rte_flow_attr *attr, 4589 const struct rte_flow_action actions[]) 4590 { 4591 int queue_action = 0; 4592 int action_n = 0; 4593 int split = 0; 4594 int push_vlan = 0; 4595 const struct rte_flow_action_queue *queue; 4596 const struct rte_flow_action_rss *rss; 4597 const struct rte_flow_action_raw_encap *raw_encap; 4598 const struct rte_eth_hairpin_conf *conf; 4599 4600 if (!attr->ingress) 4601 return 0; 4602 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4603 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4604 push_vlan = 1; 4605 switch (actions->type) { 4606 case RTE_FLOW_ACTION_TYPE_QUEUE: 4607 queue = actions->conf; 4608 if (queue == NULL) 4609 return 0; 4610 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4611 if (conf == NULL || conf->tx_explicit != 0) 4612 return 0; 4613 queue_action = 1; 4614 action_n++; 4615 break; 4616 case RTE_FLOW_ACTION_TYPE_RSS: 4617 rss = actions->conf; 4618 if (rss == NULL || rss->queue_num == 0) 4619 return 0; 4620 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4621 if (conf == NULL || conf->tx_explicit != 0) 4622 return 0; 4623 queue_action = 1; 4624 action_n++; 4625 break; 4626 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4627 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4628 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4629 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4630 split++; 4631 action_n++; 4632 break; 4633 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4634 if (push_vlan) 4635 split++; 4636 action_n++; 4637 break; 4638 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4639 raw_encap = actions->conf; 4640 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4641 split++; 4642 action_n++; 4643 break; 4644 default: 4645 action_n++; 4646 break; 4647 } 4648 } 4649 if (split && queue_action) 4650 return action_n; 4651 return 0; 4652 } 4653 4654 /* Declare flow create/destroy prototype in advance. */ 4655 static uint32_t 4656 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4657 const struct rte_flow_attr *attr, 4658 const struct rte_flow_item items[], 4659 const struct rte_flow_action actions[], 4660 bool external, struct rte_flow_error *error); 4661 4662 static void 4663 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4664 uint32_t flow_idx); 4665 4666 int 4667 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4668 struct mlx5_list_entry *entry, void *cb_ctx) 4669 { 4670 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4671 struct mlx5_flow_mreg_copy_resource *mcp_res = 4672 container_of(entry, typeof(*mcp_res), hlist_ent); 4673 4674 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4675 } 4676 4677 struct mlx5_list_entry * 4678 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4679 { 4680 struct rte_eth_dev *dev = tool_ctx; 4681 struct mlx5_priv *priv = dev->data->dev_private; 4682 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4683 struct mlx5_flow_mreg_copy_resource *mcp_res; 4684 struct rte_flow_error *error = ctx->error; 4685 uint32_t idx = 0; 4686 int ret; 4687 uint32_t mark_id = *(uint32_t *)(ctx->data); 4688 struct rte_flow_attr attr = { 4689 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4690 .ingress = 1, 4691 }; 4692 struct mlx5_rte_flow_item_tag tag_spec = { 4693 .data = mark_id, 4694 }; 4695 struct rte_flow_item items[] = { 4696 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4697 }; 4698 struct rte_flow_action_mark ftag = { 4699 .id = mark_id, 4700 }; 4701 struct mlx5_flow_action_copy_mreg cp_mreg = { 4702 .dst = REG_B, 4703 .src = REG_NON, 4704 }; 4705 struct rte_flow_action_jump jump = { 4706 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4707 }; 4708 struct rte_flow_action actions[] = { 4709 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4710 }; 4711 4712 /* Fill the register fields in the flow. */ 4713 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4714 if (ret < 0) 4715 return NULL; 4716 tag_spec.id = ret; 4717 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4718 if (ret < 0) 4719 return NULL; 4720 cp_mreg.src = ret; 4721 /* Provide the full width of FLAG specific value. */ 4722 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4723 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4724 /* Build a new flow. */ 4725 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4726 items[0] = (struct rte_flow_item){ 4727 .type = (enum rte_flow_item_type) 4728 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4729 .spec = &tag_spec, 4730 }; 4731 items[1] = (struct rte_flow_item){ 4732 .type = RTE_FLOW_ITEM_TYPE_END, 4733 }; 4734 actions[0] = (struct rte_flow_action){ 4735 .type = (enum rte_flow_action_type) 4736 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4737 .conf = &ftag, 4738 }; 4739 actions[1] = (struct rte_flow_action){ 4740 .type = (enum rte_flow_action_type) 4741 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4742 .conf = &cp_mreg, 4743 }; 4744 actions[2] = (struct rte_flow_action){ 4745 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4746 .conf = &jump, 4747 }; 4748 actions[3] = (struct rte_flow_action){ 4749 .type = RTE_FLOW_ACTION_TYPE_END, 4750 }; 4751 } else { 4752 /* Default rule, wildcard match. */ 4753 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4754 items[0] = (struct rte_flow_item){ 4755 .type = RTE_FLOW_ITEM_TYPE_END, 4756 }; 4757 actions[0] = (struct rte_flow_action){ 4758 .type = (enum rte_flow_action_type) 4759 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4760 .conf = &cp_mreg, 4761 }; 4762 actions[1] = (struct rte_flow_action){ 4763 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4764 .conf = &jump, 4765 }; 4766 actions[2] = (struct rte_flow_action){ 4767 .type = RTE_FLOW_ACTION_TYPE_END, 4768 }; 4769 } 4770 /* Build a new entry. */ 4771 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4772 if (!mcp_res) { 4773 rte_errno = ENOMEM; 4774 return NULL; 4775 } 4776 mcp_res->idx = idx; 4777 mcp_res->mark_id = mark_id; 4778 /* 4779 * The copy Flows are not included in any list. There 4780 * ones are referenced from other Flows and can not 4781 * be applied, removed, deleted in arbitrary order 4782 * by list traversing. 4783 */ 4784 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4785 &attr, items, actions, false, error); 4786 if (!mcp_res->rix_flow) { 4787 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4788 return NULL; 4789 } 4790 return &mcp_res->hlist_ent; 4791 } 4792 4793 struct mlx5_list_entry * 4794 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4795 void *cb_ctx __rte_unused) 4796 { 4797 struct rte_eth_dev *dev = tool_ctx; 4798 struct mlx5_priv *priv = dev->data->dev_private; 4799 struct mlx5_flow_mreg_copy_resource *mcp_res; 4800 uint32_t idx = 0; 4801 4802 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4803 if (!mcp_res) { 4804 rte_errno = ENOMEM; 4805 return NULL; 4806 } 4807 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4808 mcp_res->idx = idx; 4809 return &mcp_res->hlist_ent; 4810 } 4811 4812 void 4813 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4814 { 4815 struct mlx5_flow_mreg_copy_resource *mcp_res = 4816 container_of(entry, typeof(*mcp_res), hlist_ent); 4817 struct rte_eth_dev *dev = tool_ctx; 4818 struct mlx5_priv *priv = dev->data->dev_private; 4819 4820 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4821 } 4822 4823 /** 4824 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4825 * 4826 * As mark_id is unique, if there's already a registered flow for the mark_id, 4827 * return by increasing the reference counter of the resource. Otherwise, create 4828 * the resource (mcp_res) and flow. 4829 * 4830 * Flow looks like, 4831 * - If ingress port is ANY and reg_c[1] is mark_id, 4832 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4833 * 4834 * For default flow (zero mark_id), flow is like, 4835 * - If ingress port is ANY, 4836 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4837 * 4838 * @param dev 4839 * Pointer to Ethernet device. 4840 * @param mark_id 4841 * ID of MARK action, zero means default flow for META. 4842 * @param[out] error 4843 * Perform verbose error reporting if not NULL. 4844 * 4845 * @return 4846 * Associated resource on success, NULL otherwise and rte_errno is set. 4847 */ 4848 static struct mlx5_flow_mreg_copy_resource * 4849 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4850 struct rte_flow_error *error) 4851 { 4852 struct mlx5_priv *priv = dev->data->dev_private; 4853 struct mlx5_list_entry *entry; 4854 struct mlx5_flow_cb_ctx ctx = { 4855 .dev = dev, 4856 .error = error, 4857 .data = &mark_id, 4858 }; 4859 4860 /* Check if already registered. */ 4861 MLX5_ASSERT(priv->mreg_cp_tbl); 4862 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4863 if (!entry) 4864 return NULL; 4865 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4866 hlist_ent); 4867 } 4868 4869 void 4870 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4871 { 4872 struct mlx5_flow_mreg_copy_resource *mcp_res = 4873 container_of(entry, typeof(*mcp_res), hlist_ent); 4874 struct rte_eth_dev *dev = tool_ctx; 4875 struct mlx5_priv *priv = dev->data->dev_private; 4876 4877 MLX5_ASSERT(mcp_res->rix_flow); 4878 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4879 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4880 } 4881 4882 /** 4883 * Release flow in RX_CP_TBL. 4884 * 4885 * @param dev 4886 * Pointer to Ethernet device. 4887 * @flow 4888 * Parent flow for wich copying is provided. 4889 */ 4890 static void 4891 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4892 struct rte_flow *flow) 4893 { 4894 struct mlx5_flow_mreg_copy_resource *mcp_res; 4895 struct mlx5_priv *priv = dev->data->dev_private; 4896 4897 if (!flow->rix_mreg_copy) 4898 return; 4899 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4900 flow->rix_mreg_copy); 4901 if (!mcp_res || !priv->mreg_cp_tbl) 4902 return; 4903 MLX5_ASSERT(mcp_res->rix_flow); 4904 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4905 flow->rix_mreg_copy = 0; 4906 } 4907 4908 /** 4909 * Remove the default copy action from RX_CP_TBL. 4910 * 4911 * This functions is called in the mlx5_dev_start(). No thread safe 4912 * is guaranteed. 4913 * 4914 * @param dev 4915 * Pointer to Ethernet device. 4916 */ 4917 static void 4918 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4919 { 4920 struct mlx5_list_entry *entry; 4921 struct mlx5_priv *priv = dev->data->dev_private; 4922 struct mlx5_flow_cb_ctx ctx; 4923 uint32_t mark_id; 4924 4925 /* Check if default flow is registered. */ 4926 if (!priv->mreg_cp_tbl) 4927 return; 4928 mark_id = MLX5_DEFAULT_COPY_ID; 4929 ctx.data = &mark_id; 4930 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4931 if (!entry) 4932 return; 4933 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4934 } 4935 4936 /** 4937 * Add the default copy action in in RX_CP_TBL. 4938 * 4939 * This functions is called in the mlx5_dev_start(). No thread safe 4940 * is guaranteed. 4941 * 4942 * @param dev 4943 * Pointer to Ethernet device. 4944 * @param[out] error 4945 * Perform verbose error reporting if not NULL. 4946 * 4947 * @return 4948 * 0 for success, negative value otherwise and rte_errno is set. 4949 */ 4950 static int 4951 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4952 struct rte_flow_error *error) 4953 { 4954 struct mlx5_priv *priv = dev->data->dev_private; 4955 struct mlx5_flow_mreg_copy_resource *mcp_res; 4956 struct mlx5_flow_cb_ctx ctx; 4957 uint32_t mark_id; 4958 4959 /* Check whether extensive metadata feature is engaged. */ 4960 if (!priv->sh->config.dv_flow_en || 4961 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4962 !mlx5_flow_ext_mreg_supported(dev) || 4963 !priv->sh->dv_regc0_mask) 4964 return 0; 4965 /* 4966 * Add default mreg copy flow may be called multiple time, but 4967 * only be called once in stop. Avoid register it twice. 4968 */ 4969 mark_id = MLX5_DEFAULT_COPY_ID; 4970 ctx.data = &mark_id; 4971 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4972 return 0; 4973 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4974 if (!mcp_res) 4975 return -rte_errno; 4976 return 0; 4977 } 4978 4979 /** 4980 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4981 * 4982 * All the flow having Q/RSS action should be split by 4983 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4984 * performs the following, 4985 * - CQE->flow_tag := reg_c[1] (MARK) 4986 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4987 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4988 * but there should be a flow per each MARK ID set by MARK action. 4989 * 4990 * For the aforementioned reason, if there's a MARK action in flow's action 4991 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4992 * the MARK ID to CQE's flow_tag like, 4993 * - If reg_c[1] is mark_id, 4994 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4995 * 4996 * For SET_META action which stores value in reg_c[0], as the destination is 4997 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4998 * MARK ID means the default flow. The default flow looks like, 4999 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5000 * 5001 * @param dev 5002 * Pointer to Ethernet device. 5003 * @param flow 5004 * Pointer to flow structure. 5005 * @param[in] actions 5006 * Pointer to the list of actions. 5007 * @param[out] error 5008 * Perform verbose error reporting if not NULL. 5009 * 5010 * @return 5011 * 0 on success, negative value otherwise and rte_errno is set. 5012 */ 5013 static int 5014 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5015 struct rte_flow *flow, 5016 const struct rte_flow_action *actions, 5017 struct rte_flow_error *error) 5018 { 5019 struct mlx5_priv *priv = dev->data->dev_private; 5020 struct mlx5_sh_config *config = &priv->sh->config; 5021 struct mlx5_flow_mreg_copy_resource *mcp_res; 5022 const struct rte_flow_action_mark *mark; 5023 5024 /* Check whether extensive metadata feature is engaged. */ 5025 if (!config->dv_flow_en || 5026 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5027 !mlx5_flow_ext_mreg_supported(dev) || 5028 !priv->sh->dv_regc0_mask) 5029 return 0; 5030 /* Find MARK action. */ 5031 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5032 switch (actions->type) { 5033 case RTE_FLOW_ACTION_TYPE_FLAG: 5034 mcp_res = flow_mreg_add_copy_action 5035 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5036 if (!mcp_res) 5037 return -rte_errno; 5038 flow->rix_mreg_copy = mcp_res->idx; 5039 return 0; 5040 case RTE_FLOW_ACTION_TYPE_MARK: 5041 mark = (const struct rte_flow_action_mark *) 5042 actions->conf; 5043 mcp_res = 5044 flow_mreg_add_copy_action(dev, mark->id, error); 5045 if (!mcp_res) 5046 return -rte_errno; 5047 flow->rix_mreg_copy = mcp_res->idx; 5048 return 0; 5049 default: 5050 break; 5051 } 5052 } 5053 return 0; 5054 } 5055 5056 #define MLX5_MAX_SPLIT_ACTIONS 24 5057 #define MLX5_MAX_SPLIT_ITEMS 24 5058 5059 /** 5060 * Split the hairpin flow. 5061 * Since HW can't support encap and push-vlan on Rx, we move these 5062 * actions to Tx. 5063 * If the count action is after the encap then we also 5064 * move the count action. in this case the count will also measure 5065 * the outer bytes. 5066 * 5067 * @param dev 5068 * Pointer to Ethernet device. 5069 * @param[in] actions 5070 * Associated actions (list terminated by the END action). 5071 * @param[out] actions_rx 5072 * Rx flow actions. 5073 * @param[out] actions_tx 5074 * Tx flow actions.. 5075 * @param[out] pattern_tx 5076 * The pattern items for the Tx flow. 5077 * @param[out] flow_id 5078 * The flow ID connected to this flow. 5079 * 5080 * @return 5081 * 0 on success. 5082 */ 5083 static int 5084 flow_hairpin_split(struct rte_eth_dev *dev, 5085 const struct rte_flow_action actions[], 5086 struct rte_flow_action actions_rx[], 5087 struct rte_flow_action actions_tx[], 5088 struct rte_flow_item pattern_tx[], 5089 uint32_t flow_id) 5090 { 5091 const struct rte_flow_action_raw_encap *raw_encap; 5092 const struct rte_flow_action_raw_decap *raw_decap; 5093 struct mlx5_rte_flow_action_set_tag *set_tag; 5094 struct rte_flow_action *tag_action; 5095 struct mlx5_rte_flow_item_tag *tag_item; 5096 struct rte_flow_item *item; 5097 char *addr; 5098 int push_vlan = 0; 5099 int encap = 0; 5100 5101 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5102 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5103 push_vlan = 1; 5104 switch (actions->type) { 5105 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5106 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5107 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5108 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5109 rte_memcpy(actions_tx, actions, 5110 sizeof(struct rte_flow_action)); 5111 actions_tx++; 5112 break; 5113 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5114 if (push_vlan) { 5115 rte_memcpy(actions_tx, actions, 5116 sizeof(struct rte_flow_action)); 5117 actions_tx++; 5118 } else { 5119 rte_memcpy(actions_rx, actions, 5120 sizeof(struct rte_flow_action)); 5121 actions_rx++; 5122 } 5123 break; 5124 case RTE_FLOW_ACTION_TYPE_COUNT: 5125 if (encap) { 5126 rte_memcpy(actions_tx, actions, 5127 sizeof(struct rte_flow_action)); 5128 actions_tx++; 5129 } else { 5130 rte_memcpy(actions_rx, actions, 5131 sizeof(struct rte_flow_action)); 5132 actions_rx++; 5133 } 5134 break; 5135 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5136 raw_encap = actions->conf; 5137 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5138 memcpy(actions_tx, actions, 5139 sizeof(struct rte_flow_action)); 5140 actions_tx++; 5141 encap = 1; 5142 } else { 5143 rte_memcpy(actions_rx, actions, 5144 sizeof(struct rte_flow_action)); 5145 actions_rx++; 5146 } 5147 break; 5148 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5149 raw_decap = actions->conf; 5150 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5151 memcpy(actions_tx, actions, 5152 sizeof(struct rte_flow_action)); 5153 actions_tx++; 5154 } else { 5155 rte_memcpy(actions_rx, actions, 5156 sizeof(struct rte_flow_action)); 5157 actions_rx++; 5158 } 5159 break; 5160 default: 5161 rte_memcpy(actions_rx, actions, 5162 sizeof(struct rte_flow_action)); 5163 actions_rx++; 5164 break; 5165 } 5166 } 5167 /* Add set meta action and end action for the Rx flow. */ 5168 tag_action = actions_rx; 5169 tag_action->type = (enum rte_flow_action_type) 5170 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5171 actions_rx++; 5172 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5173 actions_rx++; 5174 set_tag = (void *)actions_rx; 5175 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5176 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5177 .data = flow_id, 5178 }; 5179 MLX5_ASSERT(set_tag->id > REG_NON); 5180 tag_action->conf = set_tag; 5181 /* Create Tx item list. */ 5182 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5183 addr = (void *)&pattern_tx[2]; 5184 item = pattern_tx; 5185 item->type = (enum rte_flow_item_type) 5186 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5187 tag_item = (void *)addr; 5188 tag_item->data = flow_id; 5189 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5190 MLX5_ASSERT(set_tag->id > REG_NON); 5191 item->spec = tag_item; 5192 addr += sizeof(struct mlx5_rte_flow_item_tag); 5193 tag_item = (void *)addr; 5194 tag_item->data = UINT32_MAX; 5195 tag_item->id = UINT16_MAX; 5196 item->mask = tag_item; 5197 item->last = NULL; 5198 item++; 5199 item->type = RTE_FLOW_ITEM_TYPE_END; 5200 return 0; 5201 } 5202 5203 /** 5204 * The last stage of splitting chain, just creates the subflow 5205 * without any modification. 5206 * 5207 * @param[in] dev 5208 * Pointer to Ethernet device. 5209 * @param[in] flow 5210 * Parent flow structure pointer. 5211 * @param[in, out] sub_flow 5212 * Pointer to return the created subflow, may be NULL. 5213 * @param[in] attr 5214 * Flow rule attributes. 5215 * @param[in] items 5216 * Pattern specification (list terminated by the END pattern item). 5217 * @param[in] actions 5218 * Associated actions (list terminated by the END action). 5219 * @param[in] flow_split_info 5220 * Pointer to flow split info structure. 5221 * @param[out] error 5222 * Perform verbose error reporting if not NULL. 5223 * @return 5224 * 0 on success, negative value otherwise 5225 */ 5226 static int 5227 flow_create_split_inner(struct rte_eth_dev *dev, 5228 struct rte_flow *flow, 5229 struct mlx5_flow **sub_flow, 5230 const struct rte_flow_attr *attr, 5231 const struct rte_flow_item items[], 5232 const struct rte_flow_action actions[], 5233 struct mlx5_flow_split_info *flow_split_info, 5234 struct rte_flow_error *error) 5235 { 5236 struct mlx5_flow *dev_flow; 5237 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5238 5239 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5240 flow_split_info->flow_idx, error); 5241 if (!dev_flow) 5242 return -rte_errno; 5243 dev_flow->flow = flow; 5244 dev_flow->external = flow_split_info->external; 5245 dev_flow->skip_scale = flow_split_info->skip_scale; 5246 /* Subflow object was created, we must include one in the list. */ 5247 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5248 dev_flow->handle, next); 5249 /* 5250 * If dev_flow is as one of the suffix flow, some actions in suffix 5251 * flow may need some user defined item layer flags, and pass the 5252 * Metadata rxq mark flag to suffix flow as well. 5253 */ 5254 if (flow_split_info->prefix_layers) 5255 dev_flow->handle->layers = flow_split_info->prefix_layers; 5256 if (flow_split_info->prefix_mark) { 5257 MLX5_ASSERT(wks); 5258 wks->mark = 1; 5259 } 5260 if (sub_flow) 5261 *sub_flow = dev_flow; 5262 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5263 dev_flow->dv.table_id = flow_split_info->table_id; 5264 #endif 5265 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5266 } 5267 5268 /** 5269 * Get the sub policy of a meter. 5270 * 5271 * @param[in] dev 5272 * Pointer to Ethernet device. 5273 * @param[in] flow 5274 * Parent flow structure pointer. 5275 * @param wks 5276 * Pointer to thread flow work space. 5277 * @param[in] attr 5278 * Flow rule attributes. 5279 * @param[in] items 5280 * Pattern specification (list terminated by the END pattern item). 5281 * @param[out] error 5282 * Perform verbose error reporting if not NULL. 5283 * 5284 * @return 5285 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5286 */ 5287 static struct mlx5_flow_meter_sub_policy * 5288 get_meter_sub_policy(struct rte_eth_dev *dev, 5289 struct rte_flow *flow, 5290 struct mlx5_flow_workspace *wks, 5291 const struct rte_flow_attr *attr, 5292 const struct rte_flow_item items[], 5293 struct rte_flow_error *error) 5294 { 5295 struct mlx5_flow_meter_policy *policy; 5296 struct mlx5_flow_meter_policy *final_policy; 5297 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5298 5299 policy = wks->policy; 5300 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5301 if (final_policy->is_rss || final_policy->is_queue) { 5302 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5303 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5304 uint32_t i; 5305 5306 /* 5307 * This is a tmp dev_flow, 5308 * no need to register any matcher for it in translate. 5309 */ 5310 wks->skip_matcher_reg = 1; 5311 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5312 struct mlx5_flow dev_flow = {0}; 5313 struct mlx5_flow_handle dev_handle = { {0} }; 5314 uint8_t fate = final_policy->act_cnt[i].fate_action; 5315 5316 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5317 const struct rte_flow_action_rss *rss_act = 5318 final_policy->act_cnt[i].rss->conf; 5319 struct rte_flow_action rss_actions[2] = { 5320 [0] = { 5321 .type = RTE_FLOW_ACTION_TYPE_RSS, 5322 .conf = rss_act, 5323 }, 5324 [1] = { 5325 .type = RTE_FLOW_ACTION_TYPE_END, 5326 .conf = NULL, 5327 } 5328 }; 5329 5330 dev_flow.handle = &dev_handle; 5331 dev_flow.ingress = attr->ingress; 5332 dev_flow.flow = flow; 5333 dev_flow.external = 0; 5334 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5335 dev_flow.dv.transfer = attr->transfer; 5336 #endif 5337 /** 5338 * Translate RSS action to get rss hash fields. 5339 */ 5340 if (flow_drv_translate(dev, &dev_flow, attr, 5341 items, rss_actions, error)) 5342 goto exit; 5343 rss_desc_v[i] = wks->rss_desc; 5344 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5345 rss_desc_v[i].hash_fields = 5346 dev_flow.hash_fields; 5347 rss_desc_v[i].queue_num = 5348 rss_desc_v[i].hash_fields ? 5349 rss_desc_v[i].queue_num : 1; 5350 rss_desc_v[i].tunnel = 5351 !!(dev_flow.handle->layers & 5352 MLX5_FLOW_LAYER_TUNNEL); 5353 /* Use the RSS queues in the containers. */ 5354 rss_desc_v[i].queue = 5355 (uint16_t *)(uintptr_t)rss_act->queue; 5356 rss_desc[i] = &rss_desc_v[i]; 5357 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5358 /* This is queue action. */ 5359 rss_desc_v[i] = wks->rss_desc; 5360 rss_desc_v[i].key_len = 0; 5361 rss_desc_v[i].hash_fields = 0; 5362 rss_desc_v[i].queue = 5363 &final_policy->act_cnt[i].queue; 5364 rss_desc_v[i].queue_num = 1; 5365 rss_desc[i] = &rss_desc_v[i]; 5366 } else { 5367 rss_desc[i] = NULL; 5368 } 5369 } 5370 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5371 flow, policy, rss_desc); 5372 } else { 5373 enum mlx5_meter_domain mtr_domain = 5374 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5375 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5376 MLX5_MTR_DOMAIN_INGRESS); 5377 sub_policy = policy->sub_policys[mtr_domain][0]; 5378 } 5379 if (!sub_policy) 5380 rte_flow_error_set(error, EINVAL, 5381 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5382 "Failed to get meter sub-policy."); 5383 exit: 5384 return sub_policy; 5385 } 5386 5387 /** 5388 * Split the meter flow. 5389 * 5390 * As meter flow will split to three sub flow, other than meter 5391 * action, the other actions make sense to only meter accepts 5392 * the packet. If it need to be dropped, no other additional 5393 * actions should be take. 5394 * 5395 * One kind of special action which decapsulates the L3 tunnel 5396 * header will be in the prefix sub flow, as not to take the 5397 * L3 tunnel header into account. 5398 * 5399 * @param[in] dev 5400 * Pointer to Ethernet device. 5401 * @param[in] flow 5402 * Parent flow structure pointer. 5403 * @param wks 5404 * Pointer to thread flow work space. 5405 * @param[in] attr 5406 * Flow rule attributes. 5407 * @param[in] items 5408 * Pattern specification (list terminated by the END pattern item). 5409 * @param[out] sfx_items 5410 * Suffix flow match items (list terminated by the END pattern item). 5411 * @param[in] actions 5412 * Associated actions (list terminated by the END action). 5413 * @param[out] actions_sfx 5414 * Suffix flow actions. 5415 * @param[out] actions_pre 5416 * Prefix flow actions. 5417 * @param[out] mtr_flow_id 5418 * Pointer to meter flow id. 5419 * @param[out] error 5420 * Perform verbose error reporting if not NULL. 5421 * 5422 * @return 5423 * 0 on success, a negative errno value otherwise and rte_errno is set. 5424 */ 5425 static int 5426 flow_meter_split_prep(struct rte_eth_dev *dev, 5427 struct rte_flow *flow, 5428 struct mlx5_flow_workspace *wks, 5429 const struct rte_flow_attr *attr, 5430 const struct rte_flow_item items[], 5431 struct rte_flow_item sfx_items[], 5432 const struct rte_flow_action actions[], 5433 struct rte_flow_action actions_sfx[], 5434 struct rte_flow_action actions_pre[], 5435 uint32_t *mtr_flow_id, 5436 struct rte_flow_error *error) 5437 { 5438 struct mlx5_priv *priv = dev->data->dev_private; 5439 struct mlx5_flow_meter_info *fm = wks->fm; 5440 struct rte_flow_action *tag_action = NULL; 5441 struct rte_flow_item *tag_item; 5442 struct mlx5_rte_flow_action_set_tag *set_tag; 5443 const struct rte_flow_action_raw_encap *raw_encap; 5444 const struct rte_flow_action_raw_decap *raw_decap; 5445 struct mlx5_rte_flow_item_tag *tag_item_spec; 5446 struct mlx5_rte_flow_item_tag *tag_item_mask; 5447 uint32_t tag_id = 0; 5448 struct rte_flow_item *vlan_item_dst = NULL; 5449 const struct rte_flow_item *vlan_item_src = NULL; 5450 const struct rte_flow_item *orig_items = items; 5451 struct rte_flow_action *hw_mtr_action; 5452 struct rte_flow_action *action_pre_head = NULL; 5453 uint16_t flow_src_port = priv->representor_id; 5454 bool mtr_first; 5455 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5456 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5457 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5458 uint32_t flow_id = 0; 5459 uint32_t flow_id_reversed = 0; 5460 uint8_t flow_id_bits = 0; 5461 bool after_meter = false; 5462 int shift; 5463 5464 /* Prepare the suffix subflow items. */ 5465 tag_item = sfx_items++; 5466 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5467 int item_type = items->type; 5468 5469 switch (item_type) { 5470 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5471 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5472 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5473 return -rte_errno; 5474 if (!fm->def_policy && wks->policy->is_hierarchy && 5475 flow_src_port != priv->representor_id) { 5476 if (flow_drv_mtr_hierarchy_rule_create(dev, 5477 flow, fm, 5478 flow_src_port, 5479 items, 5480 error)) 5481 return -rte_errno; 5482 } 5483 memcpy(sfx_items, items, sizeof(*sfx_items)); 5484 sfx_items++; 5485 break; 5486 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5487 flow_src_port = 0; 5488 memcpy(sfx_items, items, sizeof(*sfx_items)); 5489 sfx_items++; 5490 break; 5491 case RTE_FLOW_ITEM_TYPE_VLAN: 5492 /* Determine if copy vlan item below. */ 5493 vlan_item_src = items; 5494 vlan_item_dst = sfx_items++; 5495 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5496 break; 5497 default: 5498 break; 5499 } 5500 } 5501 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5502 sfx_items++; 5503 mtr_first = priv->sh->meter_aso_en && 5504 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5505 /* For ASO meter, meter must be before tag in TX direction. */ 5506 if (mtr_first) { 5507 action_pre_head = actions_pre++; 5508 /* Leave space for tag action. */ 5509 tag_action = actions_pre++; 5510 } 5511 /* Prepare the actions for prefix and suffix flow. */ 5512 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5513 struct rte_flow_action *action_cur = NULL; 5514 5515 switch (actions->type) { 5516 case RTE_FLOW_ACTION_TYPE_METER: 5517 if (mtr_first) { 5518 action_cur = action_pre_head; 5519 } else { 5520 /* Leave space for tag action. */ 5521 tag_action = actions_pre++; 5522 action_cur = actions_pre++; 5523 } 5524 after_meter = true; 5525 break; 5526 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5527 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5528 action_cur = actions_pre++; 5529 break; 5530 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5531 raw_encap = actions->conf; 5532 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5533 action_cur = actions_pre++; 5534 break; 5535 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5536 raw_decap = actions->conf; 5537 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5538 action_cur = actions_pre++; 5539 break; 5540 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5541 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5542 if (vlan_item_dst && vlan_item_src) { 5543 memcpy(vlan_item_dst, vlan_item_src, 5544 sizeof(*vlan_item_dst)); 5545 /* 5546 * Convert to internal match item, it is used 5547 * for vlan push and set vid. 5548 */ 5549 vlan_item_dst->type = (enum rte_flow_item_type) 5550 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5551 } 5552 break; 5553 case RTE_FLOW_ACTION_TYPE_COUNT: 5554 if (fm->def_policy) 5555 action_cur = after_meter ? 5556 actions_sfx++ : actions_pre++; 5557 break; 5558 default: 5559 break; 5560 } 5561 if (!action_cur) 5562 action_cur = (fm->def_policy) ? 5563 actions_sfx++ : actions_pre++; 5564 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5565 } 5566 /* Add end action to the actions. */ 5567 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5568 if (priv->sh->meter_aso_en) { 5569 /** 5570 * For ASO meter, need to add an extra jump action explicitly, 5571 * to jump from meter to policer table. 5572 */ 5573 struct mlx5_flow_meter_sub_policy *sub_policy; 5574 struct mlx5_flow_tbl_data_entry *tbl_data; 5575 5576 if (!fm->def_policy) { 5577 sub_policy = get_meter_sub_policy(dev, flow, wks, 5578 attr, orig_items, 5579 error); 5580 if (!sub_policy) 5581 return -rte_errno; 5582 } else { 5583 enum mlx5_meter_domain mtr_domain = 5584 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5585 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5586 MLX5_MTR_DOMAIN_INGRESS); 5587 5588 sub_policy = 5589 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5590 } 5591 tbl_data = container_of(sub_policy->tbl_rsc, 5592 struct mlx5_flow_tbl_data_entry, tbl); 5593 hw_mtr_action = actions_pre++; 5594 hw_mtr_action->type = (enum rte_flow_action_type) 5595 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5596 hw_mtr_action->conf = tbl_data->jump.action; 5597 } 5598 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5599 actions_pre++; 5600 if (!tag_action) 5601 return rte_flow_error_set(error, ENOMEM, 5602 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5603 NULL, "No tag action space."); 5604 if (!mtr_flow_id) { 5605 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5606 goto exit; 5607 } 5608 /* Only default-policy Meter creates mtr flow id. */ 5609 if (fm->def_policy) { 5610 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5611 if (!tag_id) 5612 return rte_flow_error_set(error, ENOMEM, 5613 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5614 "Failed to allocate meter flow id."); 5615 flow_id = tag_id - 1; 5616 flow_id_bits = (!flow_id) ? 1 : 5617 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5618 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5619 mtr_reg_bits) { 5620 mlx5_ipool_free(fm->flow_ipool, tag_id); 5621 return rte_flow_error_set(error, EINVAL, 5622 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5623 "Meter flow id exceeds max limit."); 5624 } 5625 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5626 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5627 } 5628 /* Build tag actions and items for meter_id/meter flow_id. */ 5629 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5630 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5631 tag_item_mask = tag_item_spec + 1; 5632 /* Both flow_id and meter_id share the same register. */ 5633 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5634 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5635 0, error), 5636 .offset = mtr_id_offset, 5637 .length = mtr_reg_bits, 5638 .data = flow->meter, 5639 }; 5640 /* 5641 * The color Reg bits used by flow_id are growing from 5642 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5643 */ 5644 for (shift = 0; shift < flow_id_bits; shift++) 5645 flow_id_reversed = (flow_id_reversed << 1) | 5646 ((flow_id >> shift) & 0x1); 5647 set_tag->data |= 5648 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5649 tag_item_spec->id = set_tag->id; 5650 tag_item_spec->data = set_tag->data << mtr_id_offset; 5651 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5652 tag_action->type = (enum rte_flow_action_type) 5653 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5654 tag_action->conf = set_tag; 5655 tag_item->type = (enum rte_flow_item_type) 5656 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5657 tag_item->spec = tag_item_spec; 5658 tag_item->last = NULL; 5659 tag_item->mask = tag_item_mask; 5660 exit: 5661 if (mtr_flow_id) 5662 *mtr_flow_id = tag_id; 5663 return 0; 5664 } 5665 5666 /** 5667 * Split action list having QUEUE/RSS for metadata register copy. 5668 * 5669 * Once Q/RSS action is detected in user's action list, the flow action 5670 * should be split in order to copy metadata registers, which will happen in 5671 * RX_CP_TBL like, 5672 * - CQE->flow_tag := reg_c[1] (MARK) 5673 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5674 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5675 * This is because the last action of each flow must be a terminal action 5676 * (QUEUE, RSS or DROP). 5677 * 5678 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5679 * stored and kept in the mlx5_flow structure per each sub_flow. 5680 * 5681 * The Q/RSS action is replaced with, 5682 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5683 * And the following JUMP action is added at the end, 5684 * - JUMP, to RX_CP_TBL. 5685 * 5686 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5687 * flow_create_split_metadata() routine. The flow will look like, 5688 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5689 * 5690 * @param dev 5691 * Pointer to Ethernet device. 5692 * @param[out] split_actions 5693 * Pointer to store split actions to jump to CP_TBL. 5694 * @param[in] actions 5695 * Pointer to the list of original flow actions. 5696 * @param[in] qrss 5697 * Pointer to the Q/RSS action. 5698 * @param[in] actions_n 5699 * Number of original actions. 5700 * @param[in] mtr_sfx 5701 * Check if it is in meter suffix table. 5702 * @param[out] error 5703 * Perform verbose error reporting if not NULL. 5704 * 5705 * @return 5706 * non-zero unique flow_id on success, otherwise 0 and 5707 * error/rte_error are set. 5708 */ 5709 static uint32_t 5710 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5711 struct rte_flow_action *split_actions, 5712 const struct rte_flow_action *actions, 5713 const struct rte_flow_action *qrss, 5714 int actions_n, int mtr_sfx, 5715 struct rte_flow_error *error) 5716 { 5717 struct mlx5_priv *priv = dev->data->dev_private; 5718 struct mlx5_rte_flow_action_set_tag *set_tag; 5719 struct rte_flow_action_jump *jump; 5720 const int qrss_idx = qrss - actions; 5721 uint32_t flow_id = 0; 5722 int ret = 0; 5723 5724 /* 5725 * Given actions will be split 5726 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5727 * - Add jump to mreg CP_TBL. 5728 * As a result, there will be one more action. 5729 */ 5730 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5731 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5732 ++actions_n; 5733 set_tag = (void *)(split_actions + actions_n); 5734 /* 5735 * If we are not the meter suffix flow, add the tag action. 5736 * Since meter suffix flow already has the tag added. 5737 */ 5738 if (!mtr_sfx) { 5739 /* 5740 * Allocate the new subflow ID. This one is unique within 5741 * device and not shared with representors. Otherwise, 5742 * we would have to resolve multi-thread access synch 5743 * issue. Each flow on the shared device is appended 5744 * with source vport identifier, so the resulting 5745 * flows will be unique in the shared (by master and 5746 * representors) domain even if they have coinciding 5747 * IDs. 5748 */ 5749 mlx5_ipool_malloc(priv->sh->ipool 5750 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5751 if (!flow_id) 5752 return rte_flow_error_set(error, ENOMEM, 5753 RTE_FLOW_ERROR_TYPE_ACTION, 5754 NULL, "can't allocate id " 5755 "for split Q/RSS subflow"); 5756 /* Internal SET_TAG action to set flow ID. */ 5757 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5758 .data = flow_id, 5759 }; 5760 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5761 if (ret < 0) 5762 return ret; 5763 set_tag->id = ret; 5764 /* Construct new actions array. */ 5765 /* Replace QUEUE/RSS action. */ 5766 split_actions[qrss_idx] = (struct rte_flow_action){ 5767 .type = (enum rte_flow_action_type) 5768 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5769 .conf = set_tag, 5770 }; 5771 } else { 5772 /* 5773 * If we are the suffix flow of meter, tag already exist. 5774 * Set the QUEUE/RSS action to void. 5775 */ 5776 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5777 } 5778 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5779 jump = (void *)(set_tag + 1); 5780 *jump = (struct rte_flow_action_jump){ 5781 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5782 }; 5783 split_actions[actions_n - 2] = (struct rte_flow_action){ 5784 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5785 .conf = jump, 5786 }; 5787 split_actions[actions_n - 1] = (struct rte_flow_action){ 5788 .type = RTE_FLOW_ACTION_TYPE_END, 5789 }; 5790 return flow_id; 5791 } 5792 5793 /** 5794 * Extend the given action list for Tx metadata copy. 5795 * 5796 * Copy the given action list to the ext_actions and add flow metadata register 5797 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5798 * 5799 * @param[out] ext_actions 5800 * Pointer to the extended action list. 5801 * @param[in] actions 5802 * Pointer to the list of actions. 5803 * @param[in] actions_n 5804 * Number of actions in the list. 5805 * @param[out] error 5806 * Perform verbose error reporting if not NULL. 5807 * @param[in] encap_idx 5808 * The encap action index. 5809 * 5810 * @return 5811 * 0 on success, negative value otherwise 5812 */ 5813 static int 5814 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5815 struct rte_flow_action *ext_actions, 5816 const struct rte_flow_action *actions, 5817 int actions_n, struct rte_flow_error *error, 5818 int encap_idx) 5819 { 5820 struct mlx5_flow_action_copy_mreg *cp_mreg = 5821 (struct mlx5_flow_action_copy_mreg *) 5822 (ext_actions + actions_n + 1); 5823 int ret; 5824 5825 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5826 if (ret < 0) 5827 return ret; 5828 cp_mreg->dst = ret; 5829 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5830 if (ret < 0) 5831 return ret; 5832 cp_mreg->src = ret; 5833 if (encap_idx != 0) 5834 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5835 if (encap_idx == actions_n - 1) { 5836 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5837 .type = (enum rte_flow_action_type) 5838 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5839 .conf = cp_mreg, 5840 }; 5841 ext_actions[actions_n] = (struct rte_flow_action){ 5842 .type = RTE_FLOW_ACTION_TYPE_END, 5843 }; 5844 } else { 5845 ext_actions[encap_idx] = (struct rte_flow_action){ 5846 .type = (enum rte_flow_action_type) 5847 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5848 .conf = cp_mreg, 5849 }; 5850 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5851 sizeof(*ext_actions) * (actions_n - encap_idx)); 5852 } 5853 return 0; 5854 } 5855 5856 /** 5857 * Check the match action from the action list. 5858 * 5859 * @param[in] actions 5860 * Pointer to the list of actions. 5861 * @param[in] attr 5862 * Flow rule attributes. 5863 * @param[in] action 5864 * The action to be check if exist. 5865 * @param[out] match_action_pos 5866 * Pointer to the position of the matched action if exists, otherwise is -1. 5867 * @param[out] qrss_action_pos 5868 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5869 * @param[out] modify_after_mirror 5870 * Pointer to the flag of modify action after FDB mirroring. 5871 * 5872 * @return 5873 * > 0 the total number of actions. 5874 * 0 if not found match action in action list. 5875 */ 5876 static int 5877 flow_check_match_action(const struct rte_flow_action actions[], 5878 const struct rte_flow_attr *attr, 5879 enum rte_flow_action_type action, 5880 int *match_action_pos, int *qrss_action_pos, 5881 int *modify_after_mirror) 5882 { 5883 const struct rte_flow_action_sample *sample; 5884 const struct rte_flow_action_raw_decap *decap; 5885 int actions_n = 0; 5886 uint32_t ratio = 0; 5887 int sub_type = 0; 5888 int flag = 0; 5889 int fdb_mirror = 0; 5890 5891 *match_action_pos = -1; 5892 *qrss_action_pos = -1; 5893 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5894 if (actions->type == action) { 5895 flag = 1; 5896 *match_action_pos = actions_n; 5897 } 5898 switch (actions->type) { 5899 case RTE_FLOW_ACTION_TYPE_QUEUE: 5900 case RTE_FLOW_ACTION_TYPE_RSS: 5901 *qrss_action_pos = actions_n; 5902 break; 5903 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5904 sample = actions->conf; 5905 ratio = sample->ratio; 5906 sub_type = ((const struct rte_flow_action *) 5907 (sample->actions))->type; 5908 if (ratio == 1 && attr->transfer && 5909 sub_type != RTE_FLOW_ACTION_TYPE_END) 5910 fdb_mirror = 1; 5911 break; 5912 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5913 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5914 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5915 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5916 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5917 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5918 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5919 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5920 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5921 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5922 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5923 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5924 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5925 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5926 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5927 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5928 case RTE_FLOW_ACTION_TYPE_FLAG: 5929 case RTE_FLOW_ACTION_TYPE_MARK: 5930 case RTE_FLOW_ACTION_TYPE_SET_META: 5931 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5932 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5933 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5934 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5935 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5936 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5937 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5938 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5939 case RTE_FLOW_ACTION_TYPE_METER: 5940 if (fdb_mirror) 5941 *modify_after_mirror = 1; 5942 break; 5943 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5944 decap = actions->conf; 5945 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5946 ; 5947 actions_n++; 5948 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5949 const struct rte_flow_action_raw_encap *encap = 5950 actions->conf; 5951 if (decap->size <= 5952 MLX5_ENCAPSULATION_DECISION_SIZE && 5953 encap->size > 5954 MLX5_ENCAPSULATION_DECISION_SIZE) 5955 /* L3 encap. */ 5956 break; 5957 } 5958 if (fdb_mirror) 5959 *modify_after_mirror = 1; 5960 break; 5961 default: 5962 break; 5963 } 5964 actions_n++; 5965 } 5966 if (flag && fdb_mirror && !*modify_after_mirror) { 5967 /* FDB mirroring uses the destination array to implement 5968 * instead of FLOW_SAMPLER object. 5969 */ 5970 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5971 flag = 0; 5972 } 5973 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5974 return flag ? actions_n + 1 : 0; 5975 } 5976 5977 #define SAMPLE_SUFFIX_ITEM 3 5978 5979 /** 5980 * Split the sample flow. 5981 * 5982 * As sample flow will split to two sub flow, sample flow with 5983 * sample action, the other actions will move to new suffix flow. 5984 * 5985 * Also add unique tag id with tag action in the sample flow, 5986 * the same tag id will be as match in the suffix flow. 5987 * 5988 * @param dev 5989 * Pointer to Ethernet device. 5990 * @param[in] add_tag 5991 * Add extra tag action flag. 5992 * @param[out] sfx_items 5993 * Suffix flow match items (list terminated by the END pattern item). 5994 * @param[in] actions 5995 * Associated actions (list terminated by the END action). 5996 * @param[out] actions_sfx 5997 * Suffix flow actions. 5998 * @param[out] actions_pre 5999 * Prefix flow actions. 6000 * @param[in] actions_n 6001 * The total number of actions. 6002 * @param[in] sample_action_pos 6003 * The sample action position. 6004 * @param[in] qrss_action_pos 6005 * The Queue/RSS action position. 6006 * @param[in] jump_table 6007 * Add extra jump action flag. 6008 * @param[out] error 6009 * Perform verbose error reporting if not NULL. 6010 * 6011 * @return 6012 * 0 on success, or unique flow_id, a negative errno value 6013 * otherwise and rte_errno is set. 6014 */ 6015 static int 6016 flow_sample_split_prep(struct rte_eth_dev *dev, 6017 int add_tag, 6018 const struct rte_flow_item items[], 6019 struct rte_flow_item sfx_items[], 6020 const struct rte_flow_action actions[], 6021 struct rte_flow_action actions_sfx[], 6022 struct rte_flow_action actions_pre[], 6023 int actions_n, 6024 int sample_action_pos, 6025 int qrss_action_pos, 6026 int jump_table, 6027 struct rte_flow_error *error) 6028 { 6029 struct mlx5_priv *priv = dev->data->dev_private; 6030 struct mlx5_rte_flow_action_set_tag *set_tag; 6031 struct mlx5_rte_flow_item_tag *tag_spec; 6032 struct mlx5_rte_flow_item_tag *tag_mask; 6033 struct rte_flow_action_jump *jump_action; 6034 uint32_t tag_id = 0; 6035 int append_index = 0; 6036 int set_tag_idx = -1; 6037 int index; 6038 int ret; 6039 6040 if (sample_action_pos < 0) 6041 return rte_flow_error_set(error, EINVAL, 6042 RTE_FLOW_ERROR_TYPE_ACTION, 6043 NULL, "invalid position of sample " 6044 "action in list"); 6045 /* Prepare the actions for prefix and suffix flow. */ 6046 if (add_tag) { 6047 /* Update the new added tag action index preceding 6048 * the PUSH_VLAN or ENCAP action. 6049 */ 6050 const struct rte_flow_action_raw_encap *raw_encap; 6051 const struct rte_flow_action *action = actions; 6052 int encap_idx; 6053 int action_idx = 0; 6054 int raw_decap_idx = -1; 6055 int push_vlan_idx = -1; 6056 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6057 switch (action->type) { 6058 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6059 raw_decap_idx = action_idx; 6060 break; 6061 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6062 raw_encap = action->conf; 6063 if (raw_encap->size > 6064 MLX5_ENCAPSULATION_DECISION_SIZE) { 6065 encap_idx = raw_decap_idx != -1 ? 6066 raw_decap_idx : action_idx; 6067 if (encap_idx < sample_action_pos && 6068 push_vlan_idx == -1) 6069 set_tag_idx = encap_idx; 6070 } 6071 break; 6072 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6073 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6074 encap_idx = action_idx; 6075 if (encap_idx < sample_action_pos && 6076 push_vlan_idx == -1) 6077 set_tag_idx = encap_idx; 6078 break; 6079 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6080 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6081 if (action_idx < sample_action_pos && 6082 push_vlan_idx == -1) { 6083 set_tag_idx = action_idx; 6084 push_vlan_idx = action_idx; 6085 } 6086 break; 6087 default: 6088 break; 6089 } 6090 action_idx++; 6091 } 6092 } 6093 /* Prepare the actions for prefix and suffix flow. */ 6094 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6095 index = qrss_action_pos; 6096 /* Put the preceding the Queue/RSS action into prefix flow. */ 6097 if (index != 0) 6098 memcpy(actions_pre, actions, 6099 sizeof(struct rte_flow_action) * index); 6100 /* Put others preceding the sample action into prefix flow. */ 6101 if (sample_action_pos > index + 1) 6102 memcpy(actions_pre + index, actions + index + 1, 6103 sizeof(struct rte_flow_action) * 6104 (sample_action_pos - index - 1)); 6105 index = sample_action_pos - 1; 6106 /* Put Queue/RSS action into Suffix flow. */ 6107 memcpy(actions_sfx, actions + qrss_action_pos, 6108 sizeof(struct rte_flow_action)); 6109 actions_sfx++; 6110 } else if (add_tag && set_tag_idx >= 0) { 6111 if (set_tag_idx > 0) 6112 memcpy(actions_pre, actions, 6113 sizeof(struct rte_flow_action) * set_tag_idx); 6114 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6115 sizeof(struct rte_flow_action) * 6116 (sample_action_pos - set_tag_idx)); 6117 index = sample_action_pos; 6118 } else { 6119 index = sample_action_pos; 6120 if (index != 0) 6121 memcpy(actions_pre, actions, 6122 sizeof(struct rte_flow_action) * index); 6123 } 6124 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6125 * For CX6DX and above, metadata registers Cx preserve their value, 6126 * add an extra tag action for NIC-RX and E-Switch Domain. 6127 */ 6128 if (add_tag) { 6129 /* Prepare the prefix tag action. */ 6130 append_index++; 6131 set_tag = (void *)(actions_pre + actions_n + append_index); 6132 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6133 /* Trust VF/SF on CX5 not supported meter so that the reserved 6134 * metadata regC is REG_NON, back to use application tag 6135 * index 0. 6136 */ 6137 if (unlikely(ret == REG_NON)) 6138 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6139 if (ret < 0) 6140 return ret; 6141 mlx5_ipool_malloc(priv->sh->ipool 6142 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6143 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6144 .id = ret, 6145 .data = tag_id, 6146 }; 6147 /* Prepare the suffix subflow items. */ 6148 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6149 tag_spec->data = tag_id; 6150 tag_spec->id = set_tag->id; 6151 tag_mask = tag_spec + 1; 6152 tag_mask->data = UINT32_MAX; 6153 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6154 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6155 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6156 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6157 memcpy(sfx_items, items, sizeof(*sfx_items)); 6158 sfx_items++; 6159 break; 6160 } 6161 } 6162 sfx_items[0] = (struct rte_flow_item){ 6163 .type = (enum rte_flow_item_type) 6164 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6165 .spec = tag_spec, 6166 .last = NULL, 6167 .mask = tag_mask, 6168 }; 6169 sfx_items[1] = (struct rte_flow_item){ 6170 .type = (enum rte_flow_item_type) 6171 RTE_FLOW_ITEM_TYPE_END, 6172 }; 6173 /* Prepare the tag action in prefix subflow. */ 6174 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6175 actions_pre[set_tag_idx] = 6176 (struct rte_flow_action){ 6177 .type = (enum rte_flow_action_type) 6178 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6179 .conf = set_tag, 6180 }; 6181 /* Update next sample position due to add one tag action */ 6182 index += 1; 6183 } 6184 /* Copy the sample action into prefix flow. */ 6185 memcpy(actions_pre + index, actions + sample_action_pos, 6186 sizeof(struct rte_flow_action)); 6187 index += 1; 6188 /* For the modify action after the sample action in E-Switch mirroring, 6189 * Add the extra jump action in prefix subflow and jump into the next 6190 * table, then do the modify action in the new table. 6191 */ 6192 if (jump_table) { 6193 /* Prepare the prefix jump action. */ 6194 append_index++; 6195 jump_action = (void *)(actions_pre + actions_n + append_index); 6196 jump_action->group = jump_table; 6197 actions_pre[index++] = 6198 (struct rte_flow_action){ 6199 .type = (enum rte_flow_action_type) 6200 RTE_FLOW_ACTION_TYPE_JUMP, 6201 .conf = jump_action, 6202 }; 6203 } 6204 actions_pre[index] = (struct rte_flow_action){ 6205 .type = (enum rte_flow_action_type) 6206 RTE_FLOW_ACTION_TYPE_END, 6207 }; 6208 /* Put the actions after sample into Suffix flow. */ 6209 memcpy(actions_sfx, actions + sample_action_pos + 1, 6210 sizeof(struct rte_flow_action) * 6211 (actions_n - sample_action_pos - 1)); 6212 return tag_id; 6213 } 6214 6215 /** 6216 * The splitting for metadata feature. 6217 * 6218 * - Q/RSS action on NIC Rx should be split in order to pass by 6219 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6220 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6221 * 6222 * - All the actions on NIC Tx should have a mreg copy action to 6223 * copy reg_a from WQE to reg_c[0]. 6224 * 6225 * @param dev 6226 * Pointer to Ethernet device. 6227 * @param[in] flow 6228 * Parent flow structure pointer. 6229 * @param[in] attr 6230 * Flow rule attributes. 6231 * @param[in] items 6232 * Pattern specification (list terminated by the END pattern item). 6233 * @param[in] actions 6234 * Associated actions (list terminated by the END action). 6235 * @param[in] flow_split_info 6236 * Pointer to flow split info structure. 6237 * @param[out] error 6238 * Perform verbose error reporting if not NULL. 6239 * @return 6240 * 0 on success, negative value otherwise 6241 */ 6242 static int 6243 flow_create_split_metadata(struct rte_eth_dev *dev, 6244 struct rte_flow *flow, 6245 const struct rte_flow_attr *attr, 6246 const struct rte_flow_item items[], 6247 const struct rte_flow_action actions[], 6248 struct mlx5_flow_split_info *flow_split_info, 6249 struct rte_flow_error *error) 6250 { 6251 struct mlx5_priv *priv = dev->data->dev_private; 6252 struct mlx5_sh_config *config = &priv->sh->config; 6253 const struct rte_flow_action *qrss = NULL; 6254 struct rte_flow_action *ext_actions = NULL; 6255 struct mlx5_flow *dev_flow = NULL; 6256 uint32_t qrss_id = 0; 6257 int mtr_sfx = 0; 6258 size_t act_size; 6259 int actions_n; 6260 int encap_idx; 6261 int ret; 6262 6263 /* Check whether extensive metadata feature is engaged. */ 6264 if (!config->dv_flow_en || 6265 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6266 !mlx5_flow_ext_mreg_supported(dev)) 6267 return flow_create_split_inner(dev, flow, NULL, attr, items, 6268 actions, flow_split_info, error); 6269 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6270 &encap_idx); 6271 if (qrss) { 6272 /* Exclude hairpin flows from splitting. */ 6273 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6274 const struct rte_flow_action_queue *queue; 6275 6276 queue = qrss->conf; 6277 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6278 qrss = NULL; 6279 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6280 const struct rte_flow_action_rss *rss; 6281 6282 rss = qrss->conf; 6283 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6284 qrss = NULL; 6285 } 6286 } 6287 if (qrss) { 6288 /* Check if it is in meter suffix table. */ 6289 mtr_sfx = attr->group == 6290 ((attr->transfer && priv->fdb_def_rule) ? 6291 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6292 MLX5_FLOW_TABLE_LEVEL_METER); 6293 /* 6294 * Q/RSS action on NIC Rx should be split in order to pass by 6295 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6296 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6297 */ 6298 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6299 sizeof(struct rte_flow_action_set_tag) + 6300 sizeof(struct rte_flow_action_jump); 6301 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6302 SOCKET_ID_ANY); 6303 if (!ext_actions) 6304 return rte_flow_error_set(error, ENOMEM, 6305 RTE_FLOW_ERROR_TYPE_ACTION, 6306 NULL, "no memory to split " 6307 "metadata flow"); 6308 /* 6309 * Create the new actions list with removed Q/RSS action 6310 * and appended set tag and jump to register copy table 6311 * (RX_CP_TBL). We should preallocate unique tag ID here 6312 * in advance, because it is needed for set tag action. 6313 */ 6314 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6315 qrss, actions_n, 6316 mtr_sfx, error); 6317 if (!mtr_sfx && !qrss_id) { 6318 ret = -rte_errno; 6319 goto exit; 6320 } 6321 } else if (attr->egress) { 6322 /* 6323 * All the actions on NIC Tx should have a metadata register 6324 * copy action to copy reg_a from WQE to reg_c[meta] 6325 */ 6326 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6327 sizeof(struct mlx5_flow_action_copy_mreg); 6328 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6329 SOCKET_ID_ANY); 6330 if (!ext_actions) 6331 return rte_flow_error_set(error, ENOMEM, 6332 RTE_FLOW_ERROR_TYPE_ACTION, 6333 NULL, "no memory to split " 6334 "metadata flow"); 6335 /* Create the action list appended with copy register. */ 6336 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6337 actions_n, error, encap_idx); 6338 if (ret < 0) 6339 goto exit; 6340 } 6341 /* Add the unmodified original or prefix subflow. */ 6342 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6343 items, ext_actions ? ext_actions : 6344 actions, flow_split_info, error); 6345 if (ret < 0) 6346 goto exit; 6347 MLX5_ASSERT(dev_flow); 6348 if (qrss) { 6349 const struct rte_flow_attr q_attr = { 6350 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6351 .ingress = 1, 6352 }; 6353 /* Internal PMD action to set register. */ 6354 struct mlx5_rte_flow_item_tag q_tag_spec = { 6355 .data = qrss_id, 6356 .id = REG_NON, 6357 }; 6358 struct rte_flow_item q_items[] = { 6359 { 6360 .type = (enum rte_flow_item_type) 6361 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6362 .spec = &q_tag_spec, 6363 .last = NULL, 6364 .mask = NULL, 6365 }, 6366 { 6367 .type = RTE_FLOW_ITEM_TYPE_END, 6368 }, 6369 }; 6370 struct rte_flow_action q_actions[] = { 6371 { 6372 .type = qrss->type, 6373 .conf = qrss->conf, 6374 }, 6375 { 6376 .type = RTE_FLOW_ACTION_TYPE_END, 6377 }, 6378 }; 6379 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6380 6381 /* 6382 * Configure the tag item only if there is no meter subflow. 6383 * Since tag is already marked in the meter suffix subflow 6384 * we can just use the meter suffix items as is. 6385 */ 6386 if (qrss_id) { 6387 /* Not meter subflow. */ 6388 MLX5_ASSERT(!mtr_sfx); 6389 /* 6390 * Put unique id in prefix flow due to it is destroyed 6391 * after suffix flow and id will be freed after there 6392 * is no actual flows with this id and identifier 6393 * reallocation becomes possible (for example, for 6394 * other flows in other threads). 6395 */ 6396 dev_flow->handle->split_flow_id = qrss_id; 6397 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6398 error); 6399 if (ret < 0) 6400 goto exit; 6401 q_tag_spec.id = ret; 6402 } 6403 dev_flow = NULL; 6404 /* Add suffix subflow to execute Q/RSS. */ 6405 flow_split_info->prefix_layers = layers; 6406 flow_split_info->prefix_mark = 0; 6407 flow_split_info->table_id = 0; 6408 ret = flow_create_split_inner(dev, flow, &dev_flow, 6409 &q_attr, mtr_sfx ? items : 6410 q_items, q_actions, 6411 flow_split_info, error); 6412 if (ret < 0) 6413 goto exit; 6414 /* qrss ID should be freed if failed. */ 6415 qrss_id = 0; 6416 MLX5_ASSERT(dev_flow); 6417 } 6418 6419 exit: 6420 /* 6421 * We do not destroy the partially created sub_flows in case of error. 6422 * These ones are included into parent flow list and will be destroyed 6423 * by flow_drv_destroy. 6424 */ 6425 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6426 qrss_id); 6427 mlx5_free(ext_actions); 6428 return ret; 6429 } 6430 6431 /** 6432 * Create meter internal drop flow with the original pattern. 6433 * 6434 * @param dev 6435 * Pointer to Ethernet device. 6436 * @param[in] flow 6437 * Parent flow structure pointer. 6438 * @param[in] attr 6439 * Flow rule attributes. 6440 * @param[in] items 6441 * Pattern specification (list terminated by the END pattern item). 6442 * @param[in] flow_split_info 6443 * Pointer to flow split info structure. 6444 * @param[in] fm 6445 * Pointer to flow meter structure. 6446 * @param[out] error 6447 * Perform verbose error reporting if not NULL. 6448 * @return 6449 * 0 on success, negative value otherwise 6450 */ 6451 static uint32_t 6452 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6453 struct rte_flow *flow, 6454 const struct rte_flow_attr *attr, 6455 const struct rte_flow_item items[], 6456 struct mlx5_flow_split_info *flow_split_info, 6457 struct mlx5_flow_meter_info *fm, 6458 struct rte_flow_error *error) 6459 { 6460 struct mlx5_flow *dev_flow = NULL; 6461 struct rte_flow_attr drop_attr = *attr; 6462 struct rte_flow_action drop_actions[3]; 6463 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6464 6465 MLX5_ASSERT(fm->drop_cnt); 6466 drop_actions[0].type = 6467 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6468 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6469 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6470 drop_actions[1].conf = NULL; 6471 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6472 drop_actions[2].conf = NULL; 6473 drop_split_info.external = false; 6474 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6475 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6476 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6477 return flow_create_split_inner(dev, flow, &dev_flow, 6478 &drop_attr, items, drop_actions, 6479 &drop_split_info, error); 6480 } 6481 6482 /** 6483 * The splitting for meter feature. 6484 * 6485 * - The meter flow will be split to two flows as prefix and 6486 * suffix flow. The packets make sense only it pass the prefix 6487 * meter action. 6488 * 6489 * - Reg_C_5 is used for the packet to match betweend prefix and 6490 * suffix flow. 6491 * 6492 * @param dev 6493 * Pointer to Ethernet device. 6494 * @param[in] flow 6495 * Parent flow structure pointer. 6496 * @param[in] attr 6497 * Flow rule attributes. 6498 * @param[in] items 6499 * Pattern specification (list terminated by the END pattern item). 6500 * @param[in] actions 6501 * Associated actions (list terminated by the END action). 6502 * @param[in] flow_split_info 6503 * Pointer to flow split info structure. 6504 * @param[out] error 6505 * Perform verbose error reporting if not NULL. 6506 * @return 6507 * 0 on success, negative value otherwise 6508 */ 6509 static int 6510 flow_create_split_meter(struct rte_eth_dev *dev, 6511 struct rte_flow *flow, 6512 const struct rte_flow_attr *attr, 6513 const struct rte_flow_item items[], 6514 const struct rte_flow_action actions[], 6515 struct mlx5_flow_split_info *flow_split_info, 6516 struct rte_flow_error *error) 6517 { 6518 struct mlx5_priv *priv = dev->data->dev_private; 6519 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6520 struct rte_flow_action *sfx_actions = NULL; 6521 struct rte_flow_action *pre_actions = NULL; 6522 struct rte_flow_item *sfx_items = NULL; 6523 struct mlx5_flow *dev_flow = NULL; 6524 struct rte_flow_attr sfx_attr = *attr; 6525 struct mlx5_flow_meter_info *fm = NULL; 6526 uint8_t skip_scale_restore; 6527 bool has_mtr = false; 6528 bool has_modify = false; 6529 bool set_mtr_reg = true; 6530 bool is_mtr_hierarchy = false; 6531 uint32_t meter_id = 0; 6532 uint32_t mtr_idx = 0; 6533 uint32_t mtr_flow_id = 0; 6534 size_t act_size; 6535 size_t item_size; 6536 int actions_n = 0; 6537 int ret = 0; 6538 6539 if (priv->mtr_en) 6540 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6541 &has_modify, &meter_id); 6542 if (has_mtr) { 6543 if (flow->meter) { 6544 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6545 if (!fm) 6546 return rte_flow_error_set(error, EINVAL, 6547 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6548 NULL, "Meter not found."); 6549 } else { 6550 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6551 if (!fm) 6552 return rte_flow_error_set(error, EINVAL, 6553 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6554 NULL, "Meter not found."); 6555 ret = mlx5_flow_meter_attach(priv, fm, 6556 &sfx_attr, error); 6557 if (ret) 6558 return -rte_errno; 6559 flow->meter = mtr_idx; 6560 } 6561 MLX5_ASSERT(wks); 6562 wks->fm = fm; 6563 if (!fm->def_policy) { 6564 wks->policy = mlx5_flow_meter_policy_find(dev, 6565 fm->policy_id, 6566 NULL); 6567 MLX5_ASSERT(wks->policy); 6568 if (wks->policy->mark) 6569 wks->mark = 1; 6570 if (wks->policy->is_hierarchy) { 6571 wks->final_policy = 6572 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6573 wks->policy); 6574 if (!wks->final_policy) 6575 return rte_flow_error_set(error, 6576 EINVAL, 6577 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6578 "Failed to find terminal policy of hierarchy."); 6579 is_mtr_hierarchy = true; 6580 } 6581 } 6582 /* 6583 * If it isn't default-policy Meter, and 6584 * 1. Not meter hierarchy and there's no action in flow to change 6585 * packet (modify/encap/decap etc.), OR 6586 * 2. No drop count needed for this meter. 6587 * Then no need to use regC to save meter id anymore. 6588 */ 6589 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6590 set_mtr_reg = false; 6591 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6592 #define METER_PREFIX_ACTION 7 6593 act_size = (sizeof(struct rte_flow_action) * 6594 (actions_n + METER_PREFIX_ACTION)) + 6595 sizeof(struct mlx5_rte_flow_action_set_tag); 6596 /* Suffix items: tag, vlan, port id, end. */ 6597 #define METER_SUFFIX_ITEM 4 6598 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6599 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6600 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6601 0, SOCKET_ID_ANY); 6602 if (!sfx_actions) 6603 return rte_flow_error_set(error, ENOMEM, 6604 RTE_FLOW_ERROR_TYPE_ACTION, 6605 NULL, "no memory to split " 6606 "meter flow"); 6607 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6608 act_size); 6609 /* There's no suffix flow for meter of non-default policy. */ 6610 if (!fm->def_policy) 6611 pre_actions = sfx_actions + 1; 6612 else 6613 pre_actions = sfx_actions + actions_n; 6614 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6615 items, sfx_items, actions, 6616 sfx_actions, pre_actions, 6617 (set_mtr_reg ? &mtr_flow_id : NULL), 6618 error); 6619 if (ret) { 6620 ret = -rte_errno; 6621 goto exit; 6622 } 6623 /* Add the prefix subflow. */ 6624 skip_scale_restore = flow_split_info->skip_scale; 6625 flow_split_info->skip_scale |= 6626 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6627 ret = flow_create_split_inner(dev, flow, &dev_flow, 6628 attr, items, pre_actions, 6629 flow_split_info, error); 6630 flow_split_info->skip_scale = skip_scale_restore; 6631 if (ret) { 6632 if (mtr_flow_id) 6633 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6634 ret = -rte_errno; 6635 goto exit; 6636 } 6637 if (mtr_flow_id) { 6638 dev_flow->handle->split_flow_id = mtr_flow_id; 6639 dev_flow->handle->is_meter_flow_id = 1; 6640 } 6641 if (!fm->def_policy) { 6642 if (!set_mtr_reg && fm->drop_cnt) 6643 ret = 6644 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6645 &sfx_attr, items, 6646 flow_split_info, 6647 fm, error); 6648 goto exit; 6649 } 6650 /* Setting the sfx group atrr. */ 6651 sfx_attr.group = sfx_attr.transfer ? 6652 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6653 MLX5_FLOW_TABLE_LEVEL_METER; 6654 flow_split_info->prefix_layers = 6655 flow_get_prefix_layer_flags(dev_flow); 6656 flow_split_info->prefix_mark |= wks->mark; 6657 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6658 } 6659 /* Add the prefix subflow. */ 6660 ret = flow_create_split_metadata(dev, flow, 6661 &sfx_attr, sfx_items ? 6662 sfx_items : items, 6663 sfx_actions ? sfx_actions : actions, 6664 flow_split_info, error); 6665 exit: 6666 if (sfx_actions) 6667 mlx5_free(sfx_actions); 6668 return ret; 6669 } 6670 6671 /** 6672 * The splitting for sample feature. 6673 * 6674 * Once Sample action is detected in the action list, the flow actions should 6675 * be split into prefix sub flow and suffix sub flow. 6676 * 6677 * The original items remain in the prefix sub flow, all actions preceding the 6678 * sample action and the sample action itself will be copied to the prefix 6679 * sub flow, the actions following the sample action will be copied to the 6680 * suffix sub flow, Queue action always be located in the suffix sub flow. 6681 * 6682 * In order to make the packet from prefix sub flow matches with suffix sub 6683 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6684 * flow uses tag item with the unique flow id. 6685 * 6686 * @param dev 6687 * Pointer to Ethernet device. 6688 * @param[in] flow 6689 * Parent flow structure pointer. 6690 * @param[in] attr 6691 * Flow rule attributes. 6692 * @param[in] items 6693 * Pattern specification (list terminated by the END pattern item). 6694 * @param[in] actions 6695 * Associated actions (list terminated by the END action). 6696 * @param[in] flow_split_info 6697 * Pointer to flow split info structure. 6698 * @param[out] error 6699 * Perform verbose error reporting if not NULL. 6700 * @return 6701 * 0 on success, negative value otherwise 6702 */ 6703 static int 6704 flow_create_split_sample(struct rte_eth_dev *dev, 6705 struct rte_flow *flow, 6706 const struct rte_flow_attr *attr, 6707 const struct rte_flow_item items[], 6708 const struct rte_flow_action actions[], 6709 struct mlx5_flow_split_info *flow_split_info, 6710 struct rte_flow_error *error) 6711 { 6712 struct mlx5_priv *priv = dev->data->dev_private; 6713 struct rte_flow_action *sfx_actions = NULL; 6714 struct rte_flow_action *pre_actions = NULL; 6715 struct rte_flow_item *sfx_items = NULL; 6716 struct mlx5_flow *dev_flow = NULL; 6717 struct rte_flow_attr sfx_attr = *attr; 6718 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6719 struct mlx5_flow_dv_sample_resource *sample_res; 6720 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6721 struct mlx5_flow_tbl_resource *sfx_tbl; 6722 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6723 #endif 6724 size_t act_size; 6725 size_t item_size; 6726 uint32_t fdb_tx = 0; 6727 int32_t tag_id = 0; 6728 int actions_n = 0; 6729 int sample_action_pos; 6730 int qrss_action_pos; 6731 int add_tag = 0; 6732 int modify_after_mirror = 0; 6733 uint16_t jump_table = 0; 6734 const uint32_t next_ft_step = 1; 6735 int ret = 0; 6736 struct mlx5_priv *item_port_priv = NULL; 6737 const struct rte_flow_item *item; 6738 6739 if (priv->sampler_en) 6740 actions_n = flow_check_match_action(actions, attr, 6741 RTE_FLOW_ACTION_TYPE_SAMPLE, 6742 &sample_action_pos, &qrss_action_pos, 6743 &modify_after_mirror); 6744 if (actions_n) { 6745 /* The prefix actions must includes sample, tag, end. */ 6746 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6747 + sizeof(struct mlx5_rte_flow_action_set_tag); 6748 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6749 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6750 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6751 item_size), 0, SOCKET_ID_ANY); 6752 if (!sfx_actions) 6753 return rte_flow_error_set(error, ENOMEM, 6754 RTE_FLOW_ERROR_TYPE_ACTION, 6755 NULL, "no memory to split " 6756 "sample flow"); 6757 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 6758 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6759 const struct rte_flow_item_port_id *spec; 6760 6761 spec = (const struct rte_flow_item_port_id *)item->spec; 6762 if (spec) 6763 item_port_priv = 6764 mlx5_port_to_eswitch_info(spec->id, true); 6765 break; 6766 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6767 const struct rte_flow_item_ethdev *spec; 6768 6769 spec = (const struct rte_flow_item_ethdev *)item->spec; 6770 if (spec) 6771 item_port_priv = 6772 mlx5_port_to_eswitch_info(spec->port_id, true); 6773 break; 6774 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 6775 const struct rte_flow_item_ethdev *spec; 6776 6777 spec = (const struct rte_flow_item_ethdev *)item->spec; 6778 if (spec) 6779 item_port_priv = 6780 mlx5_port_to_eswitch_info(spec->port_id, true); 6781 break; 6782 } 6783 } 6784 /* The representor_id is UINT16_MAX for uplink. */ 6785 fdb_tx = (attr->transfer && 6786 flow_source_vport_representor(priv, item_port_priv)); 6787 /* 6788 * When reg_c_preserve is set, metadata registers Cx preserve 6789 * their value even through packet duplication. 6790 */ 6791 add_tag = (!fdb_tx || 6792 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6793 if (add_tag) 6794 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6795 + act_size); 6796 if (modify_after_mirror) 6797 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6798 next_ft_step; 6799 pre_actions = sfx_actions + actions_n; 6800 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6801 actions, sfx_actions, 6802 pre_actions, actions_n, 6803 sample_action_pos, 6804 qrss_action_pos, jump_table, 6805 error); 6806 if (tag_id < 0 || (add_tag && !tag_id)) { 6807 ret = -rte_errno; 6808 goto exit; 6809 } 6810 if (modify_after_mirror) 6811 flow_split_info->skip_scale = 6812 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6813 /* Add the prefix subflow. */ 6814 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6815 items, pre_actions, 6816 flow_split_info, error); 6817 if (ret) { 6818 ret = -rte_errno; 6819 goto exit; 6820 } 6821 dev_flow->handle->split_flow_id = tag_id; 6822 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6823 if (!modify_after_mirror) { 6824 /* Set the sfx group attr. */ 6825 sample_res = (struct mlx5_flow_dv_sample_resource *) 6826 dev_flow->dv.sample_res; 6827 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6828 sample_res->normal_path_tbl; 6829 sfx_tbl_data = container_of(sfx_tbl, 6830 struct mlx5_flow_tbl_data_entry, 6831 tbl); 6832 sfx_attr.group = sfx_attr.transfer ? 6833 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6834 } else { 6835 MLX5_ASSERT(attr->transfer); 6836 sfx_attr.group = jump_table; 6837 } 6838 flow_split_info->prefix_layers = 6839 flow_get_prefix_layer_flags(dev_flow); 6840 MLX5_ASSERT(wks); 6841 flow_split_info->prefix_mark |= wks->mark; 6842 /* Suffix group level already be scaled with factor, set 6843 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6844 * again in translation. 6845 */ 6846 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6847 #endif 6848 } 6849 /* Add the suffix subflow. */ 6850 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6851 sfx_items ? sfx_items : items, 6852 sfx_actions ? sfx_actions : actions, 6853 flow_split_info, error); 6854 exit: 6855 if (sfx_actions) 6856 mlx5_free(sfx_actions); 6857 return ret; 6858 } 6859 6860 /** 6861 * Split the flow to subflow set. The splitters might be linked 6862 * in the chain, like this: 6863 * flow_create_split_outer() calls: 6864 * flow_create_split_meter() calls: 6865 * flow_create_split_metadata(meter_subflow_0) calls: 6866 * flow_create_split_inner(metadata_subflow_0) 6867 * flow_create_split_inner(metadata_subflow_1) 6868 * flow_create_split_inner(metadata_subflow_2) 6869 * flow_create_split_metadata(meter_subflow_1) calls: 6870 * flow_create_split_inner(metadata_subflow_0) 6871 * flow_create_split_inner(metadata_subflow_1) 6872 * flow_create_split_inner(metadata_subflow_2) 6873 * 6874 * This provide flexible way to add new levels of flow splitting. 6875 * The all of successfully created subflows are included to the 6876 * parent flow dev_flow list. 6877 * 6878 * @param dev 6879 * Pointer to Ethernet device. 6880 * @param[in] flow 6881 * Parent flow structure pointer. 6882 * @param[in] attr 6883 * Flow rule attributes. 6884 * @param[in] items 6885 * Pattern specification (list terminated by the END pattern item). 6886 * @param[in] actions 6887 * Associated actions (list terminated by the END action). 6888 * @param[in] flow_split_info 6889 * Pointer to flow split info structure. 6890 * @param[out] error 6891 * Perform verbose error reporting if not NULL. 6892 * @return 6893 * 0 on success, negative value otherwise 6894 */ 6895 static int 6896 flow_create_split_outer(struct rte_eth_dev *dev, 6897 struct rte_flow *flow, 6898 const struct rte_flow_attr *attr, 6899 const struct rte_flow_item items[], 6900 const struct rte_flow_action actions[], 6901 struct mlx5_flow_split_info *flow_split_info, 6902 struct rte_flow_error *error) 6903 { 6904 int ret; 6905 6906 ret = flow_create_split_sample(dev, flow, attr, items, 6907 actions, flow_split_info, error); 6908 MLX5_ASSERT(ret <= 0); 6909 return ret; 6910 } 6911 6912 static inline struct mlx5_flow_tunnel * 6913 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6914 { 6915 struct mlx5_flow_tunnel *tunnel; 6916 6917 #pragma GCC diagnostic push 6918 #pragma GCC diagnostic ignored "-Wcast-qual" 6919 tunnel = (typeof(tunnel))flow->tunnel; 6920 #pragma GCC diagnostic pop 6921 6922 return tunnel; 6923 } 6924 6925 /** 6926 * Adjust flow RSS workspace if needed. 6927 * 6928 * @param wks 6929 * Pointer to thread flow work space. 6930 * @param rss_desc 6931 * Pointer to RSS descriptor. 6932 * @param[in] nrssq_num 6933 * New RSS queue number. 6934 * 6935 * @return 6936 * 0 on success, -1 otherwise and rte_errno is set. 6937 */ 6938 static int 6939 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6940 struct mlx5_flow_rss_desc *rss_desc, 6941 uint32_t nrssq_num) 6942 { 6943 if (likely(nrssq_num <= wks->rssq_num)) 6944 return 0; 6945 rss_desc->queue = realloc(rss_desc->queue, 6946 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6947 if (!rss_desc->queue) { 6948 rte_errno = ENOMEM; 6949 return -1; 6950 } 6951 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6952 return 0; 6953 } 6954 6955 /** 6956 * Create a flow and add it to @p list. 6957 * 6958 * @param dev 6959 * Pointer to Ethernet device. 6960 * @param list 6961 * Pointer to a TAILQ flow list. If this parameter NULL, 6962 * no list insertion occurred, flow is just created, 6963 * this is caller's responsibility to track the 6964 * created flow. 6965 * @param[in] attr 6966 * Flow rule attributes. 6967 * @param[in] items 6968 * Pattern specification (list terminated by the END pattern item). 6969 * @param[in] actions 6970 * Associated actions (list terminated by the END action). 6971 * @param[in] external 6972 * This flow rule is created by request external to PMD. 6973 * @param[out] error 6974 * Perform verbose error reporting if not NULL. 6975 * 6976 * @return 6977 * A flow index on success, 0 otherwise and rte_errno is set. 6978 */ 6979 static uint32_t 6980 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6981 const struct rte_flow_attr *attr, 6982 const struct rte_flow_item items[], 6983 const struct rte_flow_action original_actions[], 6984 bool external, struct rte_flow_error *error) 6985 { 6986 struct mlx5_priv *priv = dev->data->dev_private; 6987 struct rte_flow *flow = NULL; 6988 struct mlx5_flow *dev_flow; 6989 const struct rte_flow_action_rss *rss = NULL; 6990 struct mlx5_translated_action_handle 6991 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6992 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6993 union { 6994 struct mlx5_flow_expand_rss buf; 6995 uint8_t buffer[8192]; 6996 } expand_buffer; 6997 union { 6998 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6999 uint8_t buffer[2048]; 7000 } actions_rx; 7001 union { 7002 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7003 uint8_t buffer[2048]; 7004 } actions_hairpin_tx; 7005 union { 7006 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7007 uint8_t buffer[2048]; 7008 } items_tx; 7009 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7010 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7011 struct mlx5_flow_rss_desc *rss_desc; 7012 const struct rte_flow_action *p_actions_rx; 7013 uint32_t i; 7014 uint32_t idx = 0; 7015 int hairpin_flow; 7016 struct rte_flow_attr attr_tx = { .priority = 0 }; 7017 const struct rte_flow_action *actions; 7018 struct rte_flow_action *translated_actions = NULL; 7019 struct mlx5_flow_tunnel *tunnel; 7020 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7021 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7022 struct mlx5_flow_split_info flow_split_info = { 7023 .external = !!external, 7024 .skip_scale = 0, 7025 .flow_idx = 0, 7026 .prefix_mark = 0, 7027 .prefix_layers = 0, 7028 .table_id = 0 7029 }; 7030 int ret; 7031 7032 MLX5_ASSERT(wks); 7033 rss_desc = &wks->rss_desc; 7034 ret = flow_action_handles_translate(dev, original_actions, 7035 indir_actions, 7036 &indir_actions_n, 7037 &translated_actions, error); 7038 if (ret < 0) { 7039 MLX5_ASSERT(translated_actions == NULL); 7040 return 0; 7041 } 7042 actions = translated_actions ? translated_actions : original_actions; 7043 p_actions_rx = actions; 7044 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7045 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7046 external, hairpin_flow, error); 7047 if (ret < 0) 7048 goto error_before_hairpin_split; 7049 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7050 if (!flow) { 7051 rte_errno = ENOMEM; 7052 goto error_before_hairpin_split; 7053 } 7054 if (hairpin_flow > 0) { 7055 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7056 rte_errno = EINVAL; 7057 goto error_before_hairpin_split; 7058 } 7059 flow_hairpin_split(dev, actions, actions_rx.actions, 7060 actions_hairpin_tx.actions, items_tx.items, 7061 idx); 7062 p_actions_rx = actions_rx.actions; 7063 } 7064 flow_split_info.flow_idx = idx; 7065 flow->drv_type = flow_get_drv_type(dev, attr); 7066 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7067 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7068 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7069 /* RSS Action only works on NIC RX domain */ 7070 if (attr->ingress) 7071 rss = flow_get_rss_action(dev, p_actions_rx); 7072 if (rss) { 7073 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 7074 return 0; 7075 /* 7076 * The following information is required by 7077 * mlx5_flow_hashfields_adjust() in advance. 7078 */ 7079 rss_desc->level = rss->level; 7080 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7081 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7082 } 7083 flow->dev_handles = 0; 7084 if (rss && rss->types) { 7085 unsigned int graph_root; 7086 7087 graph_root = find_graph_root(rss->level); 7088 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7089 items, rss->types, 7090 mlx5_support_expansion, graph_root); 7091 MLX5_ASSERT(ret > 0 && 7092 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7093 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7094 for (i = 0; i < buf->entries; ++i) 7095 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7096 } 7097 } else { 7098 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7099 sizeof(expand_buffer.buffer), 7100 items, sq_specs); 7101 if (ret) { 7102 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7103 NULL, "not enough memory for rte_flow"); 7104 goto error; 7105 } 7106 if (buf->entries == 0) { 7107 buf->entries = 1; 7108 buf->entry[0].pattern = (void *)(uintptr_t)items; 7109 } 7110 } 7111 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7112 indir_actions_n); 7113 for (i = 0; i < buf->entries; ++i) { 7114 /* Initialize flow split data. */ 7115 flow_split_info.prefix_layers = 0; 7116 flow_split_info.prefix_mark = 0; 7117 flow_split_info.skip_scale = 0; 7118 /* 7119 * The splitter may create multiple dev_flows, 7120 * depending on configuration. In the simplest 7121 * case it just creates unmodified original flow. 7122 */ 7123 ret = flow_create_split_outer(dev, flow, attr, 7124 buf->entry[i].pattern, 7125 p_actions_rx, &flow_split_info, 7126 error); 7127 if (ret < 0) 7128 goto error; 7129 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7130 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7131 p_actions_rx, 7132 idx, 7133 wks->flows[0].tunnel, 7134 &default_miss_ctx, 7135 error); 7136 if (ret < 0) { 7137 mlx5_free(default_miss_ctx.queue); 7138 goto error; 7139 } 7140 } 7141 } 7142 /* Create the tx flow. */ 7143 if (hairpin_flow) { 7144 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7145 attr_tx.ingress = 0; 7146 attr_tx.egress = 1; 7147 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7148 actions_hairpin_tx.actions, 7149 idx, error); 7150 if (!dev_flow) 7151 goto error; 7152 dev_flow->flow = flow; 7153 dev_flow->external = 0; 7154 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7155 dev_flow->handle, next); 7156 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7157 items_tx.items, 7158 actions_hairpin_tx.actions, error); 7159 if (ret < 0) 7160 goto error; 7161 } 7162 /* 7163 * Update the metadata register copy table. If extensive 7164 * metadata feature is enabled and registers are supported 7165 * we might create the extra rte_flow for each unique 7166 * MARK/FLAG action ID. 7167 * 7168 * The table is updated for ingress and transfer flows only, because 7169 * the egress Flows belong to the different device and 7170 * copy table should be updated in peer NIC Rx domain. 7171 */ 7172 if ((attr->ingress || attr->transfer) && 7173 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7174 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7175 if (ret) 7176 goto error; 7177 } 7178 /* 7179 * If the flow is external (from application) OR device is started, 7180 * OR mreg discover, then apply immediately. 7181 */ 7182 if (external || dev->data->dev_started || 7183 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7184 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7185 ret = flow_drv_apply(dev, flow, error); 7186 if (ret < 0) 7187 goto error; 7188 } 7189 flow->type = type; 7190 flow_rxq_flags_set(dev, flow); 7191 rte_free(translated_actions); 7192 tunnel = flow_tunnel_from_rule(wks->flows); 7193 if (tunnel) { 7194 flow->tunnel = 1; 7195 flow->tunnel_id = tunnel->tunnel_id; 7196 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7197 mlx5_free(default_miss_ctx.queue); 7198 } 7199 mlx5_flow_pop_thread_workspace(); 7200 return idx; 7201 error: 7202 MLX5_ASSERT(flow); 7203 ret = rte_errno; /* Save rte_errno before cleanup. */ 7204 flow_mreg_del_copy_action(dev, flow); 7205 flow_drv_destroy(dev, flow); 7206 if (rss_desc->shared_rss) 7207 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7208 mlx5_ipool_get 7209 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7210 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7211 mlx5_ipool_free(priv->flows[type], idx); 7212 rte_errno = ret; /* Restore rte_errno. */ 7213 ret = rte_errno; 7214 rte_errno = ret; 7215 error_before_hairpin_split: 7216 mlx5_flow_pop_thread_workspace(); 7217 rte_free(translated_actions); 7218 return 0; 7219 } 7220 7221 /** 7222 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7223 * incoming packets to table 1. 7224 * 7225 * Other flow rules, requested for group n, will be created in 7226 * e-switch table n+1. 7227 * Jump action to e-switch group n will be created to group n+1. 7228 * 7229 * Used when working in switchdev mode, to utilise advantages of table 1 7230 * and above. 7231 * 7232 * @param dev 7233 * Pointer to Ethernet device. 7234 * 7235 * @return 7236 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7237 */ 7238 struct rte_flow * 7239 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7240 { 7241 const struct rte_flow_attr attr = { 7242 .group = 0, 7243 .priority = 0, 7244 .ingress = 0, 7245 .egress = 0, 7246 .transfer = 1, 7247 }; 7248 const struct rte_flow_item pattern = { 7249 .type = RTE_FLOW_ITEM_TYPE_END, 7250 }; 7251 struct rte_flow_action_jump jump = { 7252 .group = 1, 7253 }; 7254 const struct rte_flow_action actions[] = { 7255 { 7256 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7257 .conf = &jump, 7258 }, 7259 { 7260 .type = RTE_FLOW_ACTION_TYPE_END, 7261 }, 7262 }; 7263 struct rte_flow_error error; 7264 7265 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7266 &attr, &pattern, 7267 actions, false, &error); 7268 } 7269 7270 /** 7271 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7272 * and sq number, directs all packets to peer vport. 7273 * 7274 * @param dev 7275 * Pointer to Ethernet device. 7276 * @param sq_num 7277 * SQ number. 7278 * 7279 * @return 7280 * Flow ID on success, 0 otherwise and rte_errno is set. 7281 */ 7282 uint32_t 7283 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7284 { 7285 struct rte_flow_attr attr = { 7286 .group = 0, 7287 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7288 .ingress = 0, 7289 .egress = 0, 7290 .transfer = 1, 7291 }; 7292 struct rte_flow_item_port_id port_spec = { 7293 .id = MLX5_PORT_ESW_MGR, 7294 }; 7295 struct mlx5_rte_flow_item_sq sq_spec = { 7296 .queue = sq_num, 7297 }; 7298 struct rte_flow_item pattern[] = { 7299 { 7300 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7301 .spec = &port_spec, 7302 }, 7303 { 7304 .type = (enum rte_flow_item_type) 7305 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7306 .spec = &sq_spec, 7307 }, 7308 { 7309 .type = RTE_FLOW_ITEM_TYPE_END, 7310 }, 7311 }; 7312 struct rte_flow_action_jump jump = { 7313 .group = 1, 7314 }; 7315 struct rte_flow_action_port_id port = { 7316 .id = dev->data->port_id, 7317 }; 7318 struct rte_flow_action actions[] = { 7319 { 7320 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7321 .conf = &jump, 7322 }, 7323 { 7324 .type = RTE_FLOW_ACTION_TYPE_END, 7325 }, 7326 }; 7327 struct rte_flow_error error; 7328 7329 /* 7330 * Creates group 0, highest priority jump flow. 7331 * Matches txq to bypass kernel packets. 7332 */ 7333 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7334 false, &error) == 0) 7335 return 0; 7336 /* Create group 1, lowest priority redirect flow for txq. */ 7337 attr.group = 1; 7338 actions[0].conf = &port; 7339 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7340 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7341 actions, false, &error); 7342 } 7343 7344 /** 7345 * Validate a flow supported by the NIC. 7346 * 7347 * @see rte_flow_validate() 7348 * @see rte_flow_ops 7349 */ 7350 int 7351 mlx5_flow_validate(struct rte_eth_dev *dev, 7352 const struct rte_flow_attr *attr, 7353 const struct rte_flow_item items[], 7354 const struct rte_flow_action original_actions[], 7355 struct rte_flow_error *error) 7356 { 7357 int hairpin_flow; 7358 struct mlx5_translated_action_handle 7359 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7360 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7361 const struct rte_flow_action *actions; 7362 struct rte_flow_action *translated_actions = NULL; 7363 int ret = flow_action_handles_translate(dev, original_actions, 7364 indir_actions, 7365 &indir_actions_n, 7366 &translated_actions, error); 7367 7368 if (ret) 7369 return ret; 7370 actions = translated_actions ? translated_actions : original_actions; 7371 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7372 ret = flow_drv_validate(dev, attr, items, actions, 7373 true, hairpin_flow, error); 7374 rte_free(translated_actions); 7375 return ret; 7376 } 7377 7378 /** 7379 * Create a flow. 7380 * 7381 * @see rte_flow_create() 7382 * @see rte_flow_ops 7383 */ 7384 struct rte_flow * 7385 mlx5_flow_create(struct rte_eth_dev *dev, 7386 const struct rte_flow_attr *attr, 7387 const struct rte_flow_item items[], 7388 const struct rte_flow_action actions[], 7389 struct rte_flow_error *error) 7390 { 7391 struct mlx5_priv *priv = dev->data->dev_private; 7392 7393 if (priv->sh->config.dv_flow_en == 2) { 7394 rte_flow_error_set(error, ENOTSUP, 7395 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7396 NULL, 7397 "Flow non-Q creation not supported"); 7398 return NULL; 7399 } 7400 /* 7401 * If the device is not started yet, it is not allowed to created a 7402 * flow from application. PMD default flows and traffic control flows 7403 * are not affected. 7404 */ 7405 if (unlikely(!dev->data->dev_started)) { 7406 DRV_LOG(DEBUG, "port %u is not started when " 7407 "inserting a flow", dev->data->port_id); 7408 rte_flow_error_set(error, ENODEV, 7409 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7410 NULL, 7411 "port not started"); 7412 return NULL; 7413 } 7414 7415 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7416 attr, items, actions, 7417 true, error); 7418 } 7419 7420 /** 7421 * Destroy a flow in a list. 7422 * 7423 * @param dev 7424 * Pointer to Ethernet device. 7425 * @param[in] flow_idx 7426 * Index of flow to destroy. 7427 */ 7428 static void 7429 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7430 uint32_t flow_idx) 7431 { 7432 struct mlx5_priv *priv = dev->data->dev_private; 7433 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7434 7435 if (!flow) 7436 return; 7437 MLX5_ASSERT(flow->type == type); 7438 /* 7439 * Update RX queue flags only if port is started, otherwise it is 7440 * already clean. 7441 */ 7442 if (dev->data->dev_started) 7443 flow_rxq_flags_trim(dev, flow); 7444 flow_drv_destroy(dev, flow); 7445 if (flow->tunnel) { 7446 struct mlx5_flow_tunnel *tunnel; 7447 7448 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7449 RTE_VERIFY(tunnel); 7450 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7451 mlx5_flow_tunnel_free(dev, tunnel); 7452 } 7453 flow_mreg_del_copy_action(dev, flow); 7454 mlx5_ipool_free(priv->flows[type], flow_idx); 7455 } 7456 7457 /** 7458 * Destroy all flows. 7459 * 7460 * @param dev 7461 * Pointer to Ethernet device. 7462 * @param type 7463 * Flow type to be flushed. 7464 * @param active 7465 * If flushing is called actively. 7466 */ 7467 void 7468 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7469 bool active) 7470 { 7471 struct mlx5_priv *priv = dev->data->dev_private; 7472 uint32_t num_flushed = 0, fidx = 1; 7473 struct rte_flow *flow; 7474 7475 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7476 if (priv->sh->config.dv_flow_en == 2 && 7477 type == MLX5_FLOW_TYPE_GEN) { 7478 flow_hw_q_flow_flush(dev, NULL); 7479 return; 7480 } 7481 #endif 7482 7483 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7484 flow_list_destroy(dev, type, fidx); 7485 num_flushed++; 7486 } 7487 if (active) { 7488 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7489 dev->data->port_id, num_flushed); 7490 } 7491 } 7492 7493 /** 7494 * Stop all default actions for flows. 7495 * 7496 * @param dev 7497 * Pointer to Ethernet device. 7498 */ 7499 void 7500 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7501 { 7502 flow_mreg_del_default_copy_action(dev); 7503 flow_rxq_flags_clear(dev); 7504 } 7505 7506 /** 7507 * Start all default actions for flows. 7508 * 7509 * @param dev 7510 * Pointer to Ethernet device. 7511 * @return 7512 * 0 on success, a negative errno value otherwise and rte_errno is set. 7513 */ 7514 int 7515 mlx5_flow_start_default(struct rte_eth_dev *dev) 7516 { 7517 struct rte_flow_error error; 7518 7519 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7520 return flow_mreg_add_default_copy_action(dev, &error); 7521 } 7522 7523 /** 7524 * Release key of thread specific flow workspace data. 7525 */ 7526 void 7527 flow_release_workspace(void *data) 7528 { 7529 struct mlx5_flow_workspace *wks = data; 7530 struct mlx5_flow_workspace *next; 7531 7532 while (wks) { 7533 next = wks->next; 7534 free(wks->rss_desc.queue); 7535 free(wks); 7536 wks = next; 7537 } 7538 } 7539 7540 /** 7541 * Get thread specific current flow workspace. 7542 * 7543 * @return pointer to thread specific flow workspace data, NULL on error. 7544 */ 7545 struct mlx5_flow_workspace* 7546 mlx5_flow_get_thread_workspace(void) 7547 { 7548 struct mlx5_flow_workspace *data; 7549 7550 data = mlx5_flow_os_get_specific_workspace(); 7551 MLX5_ASSERT(data && data->inuse); 7552 if (!data || !data->inuse) 7553 DRV_LOG(ERR, "flow workspace not initialized."); 7554 return data; 7555 } 7556 7557 /** 7558 * Allocate and init new flow workspace. 7559 * 7560 * @return pointer to flow workspace data, NULL on error. 7561 */ 7562 static struct mlx5_flow_workspace* 7563 flow_alloc_thread_workspace(void) 7564 { 7565 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7566 7567 if (!data) { 7568 DRV_LOG(ERR, "Failed to allocate flow workspace " 7569 "memory."); 7570 return NULL; 7571 } 7572 data->rss_desc.queue = calloc(1, 7573 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7574 if (!data->rss_desc.queue) 7575 goto err; 7576 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7577 return data; 7578 err: 7579 free(data->rss_desc.queue); 7580 free(data); 7581 return NULL; 7582 } 7583 7584 /** 7585 * Get new thread specific flow workspace. 7586 * 7587 * If current workspace inuse, create new one and set as current. 7588 * 7589 * @return pointer to thread specific flow workspace data, NULL on error. 7590 */ 7591 struct mlx5_flow_workspace* 7592 mlx5_flow_push_thread_workspace(void) 7593 { 7594 struct mlx5_flow_workspace *curr; 7595 struct mlx5_flow_workspace *data; 7596 7597 curr = mlx5_flow_os_get_specific_workspace(); 7598 if (!curr) { 7599 data = flow_alloc_thread_workspace(); 7600 if (!data) 7601 return NULL; 7602 } else if (!curr->inuse) { 7603 data = curr; 7604 } else if (curr->next) { 7605 data = curr->next; 7606 } else { 7607 data = flow_alloc_thread_workspace(); 7608 if (!data) 7609 return NULL; 7610 curr->next = data; 7611 data->prev = curr; 7612 } 7613 data->inuse = 1; 7614 data->flow_idx = 0; 7615 /* Set as current workspace */ 7616 if (mlx5_flow_os_set_specific_workspace(data)) 7617 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7618 return data; 7619 } 7620 7621 /** 7622 * Close current thread specific flow workspace. 7623 * 7624 * If previous workspace available, set it as current. 7625 * 7626 * @return pointer to thread specific flow workspace data, NULL on error. 7627 */ 7628 void 7629 mlx5_flow_pop_thread_workspace(void) 7630 { 7631 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7632 7633 if (!data) 7634 return; 7635 if (!data->inuse) { 7636 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7637 return; 7638 } 7639 data->inuse = 0; 7640 if (!data->prev) 7641 return; 7642 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7643 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7644 } 7645 7646 /** 7647 * Verify the flow list is empty 7648 * 7649 * @param dev 7650 * Pointer to Ethernet device. 7651 * 7652 * @return the number of flows not released. 7653 */ 7654 int 7655 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7656 { 7657 struct mlx5_priv *priv = dev->data->dev_private; 7658 struct rte_flow *flow; 7659 uint32_t idx = 0; 7660 int ret = 0, i; 7661 7662 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7663 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7664 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7665 dev->data->port_id, (void *)flow); 7666 ret++; 7667 } 7668 } 7669 return ret; 7670 } 7671 7672 /** 7673 * Enable default hairpin egress flow. 7674 * 7675 * @param dev 7676 * Pointer to Ethernet device. 7677 * @param sq_num 7678 * The SQ hw number. 7679 * 7680 * @return 7681 * 0 on success, a negative errno value otherwise and rte_errno is set. 7682 */ 7683 int 7684 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7685 uint32_t sq_num) 7686 { 7687 const struct rte_flow_attr attr = { 7688 .egress = 1, 7689 .priority = 0, 7690 }; 7691 struct mlx5_rte_flow_item_sq queue_spec = { 7692 .queue = sq_num, 7693 }; 7694 struct mlx5_rte_flow_item_sq queue_mask = { 7695 .queue = UINT32_MAX, 7696 }; 7697 struct rte_flow_item items[] = { 7698 { 7699 .type = (enum rte_flow_item_type) 7700 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7701 .spec = &queue_spec, 7702 .last = NULL, 7703 .mask = &queue_mask, 7704 }, 7705 { 7706 .type = RTE_FLOW_ITEM_TYPE_END, 7707 }, 7708 }; 7709 struct rte_flow_action_jump jump = { 7710 .group = MLX5_HAIRPIN_TX_TABLE, 7711 }; 7712 struct rte_flow_action actions[2]; 7713 uint32_t flow_idx; 7714 struct rte_flow_error error; 7715 7716 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7717 actions[0].conf = &jump; 7718 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7719 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7720 &attr, items, actions, false, &error); 7721 if (!flow_idx) { 7722 DRV_LOG(DEBUG, 7723 "Failed to create ctrl flow: rte_errno(%d)," 7724 " type(%d), message(%s)", 7725 rte_errno, error.type, 7726 error.message ? error.message : " (no stated reason)"); 7727 return -rte_errno; 7728 } 7729 return 0; 7730 } 7731 7732 /** 7733 * Enable a control flow configured from the control plane. 7734 * 7735 * @param dev 7736 * Pointer to Ethernet device. 7737 * @param eth_spec 7738 * An Ethernet flow spec to apply. 7739 * @param eth_mask 7740 * An Ethernet flow mask to apply. 7741 * @param vlan_spec 7742 * A VLAN flow spec to apply. 7743 * @param vlan_mask 7744 * A VLAN flow mask to apply. 7745 * 7746 * @return 7747 * 0 on success, a negative errno value otherwise and rte_errno is set. 7748 */ 7749 int 7750 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7751 struct rte_flow_item_eth *eth_spec, 7752 struct rte_flow_item_eth *eth_mask, 7753 struct rte_flow_item_vlan *vlan_spec, 7754 struct rte_flow_item_vlan *vlan_mask) 7755 { 7756 struct mlx5_priv *priv = dev->data->dev_private; 7757 const struct rte_flow_attr attr = { 7758 .ingress = 1, 7759 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7760 }; 7761 struct rte_flow_item items[] = { 7762 { 7763 .type = RTE_FLOW_ITEM_TYPE_ETH, 7764 .spec = eth_spec, 7765 .last = NULL, 7766 .mask = eth_mask, 7767 }, 7768 { 7769 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7770 RTE_FLOW_ITEM_TYPE_END, 7771 .spec = vlan_spec, 7772 .last = NULL, 7773 .mask = vlan_mask, 7774 }, 7775 { 7776 .type = RTE_FLOW_ITEM_TYPE_END, 7777 }, 7778 }; 7779 uint16_t queue[priv->reta_idx_n]; 7780 struct rte_flow_action_rss action_rss = { 7781 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7782 .level = 0, 7783 .types = priv->rss_conf.rss_hf, 7784 .key_len = priv->rss_conf.rss_key_len, 7785 .queue_num = priv->reta_idx_n, 7786 .key = priv->rss_conf.rss_key, 7787 .queue = queue, 7788 }; 7789 struct rte_flow_action actions[] = { 7790 { 7791 .type = RTE_FLOW_ACTION_TYPE_RSS, 7792 .conf = &action_rss, 7793 }, 7794 { 7795 .type = RTE_FLOW_ACTION_TYPE_END, 7796 }, 7797 }; 7798 uint32_t flow_idx; 7799 struct rte_flow_error error; 7800 unsigned int i; 7801 7802 if (!priv->reta_idx_n || !priv->rxqs_n) { 7803 return 0; 7804 } 7805 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7806 action_rss.types = 0; 7807 for (i = 0; i != priv->reta_idx_n; ++i) 7808 queue[i] = (*priv->reta_idx)[i]; 7809 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7810 &attr, items, actions, false, &error); 7811 if (!flow_idx) 7812 return -rte_errno; 7813 return 0; 7814 } 7815 7816 /** 7817 * Enable a flow control configured from the control plane. 7818 * 7819 * @param dev 7820 * Pointer to Ethernet device. 7821 * @param eth_spec 7822 * An Ethernet flow spec to apply. 7823 * @param eth_mask 7824 * An Ethernet flow mask to apply. 7825 * 7826 * @return 7827 * 0 on success, a negative errno value otherwise and rte_errno is set. 7828 */ 7829 int 7830 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7831 struct rte_flow_item_eth *eth_spec, 7832 struct rte_flow_item_eth *eth_mask) 7833 { 7834 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7835 } 7836 7837 /** 7838 * Create default miss flow rule matching lacp traffic 7839 * 7840 * @param dev 7841 * Pointer to Ethernet device. 7842 * @param eth_spec 7843 * An Ethernet flow spec to apply. 7844 * 7845 * @return 7846 * 0 on success, a negative errno value otherwise and rte_errno is set. 7847 */ 7848 int 7849 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7850 { 7851 /* 7852 * The LACP matching is done by only using ether type since using 7853 * a multicast dst mac causes kernel to give low priority to this flow. 7854 */ 7855 static const struct rte_flow_item_eth lacp_spec = { 7856 .type = RTE_BE16(0x8809), 7857 }; 7858 static const struct rte_flow_item_eth lacp_mask = { 7859 .type = 0xffff, 7860 }; 7861 const struct rte_flow_attr attr = { 7862 .ingress = 1, 7863 }; 7864 struct rte_flow_item items[] = { 7865 { 7866 .type = RTE_FLOW_ITEM_TYPE_ETH, 7867 .spec = &lacp_spec, 7868 .mask = &lacp_mask, 7869 }, 7870 { 7871 .type = RTE_FLOW_ITEM_TYPE_END, 7872 }, 7873 }; 7874 struct rte_flow_action actions[] = { 7875 { 7876 .type = (enum rte_flow_action_type) 7877 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7878 }, 7879 { 7880 .type = RTE_FLOW_ACTION_TYPE_END, 7881 }, 7882 }; 7883 struct rte_flow_error error; 7884 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7885 &attr, items, actions, 7886 false, &error); 7887 7888 if (!flow_idx) 7889 return -rte_errno; 7890 return 0; 7891 } 7892 7893 /** 7894 * Destroy a flow. 7895 * 7896 * @see rte_flow_destroy() 7897 * @see rte_flow_ops 7898 */ 7899 int 7900 mlx5_flow_destroy(struct rte_eth_dev *dev, 7901 struct rte_flow *flow, 7902 struct rte_flow_error *error __rte_unused) 7903 { 7904 struct mlx5_priv *priv = dev->data->dev_private; 7905 7906 if (priv->sh->config.dv_flow_en == 2) 7907 return rte_flow_error_set(error, ENOTSUP, 7908 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7909 NULL, 7910 "Flow non-Q destruction not supported"); 7911 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7912 (uintptr_t)(void *)flow); 7913 return 0; 7914 } 7915 7916 /** 7917 * Destroy all flows. 7918 * 7919 * @see rte_flow_flush() 7920 * @see rte_flow_ops 7921 */ 7922 int 7923 mlx5_flow_flush(struct rte_eth_dev *dev, 7924 struct rte_flow_error *error __rte_unused) 7925 { 7926 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7927 return 0; 7928 } 7929 7930 /** 7931 * Isolated mode. 7932 * 7933 * @see rte_flow_isolate() 7934 * @see rte_flow_ops 7935 */ 7936 int 7937 mlx5_flow_isolate(struct rte_eth_dev *dev, 7938 int enable, 7939 struct rte_flow_error *error) 7940 { 7941 struct mlx5_priv *priv = dev->data->dev_private; 7942 7943 if (dev->data->dev_started) { 7944 rte_flow_error_set(error, EBUSY, 7945 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7946 NULL, 7947 "port must be stopped first"); 7948 return -rte_errno; 7949 } 7950 priv->isolated = !!enable; 7951 if (enable) 7952 dev->dev_ops = &mlx5_dev_ops_isolate; 7953 else 7954 dev->dev_ops = &mlx5_dev_ops; 7955 7956 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7957 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7958 7959 return 0; 7960 } 7961 7962 /** 7963 * Query a flow. 7964 * 7965 * @see rte_flow_query() 7966 * @see rte_flow_ops 7967 */ 7968 static int 7969 flow_drv_query(struct rte_eth_dev *dev, 7970 struct rte_flow *eflow, 7971 const struct rte_flow_action *actions, 7972 void *data, 7973 struct rte_flow_error *error) 7974 { 7975 struct mlx5_priv *priv = dev->data->dev_private; 7976 const struct mlx5_flow_driver_ops *fops; 7977 struct rte_flow *flow = NULL; 7978 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 7979 7980 if (priv->sh->config.dv_flow_en == 2) { 7981 #ifdef HAVE_MLX5_HWS_SUPPORT 7982 flow = eflow; 7983 ftype = MLX5_FLOW_TYPE_HW; 7984 #endif 7985 } else { 7986 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7987 (uintptr_t)(void *)eflow); 7988 } 7989 if (!flow) { 7990 return rte_flow_error_set(error, ENOENT, 7991 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7992 NULL, 7993 "invalid flow handle"); 7994 } 7995 if (ftype == MLX5_FLOW_TYPE_MIN) 7996 ftype = flow->drv_type; 7997 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7998 fops = flow_get_drv_ops(ftype); 7999 8000 return fops->query(dev, flow, actions, data, error); 8001 } 8002 8003 /** 8004 * Query a flow. 8005 * 8006 * @see rte_flow_query() 8007 * @see rte_flow_ops 8008 */ 8009 int 8010 mlx5_flow_query(struct rte_eth_dev *dev, 8011 struct rte_flow *flow, 8012 const struct rte_flow_action *actions, 8013 void *data, 8014 struct rte_flow_error *error) 8015 { 8016 int ret; 8017 8018 ret = flow_drv_query(dev, flow, actions, data, 8019 error); 8020 if (ret < 0) 8021 return ret; 8022 return 0; 8023 } 8024 8025 /** 8026 * Get rte_flow callbacks. 8027 * 8028 * @param dev 8029 * Pointer to Ethernet device structure. 8030 * @param ops 8031 * Pointer to operation-specific structure. 8032 * 8033 * @return 0 8034 */ 8035 int 8036 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8037 const struct rte_flow_ops **ops) 8038 { 8039 *ops = &mlx5_flow_ops; 8040 return 0; 8041 } 8042 8043 /** 8044 * Validate meter policy actions. 8045 * Dispatcher for action type specific validation. 8046 * 8047 * @param[in] dev 8048 * Pointer to the Ethernet device structure. 8049 * @param[in] action 8050 * The meter policy action object to validate. 8051 * @param[in] attr 8052 * Attributes of flow to determine steering domain. 8053 * @param[out] is_rss 8054 * Is RSS or not. 8055 * @param[out] domain_bitmap 8056 * Domain bitmap. 8057 * @param[out] is_def_policy 8058 * Is default policy or not. 8059 * @param[out] error 8060 * Perform verbose error reporting if not NULL. Initialized in case of 8061 * error only. 8062 * 8063 * @return 8064 * 0 on success, otherwise negative errno value. 8065 */ 8066 int 8067 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8068 const struct rte_flow_action *actions[RTE_COLORS], 8069 struct rte_flow_attr *attr, 8070 bool *is_rss, 8071 uint8_t *domain_bitmap, 8072 uint8_t *policy_mode, 8073 struct rte_mtr_error *error) 8074 { 8075 const struct mlx5_flow_driver_ops *fops; 8076 8077 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8078 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8079 domain_bitmap, policy_mode, error); 8080 } 8081 8082 /** 8083 * Destroy the meter table set. 8084 * 8085 * @param[in] dev 8086 * Pointer to Ethernet device. 8087 * @param[in] mtr_policy 8088 * Meter policy struct. 8089 */ 8090 void 8091 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8092 struct mlx5_flow_meter_policy *mtr_policy) 8093 { 8094 const struct mlx5_flow_driver_ops *fops; 8095 8096 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8097 fops->destroy_mtr_acts(dev, mtr_policy); 8098 } 8099 8100 /** 8101 * Create policy action, lock free, 8102 * (mutex should be acquired by caller). 8103 * Dispatcher for action type specific call. 8104 * 8105 * @param[in] dev 8106 * Pointer to the Ethernet device structure. 8107 * @param[in] mtr_policy 8108 * Meter policy struct. 8109 * @param[in] action 8110 * Action specification used to create meter actions. 8111 * @param[in] attr 8112 * Flow rule attributes. 8113 * @param[out] error 8114 * Perform verbose error reporting if not NULL. Initialized in case of 8115 * error only. 8116 * 8117 * @return 8118 * 0 on success, otherwise negative errno value. 8119 */ 8120 int 8121 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8122 struct mlx5_flow_meter_policy *mtr_policy, 8123 const struct rte_flow_action *actions[RTE_COLORS], 8124 struct rte_flow_attr *attr, 8125 struct rte_mtr_error *error) 8126 { 8127 const struct mlx5_flow_driver_ops *fops; 8128 8129 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8130 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8131 } 8132 8133 /** 8134 * Create policy rules, lock free, 8135 * (mutex should be acquired by caller). 8136 * Dispatcher for action type specific call. 8137 * 8138 * @param[in] dev 8139 * Pointer to the Ethernet device structure. 8140 * @param[in] mtr_policy 8141 * Meter policy struct. 8142 * 8143 * @return 8144 * 0 on success, -1 otherwise. 8145 */ 8146 int 8147 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8148 struct mlx5_flow_meter_policy *mtr_policy) 8149 { 8150 const struct mlx5_flow_driver_ops *fops; 8151 8152 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8153 return fops->create_policy_rules(dev, mtr_policy); 8154 } 8155 8156 /** 8157 * Destroy policy rules, lock free, 8158 * (mutex should be acquired by caller). 8159 * Dispatcher for action type specific call. 8160 * 8161 * @param[in] dev 8162 * Pointer to the Ethernet device structure. 8163 * @param[in] mtr_policy 8164 * Meter policy struct. 8165 */ 8166 void 8167 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8168 struct mlx5_flow_meter_policy *mtr_policy) 8169 { 8170 const struct mlx5_flow_driver_ops *fops; 8171 8172 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8173 fops->destroy_policy_rules(dev, mtr_policy); 8174 } 8175 8176 /** 8177 * Destroy the default policy table set. 8178 * 8179 * @param[in] dev 8180 * Pointer to Ethernet device. 8181 */ 8182 void 8183 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8184 { 8185 const struct mlx5_flow_driver_ops *fops; 8186 8187 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8188 fops->destroy_def_policy(dev); 8189 } 8190 8191 /** 8192 * Destroy the default policy table set. 8193 * 8194 * @param[in] dev 8195 * Pointer to Ethernet device. 8196 * 8197 * @return 8198 * 0 on success, -1 otherwise. 8199 */ 8200 int 8201 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8202 { 8203 const struct mlx5_flow_driver_ops *fops; 8204 8205 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8206 return fops->create_def_policy(dev); 8207 } 8208 8209 /** 8210 * Create the needed meter and suffix tables. 8211 * 8212 * @param[in] dev 8213 * Pointer to Ethernet device. 8214 * 8215 * @return 8216 * 0 on success, -1 otherwise. 8217 */ 8218 int 8219 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8220 struct mlx5_flow_meter_info *fm, 8221 uint32_t mtr_idx, 8222 uint8_t domain_bitmap) 8223 { 8224 const struct mlx5_flow_driver_ops *fops; 8225 8226 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8227 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8228 } 8229 8230 /** 8231 * Destroy the meter table set. 8232 * 8233 * @param[in] dev 8234 * Pointer to Ethernet device. 8235 * @param[in] tbl 8236 * Pointer to the meter table set. 8237 */ 8238 void 8239 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8240 struct mlx5_flow_meter_info *fm) 8241 { 8242 const struct mlx5_flow_driver_ops *fops; 8243 8244 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8245 fops->destroy_mtr_tbls(dev, fm); 8246 } 8247 8248 /** 8249 * Destroy the global meter drop table. 8250 * 8251 * @param[in] dev 8252 * Pointer to Ethernet device. 8253 */ 8254 void 8255 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8256 { 8257 const struct mlx5_flow_driver_ops *fops; 8258 8259 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8260 fops->destroy_mtr_drop_tbls(dev); 8261 } 8262 8263 /** 8264 * Destroy the sub policy table with RX queue. 8265 * 8266 * @param[in] dev 8267 * Pointer to Ethernet device. 8268 * @param[in] mtr_policy 8269 * Pointer to meter policy table. 8270 */ 8271 void 8272 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8273 struct mlx5_flow_meter_policy *mtr_policy) 8274 { 8275 const struct mlx5_flow_driver_ops *fops; 8276 8277 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8278 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8279 } 8280 8281 /** 8282 * Allocate the needed aso flow meter id. 8283 * 8284 * @param[in] dev 8285 * Pointer to Ethernet device. 8286 * 8287 * @return 8288 * Index to aso flow meter on success, NULL otherwise. 8289 */ 8290 uint32_t 8291 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8292 { 8293 const struct mlx5_flow_driver_ops *fops; 8294 8295 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8296 return fops->create_meter(dev); 8297 } 8298 8299 /** 8300 * Free the aso flow meter id. 8301 * 8302 * @param[in] dev 8303 * Pointer to Ethernet device. 8304 * @param[in] mtr_idx 8305 * Index to aso flow meter to be free. 8306 * 8307 * @return 8308 * 0 on success. 8309 */ 8310 void 8311 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8312 { 8313 const struct mlx5_flow_driver_ops *fops; 8314 8315 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8316 fops->free_meter(dev, mtr_idx); 8317 } 8318 8319 /** 8320 * Allocate a counter. 8321 * 8322 * @param[in] dev 8323 * Pointer to Ethernet device structure. 8324 * 8325 * @return 8326 * Index to allocated counter on success, 0 otherwise. 8327 */ 8328 uint32_t 8329 mlx5_counter_alloc(struct rte_eth_dev *dev) 8330 { 8331 const struct mlx5_flow_driver_ops *fops; 8332 struct rte_flow_attr attr = { .transfer = 0 }; 8333 8334 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8335 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8336 return fops->counter_alloc(dev); 8337 } 8338 DRV_LOG(ERR, 8339 "port %u counter allocate is not supported.", 8340 dev->data->port_id); 8341 return 0; 8342 } 8343 8344 /** 8345 * Free a counter. 8346 * 8347 * @param[in] dev 8348 * Pointer to Ethernet device structure. 8349 * @param[in] cnt 8350 * Index to counter to be free. 8351 */ 8352 void 8353 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8354 { 8355 const struct mlx5_flow_driver_ops *fops; 8356 struct rte_flow_attr attr = { .transfer = 0 }; 8357 8358 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8359 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8360 fops->counter_free(dev, cnt); 8361 return; 8362 } 8363 DRV_LOG(ERR, 8364 "port %u counter free is not supported.", 8365 dev->data->port_id); 8366 } 8367 8368 /** 8369 * Query counter statistics. 8370 * 8371 * @param[in] dev 8372 * Pointer to Ethernet device structure. 8373 * @param[in] cnt 8374 * Index to counter to query. 8375 * @param[in] clear 8376 * Set to clear counter statistics. 8377 * @param[out] pkts 8378 * The counter hits packets number to save. 8379 * @param[out] bytes 8380 * The counter hits bytes number to save. 8381 * 8382 * @return 8383 * 0 on success, a negative errno value otherwise. 8384 */ 8385 int 8386 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8387 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8388 { 8389 const struct mlx5_flow_driver_ops *fops; 8390 struct rte_flow_attr attr = { .transfer = 0 }; 8391 8392 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8393 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8394 return fops->counter_query(dev, cnt, clear, pkts, 8395 bytes, action); 8396 } 8397 DRV_LOG(ERR, 8398 "port %u counter query is not supported.", 8399 dev->data->port_id); 8400 return -ENOTSUP; 8401 } 8402 8403 /** 8404 * Get information about HWS pre-configurable resources. 8405 * 8406 * @param[in] dev 8407 * Pointer to the rte_eth_dev structure. 8408 * @param[out] port_info 8409 * Pointer to port information. 8410 * @param[out] queue_info 8411 * Pointer to queue information. 8412 * @param[out] error 8413 * Pointer to error structure. 8414 * 8415 * @return 8416 * 0 on success, a negative errno value otherwise and rte_errno is set. 8417 */ 8418 static int 8419 mlx5_flow_info_get(struct rte_eth_dev *dev, 8420 struct rte_flow_port_info *port_info, 8421 struct rte_flow_queue_info *queue_info, 8422 struct rte_flow_error *error) 8423 { 8424 const struct mlx5_flow_driver_ops *fops; 8425 struct rte_flow_attr attr = {0}; 8426 8427 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8428 return rte_flow_error_set(error, ENOTSUP, 8429 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8430 NULL, 8431 "info get with incorrect steering mode"); 8432 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8433 return fops->info_get(dev, port_info, queue_info, error); 8434 } 8435 8436 /** 8437 * Configure port HWS resources. 8438 * 8439 * @param[in] dev 8440 * Pointer to the rte_eth_dev structure. 8441 * @param[in] port_attr 8442 * Port configuration attributes. 8443 * @param[in] nb_queue 8444 * Number of queue. 8445 * @param[in] queue_attr 8446 * Array that holds attributes for each flow queue. 8447 * @param[out] error 8448 * Pointer to error structure. 8449 * 8450 * @return 8451 * 0 on success, a negative errno value otherwise and rte_errno is set. 8452 */ 8453 static int 8454 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8455 const struct rte_flow_port_attr *port_attr, 8456 uint16_t nb_queue, 8457 const struct rte_flow_queue_attr *queue_attr[], 8458 struct rte_flow_error *error) 8459 { 8460 const struct mlx5_flow_driver_ops *fops; 8461 struct rte_flow_attr attr = {0}; 8462 8463 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8464 return rte_flow_error_set(error, ENOTSUP, 8465 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8466 NULL, 8467 "port configure with incorrect steering mode"); 8468 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8469 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8470 } 8471 8472 /** 8473 * Validate item template. 8474 * 8475 * @param[in] dev 8476 * Pointer to the rte_eth_dev structure. 8477 * @param[in] attr 8478 * Pointer to the item template attributes. 8479 * @param[in] items 8480 * The template item pattern. 8481 * @param[out] error 8482 * Pointer to error structure. 8483 * 8484 * @return 8485 * 0 on success, a negative errno value otherwise and rte_errno is set. 8486 */ 8487 int 8488 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8489 const struct rte_flow_pattern_template_attr *attr, 8490 const struct rte_flow_item items[], 8491 struct rte_flow_error *error) 8492 { 8493 const struct mlx5_flow_driver_ops *fops; 8494 struct rte_flow_attr fattr = {0}; 8495 8496 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8497 rte_flow_error_set(error, ENOTSUP, 8498 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8499 "pattern validate with incorrect steering mode"); 8500 return -ENOTSUP; 8501 } 8502 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8503 return fops->pattern_validate(dev, attr, items, error); 8504 } 8505 8506 /** 8507 * Create flow item template. 8508 * 8509 * @param[in] dev 8510 * Pointer to the rte_eth_dev structure. 8511 * @param[in] attr 8512 * Pointer to the item template attributes. 8513 * @param[in] items 8514 * The template item pattern. 8515 * @param[out] error 8516 * Pointer to error structure. 8517 * 8518 * @return 8519 * 0 on success, a negative errno value otherwise and rte_errno is set. 8520 */ 8521 static struct rte_flow_pattern_template * 8522 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8523 const struct rte_flow_pattern_template_attr *attr, 8524 const struct rte_flow_item items[], 8525 struct rte_flow_error *error) 8526 { 8527 const struct mlx5_flow_driver_ops *fops; 8528 struct rte_flow_attr fattr = {0}; 8529 8530 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8531 rte_flow_error_set(error, ENOTSUP, 8532 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8533 NULL, 8534 "pattern create with incorrect steering mode"); 8535 return NULL; 8536 } 8537 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8538 return fops->pattern_template_create(dev, attr, items, error); 8539 } 8540 8541 /** 8542 * Destroy flow item template. 8543 * 8544 * @param[in] dev 8545 * Pointer to the rte_eth_dev structure. 8546 * @param[in] template 8547 * Pointer to the item template to be destroyed. 8548 * @param[out] error 8549 * Pointer to error structure. 8550 * 8551 * @return 8552 * 0 on success, a negative errno value otherwise and rte_errno is set. 8553 */ 8554 static int 8555 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8556 struct rte_flow_pattern_template *template, 8557 struct rte_flow_error *error) 8558 { 8559 const struct mlx5_flow_driver_ops *fops; 8560 struct rte_flow_attr attr = {0}; 8561 8562 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8563 return rte_flow_error_set(error, ENOTSUP, 8564 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8565 NULL, 8566 "pattern destroy with incorrect steering mode"); 8567 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8568 return fops->pattern_template_destroy(dev, template, error); 8569 } 8570 8571 /** 8572 * Validate flow actions template. 8573 * 8574 * @param[in] dev 8575 * Pointer to the rte_eth_dev structure. 8576 * @param[in] attr 8577 * Pointer to the action template attributes. 8578 * @param[in] actions 8579 * Associated actions (list terminated by the END action). 8580 * @param[in] masks 8581 * List of actions that marks which of the action's member is constant. 8582 * @param[out] error 8583 * Pointer to error structure. 8584 * 8585 * @return 8586 * 0 on success, a negative errno value otherwise and rte_errno is set. 8587 */ 8588 int 8589 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8590 const struct rte_flow_actions_template_attr *attr, 8591 const struct rte_flow_action actions[], 8592 const struct rte_flow_action masks[], 8593 struct rte_flow_error *error) 8594 { 8595 const struct mlx5_flow_driver_ops *fops; 8596 struct rte_flow_attr fattr = {0}; 8597 8598 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8599 rte_flow_error_set(error, ENOTSUP, 8600 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8601 "actions validate with incorrect steering mode"); 8602 return -ENOTSUP; 8603 } 8604 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8605 return fops->actions_validate(dev, attr, actions, masks, error); 8606 } 8607 8608 /** 8609 * Create flow item template. 8610 * 8611 * @param[in] dev 8612 * Pointer to the rte_eth_dev structure. 8613 * @param[in] attr 8614 * Pointer to the action template attributes. 8615 * @param[in] actions 8616 * Associated actions (list terminated by the END action). 8617 * @param[in] masks 8618 * List of actions that marks which of the action's member is constant. 8619 * @param[out] error 8620 * Pointer to error structure. 8621 * 8622 * @return 8623 * 0 on success, a negative errno value otherwise and rte_errno is set. 8624 */ 8625 static struct rte_flow_actions_template * 8626 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8627 const struct rte_flow_actions_template_attr *attr, 8628 const struct rte_flow_action actions[], 8629 const struct rte_flow_action masks[], 8630 struct rte_flow_error *error) 8631 { 8632 const struct mlx5_flow_driver_ops *fops; 8633 struct rte_flow_attr fattr = {0}; 8634 8635 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8636 rte_flow_error_set(error, ENOTSUP, 8637 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8638 NULL, 8639 "action create with incorrect steering mode"); 8640 return NULL; 8641 } 8642 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8643 return fops->actions_template_create(dev, attr, actions, masks, error); 8644 } 8645 8646 /** 8647 * Destroy flow action template. 8648 * 8649 * @param[in] dev 8650 * Pointer to the rte_eth_dev structure. 8651 * @param[in] template 8652 * Pointer to the action template to be destroyed. 8653 * @param[out] error 8654 * Pointer to error structure. 8655 * 8656 * @return 8657 * 0 on success, a negative errno value otherwise and rte_errno is set. 8658 */ 8659 static int 8660 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8661 struct rte_flow_actions_template *template, 8662 struct rte_flow_error *error) 8663 { 8664 const struct mlx5_flow_driver_ops *fops; 8665 struct rte_flow_attr attr = {0}; 8666 8667 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8668 return rte_flow_error_set(error, ENOTSUP, 8669 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8670 NULL, 8671 "action destroy with incorrect steering mode"); 8672 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8673 return fops->actions_template_destroy(dev, template, error); 8674 } 8675 8676 /** 8677 * Create flow table. 8678 * 8679 * @param[in] dev 8680 * Pointer to the rte_eth_dev structure. 8681 * @param[in] attr 8682 * Pointer to the table attributes. 8683 * @param[in] item_templates 8684 * Item template array to be binded to the table. 8685 * @param[in] nb_item_templates 8686 * Number of item template. 8687 * @param[in] action_templates 8688 * Action template array to be binded to the table. 8689 * @param[in] nb_action_templates 8690 * Number of action template. 8691 * @param[out] error 8692 * Pointer to error structure. 8693 * 8694 * @return 8695 * Table on success, NULL otherwise and rte_errno is set. 8696 */ 8697 static struct rte_flow_template_table * 8698 mlx5_flow_table_create(struct rte_eth_dev *dev, 8699 const struct rte_flow_template_table_attr *attr, 8700 struct rte_flow_pattern_template *item_templates[], 8701 uint8_t nb_item_templates, 8702 struct rte_flow_actions_template *action_templates[], 8703 uint8_t nb_action_templates, 8704 struct rte_flow_error *error) 8705 { 8706 const struct mlx5_flow_driver_ops *fops; 8707 struct rte_flow_attr fattr = {0}; 8708 8709 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8710 rte_flow_error_set(error, ENOTSUP, 8711 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8712 NULL, 8713 "table create with incorrect steering mode"); 8714 return NULL; 8715 } 8716 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8717 return fops->template_table_create(dev, 8718 attr, 8719 item_templates, 8720 nb_item_templates, 8721 action_templates, 8722 nb_action_templates, 8723 error); 8724 } 8725 8726 /** 8727 * PMD destroy flow table. 8728 * 8729 * @param[in] dev 8730 * Pointer to the rte_eth_dev structure. 8731 * @param[in] table 8732 * Pointer to the table to be destroyed. 8733 * @param[out] error 8734 * Pointer to error structure. 8735 * 8736 * @return 8737 * 0 on success, a negative errno value otherwise and rte_errno is set. 8738 */ 8739 static int 8740 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8741 struct rte_flow_template_table *table, 8742 struct rte_flow_error *error) 8743 { 8744 const struct mlx5_flow_driver_ops *fops; 8745 struct rte_flow_attr attr = {0}; 8746 8747 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8748 return rte_flow_error_set(error, ENOTSUP, 8749 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8750 NULL, 8751 "table destroy with incorrect steering mode"); 8752 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8753 return fops->template_table_destroy(dev, table, error); 8754 } 8755 8756 /** 8757 * Enqueue flow creation. 8758 * 8759 * @param[in] dev 8760 * Pointer to the rte_eth_dev structure. 8761 * @param[in] queue_id 8762 * The queue to create the flow. 8763 * @param[in] attr 8764 * Pointer to the flow operation attributes. 8765 * @param[in] items 8766 * Items with flow spec value. 8767 * @param[in] pattern_template_index 8768 * The item pattern flow follows from the table. 8769 * @param[in] actions 8770 * Action with flow spec value. 8771 * @param[in] action_template_index 8772 * The action pattern flow follows from the table. 8773 * @param[in] user_data 8774 * Pointer to the user_data. 8775 * @param[out] error 8776 * Pointer to error structure. 8777 * 8778 * @return 8779 * Flow pointer on success, NULL otherwise and rte_errno is set. 8780 */ 8781 static struct rte_flow * 8782 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8783 uint32_t queue_id, 8784 const struct rte_flow_op_attr *attr, 8785 struct rte_flow_template_table *table, 8786 const struct rte_flow_item items[], 8787 uint8_t pattern_template_index, 8788 const struct rte_flow_action actions[], 8789 uint8_t action_template_index, 8790 void *user_data, 8791 struct rte_flow_error *error) 8792 { 8793 const struct mlx5_flow_driver_ops *fops; 8794 struct rte_flow_attr fattr = {0}; 8795 8796 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8797 rte_flow_error_set(error, ENOTSUP, 8798 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8799 NULL, 8800 "flow_q create with incorrect steering mode"); 8801 return NULL; 8802 } 8803 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8804 return fops->async_flow_create(dev, queue_id, attr, table, 8805 items, pattern_template_index, 8806 actions, action_template_index, 8807 user_data, error); 8808 } 8809 8810 /** 8811 * Enqueue flow destruction. 8812 * 8813 * @param[in] dev 8814 * Pointer to the rte_eth_dev structure. 8815 * @param[in] queue 8816 * The queue to destroy the flow. 8817 * @param[in] attr 8818 * Pointer to the flow operation attributes. 8819 * @param[in] flow 8820 * Pointer to the flow to be destroyed. 8821 * @param[in] user_data 8822 * Pointer to the user_data. 8823 * @param[out] error 8824 * Pointer to error structure. 8825 * 8826 * @return 8827 * 0 on success, negative value otherwise and rte_errno is set. 8828 */ 8829 static int 8830 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8831 uint32_t queue, 8832 const struct rte_flow_op_attr *attr, 8833 struct rte_flow *flow, 8834 void *user_data, 8835 struct rte_flow_error *error) 8836 { 8837 const struct mlx5_flow_driver_ops *fops; 8838 struct rte_flow_attr fattr = {0}; 8839 8840 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8841 return rte_flow_error_set(error, ENOTSUP, 8842 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8843 NULL, 8844 "flow_q destroy with incorrect steering mode"); 8845 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8846 return fops->async_flow_destroy(dev, queue, attr, flow, 8847 user_data, error); 8848 } 8849 8850 /** 8851 * Pull the enqueued flows. 8852 * 8853 * @param[in] dev 8854 * Pointer to the rte_eth_dev structure. 8855 * @param[in] queue 8856 * The queue to pull the result. 8857 * @param[in/out] res 8858 * Array to save the results. 8859 * @param[in] n_res 8860 * Available result with the array. 8861 * @param[out] error 8862 * Pointer to error structure. 8863 * 8864 * @return 8865 * Result number on success, negative value otherwise and rte_errno is set. 8866 */ 8867 static int 8868 mlx5_flow_pull(struct rte_eth_dev *dev, 8869 uint32_t queue, 8870 struct rte_flow_op_result res[], 8871 uint16_t n_res, 8872 struct rte_flow_error *error) 8873 { 8874 const struct mlx5_flow_driver_ops *fops; 8875 struct rte_flow_attr attr = {0}; 8876 8877 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8878 return rte_flow_error_set(error, ENOTSUP, 8879 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8880 NULL, 8881 "flow_q pull with incorrect steering mode"); 8882 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8883 return fops->pull(dev, queue, res, n_res, error); 8884 } 8885 8886 /** 8887 * Push the enqueued flows. 8888 * 8889 * @param[in] dev 8890 * Pointer to the rte_eth_dev structure. 8891 * @param[in] queue 8892 * The queue to push the flows. 8893 * @param[out] error 8894 * Pointer to error structure. 8895 * 8896 * @return 8897 * 0 on success, negative value otherwise and rte_errno is set. 8898 */ 8899 static int 8900 mlx5_flow_push(struct rte_eth_dev *dev, 8901 uint32_t queue, 8902 struct rte_flow_error *error) 8903 { 8904 const struct mlx5_flow_driver_ops *fops; 8905 struct rte_flow_attr attr = {0}; 8906 8907 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8908 return rte_flow_error_set(error, ENOTSUP, 8909 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8910 NULL, 8911 "flow_q push with incorrect steering mode"); 8912 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8913 return fops->push(dev, queue, error); 8914 } 8915 8916 /** 8917 * Create shared action. 8918 * 8919 * @param[in] dev 8920 * Pointer to the rte_eth_dev structure. 8921 * @param[in] queue 8922 * Which queue to be used.. 8923 * @param[in] attr 8924 * Operation attribute. 8925 * @param[in] conf 8926 * Indirect action configuration. 8927 * @param[in] action 8928 * rte_flow action detail. 8929 * @param[in] user_data 8930 * Pointer to the user_data. 8931 * @param[out] error 8932 * Pointer to error structure. 8933 * 8934 * @return 8935 * Action handle on success, NULL otherwise and rte_errno is set. 8936 */ 8937 static struct rte_flow_action_handle * 8938 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8939 const struct rte_flow_op_attr *attr, 8940 const struct rte_flow_indir_action_conf *conf, 8941 const struct rte_flow_action *action, 8942 void *user_data, 8943 struct rte_flow_error *error) 8944 { 8945 const struct mlx5_flow_driver_ops *fops = 8946 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8947 8948 return fops->async_action_create(dev, queue, attr, conf, action, 8949 user_data, error); 8950 } 8951 8952 /** 8953 * Update shared action. 8954 * 8955 * @param[in] dev 8956 * Pointer to the rte_eth_dev structure. 8957 * @param[in] queue 8958 * Which queue to be used.. 8959 * @param[in] attr 8960 * Operation attribute. 8961 * @param[in] handle 8962 * Action handle to be updated. 8963 * @param[in] update 8964 * Update value. 8965 * @param[in] user_data 8966 * Pointer to the user_data. 8967 * @param[out] error 8968 * Pointer to error structure. 8969 * 8970 * @return 8971 * 0 on success, negative value otherwise and rte_errno is set. 8972 */ 8973 static int 8974 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8975 const struct rte_flow_op_attr *attr, 8976 struct rte_flow_action_handle *handle, 8977 const void *update, 8978 void *user_data, 8979 struct rte_flow_error *error) 8980 { 8981 const struct mlx5_flow_driver_ops *fops = 8982 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8983 8984 return fops->async_action_update(dev, queue, attr, handle, 8985 update, user_data, error); 8986 } 8987 8988 /** 8989 * Query shared action. 8990 * 8991 * @param[in] dev 8992 * Pointer to the rte_eth_dev structure. 8993 * @param[in] queue 8994 * Which queue to be used.. 8995 * @param[in] attr 8996 * Operation attribute. 8997 * @param[in] handle 8998 * Action handle to be updated. 8999 * @param[in] data 9000 * Pointer query result data. 9001 * @param[in] user_data 9002 * Pointer to the user_data. 9003 * @param[out] error 9004 * Pointer to error structure. 9005 * 9006 * @return 9007 * 0 on success, negative value otherwise and rte_errno is set. 9008 */ 9009 static int 9010 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9011 const struct rte_flow_op_attr *attr, 9012 const struct rte_flow_action_handle *handle, 9013 void *data, 9014 void *user_data, 9015 struct rte_flow_error *error) 9016 { 9017 const struct mlx5_flow_driver_ops *fops = 9018 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9019 9020 return fops->async_action_query(dev, queue, attr, handle, 9021 data, user_data, error); 9022 } 9023 9024 /** 9025 * Destroy shared action. 9026 * 9027 * @param[in] dev 9028 * Pointer to the rte_eth_dev structure. 9029 * @param[in] queue 9030 * Which queue to be used.. 9031 * @param[in] attr 9032 * Operation attribute. 9033 * @param[in] handle 9034 * Action handle to be destroyed. 9035 * @param[in] user_data 9036 * Pointer to the user_data. 9037 * @param[out] error 9038 * Pointer to error structure. 9039 * 9040 * @return 9041 * 0 on success, negative value otherwise and rte_errno is set. 9042 */ 9043 static int 9044 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9045 const struct rte_flow_op_attr *attr, 9046 struct rte_flow_action_handle *handle, 9047 void *user_data, 9048 struct rte_flow_error *error) 9049 { 9050 const struct mlx5_flow_driver_ops *fops = 9051 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9052 9053 return fops->async_action_destroy(dev, queue, attr, handle, 9054 user_data, error); 9055 } 9056 9057 /** 9058 * Allocate a new memory for the counter values wrapped by all the needed 9059 * management. 9060 * 9061 * @param[in] sh 9062 * Pointer to mlx5_dev_ctx_shared object. 9063 * 9064 * @return 9065 * 0 on success, a negative errno value otherwise. 9066 */ 9067 static int 9068 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9069 { 9070 struct mlx5_counter_stats_mem_mng *mem_mng; 9071 volatile struct flow_counter_stats *raw_data; 9072 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9073 int size = (sizeof(struct flow_counter_stats) * 9074 MLX5_COUNTERS_PER_POOL + 9075 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9076 sizeof(struct mlx5_counter_stats_mem_mng); 9077 size_t pgsize = rte_mem_page_size(); 9078 uint8_t *mem; 9079 int ret; 9080 int i; 9081 9082 if (pgsize == (size_t)-1) { 9083 DRV_LOG(ERR, "Failed to get mem page size"); 9084 rte_errno = ENOMEM; 9085 return -ENOMEM; 9086 } 9087 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9088 if (!mem) { 9089 rte_errno = ENOMEM; 9090 return -ENOMEM; 9091 } 9092 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9093 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9094 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9095 sh->cdev->pdn, mem, size, 9096 &mem_mng->wm); 9097 if (ret) { 9098 rte_errno = errno; 9099 mlx5_free(mem); 9100 return -rte_errno; 9101 } 9102 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9103 raw_data = (volatile struct flow_counter_stats *)mem; 9104 for (i = 0; i < raws_n; ++i) { 9105 mem_mng->raws[i].mem_mng = mem_mng; 9106 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9107 } 9108 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9109 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9110 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9111 next); 9112 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9113 sh->sws_cmng.mem_mng = mem_mng; 9114 return 0; 9115 } 9116 9117 /** 9118 * Set the statistic memory to the new counter pool. 9119 * 9120 * @param[in] sh 9121 * Pointer to mlx5_dev_ctx_shared object. 9122 * @param[in] pool 9123 * Pointer to the pool to set the statistic memory. 9124 * 9125 * @return 9126 * 0 on success, a negative errno value otherwise. 9127 */ 9128 static int 9129 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9130 struct mlx5_flow_counter_pool *pool) 9131 { 9132 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9133 /* Resize statistic memory once used out. */ 9134 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9135 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9136 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9137 return -1; 9138 } 9139 rte_spinlock_lock(&pool->sl); 9140 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9141 rte_spinlock_unlock(&pool->sl); 9142 pool->raw_hw = NULL; 9143 return 0; 9144 } 9145 9146 #define MLX5_POOL_QUERY_FREQ_US 1000000 9147 9148 /** 9149 * Set the periodic procedure for triggering asynchronous batch queries for all 9150 * the counter pools. 9151 * 9152 * @param[in] sh 9153 * Pointer to mlx5_dev_ctx_shared object. 9154 */ 9155 void 9156 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9157 { 9158 uint32_t pools_n, us; 9159 9160 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9161 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9162 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9163 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9164 sh->sws_cmng.query_thread_on = 0; 9165 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9166 } else { 9167 sh->sws_cmng.query_thread_on = 1; 9168 } 9169 } 9170 9171 /** 9172 * The periodic procedure for triggering asynchronous batch queries for all the 9173 * counter pools. This function is probably called by the host thread. 9174 * 9175 * @param[in] arg 9176 * The parameter for the alarm process. 9177 */ 9178 void 9179 mlx5_flow_query_alarm(void *arg) 9180 { 9181 struct mlx5_dev_ctx_shared *sh = arg; 9182 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9183 uint16_t pool_index = cmng->pool_index; 9184 struct mlx5_flow_counter_pool *pool; 9185 uint16_t n_valid; 9186 int ret; 9187 9188 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9189 goto set_alarm; 9190 rte_spinlock_lock(&cmng->pool_update_sl); 9191 pool = cmng->pools[pool_index]; 9192 n_valid = cmng->n_valid; 9193 rte_spinlock_unlock(&cmng->pool_update_sl); 9194 /* Set the statistic memory to the new created pool. */ 9195 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9196 goto set_alarm; 9197 if (pool->raw_hw) 9198 /* There is a pool query in progress. */ 9199 goto set_alarm; 9200 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9201 if (!pool->raw_hw) 9202 /* No free counter statistics raw memory. */ 9203 goto set_alarm; 9204 /* 9205 * Identify the counters released between query trigger and query 9206 * handle more efficiently. The counter released in this gap period 9207 * should wait for a new round of query as the new arrived packets 9208 * will not be taken into account. 9209 */ 9210 pool->query_gen++; 9211 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9212 MLX5_COUNTERS_PER_POOL, 9213 NULL, NULL, 9214 pool->raw_hw->mem_mng->wm.lkey, 9215 (void *)(uintptr_t) 9216 pool->raw_hw->data, 9217 sh->devx_comp, 9218 (uint64_t)(uintptr_t)pool); 9219 if (ret) { 9220 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9221 " %d", pool->min_dcs->id); 9222 pool->raw_hw = NULL; 9223 goto set_alarm; 9224 } 9225 LIST_REMOVE(pool->raw_hw, next); 9226 cmng->pending_queries++; 9227 pool_index++; 9228 if (pool_index >= n_valid) 9229 pool_index = 0; 9230 set_alarm: 9231 cmng->pool_index = pool_index; 9232 mlx5_set_query_alarm(sh); 9233 } 9234 9235 /** 9236 * Check and callback event for new aged flow in the counter pool 9237 * 9238 * @param[in] sh 9239 * Pointer to mlx5_dev_ctx_shared object. 9240 * @param[in] pool 9241 * Pointer to Current counter pool. 9242 */ 9243 static void 9244 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9245 struct mlx5_flow_counter_pool *pool) 9246 { 9247 struct mlx5_priv *priv; 9248 struct mlx5_flow_counter *cnt; 9249 struct mlx5_age_info *age_info; 9250 struct mlx5_age_param *age_param; 9251 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9252 struct mlx5_counter_stats_raw *prev = pool->raw; 9253 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9254 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9255 uint16_t expected = AGE_CANDIDATE; 9256 uint32_t i; 9257 9258 pool->time_of_last_age_check = curr_time; 9259 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9260 cnt = MLX5_POOL_GET_CNT(pool, i); 9261 age_param = MLX5_CNT_TO_AGE(cnt); 9262 if (__atomic_load_n(&age_param->state, 9263 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9264 continue; 9265 if (cur->data[i].hits != prev->data[i].hits) { 9266 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9267 __ATOMIC_RELAXED); 9268 continue; 9269 } 9270 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9271 time_delta, 9272 __ATOMIC_RELAXED) <= age_param->timeout) 9273 continue; 9274 /** 9275 * Hold the lock first, or if between the 9276 * state AGE_TMOUT and tailq operation the 9277 * release happened, the release procedure 9278 * may delete a non-existent tailq node. 9279 */ 9280 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9281 age_info = GET_PORT_AGE_INFO(priv); 9282 rte_spinlock_lock(&age_info->aged_sl); 9283 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9284 AGE_TMOUT, false, 9285 __ATOMIC_RELAXED, 9286 __ATOMIC_RELAXED)) { 9287 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9288 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9289 } 9290 rte_spinlock_unlock(&age_info->aged_sl); 9291 } 9292 mlx5_age_event_prepare(sh); 9293 } 9294 9295 /** 9296 * Handler for the HW respond about ready values from an asynchronous batch 9297 * query. This function is probably called by the host thread. 9298 * 9299 * @param[in] sh 9300 * The pointer to the shared device context. 9301 * @param[in] async_id 9302 * The Devx async ID. 9303 * @param[in] status 9304 * The status of the completion. 9305 */ 9306 void 9307 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9308 uint64_t async_id, int status) 9309 { 9310 struct mlx5_flow_counter_pool *pool = 9311 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9312 struct mlx5_counter_stats_raw *raw_to_free; 9313 uint8_t query_gen = pool->query_gen ^ 1; 9314 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9315 enum mlx5_counter_type cnt_type = 9316 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9317 MLX5_COUNTER_TYPE_ORIGIN; 9318 9319 if (unlikely(status)) { 9320 raw_to_free = pool->raw_hw; 9321 } else { 9322 raw_to_free = pool->raw; 9323 if (pool->is_aged) 9324 mlx5_flow_aging_check(sh, pool); 9325 rte_spinlock_lock(&pool->sl); 9326 pool->raw = pool->raw_hw; 9327 rte_spinlock_unlock(&pool->sl); 9328 /* Be sure the new raw counters data is updated in memory. */ 9329 rte_io_wmb(); 9330 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9331 rte_spinlock_lock(&cmng->csl[cnt_type]); 9332 TAILQ_CONCAT(&cmng->counters[cnt_type], 9333 &pool->counters[query_gen], next); 9334 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9335 } 9336 } 9337 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9338 pool->raw_hw = NULL; 9339 sh->sws_cmng.pending_queries--; 9340 } 9341 9342 static int 9343 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9344 const struct flow_grp_info *grp_info, 9345 struct rte_flow_error *error) 9346 { 9347 if (grp_info->transfer && grp_info->external && 9348 grp_info->fdb_def_rule) { 9349 if (group == UINT32_MAX) 9350 return rte_flow_error_set 9351 (error, EINVAL, 9352 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9353 NULL, 9354 "group index not supported"); 9355 *table = group + 1; 9356 } else { 9357 *table = group; 9358 } 9359 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9360 return 0; 9361 } 9362 9363 /** 9364 * Translate the rte_flow group index to HW table value. 9365 * 9366 * If tunnel offload is disabled, all group ids converted to flow table 9367 * id using the standard method. 9368 * If tunnel offload is enabled, group id can be converted using the 9369 * standard or tunnel conversion method. Group conversion method 9370 * selection depends on flags in `grp_info` parameter: 9371 * - Internal (grp_info.external == 0) groups conversion uses the 9372 * standard method. 9373 * - Group ids in JUMP action converted with the tunnel conversion. 9374 * - Group id in rule attribute conversion depends on a rule type and 9375 * group id value: 9376 * ** non zero group attributes converted with the tunnel method 9377 * ** zero group attribute in non-tunnel rule is converted using the 9378 * standard method - there's only one root table 9379 * ** zero group attribute in steer tunnel rule is converted with the 9380 * standard method - single root table 9381 * ** zero group attribute in match tunnel rule is a special OvS 9382 * case: that value is used for portability reasons. That group 9383 * id is converted with the tunnel conversion method. 9384 * 9385 * @param[in] dev 9386 * Port device 9387 * @param[in] tunnel 9388 * PMD tunnel offload object 9389 * @param[in] group 9390 * rte_flow group index value. 9391 * @param[out] table 9392 * HW table value. 9393 * @param[in] grp_info 9394 * flags used for conversion 9395 * @param[out] error 9396 * Pointer to error structure. 9397 * 9398 * @return 9399 * 0 on success, a negative errno value otherwise and rte_errno is set. 9400 */ 9401 int 9402 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9403 const struct mlx5_flow_tunnel *tunnel, 9404 uint32_t group, uint32_t *table, 9405 const struct flow_grp_info *grp_info, 9406 struct rte_flow_error *error) 9407 { 9408 int ret; 9409 bool standard_translation; 9410 9411 if (!grp_info->skip_scale && grp_info->external && 9412 group < MLX5_MAX_TABLES_EXTERNAL) 9413 group *= MLX5_FLOW_TABLE_FACTOR; 9414 if (is_tunnel_offload_active(dev)) { 9415 standard_translation = !grp_info->external || 9416 grp_info->std_tbl_fix; 9417 } else { 9418 standard_translation = true; 9419 } 9420 DRV_LOG(DEBUG, 9421 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9422 dev->data->port_id, group, grp_info->transfer, 9423 grp_info->external, grp_info->fdb_def_rule, 9424 standard_translation ? "STANDARD" : "TUNNEL"); 9425 if (standard_translation) 9426 ret = flow_group_to_table(dev->data->port_id, group, table, 9427 grp_info, error); 9428 else 9429 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9430 table, error); 9431 9432 return ret; 9433 } 9434 9435 /** 9436 * Discover availability of metadata reg_c's. 9437 * 9438 * Iteratively use test flows to check availability. 9439 * 9440 * @param[in] dev 9441 * Pointer to the Ethernet device structure. 9442 * 9443 * @return 9444 * 0 on success, a negative errno value otherwise and rte_errno is set. 9445 */ 9446 int 9447 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9448 { 9449 struct mlx5_priv *priv = dev->data->dev_private; 9450 enum modify_reg idx; 9451 int n = 0; 9452 9453 /* reg_c[0] and reg_c[1] are reserved. */ 9454 priv->sh->flow_mreg_c[n++] = REG_C_0; 9455 priv->sh->flow_mreg_c[n++] = REG_C_1; 9456 /* Discover availability of other reg_c's. */ 9457 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9458 struct rte_flow_attr attr = { 9459 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9460 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9461 .ingress = 1, 9462 }; 9463 struct rte_flow_item items[] = { 9464 [0] = { 9465 .type = RTE_FLOW_ITEM_TYPE_END, 9466 }, 9467 }; 9468 struct rte_flow_action actions[] = { 9469 [0] = { 9470 .type = (enum rte_flow_action_type) 9471 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9472 .conf = &(struct mlx5_flow_action_copy_mreg){ 9473 .src = REG_C_1, 9474 .dst = idx, 9475 }, 9476 }, 9477 [1] = { 9478 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9479 .conf = &(struct rte_flow_action_jump){ 9480 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9481 }, 9482 }, 9483 [2] = { 9484 .type = RTE_FLOW_ACTION_TYPE_END, 9485 }, 9486 }; 9487 uint32_t flow_idx; 9488 struct rte_flow *flow; 9489 struct rte_flow_error error; 9490 9491 if (!priv->sh->config.dv_flow_en) 9492 break; 9493 /* Create internal flow, validation skips copy action. */ 9494 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9495 items, actions, false, &error); 9496 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9497 flow_idx); 9498 if (!flow) 9499 continue; 9500 priv->sh->flow_mreg_c[n++] = idx; 9501 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9502 } 9503 for (; n < MLX5_MREG_C_NUM; ++n) 9504 priv->sh->flow_mreg_c[n] = REG_NON; 9505 priv->sh->metadata_regc_check_flag = 1; 9506 return 0; 9507 } 9508 9509 int 9510 save_dump_file(const uint8_t *data, uint32_t size, 9511 uint32_t type, uint64_t id, void *arg, FILE *file) 9512 { 9513 char line[BUF_SIZE]; 9514 uint32_t out = 0; 9515 uint32_t k; 9516 uint32_t actions_num; 9517 struct rte_flow_query_count *count; 9518 9519 memset(line, 0, BUF_SIZE); 9520 switch (type) { 9521 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9522 actions_num = *(uint32_t *)(arg); 9523 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9524 type, id, actions_num); 9525 break; 9526 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9527 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9528 type, id); 9529 break; 9530 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9531 count = (struct rte_flow_query_count *)arg; 9532 fprintf(file, 9533 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9534 type, id, count->hits, count->bytes); 9535 return 0; 9536 default: 9537 return -1; 9538 } 9539 9540 for (k = 0; k < size; k++) { 9541 /* Make sure we do not overrun the line buffer length. */ 9542 if (out >= BUF_SIZE - 4) { 9543 line[out] = '\0'; 9544 break; 9545 } 9546 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9547 (data[k]) & 0xff); 9548 } 9549 fprintf(file, "%s\n", line); 9550 return 0; 9551 } 9552 9553 int 9554 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9555 struct rte_flow_query_count *count, struct rte_flow_error *error) 9556 { 9557 struct rte_flow_action action[2]; 9558 enum mlx5_flow_drv_type ftype; 9559 const struct mlx5_flow_driver_ops *fops; 9560 9561 if (!flow) { 9562 return rte_flow_error_set(error, ENOENT, 9563 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9564 NULL, 9565 "invalid flow handle"); 9566 } 9567 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9568 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9569 if (flow->counter) { 9570 memset(count, 0, sizeof(struct rte_flow_query_count)); 9571 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9572 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9573 ftype < MLX5_FLOW_TYPE_MAX); 9574 fops = flow_get_drv_ops(ftype); 9575 return fops->query(dev, flow, action, count, error); 9576 } 9577 return -1; 9578 } 9579 9580 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9581 /** 9582 * Dump flow ipool data to file 9583 * 9584 * @param[in] dev 9585 * The pointer to Ethernet device. 9586 * @param[in] file 9587 * A pointer to a file for output. 9588 * @param[out] error 9589 * Perform verbose error reporting if not NULL. PMDs initialize this 9590 * structure in case of error only. 9591 * @return 9592 * 0 on success, a negative value otherwise. 9593 */ 9594 int 9595 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9596 struct rte_flow *flow, FILE *file, 9597 struct rte_flow_error *error) 9598 { 9599 struct mlx5_priv *priv = dev->data->dev_private; 9600 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9601 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9602 uint32_t handle_idx; 9603 struct mlx5_flow_handle *dh; 9604 struct rte_flow_query_count count; 9605 uint32_t actions_num; 9606 const uint8_t *data; 9607 size_t size; 9608 uint64_t id; 9609 uint32_t type; 9610 void *action = NULL; 9611 9612 if (!flow) { 9613 return rte_flow_error_set(error, ENOENT, 9614 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9615 NULL, 9616 "invalid flow handle"); 9617 } 9618 handle_idx = flow->dev_handles; 9619 /* query counter */ 9620 if (flow->counter && 9621 (!mlx5_counter_query(dev, flow->counter, false, 9622 &count.hits, &count.bytes, &action)) && action) { 9623 id = (uint64_t)(uintptr_t)action; 9624 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9625 save_dump_file(NULL, 0, type, 9626 id, (void *)&count, file); 9627 } 9628 9629 while (handle_idx) { 9630 dh = mlx5_ipool_get(priv->sh->ipool 9631 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9632 if (!dh) 9633 continue; 9634 handle_idx = dh->next.next; 9635 9636 /* Get modify_hdr and encap_decap buf from ipools. */ 9637 encap_decap = NULL; 9638 modify_hdr = dh->dvh.modify_hdr; 9639 9640 if (dh->dvh.rix_encap_decap) { 9641 encap_decap = mlx5_ipool_get(priv->sh->ipool 9642 [MLX5_IPOOL_DECAP_ENCAP], 9643 dh->dvh.rix_encap_decap); 9644 } 9645 if (modify_hdr) { 9646 data = (const uint8_t *)modify_hdr->actions; 9647 size = (size_t)(modify_hdr->actions_num) * 8; 9648 id = (uint64_t)(uintptr_t)modify_hdr->action; 9649 actions_num = modify_hdr->actions_num; 9650 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9651 save_dump_file(data, size, type, id, 9652 (void *)(&actions_num), file); 9653 } 9654 if (encap_decap) { 9655 data = encap_decap->buf; 9656 size = encap_decap->size; 9657 id = (uint64_t)(uintptr_t)encap_decap->action; 9658 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9659 save_dump_file(data, size, type, 9660 id, NULL, file); 9661 } 9662 } 9663 return 0; 9664 } 9665 9666 /** 9667 * Dump all flow's encap_decap/modify_hdr/counter data to file 9668 * 9669 * @param[in] dev 9670 * The pointer to Ethernet device. 9671 * @param[in] file 9672 * A pointer to a file for output. 9673 * @param[out] error 9674 * Perform verbose error reporting if not NULL. PMDs initialize this 9675 * structure in case of error only. 9676 * @return 9677 * 0 on success, a negative value otherwise. 9678 */ 9679 static int 9680 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9681 FILE *file, struct rte_flow_error *error __rte_unused) 9682 { 9683 struct mlx5_priv *priv = dev->data->dev_private; 9684 struct mlx5_dev_ctx_shared *sh = priv->sh; 9685 struct mlx5_hlist *h; 9686 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9687 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9688 struct rte_flow_query_count count; 9689 uint32_t actions_num; 9690 const uint8_t *data; 9691 size_t size; 9692 uint64_t id; 9693 uint32_t type; 9694 uint32_t i; 9695 uint32_t j; 9696 struct mlx5_list_inconst *l_inconst; 9697 struct mlx5_list_entry *e; 9698 int lcore_index; 9699 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9700 uint32_t max; 9701 void *action; 9702 9703 /* encap_decap hlist is lcore_share, get global core cache. */ 9704 i = MLX5_LIST_GLOBAL; 9705 h = sh->encaps_decaps; 9706 if (h) { 9707 for (j = 0; j <= h->mask; j++) { 9708 l_inconst = &h->buckets[j].l; 9709 if (!l_inconst || !l_inconst->cache[i]) 9710 continue; 9711 9712 e = LIST_FIRST(&l_inconst->cache[i]->h); 9713 while (e) { 9714 encap_decap = 9715 (struct mlx5_flow_dv_encap_decap_resource *)e; 9716 data = encap_decap->buf; 9717 size = encap_decap->size; 9718 id = (uint64_t)(uintptr_t)encap_decap->action; 9719 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9720 save_dump_file(data, size, type, 9721 id, NULL, file); 9722 e = LIST_NEXT(e, next); 9723 } 9724 } 9725 } 9726 9727 /* get modify_hdr */ 9728 h = sh->modify_cmds; 9729 if (h) { 9730 lcore_index = rte_lcore_index(rte_lcore_id()); 9731 if (unlikely(lcore_index == -1)) { 9732 lcore_index = MLX5_LIST_NLCORE; 9733 rte_spinlock_lock(&h->l_const.lcore_lock); 9734 } 9735 i = lcore_index; 9736 9737 for (j = 0; j <= h->mask; j++) { 9738 l_inconst = &h->buckets[j].l; 9739 if (!l_inconst || !l_inconst->cache[i]) 9740 continue; 9741 9742 e = LIST_FIRST(&l_inconst->cache[i]->h); 9743 while (e) { 9744 modify_hdr = 9745 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9746 data = (const uint8_t *)modify_hdr->actions; 9747 size = (size_t)(modify_hdr->actions_num) * 8; 9748 actions_num = modify_hdr->actions_num; 9749 id = (uint64_t)(uintptr_t)modify_hdr->action; 9750 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9751 save_dump_file(data, size, type, id, 9752 (void *)(&actions_num), file); 9753 e = LIST_NEXT(e, next); 9754 } 9755 } 9756 9757 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9758 rte_spinlock_unlock(&h->l_const.lcore_lock); 9759 } 9760 9761 /* get counter */ 9762 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 9763 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9764 for (j = 1; j <= max; j++) { 9765 action = NULL; 9766 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9767 &count.bytes, &action)) && action) { 9768 id = (uint64_t)(uintptr_t)action; 9769 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9770 save_dump_file(NULL, 0, type, 9771 id, (void *)&count, file); 9772 } 9773 } 9774 return 0; 9775 } 9776 #endif 9777 9778 /** 9779 * Dump flow raw hw data to file 9780 * 9781 * @param[in] dev 9782 * The pointer to Ethernet device. 9783 * @param[in] file 9784 * A pointer to a file for output. 9785 * @param[out] error 9786 * Perform verbose error reporting if not NULL. PMDs initialize this 9787 * structure in case of error only. 9788 * @return 9789 * 0 on success, a negative value otherwise. 9790 */ 9791 int 9792 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9793 FILE *file, 9794 struct rte_flow_error *error __rte_unused) 9795 { 9796 struct mlx5_priv *priv = dev->data->dev_private; 9797 struct mlx5_dev_ctx_shared *sh = priv->sh; 9798 uint32_t handle_idx; 9799 int ret; 9800 struct mlx5_flow_handle *dh; 9801 struct rte_flow *flow; 9802 9803 if (!sh->config.dv_flow_en) { 9804 if (fputs("device dv flow disabled\n", file) <= 0) 9805 return -errno; 9806 return -ENOTSUP; 9807 } 9808 9809 /* dump all */ 9810 if (!flow_idx) { 9811 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9812 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9813 return -EINVAL; 9814 #endif 9815 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9816 sh->rx_domain, 9817 sh->tx_domain, file); 9818 } 9819 /* dump one */ 9820 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9821 (uintptr_t)(void *)flow_idx); 9822 if (!flow) 9823 return -EINVAL; 9824 9825 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9826 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9827 #endif 9828 handle_idx = flow->dev_handles; 9829 while (handle_idx) { 9830 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9831 handle_idx); 9832 if (!dh) 9833 return -ENOENT; 9834 if (dh->drv_flow) { 9835 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9836 file); 9837 if (ret) 9838 return -ENOENT; 9839 } 9840 handle_idx = dh->next.next; 9841 } 9842 return 0; 9843 } 9844 9845 /** 9846 * Get aged-out flows. 9847 * 9848 * @param[in] dev 9849 * Pointer to the Ethernet device structure. 9850 * @param[in] context 9851 * The address of an array of pointers to the aged-out flows contexts. 9852 * @param[in] nb_countexts 9853 * The length of context array pointers. 9854 * @param[out] error 9855 * Perform verbose error reporting if not NULL. Initialized in case of 9856 * error only. 9857 * 9858 * @return 9859 * how many contexts get in success, otherwise negative errno value. 9860 * if nb_contexts is 0, return the amount of all aged contexts. 9861 * if nb_contexts is not 0 , return the amount of aged flows reported 9862 * in the context array. 9863 */ 9864 int 9865 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9866 uint32_t nb_contexts, struct rte_flow_error *error) 9867 { 9868 const struct mlx5_flow_driver_ops *fops; 9869 struct rte_flow_attr attr = { .transfer = 0 }; 9870 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, &attr); 9871 9872 if (type == MLX5_FLOW_TYPE_DV || type == MLX5_FLOW_TYPE_HW) { 9873 fops = flow_get_drv_ops(type); 9874 return fops->get_aged_flows(dev, contexts, nb_contexts, error); 9875 } 9876 DRV_LOG(ERR, "port %u get aged flows is not supported.", 9877 dev->data->port_id); 9878 return -ENOTSUP; 9879 } 9880 9881 /** 9882 * Get aged-out flows per HWS queue. 9883 * 9884 * @param[in] dev 9885 * Pointer to the Ethernet device structure. 9886 * @param[in] queue_id 9887 * Flow queue to query. 9888 * @param[in] context 9889 * The address of an array of pointers to the aged-out flows contexts. 9890 * @param[in] nb_countexts 9891 * The length of context array pointers. 9892 * @param[out] error 9893 * Perform verbose error reporting if not NULL. Initialized in case of 9894 * error only. 9895 * 9896 * @return 9897 * how many contexts get in success, otherwise negative errno value. 9898 * if nb_contexts is 0, return the amount of all aged contexts. 9899 * if nb_contexts is not 0 , return the amount of aged flows reported 9900 * in the context array. 9901 */ 9902 int 9903 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 9904 void **contexts, uint32_t nb_contexts, 9905 struct rte_flow_error *error) 9906 { 9907 const struct mlx5_flow_driver_ops *fops; 9908 struct rte_flow_attr attr = { 0 }; 9909 9910 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 9911 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9912 return fops->get_q_aged_flows(dev, queue_id, contexts, 9913 nb_contexts, error); 9914 } 9915 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 9916 dev->data->port_id, queue_id); 9917 return rte_flow_error_set(error, ENOTSUP, 9918 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9919 "get Q aged flows with incorrect steering mode"); 9920 } 9921 9922 /* Wrapper for driver action_validate op callback */ 9923 static int 9924 flow_drv_action_validate(struct rte_eth_dev *dev, 9925 const struct rte_flow_indir_action_conf *conf, 9926 const struct rte_flow_action *action, 9927 const struct mlx5_flow_driver_ops *fops, 9928 struct rte_flow_error *error) 9929 { 9930 static const char err_msg[] = "indirect action validation unsupported"; 9931 9932 if (!fops->action_validate) { 9933 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9934 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9935 NULL, err_msg); 9936 return -rte_errno; 9937 } 9938 return fops->action_validate(dev, conf, action, error); 9939 } 9940 9941 /** 9942 * Destroys the shared action by handle. 9943 * 9944 * @param dev 9945 * Pointer to Ethernet device structure. 9946 * @param[in] handle 9947 * Handle for the indirect action object to be destroyed. 9948 * @param[out] error 9949 * Perform verbose error reporting if not NULL. PMDs initialize this 9950 * structure in case of error only. 9951 * 9952 * @return 9953 * 0 on success, a negative errno value otherwise and rte_errno is set. 9954 * 9955 * @note: wrapper for driver action_create op callback. 9956 */ 9957 static int 9958 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9959 struct rte_flow_action_handle *handle, 9960 struct rte_flow_error *error) 9961 { 9962 static const char err_msg[] = "indirect action destruction unsupported"; 9963 struct rte_flow_attr attr = { .transfer = 0 }; 9964 const struct mlx5_flow_driver_ops *fops = 9965 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9966 9967 if (!fops->action_destroy) { 9968 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9969 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9970 NULL, err_msg); 9971 return -rte_errno; 9972 } 9973 return fops->action_destroy(dev, handle, error); 9974 } 9975 9976 /* Wrapper for driver action_destroy op callback */ 9977 static int 9978 flow_drv_action_update(struct rte_eth_dev *dev, 9979 struct rte_flow_action_handle *handle, 9980 const void *update, 9981 const struct mlx5_flow_driver_ops *fops, 9982 struct rte_flow_error *error) 9983 { 9984 static const char err_msg[] = "indirect action update unsupported"; 9985 9986 if (!fops->action_update) { 9987 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9988 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9989 NULL, err_msg); 9990 return -rte_errno; 9991 } 9992 return fops->action_update(dev, handle, update, error); 9993 } 9994 9995 /* Wrapper for driver action_destroy op callback */ 9996 static int 9997 flow_drv_action_query(struct rte_eth_dev *dev, 9998 const struct rte_flow_action_handle *handle, 9999 void *data, 10000 const struct mlx5_flow_driver_ops *fops, 10001 struct rte_flow_error *error) 10002 { 10003 static const char err_msg[] = "indirect action query unsupported"; 10004 10005 if (!fops->action_query) { 10006 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10007 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10008 NULL, err_msg); 10009 return -rte_errno; 10010 } 10011 return fops->action_query(dev, handle, data, error); 10012 } 10013 10014 /** 10015 * Create indirect action for reuse in multiple flow rules. 10016 * 10017 * @param dev 10018 * Pointer to Ethernet device structure. 10019 * @param conf 10020 * Pointer to indirect action object configuration. 10021 * @param[in] action 10022 * Action configuration for indirect action object creation. 10023 * @param[out] error 10024 * Perform verbose error reporting if not NULL. PMDs initialize this 10025 * structure in case of error only. 10026 * @return 10027 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10028 */ 10029 static struct rte_flow_action_handle * 10030 mlx5_action_handle_create(struct rte_eth_dev *dev, 10031 const struct rte_flow_indir_action_conf *conf, 10032 const struct rte_flow_action *action, 10033 struct rte_flow_error *error) 10034 { 10035 static const char err_msg[] = "indirect action creation unsupported"; 10036 struct rte_flow_attr attr = { .transfer = 0 }; 10037 const struct mlx5_flow_driver_ops *fops = 10038 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10039 10040 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10041 return NULL; 10042 if (!fops->action_create) { 10043 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10044 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10045 NULL, err_msg); 10046 return NULL; 10047 } 10048 return fops->action_create(dev, conf, action, error); 10049 } 10050 10051 /** 10052 * Updates inplace the indirect action configuration pointed by *handle* 10053 * with the configuration provided as *update* argument. 10054 * The update of the indirect action configuration effects all flow rules 10055 * reusing the action via handle. 10056 * 10057 * @param dev 10058 * Pointer to Ethernet device structure. 10059 * @param[in] handle 10060 * Handle for the indirect action to be updated. 10061 * @param[in] update 10062 * Action specification used to modify the action pointed by handle. 10063 * *update* could be of same type with the action pointed by the *handle* 10064 * handle argument, or some other structures like a wrapper, depending on 10065 * the indirect action type. 10066 * @param[out] error 10067 * Perform verbose error reporting if not NULL. PMDs initialize this 10068 * structure in case of error only. 10069 * 10070 * @return 10071 * 0 on success, a negative errno value otherwise and rte_errno is set. 10072 */ 10073 static int 10074 mlx5_action_handle_update(struct rte_eth_dev *dev, 10075 struct rte_flow_action_handle *handle, 10076 const void *update, 10077 struct rte_flow_error *error) 10078 { 10079 struct rte_flow_attr attr = { .transfer = 0 }; 10080 const struct mlx5_flow_driver_ops *fops = 10081 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10082 int ret; 10083 10084 ret = flow_drv_action_validate(dev, NULL, 10085 (const struct rte_flow_action *)update, fops, error); 10086 if (ret) 10087 return ret; 10088 return flow_drv_action_update(dev, handle, update, fops, 10089 error); 10090 } 10091 10092 /** 10093 * Query the indirect action by handle. 10094 * 10095 * This function allows retrieving action-specific data such as counters. 10096 * Data is gathered by special action which may be present/referenced in 10097 * more than one flow rule definition. 10098 * 10099 * see @RTE_FLOW_ACTION_TYPE_COUNT 10100 * 10101 * @param dev 10102 * Pointer to Ethernet device structure. 10103 * @param[in] handle 10104 * Handle for the indirect action to query. 10105 * @param[in, out] data 10106 * Pointer to storage for the associated query data type. 10107 * @param[out] error 10108 * Perform verbose error reporting if not NULL. PMDs initialize this 10109 * structure in case of error only. 10110 * 10111 * @return 10112 * 0 on success, a negative errno value otherwise and rte_errno is set. 10113 */ 10114 static int 10115 mlx5_action_handle_query(struct rte_eth_dev *dev, 10116 const struct rte_flow_action_handle *handle, 10117 void *data, 10118 struct rte_flow_error *error) 10119 { 10120 struct rte_flow_attr attr = { .transfer = 0 }; 10121 const struct mlx5_flow_driver_ops *fops = 10122 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10123 10124 return flow_drv_action_query(dev, handle, data, fops, error); 10125 } 10126 10127 /** 10128 * Destroy all indirect actions (shared RSS). 10129 * 10130 * @param dev 10131 * Pointer to Ethernet device. 10132 * 10133 * @return 10134 * 0 on success, a negative errno value otherwise and rte_errno is set. 10135 */ 10136 int 10137 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10138 { 10139 struct rte_flow_error error; 10140 struct mlx5_priv *priv = dev->data->dev_private; 10141 struct mlx5_shared_action_rss *shared_rss; 10142 int ret = 0; 10143 uint32_t idx; 10144 10145 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10146 priv->rss_shared_actions, idx, shared_rss, next) { 10147 ret |= mlx5_action_handle_destroy(dev, 10148 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10149 } 10150 return ret; 10151 } 10152 10153 /** 10154 * Validate existing indirect actions against current device configuration 10155 * and attach them to device resources. 10156 * 10157 * @param dev 10158 * Pointer to Ethernet device. 10159 * 10160 * @return 10161 * 0 on success, a negative errno value otherwise and rte_errno is set. 10162 */ 10163 int 10164 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10165 { 10166 struct mlx5_priv *priv = dev->data->dev_private; 10167 int ret = 0; 10168 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10169 10170 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10171 const char *message; 10172 uint32_t queue_idx; 10173 10174 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10175 ind_tbl->queues_n, 10176 &message, &queue_idx); 10177 if (ret != 0) { 10178 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10179 dev->data->port_id, ind_tbl->queues[queue_idx], 10180 message); 10181 break; 10182 } 10183 } 10184 if (ret != 0) 10185 return ret; 10186 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10187 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10188 if (ret != 0) { 10189 DRV_LOG(ERR, "Port %u could not attach " 10190 "indirection table obj %p", 10191 dev->data->port_id, (void *)ind_tbl); 10192 goto error; 10193 } 10194 } 10195 10196 return 0; 10197 error: 10198 ind_tbl_last = ind_tbl; 10199 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10200 if (ind_tbl == ind_tbl_last) 10201 break; 10202 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10203 DRV_LOG(CRIT, "Port %u could not detach " 10204 "indirection table obj %p on rollback", 10205 dev->data->port_id, (void *)ind_tbl); 10206 } 10207 return ret; 10208 } 10209 10210 /** 10211 * Detach indirect actions of the device from its resources. 10212 * 10213 * @param dev 10214 * Pointer to Ethernet device. 10215 * 10216 * @return 10217 * 0 on success, a negative errno value otherwise and rte_errno is set. 10218 */ 10219 int 10220 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10221 { 10222 struct mlx5_priv *priv = dev->data->dev_private; 10223 int ret = 0; 10224 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10225 10226 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10227 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10228 if (ret != 0) { 10229 DRV_LOG(ERR, "Port %u could not detach " 10230 "indirection table obj %p", 10231 dev->data->port_id, (void *)ind_tbl); 10232 goto error; 10233 } 10234 } 10235 return 0; 10236 error: 10237 ind_tbl_last = ind_tbl; 10238 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10239 if (ind_tbl == ind_tbl_last) 10240 break; 10241 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10242 DRV_LOG(CRIT, "Port %u could not attach " 10243 "indirection table obj %p on rollback", 10244 dev->data->port_id, (void *)ind_tbl); 10245 } 10246 return ret; 10247 } 10248 10249 #ifndef HAVE_MLX5DV_DR 10250 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10251 #else 10252 #define MLX5_DOMAIN_SYNC_FLOW \ 10253 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10254 #endif 10255 10256 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10257 { 10258 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10259 const struct mlx5_flow_driver_ops *fops; 10260 int ret; 10261 struct rte_flow_attr attr = { .transfer = 0 }; 10262 10263 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10264 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10265 if (ret > 0) 10266 ret = -ret; 10267 return ret; 10268 } 10269 10270 const struct mlx5_flow_tunnel * 10271 mlx5_get_tof(const struct rte_flow_item *item, 10272 const struct rte_flow_action *action, 10273 enum mlx5_tof_rule_type *rule_type) 10274 { 10275 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10276 if (item->type == (typeof(item->type)) 10277 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10278 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10279 return flow_items_to_tunnel(item); 10280 } 10281 } 10282 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10283 if (action->type == (typeof(action->type)) 10284 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10285 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10286 return flow_actions_to_tunnel(action); 10287 } 10288 } 10289 return NULL; 10290 } 10291 10292 /** 10293 * tunnel offload functionality is defined for DV environment only 10294 */ 10295 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10296 __extension__ 10297 union tunnel_offload_mark { 10298 uint32_t val; 10299 struct { 10300 uint32_t app_reserve:8; 10301 uint32_t table_id:15; 10302 uint32_t transfer:1; 10303 uint32_t _unused_:8; 10304 }; 10305 }; 10306 10307 static bool 10308 mlx5_access_tunnel_offload_db 10309 (struct rte_eth_dev *dev, 10310 bool (*match)(struct rte_eth_dev *, 10311 struct mlx5_flow_tunnel *, const void *), 10312 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10313 void (*miss)(struct rte_eth_dev *, void *), 10314 void *ctx, bool lock_op); 10315 10316 static int 10317 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10318 struct rte_flow *flow, 10319 const struct rte_flow_attr *attr, 10320 const struct rte_flow_action *app_actions, 10321 uint32_t flow_idx, 10322 const struct mlx5_flow_tunnel *tunnel, 10323 struct tunnel_default_miss_ctx *ctx, 10324 struct rte_flow_error *error) 10325 { 10326 struct mlx5_priv *priv = dev->data->dev_private; 10327 struct mlx5_flow *dev_flow; 10328 struct rte_flow_attr miss_attr = *attr; 10329 const struct rte_flow_item miss_items[2] = { 10330 { 10331 .type = RTE_FLOW_ITEM_TYPE_ETH, 10332 .spec = NULL, 10333 .last = NULL, 10334 .mask = NULL 10335 }, 10336 { 10337 .type = RTE_FLOW_ITEM_TYPE_END, 10338 .spec = NULL, 10339 .last = NULL, 10340 .mask = NULL 10341 } 10342 }; 10343 union tunnel_offload_mark mark_id; 10344 struct rte_flow_action_mark miss_mark; 10345 struct rte_flow_action miss_actions[3] = { 10346 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10347 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10348 }; 10349 const struct rte_flow_action_jump *jump_data; 10350 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10351 struct flow_grp_info grp_info = { 10352 .external = 1, 10353 .transfer = attr->transfer, 10354 .fdb_def_rule = !!priv->fdb_def_rule, 10355 .std_tbl_fix = 0, 10356 }; 10357 int ret; 10358 10359 if (!attr->transfer) { 10360 uint32_t q_size; 10361 10362 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10363 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10364 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10365 0, SOCKET_ID_ANY); 10366 if (!ctx->queue) 10367 return rte_flow_error_set 10368 (error, ENOMEM, 10369 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10370 NULL, "invalid default miss RSS"); 10371 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10372 ctx->action_rss.level = 0, 10373 ctx->action_rss.types = priv->rss_conf.rss_hf, 10374 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10375 ctx->action_rss.queue_num = priv->reta_idx_n, 10376 ctx->action_rss.key = priv->rss_conf.rss_key, 10377 ctx->action_rss.queue = ctx->queue; 10378 if (!priv->reta_idx_n || !priv->rxqs_n) 10379 return rte_flow_error_set 10380 (error, EINVAL, 10381 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10382 NULL, "invalid port configuration"); 10383 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10384 ctx->action_rss.types = 0; 10385 for (i = 0; i != priv->reta_idx_n; ++i) 10386 ctx->queue[i] = (*priv->reta_idx)[i]; 10387 } else { 10388 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10389 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10390 } 10391 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10392 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10393 jump_data = app_actions->conf; 10394 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10395 miss_attr.group = jump_data->group; 10396 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10397 &flow_table, &grp_info, error); 10398 if (ret) 10399 return rte_flow_error_set(error, EINVAL, 10400 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10401 NULL, "invalid tunnel id"); 10402 mark_id.app_reserve = 0; 10403 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10404 mark_id.transfer = !!attr->transfer; 10405 mark_id._unused_ = 0; 10406 miss_mark.id = mark_id.val; 10407 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10408 miss_items, miss_actions, flow_idx, error); 10409 if (!dev_flow) 10410 return -rte_errno; 10411 dev_flow->flow = flow; 10412 dev_flow->external = true; 10413 dev_flow->tunnel = tunnel; 10414 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10415 /* Subflow object was created, we must include one in the list. */ 10416 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10417 dev_flow->handle, next); 10418 DRV_LOG(DEBUG, 10419 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10420 dev->data->port_id, tunnel->app_tunnel.type, 10421 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10422 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10423 miss_actions, error); 10424 if (!ret) 10425 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10426 error); 10427 10428 return ret; 10429 } 10430 10431 static const struct mlx5_flow_tbl_data_entry * 10432 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10433 { 10434 struct mlx5_priv *priv = dev->data->dev_private; 10435 struct mlx5_dev_ctx_shared *sh = priv->sh; 10436 struct mlx5_list_entry *he; 10437 union tunnel_offload_mark mbits = { .val = mark }; 10438 union mlx5_flow_tbl_key table_key = { 10439 { 10440 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10441 .id = 0, 10442 .reserved = 0, 10443 .dummy = 0, 10444 .is_fdb = !!mbits.transfer, 10445 .is_egress = 0, 10446 } 10447 }; 10448 struct mlx5_flow_cb_ctx ctx = { 10449 .data = &table_key.v64, 10450 }; 10451 10452 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10453 return he ? 10454 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10455 } 10456 10457 static void 10458 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10459 struct mlx5_list_entry *entry) 10460 { 10461 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10462 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10463 10464 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10465 tunnel_flow_tbl_to_id(tte->flow_table)); 10466 mlx5_free(tte); 10467 } 10468 10469 static int 10470 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10471 struct mlx5_list_entry *entry, void *cb_ctx) 10472 { 10473 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10474 union tunnel_tbl_key tbl = { 10475 .val = *(uint64_t *)(ctx->data), 10476 }; 10477 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10478 10479 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10480 } 10481 10482 static struct mlx5_list_entry * 10483 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10484 { 10485 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10486 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10487 struct tunnel_tbl_entry *tte; 10488 union tunnel_tbl_key tbl = { 10489 .val = *(uint64_t *)(ctx->data), 10490 }; 10491 10492 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10493 sizeof(*tte), 0, 10494 SOCKET_ID_ANY); 10495 if (!tte) 10496 goto err; 10497 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10498 &tte->flow_table); 10499 if (tte->flow_table >= MLX5_MAX_TABLES) { 10500 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10501 tte->flow_table); 10502 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10503 tte->flow_table); 10504 goto err; 10505 } else if (!tte->flow_table) { 10506 goto err; 10507 } 10508 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10509 tte->tunnel_id = tbl.tunnel_id; 10510 tte->group = tbl.group; 10511 return &tte->hash; 10512 err: 10513 if (tte) 10514 mlx5_free(tte); 10515 return NULL; 10516 } 10517 10518 static struct mlx5_list_entry * 10519 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10520 struct mlx5_list_entry *oentry, 10521 void *cb_ctx __rte_unused) 10522 { 10523 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10524 0, SOCKET_ID_ANY); 10525 10526 if (!tte) 10527 return NULL; 10528 memcpy(tte, oentry, sizeof(*tte)); 10529 return &tte->hash; 10530 } 10531 10532 static void 10533 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10534 struct mlx5_list_entry *entry) 10535 { 10536 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10537 10538 mlx5_free(tte); 10539 } 10540 10541 static uint32_t 10542 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10543 const struct mlx5_flow_tunnel *tunnel, 10544 uint32_t group, uint32_t *table, 10545 struct rte_flow_error *error) 10546 { 10547 struct mlx5_list_entry *he; 10548 struct tunnel_tbl_entry *tte; 10549 union tunnel_tbl_key key = { 10550 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10551 .group = group 10552 }; 10553 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10554 struct mlx5_hlist *group_hash; 10555 struct mlx5_flow_cb_ctx ctx = { 10556 .data = &key.val, 10557 }; 10558 10559 group_hash = tunnel ? tunnel->groups : thub->groups; 10560 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10561 if (!he) 10562 return rte_flow_error_set(error, EINVAL, 10563 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10564 NULL, 10565 "tunnel group index not supported"); 10566 tte = container_of(he, typeof(*tte), hash); 10567 *table = tte->flow_table; 10568 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10569 dev->data->port_id, key.tunnel_id, group, *table); 10570 return 0; 10571 } 10572 10573 static void 10574 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10575 struct mlx5_flow_tunnel *tunnel) 10576 { 10577 struct mlx5_priv *priv = dev->data->dev_private; 10578 struct mlx5_indexed_pool *ipool; 10579 10580 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10581 dev->data->port_id, tunnel->tunnel_id); 10582 LIST_REMOVE(tunnel, chain); 10583 mlx5_hlist_destroy(tunnel->groups); 10584 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10585 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10586 } 10587 10588 static bool 10589 mlx5_access_tunnel_offload_db 10590 (struct rte_eth_dev *dev, 10591 bool (*match)(struct rte_eth_dev *, 10592 struct mlx5_flow_tunnel *, const void *), 10593 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10594 void (*miss)(struct rte_eth_dev *, void *), 10595 void *ctx, bool lock_op) 10596 { 10597 bool verdict = false; 10598 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10599 struct mlx5_flow_tunnel *tunnel; 10600 10601 rte_spinlock_lock(&thub->sl); 10602 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10603 verdict = match(dev, tunnel, (const void *)ctx); 10604 if (verdict) 10605 break; 10606 } 10607 if (!lock_op) 10608 rte_spinlock_unlock(&thub->sl); 10609 if (verdict && hit) 10610 hit(dev, tunnel, ctx); 10611 if (!verdict && miss) 10612 miss(dev, ctx); 10613 if (lock_op) 10614 rte_spinlock_unlock(&thub->sl); 10615 10616 return verdict; 10617 } 10618 10619 struct tunnel_db_find_tunnel_id_ctx { 10620 uint32_t tunnel_id; 10621 struct mlx5_flow_tunnel *tunnel; 10622 }; 10623 10624 static bool 10625 find_tunnel_id_match(struct rte_eth_dev *dev, 10626 struct mlx5_flow_tunnel *tunnel, const void *x) 10627 { 10628 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10629 10630 RTE_SET_USED(dev); 10631 return tunnel->tunnel_id == ctx->tunnel_id; 10632 } 10633 10634 static void 10635 find_tunnel_id_hit(struct rte_eth_dev *dev, 10636 struct mlx5_flow_tunnel *tunnel, void *x) 10637 { 10638 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10639 RTE_SET_USED(dev); 10640 ctx->tunnel = tunnel; 10641 } 10642 10643 static struct mlx5_flow_tunnel * 10644 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10645 { 10646 struct tunnel_db_find_tunnel_id_ctx ctx = { 10647 .tunnel_id = id, 10648 }; 10649 10650 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10651 find_tunnel_id_hit, NULL, &ctx, true); 10652 10653 return ctx.tunnel; 10654 } 10655 10656 static struct mlx5_flow_tunnel * 10657 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10658 const struct rte_flow_tunnel *app_tunnel) 10659 { 10660 struct mlx5_priv *priv = dev->data->dev_private; 10661 struct mlx5_indexed_pool *ipool; 10662 struct mlx5_flow_tunnel *tunnel; 10663 uint32_t id; 10664 10665 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10666 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10667 if (!tunnel) 10668 return NULL; 10669 if (id >= MLX5_MAX_TUNNELS) { 10670 mlx5_ipool_free(ipool, id); 10671 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10672 return NULL; 10673 } 10674 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10675 priv->sh, 10676 mlx5_flow_tunnel_grp2tbl_create_cb, 10677 mlx5_flow_tunnel_grp2tbl_match_cb, 10678 mlx5_flow_tunnel_grp2tbl_remove_cb, 10679 mlx5_flow_tunnel_grp2tbl_clone_cb, 10680 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10681 if (!tunnel->groups) { 10682 mlx5_ipool_free(ipool, id); 10683 return NULL; 10684 } 10685 /* initiate new PMD tunnel */ 10686 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10687 tunnel->tunnel_id = id; 10688 tunnel->action.type = (typeof(tunnel->action.type)) 10689 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10690 tunnel->action.conf = tunnel; 10691 tunnel->item.type = (typeof(tunnel->item.type)) 10692 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10693 tunnel->item.spec = tunnel; 10694 tunnel->item.last = NULL; 10695 tunnel->item.mask = NULL; 10696 10697 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10698 dev->data->port_id, tunnel->tunnel_id); 10699 10700 return tunnel; 10701 } 10702 10703 struct tunnel_db_get_tunnel_ctx { 10704 const struct rte_flow_tunnel *app_tunnel; 10705 struct mlx5_flow_tunnel *tunnel; 10706 }; 10707 10708 static bool get_tunnel_match(struct rte_eth_dev *dev, 10709 struct mlx5_flow_tunnel *tunnel, const void *x) 10710 { 10711 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10712 10713 RTE_SET_USED(dev); 10714 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10715 sizeof(*ctx->app_tunnel)); 10716 } 10717 10718 static void get_tunnel_hit(struct rte_eth_dev *dev, 10719 struct mlx5_flow_tunnel *tunnel, void *x) 10720 { 10721 /* called under tunnel spinlock protection */ 10722 struct tunnel_db_get_tunnel_ctx *ctx = x; 10723 10724 RTE_SET_USED(dev); 10725 tunnel->refctn++; 10726 ctx->tunnel = tunnel; 10727 } 10728 10729 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10730 { 10731 /* called under tunnel spinlock protection */ 10732 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10733 struct tunnel_db_get_tunnel_ctx *ctx = x; 10734 10735 rte_spinlock_unlock(&thub->sl); 10736 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10737 rte_spinlock_lock(&thub->sl); 10738 if (ctx->tunnel) { 10739 ctx->tunnel->refctn = 1; 10740 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10741 } 10742 } 10743 10744 10745 static int 10746 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10747 const struct rte_flow_tunnel *app_tunnel, 10748 struct mlx5_flow_tunnel **tunnel) 10749 { 10750 struct tunnel_db_get_tunnel_ctx ctx = { 10751 .app_tunnel = app_tunnel, 10752 }; 10753 10754 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10755 get_tunnel_miss, &ctx, true); 10756 *tunnel = ctx.tunnel; 10757 return ctx.tunnel ? 0 : -ENOMEM; 10758 } 10759 10760 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10761 { 10762 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10763 10764 if (!thub) 10765 return; 10766 if (!LIST_EMPTY(&thub->tunnels)) 10767 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10768 mlx5_hlist_destroy(thub->groups); 10769 mlx5_free(thub); 10770 } 10771 10772 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10773 { 10774 int err; 10775 struct mlx5_flow_tunnel_hub *thub; 10776 10777 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10778 0, SOCKET_ID_ANY); 10779 if (!thub) 10780 return -ENOMEM; 10781 LIST_INIT(&thub->tunnels); 10782 rte_spinlock_init(&thub->sl); 10783 thub->groups = mlx5_hlist_create("flow groups", 64, 10784 false, true, sh, 10785 mlx5_flow_tunnel_grp2tbl_create_cb, 10786 mlx5_flow_tunnel_grp2tbl_match_cb, 10787 mlx5_flow_tunnel_grp2tbl_remove_cb, 10788 mlx5_flow_tunnel_grp2tbl_clone_cb, 10789 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10790 if (!thub->groups) { 10791 err = -rte_errno; 10792 goto err; 10793 } 10794 sh->tunnel_hub = thub; 10795 10796 return 0; 10797 10798 err: 10799 if (thub->groups) 10800 mlx5_hlist_destroy(thub->groups); 10801 if (thub) 10802 mlx5_free(thub); 10803 return err; 10804 } 10805 10806 static inline int 10807 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10808 struct rte_flow_tunnel *tunnel, 10809 struct rte_flow_error *error) 10810 { 10811 struct mlx5_priv *priv = dev->data->dev_private; 10812 10813 if (!priv->sh->config.dv_flow_en) 10814 return rte_flow_error_set(error, ENOTSUP, 10815 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10816 "flow DV interface is off"); 10817 if (!is_tunnel_offload_active(dev)) 10818 return rte_flow_error_set(error, ENOTSUP, 10819 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10820 "tunnel offload was not activated"); 10821 if (!tunnel) 10822 return rte_flow_error_set(error, EINVAL, 10823 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10824 "no application tunnel"); 10825 switch (tunnel->type) { 10826 default: 10827 return rte_flow_error_set(error, EINVAL, 10828 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10829 "unsupported tunnel type"); 10830 case RTE_FLOW_ITEM_TYPE_VXLAN: 10831 case RTE_FLOW_ITEM_TYPE_GRE: 10832 case RTE_FLOW_ITEM_TYPE_NVGRE: 10833 case RTE_FLOW_ITEM_TYPE_GENEVE: 10834 break; 10835 } 10836 return 0; 10837 } 10838 10839 static int 10840 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10841 struct rte_flow_tunnel *app_tunnel, 10842 struct rte_flow_action **actions, 10843 uint32_t *num_of_actions, 10844 struct rte_flow_error *error) 10845 { 10846 struct mlx5_flow_tunnel *tunnel; 10847 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10848 10849 if (ret) 10850 return ret; 10851 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10852 if (ret < 0) { 10853 return rte_flow_error_set(error, ret, 10854 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10855 "failed to initialize pmd tunnel"); 10856 } 10857 *actions = &tunnel->action; 10858 *num_of_actions = 1; 10859 return 0; 10860 } 10861 10862 static int 10863 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10864 struct rte_flow_tunnel *app_tunnel, 10865 struct rte_flow_item **items, 10866 uint32_t *num_of_items, 10867 struct rte_flow_error *error) 10868 { 10869 struct mlx5_flow_tunnel *tunnel; 10870 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10871 10872 if (ret) 10873 return ret; 10874 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10875 if (ret < 0) { 10876 return rte_flow_error_set(error, ret, 10877 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10878 "failed to initialize pmd tunnel"); 10879 } 10880 *items = &tunnel->item; 10881 *num_of_items = 1; 10882 return 0; 10883 } 10884 10885 struct tunnel_db_element_release_ctx { 10886 struct rte_flow_item *items; 10887 struct rte_flow_action *actions; 10888 uint32_t num_elements; 10889 struct rte_flow_error *error; 10890 int ret; 10891 }; 10892 10893 static bool 10894 tunnel_element_release_match(struct rte_eth_dev *dev, 10895 struct mlx5_flow_tunnel *tunnel, const void *x) 10896 { 10897 const struct tunnel_db_element_release_ctx *ctx = x; 10898 10899 RTE_SET_USED(dev); 10900 if (ctx->num_elements != 1) 10901 return false; 10902 else if (ctx->items) 10903 return ctx->items == &tunnel->item; 10904 else if (ctx->actions) 10905 return ctx->actions == &tunnel->action; 10906 10907 return false; 10908 } 10909 10910 static void 10911 tunnel_element_release_hit(struct rte_eth_dev *dev, 10912 struct mlx5_flow_tunnel *tunnel, void *x) 10913 { 10914 struct tunnel_db_element_release_ctx *ctx = x; 10915 ctx->ret = 0; 10916 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10917 mlx5_flow_tunnel_free(dev, tunnel); 10918 } 10919 10920 static void 10921 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10922 { 10923 struct tunnel_db_element_release_ctx *ctx = x; 10924 RTE_SET_USED(dev); 10925 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10926 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10927 "invalid argument"); 10928 } 10929 10930 static int 10931 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10932 struct rte_flow_item *pmd_items, 10933 uint32_t num_items, struct rte_flow_error *err) 10934 { 10935 struct tunnel_db_element_release_ctx ctx = { 10936 .items = pmd_items, 10937 .actions = NULL, 10938 .num_elements = num_items, 10939 .error = err, 10940 }; 10941 10942 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10943 tunnel_element_release_hit, 10944 tunnel_element_release_miss, &ctx, false); 10945 10946 return ctx.ret; 10947 } 10948 10949 static int 10950 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10951 struct rte_flow_action *pmd_actions, 10952 uint32_t num_actions, struct rte_flow_error *err) 10953 { 10954 struct tunnel_db_element_release_ctx ctx = { 10955 .items = NULL, 10956 .actions = pmd_actions, 10957 .num_elements = num_actions, 10958 .error = err, 10959 }; 10960 10961 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10962 tunnel_element_release_hit, 10963 tunnel_element_release_miss, &ctx, false); 10964 10965 return ctx.ret; 10966 } 10967 10968 static int 10969 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10970 struct rte_mbuf *m, 10971 struct rte_flow_restore_info *info, 10972 struct rte_flow_error *err) 10973 { 10974 uint64_t ol_flags = m->ol_flags; 10975 const struct mlx5_flow_tbl_data_entry *tble; 10976 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10977 10978 if (!is_tunnel_offload_active(dev)) { 10979 info->flags = 0; 10980 return 0; 10981 } 10982 10983 if ((ol_flags & mask) != mask) 10984 goto err; 10985 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10986 if (!tble) { 10987 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10988 dev->data->port_id, m->hash.fdir.hi); 10989 goto err; 10990 } 10991 MLX5_ASSERT(tble->tunnel); 10992 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10993 info->group_id = tble->group_id; 10994 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10995 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10996 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10997 10998 return 0; 10999 11000 err: 11001 return rte_flow_error_set(err, EINVAL, 11002 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11003 "failed to get restore info"); 11004 } 11005 11006 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11007 static int 11008 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11009 __rte_unused struct rte_flow_tunnel *app_tunnel, 11010 __rte_unused struct rte_flow_action **actions, 11011 __rte_unused uint32_t *num_of_actions, 11012 __rte_unused struct rte_flow_error *error) 11013 { 11014 return -ENOTSUP; 11015 } 11016 11017 static int 11018 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11019 __rte_unused struct rte_flow_tunnel *app_tunnel, 11020 __rte_unused struct rte_flow_item **items, 11021 __rte_unused uint32_t *num_of_items, 11022 __rte_unused struct rte_flow_error *error) 11023 { 11024 return -ENOTSUP; 11025 } 11026 11027 static int 11028 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11029 __rte_unused struct rte_flow_item *pmd_items, 11030 __rte_unused uint32_t num_items, 11031 __rte_unused struct rte_flow_error *err) 11032 { 11033 return -ENOTSUP; 11034 } 11035 11036 static int 11037 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11038 __rte_unused struct rte_flow_action *pmd_action, 11039 __rte_unused uint32_t num_actions, 11040 __rte_unused struct rte_flow_error *err) 11041 { 11042 return -ENOTSUP; 11043 } 11044 11045 static int 11046 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11047 __rte_unused struct rte_mbuf *m, 11048 __rte_unused struct rte_flow_restore_info *i, 11049 __rte_unused struct rte_flow_error *err) 11050 { 11051 return -ENOTSUP; 11052 } 11053 11054 static int 11055 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11056 __rte_unused struct rte_flow *flow, 11057 __rte_unused const struct rte_flow_attr *attr, 11058 __rte_unused const struct rte_flow_action *actions, 11059 __rte_unused uint32_t flow_idx, 11060 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11061 __rte_unused struct tunnel_default_miss_ctx *ctx, 11062 __rte_unused struct rte_flow_error *error) 11063 { 11064 return -ENOTSUP; 11065 } 11066 11067 static struct mlx5_flow_tunnel * 11068 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11069 __rte_unused uint32_t id) 11070 { 11071 return NULL; 11072 } 11073 11074 static void 11075 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11076 __rte_unused struct mlx5_flow_tunnel *tunnel) 11077 { 11078 } 11079 11080 static uint32_t 11081 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11082 __rte_unused const struct mlx5_flow_tunnel *t, 11083 __rte_unused uint32_t group, 11084 __rte_unused uint32_t *table, 11085 struct rte_flow_error *error) 11086 { 11087 return rte_flow_error_set(error, ENOTSUP, 11088 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11089 "tunnel offload requires DV support"); 11090 } 11091 11092 void 11093 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11094 __rte_unused uint16_t port_id) 11095 { 11096 } 11097 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11098 11099 /* Flex flow item API */ 11100 static struct rte_flow_item_flex_handle * 11101 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11102 const struct rte_flow_item_flex_conf *conf, 11103 struct rte_flow_error *error) 11104 { 11105 static const char err_msg[] = "flex item creation unsupported"; 11106 struct mlx5_priv *priv = dev->data->dev_private; 11107 struct rte_flow_attr attr = { .transfer = 0 }; 11108 const struct mlx5_flow_driver_ops *fops = 11109 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11110 11111 if (!priv->pci_dev) { 11112 rte_flow_error_set(error, ENOTSUP, 11113 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11114 "create flex item on PF only"); 11115 return NULL; 11116 } 11117 switch (priv->pci_dev->id.device_id) { 11118 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11119 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11120 break; 11121 default: 11122 rte_flow_error_set(error, ENOTSUP, 11123 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11124 "flex item available on BlueField ports only"); 11125 return NULL; 11126 } 11127 if (!fops->item_create) { 11128 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11129 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11130 NULL, err_msg); 11131 return NULL; 11132 } 11133 return fops->item_create(dev, conf, error); 11134 } 11135 11136 static int 11137 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11138 const struct rte_flow_item_flex_handle *handle, 11139 struct rte_flow_error *error) 11140 { 11141 static const char err_msg[] = "flex item release unsupported"; 11142 struct rte_flow_attr attr = { .transfer = 0 }; 11143 const struct mlx5_flow_driver_ops *fops = 11144 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11145 11146 if (!fops->item_release) { 11147 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11148 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11149 NULL, err_msg); 11150 return -rte_errno; 11151 } 11152 return fops->item_release(dev, handle, error); 11153 } 11154 11155 static void 11156 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11157 { 11158 int ret; 11159 struct rte_flow_error error; 11160 11161 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11162 char *item_name; 11163 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11164 sizeof(item_name), 11165 (void *)(uintptr_t)item->type, &error); 11166 if (ret > 0) 11167 printf("%s ", item_name); 11168 else 11169 printf("%d\n", (int)item->type); 11170 } 11171 printf("END\n"); 11172 } 11173 11174 static int 11175 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11176 { 11177 const struct rte_flow_item_udp *spec = udp_item->spec; 11178 const struct rte_flow_item_udp *mask = udp_item->mask; 11179 uint16_t udp_dport = 0; 11180 11181 if (spec != NULL) { 11182 if (!mask) 11183 mask = &rte_flow_item_udp_mask; 11184 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11185 mask->hdr.dst_port); 11186 } 11187 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11188 } 11189 11190 static const struct mlx5_flow_expand_node * 11191 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11192 unsigned int item_idx, 11193 const struct mlx5_flow_expand_node graph[], 11194 const struct mlx5_flow_expand_node *node) 11195 { 11196 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11197 11198 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11199 node != NULL && 11200 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11201 /* 11202 * The expansion node is VXLAN and it is also the last 11203 * expandable item in the pattern, so need to continue 11204 * expansion of the inner tunnel. 11205 */ 11206 MLX5_ASSERT(item_idx > 0); 11207 prev_item = pattern + item_idx - 1; 11208 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11209 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11210 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11211 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11212 } 11213 return node; 11214 } 11215 11216 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11217 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11218 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11219 }; 11220 11221 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11222 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11223 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11224 { 9, 10, 11 }, { 12, 13, 14 }, 11225 }; 11226 11227 /** 11228 * Discover the number of available flow priorities. 11229 * 11230 * @param dev 11231 * Ethernet device. 11232 * 11233 * @return 11234 * On success, number of available flow priorities. 11235 * On failure, a negative errno-style code and rte_errno is set. 11236 */ 11237 int 11238 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11239 { 11240 static const uint16_t vprio[] = {8, 16}; 11241 const struct mlx5_priv *priv = dev->data->dev_private; 11242 const struct mlx5_flow_driver_ops *fops; 11243 enum mlx5_flow_drv_type type; 11244 int ret; 11245 11246 type = mlx5_flow_os_get_type(); 11247 if (type == MLX5_FLOW_TYPE_MAX) { 11248 type = MLX5_FLOW_TYPE_VERBS; 11249 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11250 type = MLX5_FLOW_TYPE_DV; 11251 } 11252 fops = flow_get_drv_ops(type); 11253 if (fops->discover_priorities == NULL) { 11254 DRV_LOG(ERR, "Priority discovery not supported"); 11255 rte_errno = ENOTSUP; 11256 return -rte_errno; 11257 } 11258 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11259 if (ret < 0) 11260 return ret; 11261 switch (ret) { 11262 case 8: 11263 ret = RTE_DIM(priority_map_3); 11264 break; 11265 case 16: 11266 ret = RTE_DIM(priority_map_5); 11267 break; 11268 default: 11269 rte_errno = ENOTSUP; 11270 DRV_LOG(ERR, 11271 "port %u maximum priority: %d expected 8/16", 11272 dev->data->port_id, ret); 11273 return -rte_errno; 11274 } 11275 DRV_LOG(INFO, "port %u supported flow priorities:" 11276 " 0-%d for ingress or egress root table," 11277 " 0-%d for non-root table or transfer root table.", 11278 dev->data->port_id, ret - 2, 11279 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11280 return ret; 11281 } 11282 11283 /** 11284 * Adjust flow priority based on the highest layer and the request priority. 11285 * 11286 * @param[in] dev 11287 * Pointer to the Ethernet device structure. 11288 * @param[in] priority 11289 * The rule base priority. 11290 * @param[in] subpriority 11291 * The priority based on the items. 11292 * 11293 * @return 11294 * The new priority. 11295 */ 11296 uint32_t 11297 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11298 uint32_t subpriority) 11299 { 11300 uint32_t res = 0; 11301 struct mlx5_priv *priv = dev->data->dev_private; 11302 11303 switch (priv->sh->flow_max_priority) { 11304 case RTE_DIM(priority_map_3): 11305 res = priority_map_3[priority][subpriority]; 11306 break; 11307 case RTE_DIM(priority_map_5): 11308 res = priority_map_5[priority][subpriority]; 11309 break; 11310 } 11311 return res; 11312 } 11313 11314 /** 11315 * Get the priority for sending traffic to kernel table. 11316 * 11317 * @param[in] dev 11318 * Pointer to the Ethernet device structure. 11319 * 11320 * @return 11321 * On success: the value of priority for sending traffic to kernel table 11322 * On failure: -1 11323 */ 11324 uint32_t 11325 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11326 { 11327 struct mlx5_priv *priv = dev->data->dev_private; 11328 uint32_t res; 11329 11330 switch (priv->sh->flow_max_priority) { 11331 case RTE_DIM(priority_map_5): 11332 res = 15; 11333 break; 11334 case RTE_DIM(priority_map_3): 11335 res = 7; 11336 break; 11337 default: 11338 DRV_LOG(ERR, 11339 "port %u maximum priority: %d expected 8/16", 11340 dev->data->port_id, priv->sh->flow_max_priority); 11341 res = (uint32_t)-1; 11342 } 11343 return res; 11344 } 11345 11346 /** 11347 * Get the E-Switch Manager vport id. 11348 * 11349 * @param[in] dev 11350 * Pointer to the Ethernet device structure. 11351 * 11352 * @return 11353 * The vport id. 11354 */ 11355 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11356 { 11357 struct mlx5_priv *priv = dev->data->dev_private; 11358 struct mlx5_common_device *cdev = priv->sh->cdev; 11359 11360 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11361 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11362 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11363 11364 if (priv->pci_dev == NULL) 11365 return 0; 11366 switch (priv->pci_dev->id.device_id) { 11367 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 11368 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11369 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11370 /* 11371 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11372 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11373 * ECPF vport ID is selected and not the host port (0) in any BF case. 11374 */ 11375 return (int16_t)MLX5_ECPF_VPORT_ID; 11376 default: 11377 return MLX5_PF_VPORT_ID; 11378 } 11379 } 11380 11381 /** 11382 * Parse item to get the vport id. 11383 * 11384 * @param[in] dev 11385 * Pointer to the Ethernet device structure. 11386 * @param[in] item 11387 * The src port id match item. 11388 * @param[out] vport_id 11389 * Pointer to put the vport id. 11390 * @param[out] all_ports 11391 * Indicate if the item matches all ports. 11392 * @param[out] error 11393 * Pointer to error structure. 11394 * 11395 * @return 11396 * 0 on success, a negative errno value otherwise and rte_errno is set. 11397 */ 11398 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11399 const struct rte_flow_item *item, 11400 uint16_t *vport_id, 11401 bool *all_ports, 11402 struct rte_flow_error *error) 11403 { 11404 struct mlx5_priv *port_priv; 11405 const struct rte_flow_item_port_id *pid_v; 11406 uint32_t esw_mgr_port; 11407 11408 if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID && 11409 item->type != RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) 11410 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11411 NULL, "Incorrect item type."); 11412 pid_v = item->spec; 11413 if (!pid_v) { 11414 if (all_ports) 11415 *all_ports = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT); 11416 return 0; 11417 } 11418 if (all_ports) 11419 *all_ports = false; 11420 esw_mgr_port = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) ? 11421 MLX5_REPRESENTED_PORT_ESW_MGR : MLX5_PORT_ESW_MGR; 11422 if (pid_v->id == esw_mgr_port) { 11423 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11424 } else { 11425 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 11426 if (!port_priv) 11427 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11428 NULL, "Failed to get port info."); 11429 *vport_id = port_priv->representor_id; 11430 } 11431 11432 return 0; 11433 } 11434 11435 int 11436 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11437 uint16_t *proxy_port_id, 11438 struct rte_flow_error *error) 11439 { 11440 const struct mlx5_priv *priv = dev->data->dev_private; 11441 uint16_t port_id; 11442 11443 if (!priv->sh->config.dv_esw_en) 11444 return rte_flow_error_set(error, EINVAL, 11445 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11446 NULL, 11447 "unable to provide a proxy port" 11448 " without E-Switch configured"); 11449 if (!priv->master && !priv->representor) 11450 return rte_flow_error_set(error, EINVAL, 11451 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11452 NULL, 11453 "unable to provide a proxy port" 11454 " for port which is not a master" 11455 " or a representor port"); 11456 if (priv->master) { 11457 *proxy_port_id = dev->data->port_id; 11458 return 0; 11459 } 11460 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11461 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11462 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11463 11464 if (port_priv->master && 11465 port_priv->domain_id == priv->domain_id) { 11466 *proxy_port_id = port_id; 11467 return 0; 11468 } 11469 } 11470 return rte_flow_error_set(error, ENODEV, 11471 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11472 NULL, "unable to find a proxy port"); 11473 } 11474