1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static bool 168 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 169 { 170 switch (item->type) { 171 case RTE_FLOW_ITEM_TYPE_ETH: 172 case RTE_FLOW_ITEM_TYPE_VLAN: 173 case RTE_FLOW_ITEM_TYPE_IPV4: 174 case RTE_FLOW_ITEM_TYPE_IPV6: 175 case RTE_FLOW_ITEM_TYPE_UDP: 176 case RTE_FLOW_ITEM_TYPE_TCP: 177 case RTE_FLOW_ITEM_TYPE_ESP: 178 case RTE_FLOW_ITEM_TYPE_ICMP: 179 case RTE_FLOW_ITEM_TYPE_ICMP6: 180 case RTE_FLOW_ITEM_TYPE_VXLAN: 181 case RTE_FLOW_ITEM_TYPE_NVGRE: 182 case RTE_FLOW_ITEM_TYPE_GRE: 183 case RTE_FLOW_ITEM_TYPE_GENEVE: 184 case RTE_FLOW_ITEM_TYPE_MPLS: 185 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 186 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 187 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 188 case RTE_FLOW_ITEM_TYPE_GTP: 189 return true; 190 default: 191 break; 192 } 193 return false; 194 } 195 196 /** 197 * Network Service Header (NSH) and its next protocol values 198 * are described in RFC-8393. 199 */ 200 static enum rte_flow_item_type 201 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 202 { 203 enum rte_flow_item_type type; 204 205 switch (proto_mask & proto_spec) { 206 case 0: 207 type = RTE_FLOW_ITEM_TYPE_VOID; 208 break; 209 case RTE_VXLAN_GPE_TYPE_IPV4: 210 type = RTE_FLOW_ITEM_TYPE_IPV4; 211 break; 212 case RTE_VXLAN_GPE_TYPE_IPV6: 213 type = RTE_VXLAN_GPE_TYPE_IPV6; 214 break; 215 case RTE_VXLAN_GPE_TYPE_ETH: 216 type = RTE_FLOW_ITEM_TYPE_ETH; 217 break; 218 default: 219 type = RTE_FLOW_ITEM_TYPE_END; 220 } 221 return type; 222 } 223 224 static enum rte_flow_item_type 225 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 226 { 227 enum rte_flow_item_type type; 228 229 switch (proto_mask & proto_spec) { 230 case 0: 231 type = RTE_FLOW_ITEM_TYPE_VOID; 232 break; 233 case IPPROTO_UDP: 234 type = RTE_FLOW_ITEM_TYPE_UDP; 235 break; 236 case IPPROTO_TCP: 237 type = RTE_FLOW_ITEM_TYPE_TCP; 238 break; 239 case IPPROTO_IPIP: 240 type = RTE_FLOW_ITEM_TYPE_IPV4; 241 break; 242 case IPPROTO_IPV6: 243 type = RTE_FLOW_ITEM_TYPE_IPV6; 244 break; 245 case IPPROTO_ESP: 246 type = RTE_FLOW_ITEM_TYPE_ESP; 247 break; 248 default: 249 type = RTE_FLOW_ITEM_TYPE_END; 250 } 251 return type; 252 } 253 254 static enum rte_flow_item_type 255 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 256 rte_be16_t type_mask, bool is_tunnel) 257 { 258 enum rte_flow_item_type type; 259 260 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 261 case 0: 262 type = RTE_FLOW_ITEM_TYPE_VOID; 263 break; 264 case RTE_ETHER_TYPE_TEB: 265 type = is_tunnel ? 266 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 267 break; 268 case RTE_ETHER_TYPE_VLAN: 269 type = !is_tunnel ? 270 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 271 break; 272 case RTE_ETHER_TYPE_IPV4: 273 type = RTE_FLOW_ITEM_TYPE_IPV4; 274 break; 275 case RTE_ETHER_TYPE_IPV6: 276 type = RTE_FLOW_ITEM_TYPE_IPV6; 277 break; 278 default: 279 type = RTE_FLOW_ITEM_TYPE_END; 280 } 281 return type; 282 } 283 284 static enum rte_flow_item_type 285 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 286 { 287 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 288 do { \ 289 const void *m = item->mask; \ 290 const void *s = item->spec; \ 291 mask = m ? \ 292 ((const struct rte_flow_item_##type *)m)->fld : \ 293 rte_flow_item_##type##_mask.fld; \ 294 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 295 } while (0) 296 297 enum rte_flow_item_type ret; 298 uint16_t spec, mask; 299 300 if (item == NULL || item->spec == NULL) 301 return RTE_FLOW_ITEM_TYPE_VOID; 302 switch (item->type) { 303 case RTE_FLOW_ITEM_TYPE_ETH: 304 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 305 if (!mask) 306 return RTE_FLOW_ITEM_TYPE_VOID; 307 ret = mlx5_ethertype_to_item_type(spec, mask, false); 308 break; 309 case RTE_FLOW_ITEM_TYPE_VLAN: 310 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 311 if (!mask) 312 return RTE_FLOW_ITEM_TYPE_VOID; 313 ret = mlx5_ethertype_to_item_type(spec, mask, false); 314 break; 315 case RTE_FLOW_ITEM_TYPE_IPV4: 316 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 317 if (!mask) 318 return RTE_FLOW_ITEM_TYPE_VOID; 319 ret = mlx5_inet_proto_to_item_type(spec, mask); 320 break; 321 case RTE_FLOW_ITEM_TYPE_IPV6: 322 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 323 if (!mask) 324 return RTE_FLOW_ITEM_TYPE_VOID; 325 ret = mlx5_inet_proto_to_item_type(spec, mask); 326 break; 327 case RTE_FLOW_ITEM_TYPE_GENEVE: 328 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 329 ret = mlx5_ethertype_to_item_type(spec, mask, true); 330 break; 331 case RTE_FLOW_ITEM_TYPE_GRE: 332 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 333 ret = mlx5_ethertype_to_item_type(spec, mask, true); 334 break; 335 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 336 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 337 ret = mlx5_nsh_proto_to_item_type(spec, mask); 338 break; 339 default: 340 ret = RTE_FLOW_ITEM_TYPE_VOID; 341 break; 342 } 343 return ret; 344 #undef MLX5_XSET_ITEM_MASK_SPEC 345 } 346 347 static const int * 348 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 349 const int *next_node) 350 { 351 const struct mlx5_flow_expand_node *node = NULL; 352 const int *next = next_node; 353 354 while (next && *next) { 355 /* 356 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 357 * flag set, because they were not found in the flow pattern. 358 */ 359 node = &graph[*next]; 360 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 361 break; 362 next = node->next; 363 } 364 return next; 365 } 366 367 #define MLX5_RSS_EXP_ELT_N 16 368 369 /** 370 * Expand RSS flows into several possible flows according to the RSS hash 371 * fields requested and the driver capabilities. 372 * 373 * @param[out] buf 374 * Buffer to store the result expansion. 375 * @param[in] size 376 * Buffer size in bytes. If 0, @p buf can be NULL. 377 * @param[in] pattern 378 * User flow pattern. 379 * @param[in] types 380 * RSS types to expand (see RTE_ETH_RSS_* definitions). 381 * @param[in] graph 382 * Input graph to expand @p pattern according to @p types. 383 * @param[in] graph_root_index 384 * Index of root node in @p graph, typically 0. 385 * 386 * @return 387 * A positive value representing the size of @p buf in bytes regardless of 388 * @p size on success, a negative errno value otherwise and rte_errno is 389 * set, the following errors are defined: 390 * 391 * -E2BIG: graph-depth @p graph is too deep. 392 * -EINVAL: @p size has not enough space for expanded pattern. 393 */ 394 static int 395 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 396 const struct rte_flow_item *pattern, uint64_t types, 397 const struct mlx5_flow_expand_node graph[], 398 int graph_root_index) 399 { 400 const struct rte_flow_item *item; 401 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 402 const int *next_node; 403 const int *stack[MLX5_RSS_EXP_ELT_N]; 404 int stack_pos = 0; 405 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 406 unsigned int i, item_idx, last_expand_item_idx = 0; 407 size_t lsize; 408 size_t user_pattern_size = 0; 409 void *addr = NULL; 410 const struct mlx5_flow_expand_node *next = NULL; 411 struct rte_flow_item missed_item; 412 int missed = 0; 413 int elt = 0; 414 const struct rte_flow_item *last_expand_item = NULL; 415 416 memset(&missed_item, 0, sizeof(missed_item)); 417 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 418 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 419 if (lsize > size) 420 return -EINVAL; 421 buf->entry[0].priority = 0; 422 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 423 buf->entries = 0; 424 addr = buf->entry[0].pattern; 425 for (item = pattern, item_idx = 0; 426 item->type != RTE_FLOW_ITEM_TYPE_END; 427 item++, item_idx++) { 428 if (!mlx5_flow_is_rss_expandable_item(item)) { 429 user_pattern_size += sizeof(*item); 430 continue; 431 } 432 last_expand_item = item; 433 last_expand_item_idx = item_idx; 434 i = 0; 435 while (node->next && node->next[i]) { 436 next = &graph[node->next[i]]; 437 if (next->type == item->type) 438 break; 439 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 440 node = next; 441 i = 0; 442 } else { 443 ++i; 444 } 445 } 446 if (next) 447 node = next; 448 user_pattern_size += sizeof(*item); 449 } 450 user_pattern_size += sizeof(*item); /* Handle END item. */ 451 lsize += user_pattern_size; 452 if (lsize > size) 453 return -EINVAL; 454 /* Copy the user pattern in the first entry of the buffer. */ 455 rte_memcpy(addr, pattern, user_pattern_size); 456 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 457 buf->entries = 1; 458 /* Start expanding. */ 459 memset(flow_items, 0, sizeof(flow_items)); 460 user_pattern_size -= sizeof(*item); 461 /* 462 * Check if the last valid item has spec set, need complete pattern, 463 * and the pattern can be used for expansion. 464 */ 465 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 466 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 467 /* Item type END indicates expansion is not required. */ 468 return lsize; 469 } 470 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 471 next = NULL; 472 missed = 1; 473 i = 0; 474 while (node->next && node->next[i]) { 475 next = &graph[node->next[i]]; 476 if (next->type == missed_item.type) { 477 flow_items[0].type = missed_item.type; 478 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 479 break; 480 } 481 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 482 node = next; 483 i = 0; 484 } else { 485 ++i; 486 } 487 next = NULL; 488 } 489 } 490 if (next && missed) { 491 elt = 2; /* missed item + item end. */ 492 node = next; 493 lsize += elt * sizeof(*item) + user_pattern_size; 494 if (lsize > size) 495 return -EINVAL; 496 if (node->rss_types & types) { 497 buf->entry[buf->entries].priority = 1; 498 buf->entry[buf->entries].pattern = addr; 499 buf->entries++; 500 rte_memcpy(addr, buf->entry[0].pattern, 501 user_pattern_size); 502 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 503 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 504 addr = (void *)(((uintptr_t)addr) + 505 elt * sizeof(*item)); 506 } 507 } else if (last_expand_item != NULL) { 508 node = mlx5_flow_expand_rss_adjust_node(pattern, 509 last_expand_item_idx, graph, node); 510 } 511 memset(flow_items, 0, sizeof(flow_items)); 512 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 513 node->next); 514 stack[stack_pos] = next_node; 515 node = next_node ? &graph[*next_node] : NULL; 516 while (node) { 517 flow_items[stack_pos].type = node->type; 518 if (node->rss_types & types) { 519 size_t n; 520 /* 521 * compute the number of items to copy from the 522 * expansion and copy it. 523 * When the stack_pos is 0, there are 1 element in it, 524 * plus the addition END item. 525 */ 526 elt = stack_pos + 2; 527 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 528 lsize += elt * sizeof(*item) + user_pattern_size; 529 if (lsize > size) 530 return -EINVAL; 531 n = elt * sizeof(*item); 532 buf->entry[buf->entries].priority = 533 stack_pos + 1 + missed; 534 buf->entry[buf->entries].pattern = addr; 535 buf->entries++; 536 rte_memcpy(addr, buf->entry[0].pattern, 537 user_pattern_size); 538 addr = (void *)(((uintptr_t)addr) + 539 user_pattern_size); 540 rte_memcpy(addr, &missed_item, 541 missed * sizeof(*item)); 542 addr = (void *)(((uintptr_t)addr) + 543 missed * sizeof(*item)); 544 rte_memcpy(addr, flow_items, n); 545 addr = (void *)(((uintptr_t)addr) + n); 546 } 547 /* Go deeper. */ 548 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 549 node->next) { 550 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 551 node->next); 552 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 553 rte_errno = E2BIG; 554 return -rte_errno; 555 } 556 stack[stack_pos] = next_node; 557 } else if (*(next_node + 1)) { 558 /* Follow up with the next possibility. */ 559 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 560 ++next_node); 561 } else if (!stack_pos) { 562 /* 563 * Completing the traverse over the different paths. 564 * The next_node is advanced to the terminator. 565 */ 566 ++next_node; 567 } else { 568 /* Move to the next path. */ 569 while (stack_pos) { 570 next_node = stack[--stack_pos]; 571 next_node++; 572 if (*next_node) 573 break; 574 } 575 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 576 next_node); 577 stack[stack_pos] = next_node; 578 } 579 node = next_node && *next_node ? &graph[*next_node] : NULL; 580 }; 581 return lsize; 582 } 583 584 /** 585 * Expand SQN flows into several possible flows according to the Tx queue 586 * number 587 * 588 * @param[in] buf 589 * Buffer to store the result expansion. 590 * @param[in] size 591 * Buffer size in bytes. If 0, @p buf can be NULL. 592 * @param[in] pattern 593 * User flow pattern. 594 * @param[in] sq_specs 595 * Buffer to store sq spec. 596 * 597 * @return 598 * 0 for success and negative value for failure 599 * 600 */ 601 static int 602 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 603 const struct rte_flow_item *pattern, 604 struct mlx5_rte_flow_item_sq *sq_specs) 605 { 606 const struct rte_flow_item *item; 607 bool port_representor = false; 608 size_t user_pattern_size = 0; 609 struct rte_eth_dev *dev; 610 struct mlx5_priv *priv; 611 void *addr = NULL; 612 uint16_t port_id; 613 size_t lsize; 614 int elt = 2; 615 uint16_t i; 616 617 buf->entries = 0; 618 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 619 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 620 const struct rte_flow_item_ethdev *pid_v = item->spec; 621 622 if (!pid_v) 623 return 0; 624 port_id = pid_v->port_id; 625 port_representor = true; 626 } 627 user_pattern_size += sizeof(*item); 628 } 629 if (!port_representor) 630 return 0; 631 dev = &rte_eth_devices[port_id]; 632 priv = dev->data->dev_private; 633 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 634 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 635 sizeof(buf->entry[0]) * priv->txqs_n; 636 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 637 return -EINVAL; 638 addr = buf->entry[0].pattern; 639 for (i = 0; i != priv->txqs_n; ++i) { 640 struct rte_flow_item pattern_add[] = { 641 { 642 .type = (enum rte_flow_item_type) 643 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 644 .spec = &sq_specs[i], 645 }, 646 { 647 .type = RTE_FLOW_ITEM_TYPE_END, 648 }, 649 }; 650 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 651 652 if (txq == NULL) 653 return -EINVAL; 654 buf->entry[i].pattern = addr; 655 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 656 mlx5_txq_release(dev, i); 657 rte_memcpy(addr, pattern, user_pattern_size); 658 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 659 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 660 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 661 buf->entries++; 662 } 663 return 0; 664 } 665 666 enum mlx5_expansion { 667 MLX5_EXPANSION_ROOT, 668 MLX5_EXPANSION_ROOT_OUTER, 669 MLX5_EXPANSION_OUTER_ETH, 670 MLX5_EXPANSION_OUTER_VLAN, 671 MLX5_EXPANSION_OUTER_IPV4, 672 MLX5_EXPANSION_OUTER_IPV4_UDP, 673 MLX5_EXPANSION_OUTER_IPV4_TCP, 674 MLX5_EXPANSION_OUTER_IPV4_ESP, 675 MLX5_EXPANSION_OUTER_IPV4_ICMP, 676 MLX5_EXPANSION_OUTER_IPV6, 677 MLX5_EXPANSION_OUTER_IPV6_UDP, 678 MLX5_EXPANSION_OUTER_IPV6_TCP, 679 MLX5_EXPANSION_OUTER_IPV6_ESP, 680 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 681 MLX5_EXPANSION_VXLAN, 682 MLX5_EXPANSION_STD_VXLAN, 683 MLX5_EXPANSION_L3_VXLAN, 684 MLX5_EXPANSION_VXLAN_GPE, 685 MLX5_EXPANSION_GRE, 686 MLX5_EXPANSION_NVGRE, 687 MLX5_EXPANSION_GRE_KEY, 688 MLX5_EXPANSION_MPLS, 689 MLX5_EXPANSION_ETH, 690 MLX5_EXPANSION_VLAN, 691 MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV4_UDP, 693 MLX5_EXPANSION_IPV4_TCP, 694 MLX5_EXPANSION_IPV4_ESP, 695 MLX5_EXPANSION_IPV4_ICMP, 696 MLX5_EXPANSION_IPV6, 697 MLX5_EXPANSION_IPV6_UDP, 698 MLX5_EXPANSION_IPV6_TCP, 699 MLX5_EXPANSION_IPV6_ESP, 700 MLX5_EXPANSION_IPV6_ICMP6, 701 MLX5_EXPANSION_IPV6_FRAG_EXT, 702 MLX5_EXPANSION_GTP, 703 MLX5_EXPANSION_GENEVE, 704 }; 705 706 /** Supported expansion of items. */ 707 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 708 [MLX5_EXPANSION_ROOT] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 710 MLX5_EXPANSION_IPV4, 711 MLX5_EXPANSION_IPV6), 712 .type = RTE_FLOW_ITEM_TYPE_END, 713 }, 714 [MLX5_EXPANSION_ROOT_OUTER] = { 715 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 716 MLX5_EXPANSION_OUTER_IPV4, 717 MLX5_EXPANSION_OUTER_IPV6), 718 .type = RTE_FLOW_ITEM_TYPE_END, 719 }, 720 [MLX5_EXPANSION_OUTER_ETH] = { 721 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 722 .type = RTE_FLOW_ITEM_TYPE_ETH, 723 .rss_types = 0, 724 }, 725 [MLX5_EXPANSION_OUTER_VLAN] = { 726 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 727 MLX5_EXPANSION_OUTER_IPV6), 728 .type = RTE_FLOW_ITEM_TYPE_VLAN, 729 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 730 }, 731 [MLX5_EXPANSION_OUTER_IPV4] = { 732 .next = MLX5_FLOW_EXPAND_RSS_NEXT 733 (MLX5_EXPANSION_OUTER_IPV4_UDP, 734 MLX5_EXPANSION_OUTER_IPV4_TCP, 735 MLX5_EXPANSION_OUTER_IPV4_ESP, 736 MLX5_EXPANSION_OUTER_IPV4_ICMP, 737 MLX5_EXPANSION_GRE, 738 MLX5_EXPANSION_NVGRE, 739 MLX5_EXPANSION_IPV4, 740 MLX5_EXPANSION_IPV6), 741 .type = RTE_FLOW_ITEM_TYPE_IPV4, 742 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 743 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 744 }, 745 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 746 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 747 MLX5_EXPANSION_VXLAN_GPE, 748 MLX5_EXPANSION_MPLS, 749 MLX5_EXPANSION_GENEVE, 750 MLX5_EXPANSION_GTP), 751 .type = RTE_FLOW_ITEM_TYPE_UDP, 752 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 753 }, 754 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 755 .type = RTE_FLOW_ITEM_TYPE_TCP, 756 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 757 }, 758 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 759 .type = RTE_FLOW_ITEM_TYPE_ESP, 760 .rss_types = RTE_ETH_RSS_ESP, 761 }, 762 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 763 .type = RTE_FLOW_ITEM_TYPE_ICMP, 764 }, 765 [MLX5_EXPANSION_OUTER_IPV6] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT 767 (MLX5_EXPANSION_OUTER_IPV6_UDP, 768 MLX5_EXPANSION_OUTER_IPV6_TCP, 769 MLX5_EXPANSION_OUTER_IPV6_ESP, 770 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 771 MLX5_EXPANSION_IPV4, 772 MLX5_EXPANSION_IPV6, 773 MLX5_EXPANSION_GRE, 774 MLX5_EXPANSION_NVGRE), 775 .type = RTE_FLOW_ITEM_TYPE_IPV6, 776 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 777 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 778 }, 779 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 780 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 781 MLX5_EXPANSION_VXLAN_GPE, 782 MLX5_EXPANSION_MPLS, 783 MLX5_EXPANSION_GENEVE, 784 MLX5_EXPANSION_GTP), 785 .type = RTE_FLOW_ITEM_TYPE_UDP, 786 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 787 }, 788 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 789 .type = RTE_FLOW_ITEM_TYPE_TCP, 790 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 791 }, 792 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 793 .type = RTE_FLOW_ITEM_TYPE_ESP, 794 .rss_types = RTE_ETH_RSS_ESP, 795 }, 796 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 797 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 798 }, 799 [MLX5_EXPANSION_VXLAN] = { 800 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 801 MLX5_EXPANSION_IPV4, 802 MLX5_EXPANSION_IPV6), 803 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 804 }, 805 [MLX5_EXPANSION_STD_VXLAN] = { 806 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 807 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 808 }, 809 [MLX5_EXPANSION_L3_VXLAN] = { 810 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 811 MLX5_EXPANSION_IPV6), 812 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 813 }, 814 [MLX5_EXPANSION_VXLAN_GPE] = { 815 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 816 MLX5_EXPANSION_IPV4, 817 MLX5_EXPANSION_IPV6), 818 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 819 }, 820 [MLX5_EXPANSION_GRE] = { 821 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 822 MLX5_EXPANSION_IPV4, 823 MLX5_EXPANSION_IPV6, 824 MLX5_EXPANSION_GRE_KEY, 825 MLX5_EXPANSION_MPLS), 826 .type = RTE_FLOW_ITEM_TYPE_GRE, 827 }, 828 [MLX5_EXPANSION_GRE_KEY] = { 829 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 830 MLX5_EXPANSION_IPV6, 831 MLX5_EXPANSION_MPLS), 832 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 833 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 834 }, 835 [MLX5_EXPANSION_NVGRE] = { 836 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 837 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 838 }, 839 [MLX5_EXPANSION_MPLS] = { 840 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 841 MLX5_EXPANSION_IPV6, 842 MLX5_EXPANSION_ETH), 843 .type = RTE_FLOW_ITEM_TYPE_MPLS, 844 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 845 }, 846 [MLX5_EXPANSION_ETH] = { 847 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 848 .type = RTE_FLOW_ITEM_TYPE_ETH, 849 }, 850 [MLX5_EXPANSION_VLAN] = { 851 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 852 MLX5_EXPANSION_IPV6), 853 .type = RTE_FLOW_ITEM_TYPE_VLAN, 854 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 855 }, 856 [MLX5_EXPANSION_IPV4] = { 857 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 858 MLX5_EXPANSION_IPV4_TCP, 859 MLX5_EXPANSION_IPV4_ESP, 860 MLX5_EXPANSION_IPV4_ICMP), 861 .type = RTE_FLOW_ITEM_TYPE_IPV4, 862 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 863 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 864 }, 865 [MLX5_EXPANSION_IPV4_UDP] = { 866 .type = RTE_FLOW_ITEM_TYPE_UDP, 867 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 868 }, 869 [MLX5_EXPANSION_IPV4_TCP] = { 870 .type = RTE_FLOW_ITEM_TYPE_TCP, 871 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 872 }, 873 [MLX5_EXPANSION_IPV4_ESP] = { 874 .type = RTE_FLOW_ITEM_TYPE_ESP, 875 .rss_types = RTE_ETH_RSS_ESP, 876 }, 877 [MLX5_EXPANSION_IPV4_ICMP] = { 878 .type = RTE_FLOW_ITEM_TYPE_ICMP, 879 }, 880 [MLX5_EXPANSION_IPV6] = { 881 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 882 MLX5_EXPANSION_IPV6_TCP, 883 MLX5_EXPANSION_IPV6_ESP, 884 MLX5_EXPANSION_IPV6_ICMP6, 885 MLX5_EXPANSION_IPV6_FRAG_EXT), 886 .type = RTE_FLOW_ITEM_TYPE_IPV6, 887 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 888 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 889 }, 890 [MLX5_EXPANSION_IPV6_UDP] = { 891 .type = RTE_FLOW_ITEM_TYPE_UDP, 892 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 893 }, 894 [MLX5_EXPANSION_IPV6_TCP] = { 895 .type = RTE_FLOW_ITEM_TYPE_TCP, 896 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 897 }, 898 [MLX5_EXPANSION_IPV6_ESP] = { 899 .type = RTE_FLOW_ITEM_TYPE_ESP, 900 .rss_types = RTE_ETH_RSS_ESP, 901 }, 902 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 903 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 904 }, 905 [MLX5_EXPANSION_IPV6_ICMP6] = { 906 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 907 }, 908 [MLX5_EXPANSION_GTP] = { 909 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 910 MLX5_EXPANSION_IPV6), 911 .type = RTE_FLOW_ITEM_TYPE_GTP, 912 }, 913 [MLX5_EXPANSION_GENEVE] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 915 MLX5_EXPANSION_IPV4, 916 MLX5_EXPANSION_IPV6), 917 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 918 }, 919 }; 920 921 static struct rte_flow_action_handle * 922 mlx5_action_handle_create(struct rte_eth_dev *dev, 923 const struct rte_flow_indir_action_conf *conf, 924 const struct rte_flow_action *action, 925 struct rte_flow_error *error); 926 static int mlx5_action_handle_destroy 927 (struct rte_eth_dev *dev, 928 struct rte_flow_action_handle *handle, 929 struct rte_flow_error *error); 930 static int mlx5_action_handle_update 931 (struct rte_eth_dev *dev, 932 struct rte_flow_action_handle *handle, 933 const void *update, 934 struct rte_flow_error *error); 935 static int mlx5_action_handle_query 936 (struct rte_eth_dev *dev, 937 const struct rte_flow_action_handle *handle, 938 void *data, 939 struct rte_flow_error *error); 940 static int 941 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 942 struct rte_flow_tunnel *app_tunnel, 943 struct rte_flow_action **actions, 944 uint32_t *num_of_actions, 945 struct rte_flow_error *error); 946 static int 947 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 948 struct rte_flow_tunnel *app_tunnel, 949 struct rte_flow_item **items, 950 uint32_t *num_of_items, 951 struct rte_flow_error *error); 952 static int 953 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 954 struct rte_flow_item *pmd_items, 955 uint32_t num_items, struct rte_flow_error *err); 956 static int 957 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 958 struct rte_flow_action *pmd_actions, 959 uint32_t num_actions, 960 struct rte_flow_error *err); 961 static int 962 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 963 struct rte_mbuf *m, 964 struct rte_flow_restore_info *info, 965 struct rte_flow_error *err); 966 static struct rte_flow_item_flex_handle * 967 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 968 const struct rte_flow_item_flex_conf *conf, 969 struct rte_flow_error *error); 970 static int 971 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 972 const struct rte_flow_item_flex_handle *handle, 973 struct rte_flow_error *error); 974 static int 975 mlx5_flow_info_get(struct rte_eth_dev *dev, 976 struct rte_flow_port_info *port_info, 977 struct rte_flow_queue_info *queue_info, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_port_configure(struct rte_eth_dev *dev, 981 const struct rte_flow_port_attr *port_attr, 982 uint16_t nb_queue, 983 const struct rte_flow_queue_attr *queue_attr[], 984 struct rte_flow_error *err); 985 986 static struct rte_flow_pattern_template * 987 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 988 const struct rte_flow_pattern_template_attr *attr, 989 const struct rte_flow_item items[], 990 struct rte_flow_error *error); 991 992 static int 993 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 994 struct rte_flow_pattern_template *template, 995 struct rte_flow_error *error); 996 static struct rte_flow_actions_template * 997 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 998 const struct rte_flow_actions_template_attr *attr, 999 const struct rte_flow_action actions[], 1000 const struct rte_flow_action masks[], 1001 struct rte_flow_error *error); 1002 static int 1003 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1004 struct rte_flow_actions_template *template, 1005 struct rte_flow_error *error); 1006 1007 static struct rte_flow_template_table * 1008 mlx5_flow_table_create(struct rte_eth_dev *dev, 1009 const struct rte_flow_template_table_attr *attr, 1010 struct rte_flow_pattern_template *item_templates[], 1011 uint8_t nb_item_templates, 1012 struct rte_flow_actions_template *action_templates[], 1013 uint8_t nb_action_templates, 1014 struct rte_flow_error *error); 1015 static int 1016 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1017 struct rte_flow_template_table *table, 1018 struct rte_flow_error *error); 1019 static struct rte_flow * 1020 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1021 uint32_t queue, 1022 const struct rte_flow_op_attr *attr, 1023 struct rte_flow_template_table *table, 1024 const struct rte_flow_item items[], 1025 uint8_t pattern_template_index, 1026 const struct rte_flow_action actions[], 1027 uint8_t action_template_index, 1028 void *user_data, 1029 struct rte_flow_error *error); 1030 static int 1031 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1032 uint32_t queue, 1033 const struct rte_flow_op_attr *attr, 1034 struct rte_flow *flow, 1035 void *user_data, 1036 struct rte_flow_error *error); 1037 static int 1038 mlx5_flow_pull(struct rte_eth_dev *dev, 1039 uint32_t queue, 1040 struct rte_flow_op_result res[], 1041 uint16_t n_res, 1042 struct rte_flow_error *error); 1043 static int 1044 mlx5_flow_push(struct rte_eth_dev *dev, 1045 uint32_t queue, 1046 struct rte_flow_error *error); 1047 1048 static struct rte_flow_action_handle * 1049 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1050 const struct rte_flow_op_attr *attr, 1051 const struct rte_flow_indir_action_conf *conf, 1052 const struct rte_flow_action *action, 1053 void *user_data, 1054 struct rte_flow_error *error); 1055 1056 static int 1057 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1058 const struct rte_flow_op_attr *attr, 1059 struct rte_flow_action_handle *handle, 1060 const void *update, 1061 void *user_data, 1062 struct rte_flow_error *error); 1063 1064 static int 1065 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1066 const struct rte_flow_op_attr *attr, 1067 struct rte_flow_action_handle *handle, 1068 void *user_data, 1069 struct rte_flow_error *error); 1070 1071 static int 1072 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1073 const struct rte_flow_op_attr *attr, 1074 const struct rte_flow_action_handle *handle, 1075 void *data, 1076 void *user_data, 1077 struct rte_flow_error *error); 1078 1079 static const struct rte_flow_ops mlx5_flow_ops = { 1080 .validate = mlx5_flow_validate, 1081 .create = mlx5_flow_create, 1082 .destroy = mlx5_flow_destroy, 1083 .flush = mlx5_flow_flush, 1084 .isolate = mlx5_flow_isolate, 1085 .query = mlx5_flow_query, 1086 .dev_dump = mlx5_flow_dev_dump, 1087 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1088 .get_aged_flows = mlx5_flow_get_aged_flows, 1089 .action_handle_create = mlx5_action_handle_create, 1090 .action_handle_destroy = mlx5_action_handle_destroy, 1091 .action_handle_update = mlx5_action_handle_update, 1092 .action_handle_query = mlx5_action_handle_query, 1093 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1094 .tunnel_match = mlx5_flow_tunnel_match, 1095 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1096 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1097 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1098 .flex_item_create = mlx5_flow_flex_item_create, 1099 .flex_item_release = mlx5_flow_flex_item_release, 1100 .info_get = mlx5_flow_info_get, 1101 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1102 .configure = mlx5_flow_port_configure, 1103 .pattern_template_create = mlx5_flow_pattern_template_create, 1104 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1105 .actions_template_create = mlx5_flow_actions_template_create, 1106 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1107 .template_table_create = mlx5_flow_table_create, 1108 .template_table_destroy = mlx5_flow_table_destroy, 1109 .async_create = mlx5_flow_async_flow_create, 1110 .async_destroy = mlx5_flow_async_flow_destroy, 1111 .pull = mlx5_flow_pull, 1112 .push = mlx5_flow_push, 1113 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1114 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1115 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1116 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1117 }; 1118 1119 /* Tunnel information. */ 1120 struct mlx5_flow_tunnel_info { 1121 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1122 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1123 }; 1124 1125 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1126 { 1127 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1128 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1129 }, 1130 { 1131 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1132 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1133 }, 1134 { 1135 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1136 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1137 }, 1138 { 1139 .tunnel = MLX5_FLOW_LAYER_GRE, 1140 .ptype = RTE_PTYPE_TUNNEL_GRE, 1141 }, 1142 { 1143 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1144 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1145 }, 1146 { 1147 .tunnel = MLX5_FLOW_LAYER_MPLS, 1148 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1149 }, 1150 { 1151 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1152 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1153 }, 1154 { 1155 .tunnel = MLX5_FLOW_LAYER_IPIP, 1156 .ptype = RTE_PTYPE_TUNNEL_IP, 1157 }, 1158 { 1159 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1160 .ptype = RTE_PTYPE_TUNNEL_IP, 1161 }, 1162 { 1163 .tunnel = MLX5_FLOW_LAYER_GTP, 1164 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1165 }, 1166 }; 1167 1168 1169 1170 /** 1171 * Translate tag ID to register. 1172 * 1173 * @param[in] dev 1174 * Pointer to the Ethernet device structure. 1175 * @param[in] feature 1176 * The feature that request the register. 1177 * @param[in] id 1178 * The request register ID. 1179 * @param[out] error 1180 * Error description in case of any. 1181 * 1182 * @return 1183 * The request register on success, a negative errno 1184 * value otherwise and rte_errno is set. 1185 */ 1186 int 1187 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1188 enum mlx5_feature_name feature, 1189 uint32_t id, 1190 struct rte_flow_error *error) 1191 { 1192 struct mlx5_priv *priv = dev->data->dev_private; 1193 struct mlx5_sh_config *config = &priv->sh->config; 1194 enum modify_reg start_reg; 1195 bool skip_mtr_reg = false; 1196 1197 switch (feature) { 1198 case MLX5_HAIRPIN_RX: 1199 return REG_B; 1200 case MLX5_HAIRPIN_TX: 1201 return REG_A; 1202 case MLX5_METADATA_RX: 1203 switch (config->dv_xmeta_en) { 1204 case MLX5_XMETA_MODE_LEGACY: 1205 return REG_B; 1206 case MLX5_XMETA_MODE_META16: 1207 return REG_C_0; 1208 case MLX5_XMETA_MODE_META32: 1209 return REG_C_1; 1210 case MLX5_XMETA_MODE_META32_HWS: 1211 return REG_C_1; 1212 } 1213 break; 1214 case MLX5_METADATA_TX: 1215 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1216 return REG_C_1; 1217 } else { 1218 return REG_A; 1219 } 1220 case MLX5_METADATA_FDB: 1221 switch (config->dv_xmeta_en) { 1222 case MLX5_XMETA_MODE_LEGACY: 1223 return REG_NON; 1224 case MLX5_XMETA_MODE_META16: 1225 return REG_C_0; 1226 case MLX5_XMETA_MODE_META32: 1227 return REG_C_1; 1228 case MLX5_XMETA_MODE_META32_HWS: 1229 return REG_C_1; 1230 } 1231 break; 1232 case MLX5_FLOW_MARK: 1233 switch (config->dv_xmeta_en) { 1234 case MLX5_XMETA_MODE_LEGACY: 1235 case MLX5_XMETA_MODE_META32_HWS: 1236 return REG_NON; 1237 case MLX5_XMETA_MODE_META16: 1238 return REG_C_1; 1239 case MLX5_XMETA_MODE_META32: 1240 return REG_C_0; 1241 } 1242 break; 1243 case MLX5_MTR_ID: 1244 /* 1245 * If meter color and meter id share one register, flow match 1246 * should use the meter color register for match. 1247 */ 1248 if (priv->mtr_reg_share) 1249 return priv->mtr_color_reg; 1250 else 1251 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1252 REG_C_3; 1253 case MLX5_MTR_COLOR: 1254 case MLX5_ASO_FLOW_HIT: 1255 case MLX5_ASO_CONNTRACK: 1256 case MLX5_SAMPLE_ID: 1257 /* All features use the same REG_C. */ 1258 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1259 return priv->mtr_color_reg; 1260 case MLX5_COPY_MARK: 1261 /* 1262 * Metadata COPY_MARK register using is in meter suffix sub 1263 * flow while with meter. It's safe to share the same register. 1264 */ 1265 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1266 case MLX5_APP_TAG: 1267 /* 1268 * If meter is enable, it will engage the register for color 1269 * match and flow match. If meter color match is not using the 1270 * REG_C_2, need to skip the REG_C_x be used by meter color 1271 * match. 1272 * If meter is disable, free to use all available registers. 1273 */ 1274 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1275 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1276 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1277 if (id > (uint32_t)(REG_C_7 - start_reg)) 1278 return rte_flow_error_set(error, EINVAL, 1279 RTE_FLOW_ERROR_TYPE_ITEM, 1280 NULL, "invalid tag id"); 1281 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1282 return rte_flow_error_set(error, ENOTSUP, 1283 RTE_FLOW_ERROR_TYPE_ITEM, 1284 NULL, "unsupported tag id"); 1285 /* 1286 * This case means meter is using the REG_C_x great than 2. 1287 * Take care not to conflict with meter color REG_C_x. 1288 * If the available index REG_C_y >= REG_C_x, skip the 1289 * color register. 1290 */ 1291 if (skip_mtr_reg && priv->sh->flow_mreg_c 1292 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1293 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1294 return rte_flow_error_set(error, EINVAL, 1295 RTE_FLOW_ERROR_TYPE_ITEM, 1296 NULL, "invalid tag id"); 1297 if (priv->sh->flow_mreg_c 1298 [id + 1 + start_reg - REG_C_0] != REG_NON) 1299 return priv->sh->flow_mreg_c 1300 [id + 1 + start_reg - REG_C_0]; 1301 return rte_flow_error_set(error, ENOTSUP, 1302 RTE_FLOW_ERROR_TYPE_ITEM, 1303 NULL, "unsupported tag id"); 1304 } 1305 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1306 } 1307 MLX5_ASSERT(false); 1308 return rte_flow_error_set(error, EINVAL, 1309 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1310 NULL, "invalid feature name"); 1311 } 1312 1313 /** 1314 * Check extensive flow metadata register support. 1315 * 1316 * @param dev 1317 * Pointer to rte_eth_dev structure. 1318 * 1319 * @return 1320 * True if device supports extensive flow metadata register, otherwise false. 1321 */ 1322 bool 1323 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1324 { 1325 struct mlx5_priv *priv = dev->data->dev_private; 1326 1327 /* 1328 * Having available reg_c can be regarded inclusively as supporting 1329 * extensive flow metadata register, which could mean, 1330 * - metadata register copy action by modify header. 1331 * - 16 modify header actions is supported. 1332 * - reg_c's are preserved across different domain (FDB and NIC) on 1333 * packet loopback by flow lookup miss. 1334 */ 1335 return priv->sh->flow_mreg_c[2] != REG_NON; 1336 } 1337 1338 /** 1339 * Get the lowest priority. 1340 * 1341 * @param[in] dev 1342 * Pointer to the Ethernet device structure. 1343 * @param[in] attributes 1344 * Pointer to device flow rule attributes. 1345 * 1346 * @return 1347 * The value of lowest priority of flow. 1348 */ 1349 uint32_t 1350 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1351 const struct rte_flow_attr *attr) 1352 { 1353 struct mlx5_priv *priv = dev->data->dev_private; 1354 1355 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1356 return priv->sh->flow_max_priority - 2; 1357 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1358 } 1359 1360 /** 1361 * Calculate matcher priority of the flow. 1362 * 1363 * @param[in] dev 1364 * Pointer to the Ethernet device structure. 1365 * @param[in] attr 1366 * Pointer to device flow rule attributes. 1367 * @param[in] subpriority 1368 * The priority based on the items. 1369 * @param[in] external 1370 * Flow is user flow. 1371 * @return 1372 * The matcher priority of the flow. 1373 */ 1374 uint16_t 1375 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1376 const struct rte_flow_attr *attr, 1377 uint32_t subpriority, bool external) 1378 { 1379 uint16_t priority = (uint16_t)attr->priority; 1380 struct mlx5_priv *priv = dev->data->dev_private; 1381 1382 /* NIC root rules */ 1383 if (!attr->group && !attr->transfer) { 1384 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1385 priority = priv->sh->flow_max_priority - 1; 1386 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1387 /* FDB root rules */ 1388 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1389 attr->group == 0 && 1390 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1391 return (priv->sh->flow_max_priority - 1) * 3; 1392 } 1393 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1394 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1395 return priority * 3 + subpriority; 1396 } 1397 1398 /** 1399 * Verify the @p item specifications (spec, last, mask) are compatible with the 1400 * NIC capabilities. 1401 * 1402 * @param[in] item 1403 * Item specification. 1404 * @param[in] mask 1405 * @p item->mask or flow default bit-masks. 1406 * @param[in] nic_mask 1407 * Bit-masks covering supported fields by the NIC to compare with user mask. 1408 * @param[in] size 1409 * Bit-masks size in bytes. 1410 * @param[in] range_accepted 1411 * True if range of values is accepted for specific fields, false otherwise. 1412 * @param[out] error 1413 * Pointer to error structure. 1414 * 1415 * @return 1416 * 0 on success, a negative errno value otherwise and rte_errno is set. 1417 */ 1418 int 1419 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1420 const uint8_t *mask, 1421 const uint8_t *nic_mask, 1422 unsigned int size, 1423 bool range_accepted, 1424 struct rte_flow_error *error) 1425 { 1426 unsigned int i; 1427 1428 MLX5_ASSERT(nic_mask); 1429 for (i = 0; i < size; ++i) 1430 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1431 return rte_flow_error_set(error, ENOTSUP, 1432 RTE_FLOW_ERROR_TYPE_ITEM, 1433 item, 1434 "mask enables non supported" 1435 " bits"); 1436 if (!item->spec && (item->mask || item->last)) 1437 return rte_flow_error_set(error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ITEM, item, 1439 "mask/last without a spec is not" 1440 " supported"); 1441 if (item->spec && item->last && !range_accepted) { 1442 uint8_t spec[size]; 1443 uint8_t last[size]; 1444 unsigned int i; 1445 int ret; 1446 1447 for (i = 0; i < size; ++i) { 1448 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1449 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1450 } 1451 ret = memcmp(spec, last, size); 1452 if (ret != 0) 1453 return rte_flow_error_set(error, EINVAL, 1454 RTE_FLOW_ERROR_TYPE_ITEM, 1455 item, 1456 "range is not valid"); 1457 } 1458 return 0; 1459 } 1460 1461 /** 1462 * Adjust the hash fields according to the @p flow information. 1463 * 1464 * @param[in] dev_flow. 1465 * Pointer to the mlx5_flow. 1466 * @param[in] tunnel 1467 * 1 when the hash field is for a tunnel item. 1468 * @param[in] layer_types 1469 * RTE_ETH_RSS_* types. 1470 * @param[in] hash_fields 1471 * Item hash fields. 1472 * 1473 * @return 1474 * The hash fields that should be used. 1475 */ 1476 uint64_t 1477 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1478 int tunnel __rte_unused, uint64_t layer_types, 1479 uint64_t hash_fields) 1480 { 1481 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1482 int rss_request_inner = rss_desc->level >= 2; 1483 1484 /* Check RSS hash level for tunnel. */ 1485 if (tunnel && rss_request_inner) 1486 hash_fields |= IBV_RX_HASH_INNER; 1487 else if (tunnel || rss_request_inner) 1488 return 0; 1489 #endif 1490 /* Check if requested layer matches RSS hash fields. */ 1491 if (!(rss_desc->types & layer_types)) 1492 return 0; 1493 return hash_fields; 1494 } 1495 1496 /** 1497 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1498 * if several tunnel rules are used on this queue, the tunnel ptype will be 1499 * cleared. 1500 * 1501 * @param rxq_ctrl 1502 * Rx queue to update. 1503 */ 1504 static void 1505 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1506 { 1507 unsigned int i; 1508 uint32_t tunnel_ptype = 0; 1509 1510 /* Look up for the ptype to use. */ 1511 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1512 if (!rxq_ctrl->flow_tunnels_n[i]) 1513 continue; 1514 if (!tunnel_ptype) { 1515 tunnel_ptype = tunnels_info[i].ptype; 1516 } else { 1517 tunnel_ptype = 0; 1518 break; 1519 } 1520 } 1521 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1522 } 1523 1524 /** 1525 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1526 * flow. 1527 * 1528 * @param[in] dev 1529 * Pointer to the Ethernet device structure. 1530 * @param[in] dev_handle 1531 * Pointer to device flow handle structure. 1532 */ 1533 void 1534 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1535 struct mlx5_flow_handle *dev_handle) 1536 { 1537 struct mlx5_priv *priv = dev->data->dev_private; 1538 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1539 struct mlx5_ind_table_obj *ind_tbl = NULL; 1540 unsigned int i; 1541 1542 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1543 struct mlx5_hrxq *hrxq; 1544 1545 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1546 dev_handle->rix_hrxq); 1547 if (hrxq) 1548 ind_tbl = hrxq->ind_table; 1549 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1550 struct mlx5_shared_action_rss *shared_rss; 1551 1552 shared_rss = mlx5_ipool_get 1553 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1554 dev_handle->rix_srss); 1555 if (shared_rss) 1556 ind_tbl = shared_rss->ind_tbl; 1557 } 1558 if (!ind_tbl) 1559 return; 1560 for (i = 0; i != ind_tbl->queues_n; ++i) { 1561 int idx = ind_tbl->queues[i]; 1562 struct mlx5_rxq_ctrl *rxq_ctrl; 1563 1564 if (mlx5_is_external_rxq(dev, idx)) 1565 continue; 1566 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1567 MLX5_ASSERT(rxq_ctrl != NULL); 1568 if (rxq_ctrl == NULL) 1569 continue; 1570 /* 1571 * To support metadata register copy on Tx loopback, 1572 * this must be always enabled (metadata may arive 1573 * from other port - not from local flows only. 1574 */ 1575 if (tunnel) { 1576 unsigned int j; 1577 1578 /* Increase the counter matching the flow. */ 1579 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1580 if ((tunnels_info[j].tunnel & 1581 dev_handle->layers) == 1582 tunnels_info[j].tunnel) { 1583 rxq_ctrl->flow_tunnels_n[j]++; 1584 break; 1585 } 1586 } 1587 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1588 } 1589 } 1590 } 1591 1592 static void 1593 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1594 { 1595 struct mlx5_priv *priv = dev->data->dev_private; 1596 struct mlx5_rxq_ctrl *rxq_ctrl; 1597 uint16_t port_id; 1598 1599 if (priv->sh->shared_mark_enabled) 1600 return; 1601 if (priv->master || priv->representor) { 1602 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1603 struct mlx5_priv *opriv = 1604 rte_eth_devices[port_id].data->dev_private; 1605 1606 if (!opriv || 1607 opriv->sh != priv->sh || 1608 opriv->domain_id != priv->domain_id || 1609 opriv->mark_enabled) 1610 continue; 1611 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1612 rxq_ctrl->rxq.mark = 1; 1613 } 1614 opriv->mark_enabled = 1; 1615 } 1616 } else { 1617 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1618 rxq_ctrl->rxq.mark = 1; 1619 } 1620 priv->mark_enabled = 1; 1621 } 1622 priv->sh->shared_mark_enabled = 1; 1623 } 1624 1625 /** 1626 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1627 * 1628 * @param[in] dev 1629 * Pointer to the Ethernet device structure. 1630 * @param[in] flow 1631 * Pointer to flow structure. 1632 */ 1633 static void 1634 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1635 { 1636 struct mlx5_priv *priv = dev->data->dev_private; 1637 uint32_t handle_idx; 1638 struct mlx5_flow_handle *dev_handle; 1639 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1640 1641 MLX5_ASSERT(wks); 1642 if (wks->mark) 1643 flow_rxq_mark_flag_set(dev); 1644 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1645 handle_idx, dev_handle, next) 1646 flow_drv_rxq_flags_set(dev, dev_handle); 1647 } 1648 1649 /** 1650 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1651 * device flow if no other flow uses it with the same kind of request. 1652 * 1653 * @param dev 1654 * Pointer to Ethernet device. 1655 * @param[in] dev_handle 1656 * Pointer to the device flow handle structure. 1657 */ 1658 static void 1659 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1660 struct mlx5_flow_handle *dev_handle) 1661 { 1662 struct mlx5_priv *priv = dev->data->dev_private; 1663 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1664 struct mlx5_ind_table_obj *ind_tbl = NULL; 1665 unsigned int i; 1666 1667 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1668 struct mlx5_hrxq *hrxq; 1669 1670 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1671 dev_handle->rix_hrxq); 1672 if (hrxq) 1673 ind_tbl = hrxq->ind_table; 1674 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1675 struct mlx5_shared_action_rss *shared_rss; 1676 1677 shared_rss = mlx5_ipool_get 1678 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1679 dev_handle->rix_srss); 1680 if (shared_rss) 1681 ind_tbl = shared_rss->ind_tbl; 1682 } 1683 if (!ind_tbl) 1684 return; 1685 MLX5_ASSERT(dev->data->dev_started); 1686 for (i = 0; i != ind_tbl->queues_n; ++i) { 1687 int idx = ind_tbl->queues[i]; 1688 struct mlx5_rxq_ctrl *rxq_ctrl; 1689 1690 if (mlx5_is_external_rxq(dev, idx)) 1691 continue; 1692 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1693 MLX5_ASSERT(rxq_ctrl != NULL); 1694 if (rxq_ctrl == NULL) 1695 continue; 1696 if (tunnel) { 1697 unsigned int j; 1698 1699 /* Decrease the counter matching the flow. */ 1700 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1701 if ((tunnels_info[j].tunnel & 1702 dev_handle->layers) == 1703 tunnels_info[j].tunnel) { 1704 rxq_ctrl->flow_tunnels_n[j]--; 1705 break; 1706 } 1707 } 1708 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1709 } 1710 } 1711 } 1712 1713 /** 1714 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1715 * @p flow if no other flow uses it with the same kind of request. 1716 * 1717 * @param dev 1718 * Pointer to Ethernet device. 1719 * @param[in] flow 1720 * Pointer to the flow. 1721 */ 1722 static void 1723 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1724 { 1725 struct mlx5_priv *priv = dev->data->dev_private; 1726 uint32_t handle_idx; 1727 struct mlx5_flow_handle *dev_handle; 1728 1729 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1730 handle_idx, dev_handle, next) 1731 flow_drv_rxq_flags_trim(dev, dev_handle); 1732 } 1733 1734 /** 1735 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1736 * 1737 * @param dev 1738 * Pointer to Ethernet device. 1739 */ 1740 static void 1741 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1742 { 1743 struct mlx5_priv *priv = dev->data->dev_private; 1744 unsigned int i; 1745 1746 for (i = 0; i != priv->rxqs_n; ++i) { 1747 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1748 unsigned int j; 1749 1750 if (rxq == NULL || rxq->ctrl == NULL) 1751 continue; 1752 rxq->ctrl->rxq.mark = 0; 1753 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1754 rxq->ctrl->flow_tunnels_n[j] = 0; 1755 rxq->ctrl->rxq.tunnel = 0; 1756 } 1757 priv->mark_enabled = 0; 1758 priv->sh->shared_mark_enabled = 0; 1759 } 1760 1761 /** 1762 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1763 * 1764 * @param[in] dev 1765 * Pointer to the Ethernet device structure. 1766 */ 1767 void 1768 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1769 { 1770 struct mlx5_priv *priv = dev->data->dev_private; 1771 unsigned int i; 1772 1773 for (i = 0; i != priv->rxqs_n; ++i) { 1774 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1775 struct mlx5_rxq_data *data; 1776 1777 if (rxq == NULL || rxq->ctrl == NULL) 1778 continue; 1779 data = &rxq->ctrl->rxq; 1780 if (!rte_flow_dynf_metadata_avail()) { 1781 data->dynf_meta = 0; 1782 data->flow_meta_mask = 0; 1783 data->flow_meta_offset = -1; 1784 data->flow_meta_port_mask = 0; 1785 } else { 1786 data->dynf_meta = 1; 1787 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1788 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1789 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1790 } 1791 } 1792 } 1793 1794 /* 1795 * return a pointer to the desired action in the list of actions. 1796 * 1797 * @param[in] actions 1798 * The list of actions to search the action in. 1799 * @param[in] action 1800 * The action to find. 1801 * 1802 * @return 1803 * Pointer to the action in the list, if found. NULL otherwise. 1804 */ 1805 const struct rte_flow_action * 1806 mlx5_flow_find_action(const struct rte_flow_action *actions, 1807 enum rte_flow_action_type action) 1808 { 1809 if (actions == NULL) 1810 return NULL; 1811 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1812 if (actions->type == action) 1813 return actions; 1814 return NULL; 1815 } 1816 1817 /* 1818 * Validate the flag action. 1819 * 1820 * @param[in] action_flags 1821 * Bit-fields that holds the actions detected until now. 1822 * @param[in] attr 1823 * Attributes of flow that includes this action. 1824 * @param[out] error 1825 * Pointer to error structure. 1826 * 1827 * @return 1828 * 0 on success, a negative errno value otherwise and rte_errno is set. 1829 */ 1830 int 1831 mlx5_flow_validate_action_flag(uint64_t action_flags, 1832 const struct rte_flow_attr *attr, 1833 struct rte_flow_error *error) 1834 { 1835 if (action_flags & MLX5_FLOW_ACTION_MARK) 1836 return rte_flow_error_set(error, EINVAL, 1837 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1838 "can't mark and flag in same flow"); 1839 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1840 return rte_flow_error_set(error, EINVAL, 1841 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1842 "can't have 2 flag" 1843 " actions in same flow"); 1844 if (attr->egress) 1845 return rte_flow_error_set(error, ENOTSUP, 1846 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1847 "flag action not supported for " 1848 "egress"); 1849 return 0; 1850 } 1851 1852 /* 1853 * Validate the mark action. 1854 * 1855 * @param[in] action 1856 * Pointer to the queue action. 1857 * @param[in] action_flags 1858 * Bit-fields that holds the actions detected until now. 1859 * @param[in] attr 1860 * Attributes of flow that includes this action. 1861 * @param[out] error 1862 * Pointer to error structure. 1863 * 1864 * @return 1865 * 0 on success, a negative errno value otherwise and rte_errno is set. 1866 */ 1867 int 1868 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1869 uint64_t action_flags, 1870 const struct rte_flow_attr *attr, 1871 struct rte_flow_error *error) 1872 { 1873 const struct rte_flow_action_mark *mark = action->conf; 1874 1875 if (!mark) 1876 return rte_flow_error_set(error, EINVAL, 1877 RTE_FLOW_ERROR_TYPE_ACTION, 1878 action, 1879 "configuration cannot be null"); 1880 if (mark->id >= MLX5_FLOW_MARK_MAX) 1881 return rte_flow_error_set(error, EINVAL, 1882 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1883 &mark->id, 1884 "mark id must in 0 <= id < " 1885 RTE_STR(MLX5_FLOW_MARK_MAX)); 1886 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1887 return rte_flow_error_set(error, EINVAL, 1888 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1889 "can't flag and mark in same flow"); 1890 if (action_flags & MLX5_FLOW_ACTION_MARK) 1891 return rte_flow_error_set(error, EINVAL, 1892 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1893 "can't have 2 mark actions in same" 1894 " flow"); 1895 if (attr->egress) 1896 return rte_flow_error_set(error, ENOTSUP, 1897 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1898 "mark action not supported for " 1899 "egress"); 1900 return 0; 1901 } 1902 1903 /* 1904 * Validate the drop action. 1905 * 1906 * @param[in] action_flags 1907 * Bit-fields that holds the actions detected until now. 1908 * @param[in] attr 1909 * Attributes of flow that includes this action. 1910 * @param[out] error 1911 * Pointer to error structure. 1912 * 1913 * @return 1914 * 0 on success, a negative errno value otherwise and rte_errno is set. 1915 */ 1916 int 1917 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1918 const struct rte_flow_attr *attr, 1919 struct rte_flow_error *error) 1920 { 1921 if (attr->egress) 1922 return rte_flow_error_set(error, ENOTSUP, 1923 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1924 "drop action not supported for " 1925 "egress"); 1926 return 0; 1927 } 1928 1929 /* 1930 * Validate the queue action. 1931 * 1932 * @param[in] action 1933 * Pointer to the queue action. 1934 * @param[in] action_flags 1935 * Bit-fields that holds the actions detected until now. 1936 * @param[in] dev 1937 * Pointer to the Ethernet device structure. 1938 * @param[in] attr 1939 * Attributes of flow that includes this action. 1940 * @param[out] error 1941 * Pointer to error structure. 1942 * 1943 * @return 1944 * 0 on success, a negative errno value otherwise and rte_errno is set. 1945 */ 1946 int 1947 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1948 uint64_t action_flags, 1949 struct rte_eth_dev *dev, 1950 const struct rte_flow_attr *attr, 1951 struct rte_flow_error *error) 1952 { 1953 struct mlx5_priv *priv = dev->data->dev_private; 1954 const struct rte_flow_action_queue *queue = action->conf; 1955 1956 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1957 return rte_flow_error_set(error, EINVAL, 1958 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1959 "can't have 2 fate actions in" 1960 " same flow"); 1961 if (attr->egress) 1962 return rte_flow_error_set(error, ENOTSUP, 1963 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1964 "queue action not supported for egress."); 1965 if (mlx5_is_external_rxq(dev, queue->index)) 1966 return 0; 1967 if (!priv->rxqs_n) 1968 return rte_flow_error_set(error, EINVAL, 1969 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1970 NULL, "No Rx queues configured"); 1971 if (queue->index >= priv->rxqs_n) 1972 return rte_flow_error_set(error, EINVAL, 1973 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1974 &queue->index, 1975 "queue index out of range"); 1976 if (mlx5_rxq_get(dev, queue->index) == NULL) 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1979 &queue->index, 1980 "queue is not configured"); 1981 return 0; 1982 } 1983 1984 /** 1985 * Validate queue numbers for device RSS. 1986 * 1987 * @param[in] dev 1988 * Configured device. 1989 * @param[in] queues 1990 * Array of queue numbers. 1991 * @param[in] queues_n 1992 * Size of the @p queues array. 1993 * @param[out] error 1994 * On error, filled with a textual error description. 1995 * @param[out] queue_idx 1996 * On error, filled with an offending queue index in @p queues array. 1997 * 1998 * @return 1999 * 0 on success, a negative errno code on error. 2000 */ 2001 static int 2002 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2003 const uint16_t *queues, uint32_t queues_n, 2004 const char **error, uint32_t *queue_idx) 2005 { 2006 const struct mlx5_priv *priv = dev->data->dev_private; 2007 bool is_hairpin = false; 2008 bool is_ext_rss = false; 2009 uint32_t i; 2010 2011 for (i = 0; i != queues_n; ++i) { 2012 struct mlx5_rxq_ctrl *rxq_ctrl; 2013 2014 if (mlx5_is_external_rxq(dev, queues[0])) { 2015 is_ext_rss = true; 2016 continue; 2017 } 2018 if (is_ext_rss) { 2019 *error = "Combining external and regular RSS queues is not supported"; 2020 *queue_idx = i; 2021 return -ENOTSUP; 2022 } 2023 if (queues[i] >= priv->rxqs_n) { 2024 *error = "queue index out of range"; 2025 *queue_idx = i; 2026 return -EINVAL; 2027 } 2028 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2029 if (rxq_ctrl == NULL) { 2030 *error = "queue is not configured"; 2031 *queue_idx = i; 2032 return -EINVAL; 2033 } 2034 if (i == 0 && rxq_ctrl->is_hairpin) 2035 is_hairpin = true; 2036 if (is_hairpin != rxq_ctrl->is_hairpin) { 2037 *error = "combining hairpin and regular RSS queues is not supported"; 2038 *queue_idx = i; 2039 return -ENOTSUP; 2040 } 2041 } 2042 return 0; 2043 } 2044 2045 /* 2046 * Validate the rss action. 2047 * 2048 * @param[in] dev 2049 * Pointer to the Ethernet device structure. 2050 * @param[in] action 2051 * Pointer to the queue action. 2052 * @param[out] error 2053 * Pointer to error structure. 2054 * 2055 * @return 2056 * 0 on success, a negative errno value otherwise and rte_errno is set. 2057 */ 2058 int 2059 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2060 const struct rte_flow_action *action, 2061 struct rte_flow_error *error) 2062 { 2063 struct mlx5_priv *priv = dev->data->dev_private; 2064 const struct rte_flow_action_rss *rss = action->conf; 2065 int ret; 2066 const char *message; 2067 uint32_t queue_idx; 2068 2069 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2070 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2071 return rte_flow_error_set(error, ENOTSUP, 2072 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2073 &rss->func, 2074 "RSS hash function not supported"); 2075 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2076 if (rss->level > 2) 2077 #else 2078 if (rss->level > 1) 2079 #endif 2080 return rte_flow_error_set(error, ENOTSUP, 2081 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2082 &rss->level, 2083 "tunnel RSS is not supported"); 2084 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2085 if (rss->key_len == 0 && rss->key != NULL) 2086 return rte_flow_error_set(error, ENOTSUP, 2087 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2088 &rss->key_len, 2089 "RSS hash key length 0"); 2090 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2091 return rte_flow_error_set(error, ENOTSUP, 2092 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2093 &rss->key_len, 2094 "RSS hash key too small"); 2095 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2096 return rte_flow_error_set(error, ENOTSUP, 2097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2098 &rss->key_len, 2099 "RSS hash key too large"); 2100 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2101 return rte_flow_error_set(error, ENOTSUP, 2102 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2103 &rss->queue_num, 2104 "number of queues too large"); 2105 if (rss->types & MLX5_RSS_HF_MASK) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2108 &rss->types, 2109 "some RSS protocols are not" 2110 " supported"); 2111 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2112 !(rss->types & RTE_ETH_RSS_IP)) 2113 return rte_flow_error_set(error, EINVAL, 2114 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2115 "L3 partial RSS requested but L3 RSS" 2116 " type not specified"); 2117 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2118 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2119 return rte_flow_error_set(error, EINVAL, 2120 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2121 "L4 partial RSS requested but L4 RSS" 2122 " type not specified"); 2123 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2124 return rte_flow_error_set(error, EINVAL, 2125 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2126 NULL, "No Rx queues configured"); 2127 if (!rss->queue_num) 2128 return rte_flow_error_set(error, EINVAL, 2129 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2130 NULL, "No queues configured"); 2131 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2132 &message, &queue_idx); 2133 if (ret != 0) { 2134 return rte_flow_error_set(error, -ret, 2135 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2136 &rss->queue[queue_idx], message); 2137 } 2138 return 0; 2139 } 2140 2141 /* 2142 * Validate the rss action. 2143 * 2144 * @param[in] action 2145 * Pointer to the queue action. 2146 * @param[in] action_flags 2147 * Bit-fields that holds the actions detected until now. 2148 * @param[in] dev 2149 * Pointer to the Ethernet device structure. 2150 * @param[in] attr 2151 * Attributes of flow that includes this action. 2152 * @param[in] item_flags 2153 * Items that were detected. 2154 * @param[out] error 2155 * Pointer to error structure. 2156 * 2157 * @return 2158 * 0 on success, a negative errno value otherwise and rte_errno is set. 2159 */ 2160 int 2161 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2162 uint64_t action_flags, 2163 struct rte_eth_dev *dev, 2164 const struct rte_flow_attr *attr, 2165 uint64_t item_flags, 2166 struct rte_flow_error *error) 2167 { 2168 const struct rte_flow_action_rss *rss = action->conf; 2169 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2170 int ret; 2171 2172 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2175 "can't have 2 fate actions" 2176 " in same flow"); 2177 ret = mlx5_validate_action_rss(dev, action, error); 2178 if (ret) 2179 return ret; 2180 if (attr->egress) 2181 return rte_flow_error_set(error, ENOTSUP, 2182 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2183 "rss action not supported for " 2184 "egress"); 2185 if (rss->level > 1 && !tunnel) 2186 return rte_flow_error_set(error, EINVAL, 2187 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2188 "inner RSS is not supported for " 2189 "non-tunnel flows"); 2190 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2191 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2192 return rte_flow_error_set(error, EINVAL, 2193 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2194 "RSS on eCPRI is not supported now"); 2195 } 2196 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2197 !(item_flags & 2198 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2199 rss->level > 1) 2200 return rte_flow_error_set(error, EINVAL, 2201 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2202 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2203 return 0; 2204 } 2205 2206 /* 2207 * Validate the default miss action. 2208 * 2209 * @param[in] action_flags 2210 * Bit-fields that holds the actions detected until now. 2211 * @param[out] error 2212 * Pointer to error structure. 2213 * 2214 * @return 2215 * 0 on success, a negative errno value otherwise and rte_errno is set. 2216 */ 2217 int 2218 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2219 const struct rte_flow_attr *attr, 2220 struct rte_flow_error *error) 2221 { 2222 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2225 "can't have 2 fate actions in" 2226 " same flow"); 2227 if (attr->egress) 2228 return rte_flow_error_set(error, ENOTSUP, 2229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2230 "default miss action not supported " 2231 "for egress"); 2232 if (attr->group) 2233 return rte_flow_error_set(error, ENOTSUP, 2234 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2235 "only group 0 is supported"); 2236 if (attr->transfer) 2237 return rte_flow_error_set(error, ENOTSUP, 2238 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2239 NULL, "transfer is not supported"); 2240 return 0; 2241 } 2242 2243 /* 2244 * Validate the count action. 2245 * 2246 * @param[in] dev 2247 * Pointer to the Ethernet device structure. 2248 * @param[in] attr 2249 * Attributes of flow that includes this action. 2250 * @param[out] error 2251 * Pointer to error structure. 2252 * 2253 * @return 2254 * 0 on success, a negative errno value otherwise and rte_errno is set. 2255 */ 2256 int 2257 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2258 const struct rte_flow_attr *attr, 2259 struct rte_flow_error *error) 2260 { 2261 if (attr->egress) 2262 return rte_flow_error_set(error, ENOTSUP, 2263 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2264 "count action not supported for " 2265 "egress"); 2266 return 0; 2267 } 2268 2269 /* 2270 * Validate the ASO CT action. 2271 * 2272 * @param[in] dev 2273 * Pointer to the Ethernet device structure. 2274 * @param[in] conntrack 2275 * Pointer to the CT action profile. 2276 * @param[out] error 2277 * Pointer to error structure. 2278 * 2279 * @return 2280 * 0 on success, a negative errno value otherwise and rte_errno is set. 2281 */ 2282 int 2283 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2284 const struct rte_flow_action_conntrack *conntrack, 2285 struct rte_flow_error *error) 2286 { 2287 RTE_SET_USED(dev); 2288 2289 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2290 return rte_flow_error_set(error, EINVAL, 2291 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2292 "Invalid CT state"); 2293 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2296 "Invalid last TCP packet flag"); 2297 return 0; 2298 } 2299 2300 /** 2301 * Verify the @p attributes will be correctly understood by the NIC and store 2302 * them in the @p flow if everything is correct. 2303 * 2304 * @param[in] dev 2305 * Pointer to the Ethernet device structure. 2306 * @param[in] attributes 2307 * Pointer to flow attributes 2308 * @param[out] error 2309 * Pointer to error structure. 2310 * 2311 * @return 2312 * 0 on success, a negative errno value otherwise and rte_errno is set. 2313 */ 2314 int 2315 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2316 const struct rte_flow_attr *attributes, 2317 struct rte_flow_error *error) 2318 { 2319 struct mlx5_priv *priv = dev->data->dev_private; 2320 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2321 2322 if (attributes->group) 2323 return rte_flow_error_set(error, ENOTSUP, 2324 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2325 NULL, "groups is not supported"); 2326 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2327 attributes->priority >= priority_max) 2328 return rte_flow_error_set(error, ENOTSUP, 2329 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2330 NULL, "priority out of range"); 2331 if (attributes->egress) 2332 return rte_flow_error_set(error, ENOTSUP, 2333 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2334 "egress is not supported"); 2335 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2336 return rte_flow_error_set(error, ENOTSUP, 2337 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2338 NULL, "transfer is not supported"); 2339 if (!attributes->ingress) 2340 return rte_flow_error_set(error, EINVAL, 2341 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2342 NULL, 2343 "ingress attribute is mandatory"); 2344 return 0; 2345 } 2346 2347 /** 2348 * Validate ICMP6 item. 2349 * 2350 * @param[in] item 2351 * Item specification. 2352 * @param[in] item_flags 2353 * Bit-fields that holds the items detected until now. 2354 * @param[in] ext_vlan_sup 2355 * Whether extended VLAN features are supported or not. 2356 * @param[out] error 2357 * Pointer to error structure. 2358 * 2359 * @return 2360 * 0 on success, a negative errno value otherwise and rte_errno is set. 2361 */ 2362 int 2363 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2364 uint64_t item_flags, 2365 uint8_t target_protocol, 2366 struct rte_flow_error *error) 2367 { 2368 const struct rte_flow_item_icmp6 *mask = item->mask; 2369 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2370 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2371 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2372 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2373 MLX5_FLOW_LAYER_OUTER_L4; 2374 int ret; 2375 2376 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2377 return rte_flow_error_set(error, EINVAL, 2378 RTE_FLOW_ERROR_TYPE_ITEM, item, 2379 "protocol filtering not compatible" 2380 " with ICMP6 layer"); 2381 if (!(item_flags & l3m)) 2382 return rte_flow_error_set(error, EINVAL, 2383 RTE_FLOW_ERROR_TYPE_ITEM, item, 2384 "IPv6 is mandatory to filter on" 2385 " ICMP6"); 2386 if (item_flags & l4m) 2387 return rte_flow_error_set(error, EINVAL, 2388 RTE_FLOW_ERROR_TYPE_ITEM, item, 2389 "multiple L4 layers not supported"); 2390 if (!mask) 2391 mask = &rte_flow_item_icmp6_mask; 2392 ret = mlx5_flow_item_acceptable 2393 (item, (const uint8_t *)mask, 2394 (const uint8_t *)&rte_flow_item_icmp6_mask, 2395 sizeof(struct rte_flow_item_icmp6), 2396 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2397 if (ret < 0) 2398 return ret; 2399 return 0; 2400 } 2401 2402 /** 2403 * Validate ICMP item. 2404 * 2405 * @param[in] item 2406 * Item specification. 2407 * @param[in] item_flags 2408 * Bit-fields that holds the items detected until now. 2409 * @param[out] error 2410 * Pointer to error structure. 2411 * 2412 * @return 2413 * 0 on success, a negative errno value otherwise and rte_errno is set. 2414 */ 2415 int 2416 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2417 uint64_t item_flags, 2418 uint8_t target_protocol, 2419 struct rte_flow_error *error) 2420 { 2421 const struct rte_flow_item_icmp *mask = item->mask; 2422 const struct rte_flow_item_icmp nic_mask = { 2423 .hdr.icmp_type = 0xff, 2424 .hdr.icmp_code = 0xff, 2425 .hdr.icmp_ident = RTE_BE16(0xffff), 2426 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2427 }; 2428 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2429 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2430 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2431 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2432 MLX5_FLOW_LAYER_OUTER_L4; 2433 int ret; 2434 2435 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2436 return rte_flow_error_set(error, EINVAL, 2437 RTE_FLOW_ERROR_TYPE_ITEM, item, 2438 "protocol filtering not compatible" 2439 " with ICMP layer"); 2440 if (!(item_flags & l3m)) 2441 return rte_flow_error_set(error, EINVAL, 2442 RTE_FLOW_ERROR_TYPE_ITEM, item, 2443 "IPv4 is mandatory to filter" 2444 " on ICMP"); 2445 if (item_flags & l4m) 2446 return rte_flow_error_set(error, EINVAL, 2447 RTE_FLOW_ERROR_TYPE_ITEM, item, 2448 "multiple L4 layers not supported"); 2449 if (!mask) 2450 mask = &nic_mask; 2451 ret = mlx5_flow_item_acceptable 2452 (item, (const uint8_t *)mask, 2453 (const uint8_t *)&nic_mask, 2454 sizeof(struct rte_flow_item_icmp), 2455 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2456 if (ret < 0) 2457 return ret; 2458 return 0; 2459 } 2460 2461 /** 2462 * Validate Ethernet item. 2463 * 2464 * @param[in] item 2465 * Item specification. 2466 * @param[in] item_flags 2467 * Bit-fields that holds the items detected until now. 2468 * @param[out] error 2469 * Pointer to error structure. 2470 * 2471 * @return 2472 * 0 on success, a negative errno value otherwise and rte_errno is set. 2473 */ 2474 int 2475 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2476 uint64_t item_flags, bool ext_vlan_sup, 2477 struct rte_flow_error *error) 2478 { 2479 const struct rte_flow_item_eth *mask = item->mask; 2480 const struct rte_flow_item_eth nic_mask = { 2481 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2482 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2483 .type = RTE_BE16(0xffff), 2484 .has_vlan = ext_vlan_sup ? 1 : 0, 2485 }; 2486 int ret; 2487 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2488 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2489 MLX5_FLOW_LAYER_OUTER_L2; 2490 2491 if (item_flags & ethm) 2492 return rte_flow_error_set(error, ENOTSUP, 2493 RTE_FLOW_ERROR_TYPE_ITEM, item, 2494 "multiple L2 layers not supported"); 2495 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2496 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2497 return rte_flow_error_set(error, EINVAL, 2498 RTE_FLOW_ERROR_TYPE_ITEM, item, 2499 "L2 layer should not follow " 2500 "L3 layers"); 2501 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2502 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2503 return rte_flow_error_set(error, EINVAL, 2504 RTE_FLOW_ERROR_TYPE_ITEM, item, 2505 "L2 layer should not follow VLAN"); 2506 if (item_flags & MLX5_FLOW_LAYER_GTP) 2507 return rte_flow_error_set(error, EINVAL, 2508 RTE_FLOW_ERROR_TYPE_ITEM, item, 2509 "L2 layer should not follow GTP"); 2510 if (!mask) 2511 mask = &rte_flow_item_eth_mask; 2512 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2513 (const uint8_t *)&nic_mask, 2514 sizeof(struct rte_flow_item_eth), 2515 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2516 return ret; 2517 } 2518 2519 /** 2520 * Validate VLAN item. 2521 * 2522 * @param[in] item 2523 * Item specification. 2524 * @param[in] item_flags 2525 * Bit-fields that holds the items detected until now. 2526 * @param[in] dev 2527 * Ethernet device flow is being created on. 2528 * @param[out] error 2529 * Pointer to error structure. 2530 * 2531 * @return 2532 * 0 on success, a negative errno value otherwise and rte_errno is set. 2533 */ 2534 int 2535 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2536 uint64_t item_flags, 2537 struct rte_eth_dev *dev, 2538 struct rte_flow_error *error) 2539 { 2540 const struct rte_flow_item_vlan *spec = item->spec; 2541 const struct rte_flow_item_vlan *mask = item->mask; 2542 const struct rte_flow_item_vlan nic_mask = { 2543 .tci = RTE_BE16(UINT16_MAX), 2544 .inner_type = RTE_BE16(UINT16_MAX), 2545 }; 2546 uint16_t vlan_tag = 0; 2547 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2548 int ret; 2549 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2550 MLX5_FLOW_LAYER_INNER_L4) : 2551 (MLX5_FLOW_LAYER_OUTER_L3 | 2552 MLX5_FLOW_LAYER_OUTER_L4); 2553 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2554 MLX5_FLOW_LAYER_OUTER_VLAN; 2555 2556 if (item_flags & vlanm) 2557 return rte_flow_error_set(error, EINVAL, 2558 RTE_FLOW_ERROR_TYPE_ITEM, item, 2559 "multiple VLAN layers not supported"); 2560 else if ((item_flags & l34m) != 0) 2561 return rte_flow_error_set(error, EINVAL, 2562 RTE_FLOW_ERROR_TYPE_ITEM, item, 2563 "VLAN cannot follow L3/L4 layer"); 2564 if (!mask) 2565 mask = &rte_flow_item_vlan_mask; 2566 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2567 (const uint8_t *)&nic_mask, 2568 sizeof(struct rte_flow_item_vlan), 2569 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2570 if (ret) 2571 return ret; 2572 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2573 struct mlx5_priv *priv = dev->data->dev_private; 2574 2575 if (priv->vmwa_context) { 2576 /* 2577 * Non-NULL context means we have a virtual machine 2578 * and SR-IOV enabled, we have to create VLAN interface 2579 * to make hypervisor to setup E-Switch vport 2580 * context correctly. We avoid creating the multiple 2581 * VLAN interfaces, so we cannot support VLAN tag mask. 2582 */ 2583 return rte_flow_error_set(error, EINVAL, 2584 RTE_FLOW_ERROR_TYPE_ITEM, 2585 item, 2586 "VLAN tag mask is not" 2587 " supported in virtual" 2588 " environment"); 2589 } 2590 } 2591 if (spec) { 2592 vlan_tag = spec->tci; 2593 vlan_tag &= mask->tci; 2594 } 2595 /* 2596 * From verbs perspective an empty VLAN is equivalent 2597 * to a packet without VLAN layer. 2598 */ 2599 if (!vlan_tag) 2600 return rte_flow_error_set(error, EINVAL, 2601 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2602 item->spec, 2603 "VLAN cannot be empty"); 2604 return 0; 2605 } 2606 2607 /** 2608 * Validate IPV4 item. 2609 * 2610 * @param[in] item 2611 * Item specification. 2612 * @param[in] item_flags 2613 * Bit-fields that holds the items detected until now. 2614 * @param[in] last_item 2615 * Previous validated item in the pattern items. 2616 * @param[in] ether_type 2617 * Type in the ethernet layer header (including dot1q). 2618 * @param[in] acc_mask 2619 * Acceptable mask, if NULL default internal default mask 2620 * will be used to check whether item fields are supported. 2621 * @param[in] range_accepted 2622 * True if range of values is accepted for specific fields, false otherwise. 2623 * @param[out] error 2624 * Pointer to error structure. 2625 * 2626 * @return 2627 * 0 on success, a negative errno value otherwise and rte_errno is set. 2628 */ 2629 int 2630 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2631 uint64_t item_flags, 2632 uint64_t last_item, 2633 uint16_t ether_type, 2634 const struct rte_flow_item_ipv4 *acc_mask, 2635 bool range_accepted, 2636 struct rte_flow_error *error) 2637 { 2638 const struct rte_flow_item_ipv4 *mask = item->mask; 2639 const struct rte_flow_item_ipv4 *spec = item->spec; 2640 const struct rte_flow_item_ipv4 nic_mask = { 2641 .hdr = { 2642 .src_addr = RTE_BE32(0xffffffff), 2643 .dst_addr = RTE_BE32(0xffffffff), 2644 .type_of_service = 0xff, 2645 .next_proto_id = 0xff, 2646 }, 2647 }; 2648 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2649 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2650 MLX5_FLOW_LAYER_OUTER_L3; 2651 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2652 MLX5_FLOW_LAYER_OUTER_L4; 2653 int ret; 2654 uint8_t next_proto = 0xFF; 2655 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2656 MLX5_FLOW_LAYER_OUTER_VLAN | 2657 MLX5_FLOW_LAYER_INNER_VLAN); 2658 2659 if ((last_item & l2_vlan) && ether_type && 2660 ether_type != RTE_ETHER_TYPE_IPV4) 2661 return rte_flow_error_set(error, EINVAL, 2662 RTE_FLOW_ERROR_TYPE_ITEM, item, 2663 "IPv4 cannot follow L2/VLAN layer " 2664 "which ether type is not IPv4"); 2665 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2666 if (mask && spec) 2667 next_proto = mask->hdr.next_proto_id & 2668 spec->hdr.next_proto_id; 2669 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2670 return rte_flow_error_set(error, EINVAL, 2671 RTE_FLOW_ERROR_TYPE_ITEM, 2672 item, 2673 "multiple tunnel " 2674 "not supported"); 2675 } 2676 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2677 return rte_flow_error_set(error, EINVAL, 2678 RTE_FLOW_ERROR_TYPE_ITEM, item, 2679 "wrong tunnel type - IPv6 specified " 2680 "but IPv4 item provided"); 2681 if (item_flags & l3m) 2682 return rte_flow_error_set(error, ENOTSUP, 2683 RTE_FLOW_ERROR_TYPE_ITEM, item, 2684 "multiple L3 layers not supported"); 2685 else if (item_flags & l4m) 2686 return rte_flow_error_set(error, EINVAL, 2687 RTE_FLOW_ERROR_TYPE_ITEM, item, 2688 "L3 cannot follow an L4 layer."); 2689 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2690 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2691 return rte_flow_error_set(error, EINVAL, 2692 RTE_FLOW_ERROR_TYPE_ITEM, item, 2693 "L3 cannot follow an NVGRE layer."); 2694 if (!mask) 2695 mask = &rte_flow_item_ipv4_mask; 2696 else if (mask->hdr.next_proto_id != 0 && 2697 mask->hdr.next_proto_id != 0xff) 2698 return rte_flow_error_set(error, EINVAL, 2699 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2700 "partial mask is not supported" 2701 " for protocol"); 2702 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2703 acc_mask ? (const uint8_t *)acc_mask 2704 : (const uint8_t *)&nic_mask, 2705 sizeof(struct rte_flow_item_ipv4), 2706 range_accepted, error); 2707 if (ret < 0) 2708 return ret; 2709 return 0; 2710 } 2711 2712 /** 2713 * Validate IPV6 item. 2714 * 2715 * @param[in] item 2716 * Item specification. 2717 * @param[in] item_flags 2718 * Bit-fields that holds the items detected until now. 2719 * @param[in] last_item 2720 * Previous validated item in the pattern items. 2721 * @param[in] ether_type 2722 * Type in the ethernet layer header (including dot1q). 2723 * @param[in] acc_mask 2724 * Acceptable mask, if NULL default internal default mask 2725 * will be used to check whether item fields are supported. 2726 * @param[out] error 2727 * Pointer to error structure. 2728 * 2729 * @return 2730 * 0 on success, a negative errno value otherwise and rte_errno is set. 2731 */ 2732 int 2733 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2734 uint64_t item_flags, 2735 uint64_t last_item, 2736 uint16_t ether_type, 2737 const struct rte_flow_item_ipv6 *acc_mask, 2738 struct rte_flow_error *error) 2739 { 2740 const struct rte_flow_item_ipv6 *mask = item->mask; 2741 const struct rte_flow_item_ipv6 *spec = item->spec; 2742 const struct rte_flow_item_ipv6 nic_mask = { 2743 .hdr = { 2744 .src_addr = 2745 "\xff\xff\xff\xff\xff\xff\xff\xff" 2746 "\xff\xff\xff\xff\xff\xff\xff\xff", 2747 .dst_addr = 2748 "\xff\xff\xff\xff\xff\xff\xff\xff" 2749 "\xff\xff\xff\xff\xff\xff\xff\xff", 2750 .vtc_flow = RTE_BE32(0xffffffff), 2751 .proto = 0xff, 2752 }, 2753 }; 2754 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2755 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2756 MLX5_FLOW_LAYER_OUTER_L3; 2757 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2758 MLX5_FLOW_LAYER_OUTER_L4; 2759 int ret; 2760 uint8_t next_proto = 0xFF; 2761 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2762 MLX5_FLOW_LAYER_OUTER_VLAN | 2763 MLX5_FLOW_LAYER_INNER_VLAN); 2764 2765 if ((last_item & l2_vlan) && ether_type && 2766 ether_type != RTE_ETHER_TYPE_IPV6) 2767 return rte_flow_error_set(error, EINVAL, 2768 RTE_FLOW_ERROR_TYPE_ITEM, item, 2769 "IPv6 cannot follow L2/VLAN layer " 2770 "which ether type is not IPv6"); 2771 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2772 next_proto = spec->hdr.proto; 2773 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2774 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2775 return rte_flow_error_set(error, EINVAL, 2776 RTE_FLOW_ERROR_TYPE_ITEM, 2777 item, 2778 "multiple tunnel " 2779 "not supported"); 2780 } 2781 if (next_proto == IPPROTO_HOPOPTS || 2782 next_proto == IPPROTO_ROUTING || 2783 next_proto == IPPROTO_FRAGMENT || 2784 next_proto == IPPROTO_ESP || 2785 next_proto == IPPROTO_AH || 2786 next_proto == IPPROTO_DSTOPTS) 2787 return rte_flow_error_set(error, EINVAL, 2788 RTE_FLOW_ERROR_TYPE_ITEM, item, 2789 "IPv6 proto (next header) should " 2790 "not be set as extension header"); 2791 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2792 return rte_flow_error_set(error, EINVAL, 2793 RTE_FLOW_ERROR_TYPE_ITEM, item, 2794 "wrong tunnel type - IPv4 specified " 2795 "but IPv6 item provided"); 2796 if (item_flags & l3m) 2797 return rte_flow_error_set(error, ENOTSUP, 2798 RTE_FLOW_ERROR_TYPE_ITEM, item, 2799 "multiple L3 layers not supported"); 2800 else if (item_flags & l4m) 2801 return rte_flow_error_set(error, EINVAL, 2802 RTE_FLOW_ERROR_TYPE_ITEM, item, 2803 "L3 cannot follow an L4 layer."); 2804 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2805 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2806 return rte_flow_error_set(error, EINVAL, 2807 RTE_FLOW_ERROR_TYPE_ITEM, item, 2808 "L3 cannot follow an NVGRE layer."); 2809 if (!mask) 2810 mask = &rte_flow_item_ipv6_mask; 2811 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2812 acc_mask ? (const uint8_t *)acc_mask 2813 : (const uint8_t *)&nic_mask, 2814 sizeof(struct rte_flow_item_ipv6), 2815 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2816 if (ret < 0) 2817 return ret; 2818 return 0; 2819 } 2820 2821 /** 2822 * Validate UDP item. 2823 * 2824 * @param[in] item 2825 * Item specification. 2826 * @param[in] item_flags 2827 * Bit-fields that holds the items detected until now. 2828 * @param[in] target_protocol 2829 * The next protocol in the previous item. 2830 * @param[in] flow_mask 2831 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2832 * @param[out] error 2833 * Pointer to error structure. 2834 * 2835 * @return 2836 * 0 on success, a negative errno value otherwise and rte_errno is set. 2837 */ 2838 int 2839 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2840 uint64_t item_flags, 2841 uint8_t target_protocol, 2842 struct rte_flow_error *error) 2843 { 2844 const struct rte_flow_item_udp *mask = item->mask; 2845 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2846 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2847 MLX5_FLOW_LAYER_OUTER_L3; 2848 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2849 MLX5_FLOW_LAYER_OUTER_L4; 2850 int ret; 2851 2852 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2853 return rte_flow_error_set(error, EINVAL, 2854 RTE_FLOW_ERROR_TYPE_ITEM, item, 2855 "protocol filtering not compatible" 2856 " with UDP layer"); 2857 if (!(item_flags & l3m)) 2858 return rte_flow_error_set(error, EINVAL, 2859 RTE_FLOW_ERROR_TYPE_ITEM, item, 2860 "L3 is mandatory to filter on L4"); 2861 if (item_flags & l4m) 2862 return rte_flow_error_set(error, EINVAL, 2863 RTE_FLOW_ERROR_TYPE_ITEM, item, 2864 "multiple L4 layers not supported"); 2865 if (!mask) 2866 mask = &rte_flow_item_udp_mask; 2867 ret = mlx5_flow_item_acceptable 2868 (item, (const uint8_t *)mask, 2869 (const uint8_t *)&rte_flow_item_udp_mask, 2870 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2871 error); 2872 if (ret < 0) 2873 return ret; 2874 return 0; 2875 } 2876 2877 /** 2878 * Validate TCP item. 2879 * 2880 * @param[in] item 2881 * Item specification. 2882 * @param[in] item_flags 2883 * Bit-fields that holds the items detected until now. 2884 * @param[in] target_protocol 2885 * The next protocol in the previous item. 2886 * @param[out] error 2887 * Pointer to error structure. 2888 * 2889 * @return 2890 * 0 on success, a negative errno value otherwise and rte_errno is set. 2891 */ 2892 int 2893 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2894 uint64_t item_flags, 2895 uint8_t target_protocol, 2896 const struct rte_flow_item_tcp *flow_mask, 2897 struct rte_flow_error *error) 2898 { 2899 const struct rte_flow_item_tcp *mask = item->mask; 2900 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2901 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2902 MLX5_FLOW_LAYER_OUTER_L3; 2903 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2904 MLX5_FLOW_LAYER_OUTER_L4; 2905 int ret; 2906 2907 MLX5_ASSERT(flow_mask); 2908 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2909 return rte_flow_error_set(error, EINVAL, 2910 RTE_FLOW_ERROR_TYPE_ITEM, item, 2911 "protocol filtering not compatible" 2912 " with TCP layer"); 2913 if (!(item_flags & l3m)) 2914 return rte_flow_error_set(error, EINVAL, 2915 RTE_FLOW_ERROR_TYPE_ITEM, item, 2916 "L3 is mandatory to filter on L4"); 2917 if (item_flags & l4m) 2918 return rte_flow_error_set(error, EINVAL, 2919 RTE_FLOW_ERROR_TYPE_ITEM, item, 2920 "multiple L4 layers not supported"); 2921 if (!mask) 2922 mask = &rte_flow_item_tcp_mask; 2923 ret = mlx5_flow_item_acceptable 2924 (item, (const uint8_t *)mask, 2925 (const uint8_t *)flow_mask, 2926 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2927 error); 2928 if (ret < 0) 2929 return ret; 2930 return 0; 2931 } 2932 2933 /** 2934 * Validate VXLAN item. 2935 * 2936 * @param[in] dev 2937 * Pointer to the Ethernet device structure. 2938 * @param[in] udp_dport 2939 * UDP destination port 2940 * @param[in] item 2941 * Item specification. 2942 * @param[in] item_flags 2943 * Bit-fields that holds the items detected until now. 2944 * @param root 2945 * Whether action is on root table. 2946 * @param[out] error 2947 * Pointer to error structure. 2948 * 2949 * @return 2950 * 0 on success, a negative errno value otherwise and rte_errno is set. 2951 */ 2952 int 2953 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2954 uint16_t udp_dport, 2955 const struct rte_flow_item *item, 2956 uint64_t item_flags, 2957 bool root, 2958 struct rte_flow_error *error) 2959 { 2960 const struct rte_flow_item_vxlan *spec = item->spec; 2961 const struct rte_flow_item_vxlan *mask = item->mask; 2962 int ret; 2963 struct mlx5_priv *priv = dev->data->dev_private; 2964 union vni { 2965 uint32_t vlan_id; 2966 uint8_t vni[4]; 2967 } id = { .vlan_id = 0, }; 2968 const struct rte_flow_item_vxlan nic_mask = { 2969 .vni = "\xff\xff\xff", 2970 .rsvd1 = 0xff, 2971 }; 2972 const struct rte_flow_item_vxlan *valid_mask; 2973 2974 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2975 return rte_flow_error_set(error, ENOTSUP, 2976 RTE_FLOW_ERROR_TYPE_ITEM, item, 2977 "multiple tunnel layers not" 2978 " supported"); 2979 valid_mask = &rte_flow_item_vxlan_mask; 2980 /* 2981 * Verify only UDPv4 is present as defined in 2982 * https://tools.ietf.org/html/rfc7348 2983 */ 2984 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2985 return rte_flow_error_set(error, EINVAL, 2986 RTE_FLOW_ERROR_TYPE_ITEM, item, 2987 "no outer UDP layer found"); 2988 if (!mask) 2989 mask = &rte_flow_item_vxlan_mask; 2990 2991 if (priv->sh->steering_format_version != 2992 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2993 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2994 /* non-root table */ 2995 if (!root && priv->sh->misc5_cap) 2996 valid_mask = &nic_mask; 2997 /* Group zero in NIC domain */ 2998 if (!root && priv->sh->tunnel_header_0_1) 2999 valid_mask = &nic_mask; 3000 } 3001 ret = mlx5_flow_item_acceptable 3002 (item, (const uint8_t *)mask, 3003 (const uint8_t *)valid_mask, 3004 sizeof(struct rte_flow_item_vxlan), 3005 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3006 if (ret < 0) 3007 return ret; 3008 if (spec) { 3009 memcpy(&id.vni[1], spec->vni, 3); 3010 memcpy(&id.vni[1], mask->vni, 3); 3011 } 3012 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3013 return rte_flow_error_set(error, ENOTSUP, 3014 RTE_FLOW_ERROR_TYPE_ITEM, item, 3015 "VXLAN tunnel must be fully defined"); 3016 return 0; 3017 } 3018 3019 /** 3020 * Validate VXLAN_GPE item. 3021 * 3022 * @param[in] item 3023 * Item specification. 3024 * @param[in] item_flags 3025 * Bit-fields that holds the items detected until now. 3026 * @param[in] priv 3027 * Pointer to the private data structure. 3028 * @param[in] target_protocol 3029 * The next protocol in the previous item. 3030 * @param[out] error 3031 * Pointer to error structure. 3032 * 3033 * @return 3034 * 0 on success, a negative errno value otherwise and rte_errno is set. 3035 */ 3036 int 3037 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3038 uint64_t item_flags, 3039 struct rte_eth_dev *dev, 3040 struct rte_flow_error *error) 3041 { 3042 struct mlx5_priv *priv = dev->data->dev_private; 3043 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3044 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3045 int ret; 3046 union vni { 3047 uint32_t vlan_id; 3048 uint8_t vni[4]; 3049 } id = { .vlan_id = 0, }; 3050 3051 if (!priv->sh->config.l3_vxlan_en) 3052 return rte_flow_error_set(error, ENOTSUP, 3053 RTE_FLOW_ERROR_TYPE_ITEM, item, 3054 "L3 VXLAN is not enabled by device" 3055 " parameter and/or not configured in" 3056 " firmware"); 3057 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3058 return rte_flow_error_set(error, ENOTSUP, 3059 RTE_FLOW_ERROR_TYPE_ITEM, item, 3060 "multiple tunnel layers not" 3061 " supported"); 3062 /* 3063 * Verify only UDPv4 is present as defined in 3064 * https://tools.ietf.org/html/rfc7348 3065 */ 3066 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3067 return rte_flow_error_set(error, EINVAL, 3068 RTE_FLOW_ERROR_TYPE_ITEM, item, 3069 "no outer UDP layer found"); 3070 if (!mask) 3071 mask = &rte_flow_item_vxlan_gpe_mask; 3072 ret = mlx5_flow_item_acceptable 3073 (item, (const uint8_t *)mask, 3074 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 3075 sizeof(struct rte_flow_item_vxlan_gpe), 3076 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3077 if (ret < 0) 3078 return ret; 3079 if (spec) { 3080 if (spec->protocol) 3081 return rte_flow_error_set(error, ENOTSUP, 3082 RTE_FLOW_ERROR_TYPE_ITEM, 3083 item, 3084 "VxLAN-GPE protocol" 3085 " not supported"); 3086 memcpy(&id.vni[1], spec->vni, 3); 3087 memcpy(&id.vni[1], mask->vni, 3); 3088 } 3089 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3090 return rte_flow_error_set(error, ENOTSUP, 3091 RTE_FLOW_ERROR_TYPE_ITEM, item, 3092 "VXLAN-GPE tunnel must be fully" 3093 " defined"); 3094 return 0; 3095 } 3096 /** 3097 * Validate GRE Key item. 3098 * 3099 * @param[in] item 3100 * Item specification. 3101 * @param[in] item_flags 3102 * Bit flags to mark detected items. 3103 * @param[in] gre_item 3104 * Pointer to gre_item 3105 * @param[out] error 3106 * Pointer to error structure. 3107 * 3108 * @return 3109 * 0 on success, a negative errno value otherwise and rte_errno is set. 3110 */ 3111 int 3112 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3113 uint64_t item_flags, 3114 const struct rte_flow_item *gre_item, 3115 struct rte_flow_error *error) 3116 { 3117 const rte_be32_t *mask = item->mask; 3118 int ret = 0; 3119 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3120 const struct rte_flow_item_gre *gre_spec; 3121 const struct rte_flow_item_gre *gre_mask; 3122 3123 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3124 return rte_flow_error_set(error, ENOTSUP, 3125 RTE_FLOW_ERROR_TYPE_ITEM, item, 3126 "Multiple GRE key not support"); 3127 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3128 return rte_flow_error_set(error, ENOTSUP, 3129 RTE_FLOW_ERROR_TYPE_ITEM, item, 3130 "No preceding GRE header"); 3131 if (item_flags & MLX5_FLOW_LAYER_INNER) 3132 return rte_flow_error_set(error, ENOTSUP, 3133 RTE_FLOW_ERROR_TYPE_ITEM, item, 3134 "GRE key following a wrong item"); 3135 gre_mask = gre_item->mask; 3136 if (!gre_mask) 3137 gre_mask = &rte_flow_item_gre_mask; 3138 gre_spec = gre_item->spec; 3139 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3140 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3141 return rte_flow_error_set(error, EINVAL, 3142 RTE_FLOW_ERROR_TYPE_ITEM, item, 3143 "Key bit must be on"); 3144 3145 if (!mask) 3146 mask = &gre_key_default_mask; 3147 ret = mlx5_flow_item_acceptable 3148 (item, (const uint8_t *)mask, 3149 (const uint8_t *)&gre_key_default_mask, 3150 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3151 return ret; 3152 } 3153 3154 /** 3155 * Validate GRE optional item. 3156 * 3157 * @param[in] dev 3158 * Pointer to the Ethernet device structure. 3159 * @param[in] item 3160 * Item specification. 3161 * @param[in] item_flags 3162 * Bit flags to mark detected items. 3163 * @param[in] attr 3164 * Flow rule attributes. 3165 * @param[in] gre_item 3166 * Pointer to gre_item 3167 * @param[out] error 3168 * Pointer to error structure. 3169 * 3170 * @return 3171 * 0 on success, a negative errno value otherwise and rte_errno is set. 3172 */ 3173 int 3174 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3175 const struct rte_flow_item *item, 3176 uint64_t item_flags, 3177 const struct rte_flow_attr *attr, 3178 const struct rte_flow_item *gre_item, 3179 struct rte_flow_error *error) 3180 { 3181 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3182 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3183 const struct rte_flow_item_gre_opt *spec = item->spec; 3184 const struct rte_flow_item_gre_opt *mask = item->mask; 3185 struct mlx5_priv *priv = dev->data->dev_private; 3186 int ret = 0; 3187 struct rte_flow_item_gre_opt nic_mask = { 3188 .checksum_rsvd = { 3189 .checksum = RTE_BE16(UINT16_MAX), 3190 .reserved1 = 0x0, 3191 }, 3192 .key = { 3193 .key = RTE_BE32(UINT32_MAX), 3194 }, 3195 .sequence = { 3196 .sequence = RTE_BE32(UINT32_MAX), 3197 }, 3198 }; 3199 3200 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3201 return rte_flow_error_set(error, ENOTSUP, 3202 RTE_FLOW_ERROR_TYPE_ITEM, item, 3203 "No preceding GRE header"); 3204 if (item_flags & MLX5_FLOW_LAYER_INNER) 3205 return rte_flow_error_set(error, ENOTSUP, 3206 RTE_FLOW_ERROR_TYPE_ITEM, item, 3207 "GRE option following a wrong item"); 3208 if (!spec || !mask) 3209 return rte_flow_error_set(error, EINVAL, 3210 RTE_FLOW_ERROR_TYPE_ITEM, item, 3211 "At least one field gre_option(checksum/key/sequence) must be specified"); 3212 if (!gre_mask) 3213 gre_mask = &rte_flow_item_gre_mask; 3214 if (mask->checksum_rsvd.checksum) 3215 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3216 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3217 return rte_flow_error_set(error, EINVAL, 3218 RTE_FLOW_ERROR_TYPE_ITEM, 3219 item, 3220 "Checksum bit must be on"); 3221 if (mask->key.key) 3222 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3223 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3224 return rte_flow_error_set(error, EINVAL, 3225 RTE_FLOW_ERROR_TYPE_ITEM, 3226 item, "Key bit must be on"); 3227 if (mask->sequence.sequence) 3228 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3229 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3230 return rte_flow_error_set(error, EINVAL, 3231 RTE_FLOW_ERROR_TYPE_ITEM, 3232 item, 3233 "Sequence bit must be on"); 3234 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3235 if (priv->sh->steering_format_version == 3236 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3237 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3238 !priv->sh->misc5_cap) || 3239 (!(priv->sh->tunnel_header_0_1 && 3240 priv->sh->tunnel_header_2_3) && 3241 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3242 return rte_flow_error_set(error, EINVAL, 3243 RTE_FLOW_ERROR_TYPE_ITEM, 3244 item, 3245 "Checksum/Sequence not supported"); 3246 } 3247 ret = mlx5_flow_item_acceptable 3248 (item, (const uint8_t *)mask, 3249 (const uint8_t *)&nic_mask, 3250 sizeof(struct rte_flow_item_gre_opt), 3251 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3252 return ret; 3253 } 3254 3255 /** 3256 * Validate GRE item. 3257 * 3258 * @param[in] item 3259 * Item specification. 3260 * @param[in] item_flags 3261 * Bit flags to mark detected items. 3262 * @param[in] target_protocol 3263 * The next protocol in the previous item. 3264 * @param[out] error 3265 * Pointer to error structure. 3266 * 3267 * @return 3268 * 0 on success, a negative errno value otherwise and rte_errno is set. 3269 */ 3270 int 3271 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3272 uint64_t item_flags, 3273 uint8_t target_protocol, 3274 struct rte_flow_error *error) 3275 { 3276 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3277 const struct rte_flow_item_gre *mask = item->mask; 3278 int ret; 3279 const struct rte_flow_item_gre nic_mask = { 3280 .c_rsvd0_ver = RTE_BE16(0xB000), 3281 .protocol = RTE_BE16(UINT16_MAX), 3282 }; 3283 3284 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3285 return rte_flow_error_set(error, EINVAL, 3286 RTE_FLOW_ERROR_TYPE_ITEM, item, 3287 "protocol filtering not compatible" 3288 " with this GRE layer"); 3289 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3290 return rte_flow_error_set(error, ENOTSUP, 3291 RTE_FLOW_ERROR_TYPE_ITEM, item, 3292 "multiple tunnel layers not" 3293 " supported"); 3294 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3295 return rte_flow_error_set(error, ENOTSUP, 3296 RTE_FLOW_ERROR_TYPE_ITEM, item, 3297 "L3 Layer is missing"); 3298 if (!mask) 3299 mask = &rte_flow_item_gre_mask; 3300 ret = mlx5_flow_item_acceptable 3301 (item, (const uint8_t *)mask, 3302 (const uint8_t *)&nic_mask, 3303 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3304 error); 3305 if (ret < 0) 3306 return ret; 3307 #ifndef HAVE_MLX5DV_DR 3308 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3309 if (spec && (spec->protocol & mask->protocol)) 3310 return rte_flow_error_set(error, ENOTSUP, 3311 RTE_FLOW_ERROR_TYPE_ITEM, item, 3312 "without MPLS support the" 3313 " specification cannot be used for" 3314 " filtering"); 3315 #endif 3316 #endif 3317 return 0; 3318 } 3319 3320 /** 3321 * Validate Geneve item. 3322 * 3323 * @param[in] item 3324 * Item specification. 3325 * @param[in] itemFlags 3326 * Bit-fields that holds the items detected until now. 3327 * @param[in] enPriv 3328 * Pointer to the private data structure. 3329 * @param[out] error 3330 * Pointer to error structure. 3331 * 3332 * @return 3333 * 0 on success, a negative errno value otherwise and rte_errno is set. 3334 */ 3335 3336 int 3337 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3338 uint64_t item_flags, 3339 struct rte_eth_dev *dev, 3340 struct rte_flow_error *error) 3341 { 3342 struct mlx5_priv *priv = dev->data->dev_private; 3343 const struct rte_flow_item_geneve *spec = item->spec; 3344 const struct rte_flow_item_geneve *mask = item->mask; 3345 int ret; 3346 uint16_t gbhdr; 3347 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3348 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3349 const struct rte_flow_item_geneve nic_mask = { 3350 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3351 .vni = "\xff\xff\xff", 3352 .protocol = RTE_BE16(UINT16_MAX), 3353 }; 3354 3355 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3356 return rte_flow_error_set(error, ENOTSUP, 3357 RTE_FLOW_ERROR_TYPE_ITEM, item, 3358 "L3 Geneve is not enabled by device" 3359 " parameter and/or not configured in" 3360 " firmware"); 3361 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3362 return rte_flow_error_set(error, ENOTSUP, 3363 RTE_FLOW_ERROR_TYPE_ITEM, item, 3364 "multiple tunnel layers not" 3365 " supported"); 3366 /* 3367 * Verify only UDPv4 is present as defined in 3368 * https://tools.ietf.org/html/rfc7348 3369 */ 3370 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3371 return rte_flow_error_set(error, EINVAL, 3372 RTE_FLOW_ERROR_TYPE_ITEM, item, 3373 "no outer UDP layer found"); 3374 if (!mask) 3375 mask = &rte_flow_item_geneve_mask; 3376 ret = mlx5_flow_item_acceptable 3377 (item, (const uint8_t *)mask, 3378 (const uint8_t *)&nic_mask, 3379 sizeof(struct rte_flow_item_geneve), 3380 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3381 if (ret) 3382 return ret; 3383 if (spec) { 3384 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3385 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3386 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3387 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3388 return rte_flow_error_set(error, ENOTSUP, 3389 RTE_FLOW_ERROR_TYPE_ITEM, 3390 item, 3391 "Geneve protocol unsupported" 3392 " fields are being used"); 3393 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3394 return rte_flow_error_set 3395 (error, ENOTSUP, 3396 RTE_FLOW_ERROR_TYPE_ITEM, 3397 item, 3398 "Unsupported Geneve options length"); 3399 } 3400 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3401 return rte_flow_error_set 3402 (error, ENOTSUP, 3403 RTE_FLOW_ERROR_TYPE_ITEM, item, 3404 "Geneve tunnel must be fully defined"); 3405 return 0; 3406 } 3407 3408 /** 3409 * Validate Geneve TLV option item. 3410 * 3411 * @param[in] item 3412 * Item specification. 3413 * @param[in] last_item 3414 * Previous validated item in the pattern items. 3415 * @param[in] geneve_item 3416 * Previous GENEVE item specification. 3417 * @param[in] dev 3418 * Pointer to the rte_eth_dev structure. 3419 * @param[out] error 3420 * Pointer to error structure. 3421 * 3422 * @return 3423 * 0 on success, a negative errno value otherwise and rte_errno is set. 3424 */ 3425 int 3426 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3427 uint64_t last_item, 3428 const struct rte_flow_item *geneve_item, 3429 struct rte_eth_dev *dev, 3430 struct rte_flow_error *error) 3431 { 3432 struct mlx5_priv *priv = dev->data->dev_private; 3433 struct mlx5_dev_ctx_shared *sh = priv->sh; 3434 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3435 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3436 uint8_t data_max_supported = 3437 hca_attr->max_geneve_tlv_option_data_len * 4; 3438 const struct rte_flow_item_geneve *geneve_spec; 3439 const struct rte_flow_item_geneve *geneve_mask; 3440 const struct rte_flow_item_geneve_opt *spec = item->spec; 3441 const struct rte_flow_item_geneve_opt *mask = item->mask; 3442 unsigned int i; 3443 unsigned int data_len; 3444 uint8_t tlv_option_len; 3445 uint16_t optlen_m, optlen_v; 3446 const struct rte_flow_item_geneve_opt full_mask = { 3447 .option_class = RTE_BE16(0xffff), 3448 .option_type = 0xff, 3449 .option_len = 0x1f, 3450 }; 3451 3452 if (!mask) 3453 mask = &rte_flow_item_geneve_opt_mask; 3454 if (!spec) 3455 return rte_flow_error_set 3456 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3457 "Geneve TLV opt class/type/length must be specified"); 3458 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3459 return rte_flow_error_set 3460 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3461 "Geneve TLV opt length exceeds the limit (31)"); 3462 /* Check if class type and length masks are full. */ 3463 if (full_mask.option_class != mask->option_class || 3464 full_mask.option_type != mask->option_type || 3465 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3466 return rte_flow_error_set 3467 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3468 "Geneve TLV opt class/type/length masks must be full"); 3469 /* Check if length is supported */ 3470 if ((uint32_t)spec->option_len > 3471 hca_attr->max_geneve_tlv_option_data_len) 3472 return rte_flow_error_set 3473 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3474 "Geneve TLV opt length not supported"); 3475 if (hca_attr->max_geneve_tlv_options > 1) 3476 DRV_LOG(DEBUG, 3477 "max_geneve_tlv_options supports more than 1 option"); 3478 /* Check GENEVE item preceding. */ 3479 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3480 return rte_flow_error_set 3481 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3482 "Geneve opt item must be preceded with Geneve item"); 3483 geneve_spec = geneve_item->spec; 3484 geneve_mask = geneve_item->mask ? geneve_item->mask : 3485 &rte_flow_item_geneve_mask; 3486 /* Check if GENEVE TLV option size doesn't exceed option length */ 3487 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3488 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3489 tlv_option_len = spec->option_len & mask->option_len; 3490 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3491 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3492 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3493 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3494 if ((optlen_v & optlen_m) <= tlv_option_len) 3495 return rte_flow_error_set 3496 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3497 "GENEVE TLV option length exceeds optlen"); 3498 } 3499 /* Check if length is 0 or data is 0. */ 3500 if (spec->data == NULL || spec->option_len == 0) 3501 return rte_flow_error_set 3502 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3503 "Geneve TLV opt with zero data/length not supported"); 3504 /* Check not all data & mask are 0. */ 3505 data_len = spec->option_len * 4; 3506 if (mask->data == NULL) { 3507 for (i = 0; i < data_len; i++) 3508 if (spec->data[i]) 3509 break; 3510 if (i == data_len) 3511 return rte_flow_error_set(error, ENOTSUP, 3512 RTE_FLOW_ERROR_TYPE_ITEM, item, 3513 "Can't match on Geneve option data 0"); 3514 } else { 3515 for (i = 0; i < data_len; i++) 3516 if (spec->data[i] & mask->data[i]) 3517 break; 3518 if (i == data_len) 3519 return rte_flow_error_set(error, ENOTSUP, 3520 RTE_FLOW_ERROR_TYPE_ITEM, item, 3521 "Can't match on Geneve option data and mask 0"); 3522 /* Check data mask supported. */ 3523 for (i = data_max_supported; i < data_len ; i++) 3524 if (mask->data[i]) 3525 return rte_flow_error_set(error, ENOTSUP, 3526 RTE_FLOW_ERROR_TYPE_ITEM, item, 3527 "Data mask is of unsupported size"); 3528 } 3529 /* Check GENEVE option is supported in NIC. */ 3530 if (!hca_attr->geneve_tlv_opt) 3531 return rte_flow_error_set 3532 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3533 "Geneve TLV opt not supported"); 3534 /* Check if we already have geneve option with different type/class. */ 3535 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3536 geneve_opt_resource = sh->geneve_tlv_option_resource; 3537 if (geneve_opt_resource != NULL) 3538 if (geneve_opt_resource->option_class != spec->option_class || 3539 geneve_opt_resource->option_type != spec->option_type || 3540 geneve_opt_resource->length != spec->option_len) { 3541 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3542 return rte_flow_error_set(error, ENOTSUP, 3543 RTE_FLOW_ERROR_TYPE_ITEM, item, 3544 "Only one Geneve TLV option supported"); 3545 } 3546 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3547 return 0; 3548 } 3549 3550 /** 3551 * Validate MPLS item. 3552 * 3553 * @param[in] dev 3554 * Pointer to the rte_eth_dev structure. 3555 * @param[in] item 3556 * Item specification. 3557 * @param[in] item_flags 3558 * Bit-fields that holds the items detected until now. 3559 * @param[in] prev_layer 3560 * The protocol layer indicated in previous item. 3561 * @param[out] error 3562 * Pointer to error structure. 3563 * 3564 * @return 3565 * 0 on success, a negative errno value otherwise and rte_errno is set. 3566 */ 3567 int 3568 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3569 const struct rte_flow_item *item __rte_unused, 3570 uint64_t item_flags __rte_unused, 3571 uint64_t prev_layer __rte_unused, 3572 struct rte_flow_error *error) 3573 { 3574 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3575 const struct rte_flow_item_mpls *mask = item->mask; 3576 struct mlx5_priv *priv = dev->data->dev_private; 3577 int ret; 3578 3579 if (!priv->sh->dev_cap.mpls_en) 3580 return rte_flow_error_set(error, ENOTSUP, 3581 RTE_FLOW_ERROR_TYPE_ITEM, item, 3582 "MPLS not supported or" 3583 " disabled in firmware" 3584 " configuration."); 3585 /* MPLS over UDP, GRE is allowed */ 3586 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3587 MLX5_FLOW_LAYER_GRE | 3588 MLX5_FLOW_LAYER_GRE_KEY))) 3589 return rte_flow_error_set(error, EINVAL, 3590 RTE_FLOW_ERROR_TYPE_ITEM, item, 3591 "protocol filtering not compatible" 3592 " with MPLS layer"); 3593 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3594 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3595 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3596 return rte_flow_error_set(error, ENOTSUP, 3597 RTE_FLOW_ERROR_TYPE_ITEM, item, 3598 "multiple tunnel layers not" 3599 " supported"); 3600 if (!mask) 3601 mask = &rte_flow_item_mpls_mask; 3602 ret = mlx5_flow_item_acceptable 3603 (item, (const uint8_t *)mask, 3604 (const uint8_t *)&rte_flow_item_mpls_mask, 3605 sizeof(struct rte_flow_item_mpls), 3606 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3607 if (ret < 0) 3608 return ret; 3609 return 0; 3610 #else 3611 return rte_flow_error_set(error, ENOTSUP, 3612 RTE_FLOW_ERROR_TYPE_ITEM, item, 3613 "MPLS is not supported by Verbs, please" 3614 " update."); 3615 #endif 3616 } 3617 3618 /** 3619 * Validate NVGRE item. 3620 * 3621 * @param[in] item 3622 * Item specification. 3623 * @param[in] item_flags 3624 * Bit flags to mark detected items. 3625 * @param[in] target_protocol 3626 * The next protocol in the previous item. 3627 * @param[out] error 3628 * Pointer to error structure. 3629 * 3630 * @return 3631 * 0 on success, a negative errno value otherwise and rte_errno is set. 3632 */ 3633 int 3634 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3635 uint64_t item_flags, 3636 uint8_t target_protocol, 3637 struct rte_flow_error *error) 3638 { 3639 const struct rte_flow_item_nvgre *mask = item->mask; 3640 int ret; 3641 3642 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3643 return rte_flow_error_set(error, EINVAL, 3644 RTE_FLOW_ERROR_TYPE_ITEM, item, 3645 "protocol filtering not compatible" 3646 " with this GRE layer"); 3647 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3648 return rte_flow_error_set(error, ENOTSUP, 3649 RTE_FLOW_ERROR_TYPE_ITEM, item, 3650 "multiple tunnel layers not" 3651 " supported"); 3652 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3653 return rte_flow_error_set(error, ENOTSUP, 3654 RTE_FLOW_ERROR_TYPE_ITEM, item, 3655 "L3 Layer is missing"); 3656 if (!mask) 3657 mask = &rte_flow_item_nvgre_mask; 3658 ret = mlx5_flow_item_acceptable 3659 (item, (const uint8_t *)mask, 3660 (const uint8_t *)&rte_flow_item_nvgre_mask, 3661 sizeof(struct rte_flow_item_nvgre), 3662 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3663 if (ret < 0) 3664 return ret; 3665 return 0; 3666 } 3667 3668 /** 3669 * Validate eCPRI item. 3670 * 3671 * @param[in] item 3672 * Item specification. 3673 * @param[in] item_flags 3674 * Bit-fields that holds the items detected until now. 3675 * @param[in] last_item 3676 * Previous validated item in the pattern items. 3677 * @param[in] ether_type 3678 * Type in the ethernet layer header (including dot1q). 3679 * @param[in] acc_mask 3680 * Acceptable mask, if NULL default internal default mask 3681 * will be used to check whether item fields are supported. 3682 * @param[out] error 3683 * Pointer to error structure. 3684 * 3685 * @return 3686 * 0 on success, a negative errno value otherwise and rte_errno is set. 3687 */ 3688 int 3689 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3690 uint64_t item_flags, 3691 uint64_t last_item, 3692 uint16_t ether_type, 3693 const struct rte_flow_item_ecpri *acc_mask, 3694 struct rte_flow_error *error) 3695 { 3696 const struct rte_flow_item_ecpri *mask = item->mask; 3697 const struct rte_flow_item_ecpri nic_mask = { 3698 .hdr = { 3699 .common = { 3700 .u32 = 3701 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3702 .type = 0xFF, 3703 }).u32), 3704 }, 3705 .dummy[0] = 0xFFFFFFFF, 3706 }, 3707 }; 3708 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3709 MLX5_FLOW_LAYER_OUTER_VLAN); 3710 struct rte_flow_item_ecpri mask_lo; 3711 3712 if (!(last_item & outer_l2_vlan) && 3713 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3714 return rte_flow_error_set(error, EINVAL, 3715 RTE_FLOW_ERROR_TYPE_ITEM, item, 3716 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3717 if ((last_item & outer_l2_vlan) && ether_type && 3718 ether_type != RTE_ETHER_TYPE_ECPRI) 3719 return rte_flow_error_set(error, EINVAL, 3720 RTE_FLOW_ERROR_TYPE_ITEM, item, 3721 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3722 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3723 return rte_flow_error_set(error, EINVAL, 3724 RTE_FLOW_ERROR_TYPE_ITEM, item, 3725 "eCPRI with tunnel is not supported right now"); 3726 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3727 return rte_flow_error_set(error, ENOTSUP, 3728 RTE_FLOW_ERROR_TYPE_ITEM, item, 3729 "multiple L3 layers not supported"); 3730 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3731 return rte_flow_error_set(error, EINVAL, 3732 RTE_FLOW_ERROR_TYPE_ITEM, item, 3733 "eCPRI cannot coexist with a TCP layer"); 3734 /* In specification, eCPRI could be over UDP layer. */ 3735 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3736 return rte_flow_error_set(error, EINVAL, 3737 RTE_FLOW_ERROR_TYPE_ITEM, item, 3738 "eCPRI over UDP layer is not yet supported right now"); 3739 /* Mask for type field in common header could be zero. */ 3740 if (!mask) 3741 mask = &rte_flow_item_ecpri_mask; 3742 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3743 /* Input mask is in big-endian format. */ 3744 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3745 return rte_flow_error_set(error, EINVAL, 3746 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3747 "partial mask is not supported for protocol"); 3748 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3749 return rte_flow_error_set(error, EINVAL, 3750 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3751 "message header mask must be after a type mask"); 3752 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3753 acc_mask ? (const uint8_t *)acc_mask 3754 : (const uint8_t *)&nic_mask, 3755 sizeof(struct rte_flow_item_ecpri), 3756 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3757 } 3758 3759 static int 3760 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3761 const struct rte_flow_attr *attr __rte_unused, 3762 const struct rte_flow_item items[] __rte_unused, 3763 const struct rte_flow_action actions[] __rte_unused, 3764 bool external __rte_unused, 3765 int hairpin __rte_unused, 3766 struct rte_flow_error *error) 3767 { 3768 return rte_flow_error_set(error, ENOTSUP, 3769 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3770 } 3771 3772 static struct mlx5_flow * 3773 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3774 const struct rte_flow_attr *attr __rte_unused, 3775 const struct rte_flow_item items[] __rte_unused, 3776 const struct rte_flow_action actions[] __rte_unused, 3777 struct rte_flow_error *error) 3778 { 3779 rte_flow_error_set(error, ENOTSUP, 3780 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3781 return NULL; 3782 } 3783 3784 static int 3785 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3786 struct mlx5_flow *dev_flow __rte_unused, 3787 const struct rte_flow_attr *attr __rte_unused, 3788 const struct rte_flow_item items[] __rte_unused, 3789 const struct rte_flow_action actions[] __rte_unused, 3790 struct rte_flow_error *error) 3791 { 3792 return rte_flow_error_set(error, ENOTSUP, 3793 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3794 } 3795 3796 static int 3797 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3798 struct rte_flow *flow __rte_unused, 3799 struct rte_flow_error *error) 3800 { 3801 return rte_flow_error_set(error, ENOTSUP, 3802 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3803 } 3804 3805 static void 3806 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3807 struct rte_flow *flow __rte_unused) 3808 { 3809 } 3810 3811 static void 3812 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3813 struct rte_flow *flow __rte_unused) 3814 { 3815 } 3816 3817 static int 3818 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3819 struct rte_flow *flow __rte_unused, 3820 const struct rte_flow_action *actions __rte_unused, 3821 void *data __rte_unused, 3822 struct rte_flow_error *error) 3823 { 3824 return rte_flow_error_set(error, ENOTSUP, 3825 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3826 } 3827 3828 static int 3829 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3830 uint32_t domains __rte_unused, 3831 uint32_t flags __rte_unused) 3832 { 3833 return 0; 3834 } 3835 3836 /* Void driver to protect from null pointer reference. */ 3837 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3838 .validate = flow_null_validate, 3839 .prepare = flow_null_prepare, 3840 .translate = flow_null_translate, 3841 .apply = flow_null_apply, 3842 .remove = flow_null_remove, 3843 .destroy = flow_null_destroy, 3844 .query = flow_null_query, 3845 .sync_domain = flow_null_sync_domain, 3846 }; 3847 3848 /** 3849 * Select flow driver type according to flow attributes and device 3850 * configuration. 3851 * 3852 * @param[in] dev 3853 * Pointer to the dev structure. 3854 * @param[in] attr 3855 * Pointer to the flow attributes. 3856 * 3857 * @return 3858 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3859 */ 3860 static enum mlx5_flow_drv_type 3861 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3862 { 3863 struct mlx5_priv *priv = dev->data->dev_private; 3864 /* The OS can determine first a specific flow type (DV, VERBS) */ 3865 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3866 3867 if (type != MLX5_FLOW_TYPE_MAX) 3868 return type; 3869 /* 3870 * Currently when dv_flow_en == 2, only HW steering engine is 3871 * supported. New engines can also be chosen here if ready. 3872 */ 3873 if (priv->sh->config.dv_flow_en == 2) 3874 return MLX5_FLOW_TYPE_HW; 3875 if (!attr) 3876 return MLX5_FLOW_TYPE_MIN; 3877 /* If no OS specific type - continue with DV/VERBS selection */ 3878 if (attr->transfer && priv->sh->config.dv_esw_en) 3879 type = MLX5_FLOW_TYPE_DV; 3880 if (!attr->transfer) 3881 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3882 MLX5_FLOW_TYPE_VERBS; 3883 return type; 3884 } 3885 3886 #define flow_get_drv_ops(type) flow_drv_ops[type] 3887 3888 /** 3889 * Flow driver validation API. This abstracts calling driver specific functions. 3890 * The type of flow driver is determined according to flow attributes. 3891 * 3892 * @param[in] dev 3893 * Pointer to the dev structure. 3894 * @param[in] attr 3895 * Pointer to the flow attributes. 3896 * @param[in] items 3897 * Pointer to the list of items. 3898 * @param[in] actions 3899 * Pointer to the list of actions. 3900 * @param[in] external 3901 * This flow rule is created by request external to PMD. 3902 * @param[in] hairpin 3903 * Number of hairpin TX actions, 0 means classic flow. 3904 * @param[out] error 3905 * Pointer to the error structure. 3906 * 3907 * @return 3908 * 0 on success, a negative errno value otherwise and rte_errno is set. 3909 */ 3910 static inline int 3911 flow_drv_validate(struct rte_eth_dev *dev, 3912 const struct rte_flow_attr *attr, 3913 const struct rte_flow_item items[], 3914 const struct rte_flow_action actions[], 3915 bool external, int hairpin, struct rte_flow_error *error) 3916 { 3917 const struct mlx5_flow_driver_ops *fops; 3918 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3919 3920 fops = flow_get_drv_ops(type); 3921 return fops->validate(dev, attr, items, actions, external, 3922 hairpin, error); 3923 } 3924 3925 /** 3926 * Flow driver preparation API. This abstracts calling driver specific 3927 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3928 * calculates the size of memory required for device flow, allocates the memory, 3929 * initializes the device flow and returns the pointer. 3930 * 3931 * @note 3932 * This function initializes device flow structure such as dv or verbs in 3933 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3934 * rest. For example, adding returning device flow to flow->dev_flow list and 3935 * setting backward reference to the flow should be done out of this function. 3936 * layers field is not filled either. 3937 * 3938 * @param[in] dev 3939 * Pointer to the dev structure. 3940 * @param[in] attr 3941 * Pointer to the flow attributes. 3942 * @param[in] items 3943 * Pointer to the list of items. 3944 * @param[in] actions 3945 * Pointer to the list of actions. 3946 * @param[in] flow_idx 3947 * This memory pool index to the flow. 3948 * @param[out] error 3949 * Pointer to the error structure. 3950 * 3951 * @return 3952 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3953 */ 3954 static inline struct mlx5_flow * 3955 flow_drv_prepare(struct rte_eth_dev *dev, 3956 const struct rte_flow *flow, 3957 const struct rte_flow_attr *attr, 3958 const struct rte_flow_item items[], 3959 const struct rte_flow_action actions[], 3960 uint32_t flow_idx, 3961 struct rte_flow_error *error) 3962 { 3963 const struct mlx5_flow_driver_ops *fops; 3964 enum mlx5_flow_drv_type type = flow->drv_type; 3965 struct mlx5_flow *mlx5_flow = NULL; 3966 3967 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3968 fops = flow_get_drv_ops(type); 3969 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3970 if (mlx5_flow) 3971 mlx5_flow->flow_idx = flow_idx; 3972 return mlx5_flow; 3973 } 3974 3975 /** 3976 * Flow driver translation API. This abstracts calling driver specific 3977 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3978 * translates a generic flow into a driver flow. flow_drv_prepare() must 3979 * precede. 3980 * 3981 * @note 3982 * dev_flow->layers could be filled as a result of parsing during translation 3983 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3984 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3985 * flow->actions could be overwritten even though all the expanded dev_flows 3986 * have the same actions. 3987 * 3988 * @param[in] dev 3989 * Pointer to the rte dev structure. 3990 * @param[in, out] dev_flow 3991 * Pointer to the mlx5 flow. 3992 * @param[in] attr 3993 * Pointer to the flow attributes. 3994 * @param[in] items 3995 * Pointer to the list of items. 3996 * @param[in] actions 3997 * Pointer to the list of actions. 3998 * @param[out] error 3999 * Pointer to the error structure. 4000 * 4001 * @return 4002 * 0 on success, a negative errno value otherwise and rte_errno is set. 4003 */ 4004 static inline int 4005 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4006 const struct rte_flow_attr *attr, 4007 const struct rte_flow_item items[], 4008 const struct rte_flow_action actions[], 4009 struct rte_flow_error *error) 4010 { 4011 const struct mlx5_flow_driver_ops *fops; 4012 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4013 4014 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4015 fops = flow_get_drv_ops(type); 4016 return fops->translate(dev, dev_flow, attr, items, actions, error); 4017 } 4018 4019 /** 4020 * Flow driver apply API. This abstracts calling driver specific functions. 4021 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4022 * translated driver flows on to device. flow_drv_translate() must precede. 4023 * 4024 * @param[in] dev 4025 * Pointer to Ethernet device structure. 4026 * @param[in, out] flow 4027 * Pointer to flow structure. 4028 * @param[out] error 4029 * Pointer to error structure. 4030 * 4031 * @return 4032 * 0 on success, a negative errno value otherwise and rte_errno is set. 4033 */ 4034 static inline int 4035 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4036 struct rte_flow_error *error) 4037 { 4038 const struct mlx5_flow_driver_ops *fops; 4039 enum mlx5_flow_drv_type type = flow->drv_type; 4040 4041 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4042 fops = flow_get_drv_ops(type); 4043 return fops->apply(dev, flow, error); 4044 } 4045 4046 /** 4047 * Flow driver destroy API. This abstracts calling driver specific functions. 4048 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4049 * on device and releases resources of the flow. 4050 * 4051 * @param[in] dev 4052 * Pointer to Ethernet device. 4053 * @param[in, out] flow 4054 * Pointer to flow structure. 4055 */ 4056 static inline void 4057 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4058 { 4059 const struct mlx5_flow_driver_ops *fops; 4060 enum mlx5_flow_drv_type type = flow->drv_type; 4061 4062 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4063 fops = flow_get_drv_ops(type); 4064 fops->destroy(dev, flow); 4065 } 4066 4067 /** 4068 * Flow driver find RSS policy tbl API. This abstracts calling driver 4069 * specific functions. Parent flow (rte_flow) should have driver 4070 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4071 * 4072 * @param[in] dev 4073 * Pointer to Ethernet device. 4074 * @param[in, out] flow 4075 * Pointer to flow structure. 4076 * @param[in] policy 4077 * Pointer to meter policy table. 4078 * @param[in] rss_desc 4079 * Pointer to rss_desc 4080 */ 4081 static struct mlx5_flow_meter_sub_policy * 4082 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4083 struct rte_flow *flow, 4084 struct mlx5_flow_meter_policy *policy, 4085 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4086 { 4087 const struct mlx5_flow_driver_ops *fops; 4088 enum mlx5_flow_drv_type type = flow->drv_type; 4089 4090 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4091 fops = flow_get_drv_ops(type); 4092 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4093 } 4094 4095 /** 4096 * Flow driver color tag rule API. This abstracts calling driver 4097 * specific functions. Parent flow (rte_flow) should have driver 4098 * type (drv_type). It will create the color tag rules in hierarchy meter. 4099 * 4100 * @param[in] dev 4101 * Pointer to Ethernet device. 4102 * @param[in, out] flow 4103 * Pointer to flow structure. 4104 * @param[in] fm 4105 * Pointer to flow meter structure. 4106 * @param[in] src_port 4107 * The src port this extra rule should use. 4108 * @param[in] item 4109 * The src port id match item. 4110 * @param[out] error 4111 * Pointer to error structure. 4112 */ 4113 static int 4114 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4115 struct rte_flow *flow, 4116 struct mlx5_flow_meter_info *fm, 4117 int32_t src_port, 4118 const struct rte_flow_item *item, 4119 struct rte_flow_error *error) 4120 { 4121 const struct mlx5_flow_driver_ops *fops; 4122 enum mlx5_flow_drv_type type = flow->drv_type; 4123 4124 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4125 fops = flow_get_drv_ops(type); 4126 return fops->meter_hierarchy_rule_create(dev, fm, 4127 src_port, item, error); 4128 } 4129 4130 /** 4131 * Get RSS action from the action list. 4132 * 4133 * @param[in] dev 4134 * Pointer to Ethernet device. 4135 * @param[in] actions 4136 * Pointer to the list of actions. 4137 * @param[in] flow 4138 * Parent flow structure pointer. 4139 * 4140 * @return 4141 * Pointer to the RSS action if exist, else return NULL. 4142 */ 4143 static const struct rte_flow_action_rss* 4144 flow_get_rss_action(struct rte_eth_dev *dev, 4145 const struct rte_flow_action actions[]) 4146 { 4147 struct mlx5_priv *priv = dev->data->dev_private; 4148 const struct rte_flow_action_rss *rss = NULL; 4149 struct mlx5_meter_policy_action_container *acg; 4150 struct mlx5_meter_policy_action_container *acy; 4151 4152 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4153 switch (actions->type) { 4154 case RTE_FLOW_ACTION_TYPE_RSS: 4155 rss = actions->conf; 4156 break; 4157 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4158 { 4159 const struct rte_flow_action_sample *sample = 4160 actions->conf; 4161 const struct rte_flow_action *act = sample->actions; 4162 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4163 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4164 rss = act->conf; 4165 break; 4166 } 4167 case RTE_FLOW_ACTION_TYPE_METER: 4168 { 4169 uint32_t mtr_idx; 4170 struct mlx5_flow_meter_info *fm; 4171 struct mlx5_flow_meter_policy *policy; 4172 const struct rte_flow_action_meter *mtr = actions->conf; 4173 4174 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4175 if (fm && !fm->def_policy) { 4176 policy = mlx5_flow_meter_policy_find(dev, 4177 fm->policy_id, NULL); 4178 MLX5_ASSERT(policy); 4179 if (policy->is_hierarchy) { 4180 policy = 4181 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4182 policy); 4183 if (!policy) 4184 return NULL; 4185 } 4186 if (policy->is_rss) { 4187 acg = 4188 &policy->act_cnt[RTE_COLOR_GREEN]; 4189 acy = 4190 &policy->act_cnt[RTE_COLOR_YELLOW]; 4191 if (acg->fate_action == 4192 MLX5_FLOW_FATE_SHARED_RSS) 4193 rss = acg->rss->conf; 4194 else if (acy->fate_action == 4195 MLX5_FLOW_FATE_SHARED_RSS) 4196 rss = acy->rss->conf; 4197 } 4198 } 4199 break; 4200 } 4201 default: 4202 break; 4203 } 4204 } 4205 return rss; 4206 } 4207 4208 /** 4209 * Get ASO age action by index. 4210 * 4211 * @param[in] dev 4212 * Pointer to the Ethernet device structure. 4213 * @param[in] age_idx 4214 * Index to the ASO age action. 4215 * 4216 * @return 4217 * The specified ASO age action. 4218 */ 4219 struct mlx5_aso_age_action* 4220 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4221 { 4222 uint16_t pool_idx = age_idx & UINT16_MAX; 4223 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4224 struct mlx5_priv *priv = dev->data->dev_private; 4225 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4226 struct mlx5_aso_age_pool *pool; 4227 4228 rte_rwlock_read_lock(&mng->resize_rwl); 4229 pool = mng->pools[pool_idx]; 4230 rte_rwlock_read_unlock(&mng->resize_rwl); 4231 return &pool->actions[offset - 1]; 4232 } 4233 4234 /* maps indirect action to translated direct in some actions array */ 4235 struct mlx5_translated_action_handle { 4236 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4237 int index; /**< Index in related array of rte_flow_action. */ 4238 }; 4239 4240 /** 4241 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4242 * direct action if translation possible. 4243 * This functionality used to run same execution path for both direct and 4244 * indirect actions on flow create. All necessary preparations for indirect 4245 * action handling should be performed on *handle* actions list returned 4246 * from this call. 4247 * 4248 * @param[in] dev 4249 * Pointer to Ethernet device. 4250 * @param[in] actions 4251 * List of actions to translate. 4252 * @param[out] handle 4253 * List to store translated indirect action object handles. 4254 * @param[in, out] indir_n 4255 * Size of *handle* array. On return should be updated with number of 4256 * indirect actions retrieved from the *actions* list. 4257 * @param[out] translated_actions 4258 * List of actions where all indirect actions were translated to direct 4259 * if possible. NULL if no translation took place. 4260 * @param[out] error 4261 * Pointer to the error structure. 4262 * 4263 * @return 4264 * 0 on success, a negative errno value otherwise and rte_errno is set. 4265 */ 4266 static int 4267 flow_action_handles_translate(struct rte_eth_dev *dev, 4268 const struct rte_flow_action actions[], 4269 struct mlx5_translated_action_handle *handle, 4270 int *indir_n, 4271 struct rte_flow_action **translated_actions, 4272 struct rte_flow_error *error) 4273 { 4274 struct mlx5_priv *priv = dev->data->dev_private; 4275 struct rte_flow_action *translated = NULL; 4276 size_t actions_size; 4277 int n; 4278 int copied_n = 0; 4279 struct mlx5_translated_action_handle *handle_end = NULL; 4280 4281 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4282 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4283 continue; 4284 if (copied_n == *indir_n) { 4285 return rte_flow_error_set 4286 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4287 NULL, "too many shared actions"); 4288 } 4289 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4290 sizeof(actions[n].conf)); 4291 handle[copied_n].index = n; 4292 copied_n++; 4293 } 4294 n++; 4295 *indir_n = copied_n; 4296 if (!copied_n) 4297 return 0; 4298 actions_size = sizeof(struct rte_flow_action) * n; 4299 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4300 if (!translated) { 4301 rte_errno = ENOMEM; 4302 return -ENOMEM; 4303 } 4304 memcpy(translated, actions, actions_size); 4305 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4306 struct mlx5_shared_action_rss *shared_rss; 4307 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4308 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4309 uint32_t idx = act_idx & 4310 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4311 4312 switch (type) { 4313 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4314 shared_rss = mlx5_ipool_get 4315 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4316 translated[handle->index].type = 4317 RTE_FLOW_ACTION_TYPE_RSS; 4318 translated[handle->index].conf = 4319 &shared_rss->origin; 4320 break; 4321 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4322 translated[handle->index].type = 4323 (enum rte_flow_action_type) 4324 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4325 translated[handle->index].conf = (void *)(uintptr_t)idx; 4326 break; 4327 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4328 translated[handle->index].type = 4329 (enum rte_flow_action_type) 4330 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4331 translated[handle->index].conf = (void *)(uintptr_t)idx; 4332 break; 4333 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4334 if (priv->sh->flow_hit_aso_en) { 4335 translated[handle->index].type = 4336 (enum rte_flow_action_type) 4337 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4338 translated[handle->index].conf = 4339 (void *)(uintptr_t)idx; 4340 break; 4341 } 4342 /* Fall-through */ 4343 case MLX5_INDIRECT_ACTION_TYPE_CT: 4344 if (priv->sh->ct_aso_en) { 4345 translated[handle->index].type = 4346 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4347 translated[handle->index].conf = 4348 (void *)(uintptr_t)idx; 4349 break; 4350 } 4351 /* Fall-through */ 4352 default: 4353 mlx5_free(translated); 4354 return rte_flow_error_set 4355 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4356 NULL, "invalid indirect action type"); 4357 } 4358 } 4359 *translated_actions = translated; 4360 return 0; 4361 } 4362 4363 /** 4364 * Get Shared RSS action from the action list. 4365 * 4366 * @param[in] dev 4367 * Pointer to Ethernet device. 4368 * @param[in] shared 4369 * Pointer to the list of actions. 4370 * @param[in] shared_n 4371 * Actions list length. 4372 * 4373 * @return 4374 * The MLX5 RSS action ID if exists, otherwise return 0. 4375 */ 4376 static uint32_t 4377 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4378 struct mlx5_translated_action_handle *handle, 4379 int shared_n) 4380 { 4381 struct mlx5_translated_action_handle *handle_end; 4382 struct mlx5_priv *priv = dev->data->dev_private; 4383 struct mlx5_shared_action_rss *shared_rss; 4384 4385 4386 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4387 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4388 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4389 uint32_t idx = act_idx & 4390 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4391 switch (type) { 4392 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4393 shared_rss = mlx5_ipool_get 4394 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4395 idx); 4396 __atomic_add_fetch(&shared_rss->refcnt, 1, 4397 __ATOMIC_RELAXED); 4398 return idx; 4399 default: 4400 break; 4401 } 4402 } 4403 return 0; 4404 } 4405 4406 static unsigned int 4407 find_graph_root(uint32_t rss_level) 4408 { 4409 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4410 MLX5_EXPANSION_ROOT_OUTER; 4411 } 4412 4413 /** 4414 * Get layer flags from the prefix flow. 4415 * 4416 * Some flows may be split to several subflows, the prefix subflow gets the 4417 * match items and the suffix sub flow gets the actions. 4418 * Some actions need the user defined match item flags to get the detail for 4419 * the action. 4420 * This function helps the suffix flow to get the item layer flags from prefix 4421 * subflow. 4422 * 4423 * @param[in] dev_flow 4424 * Pointer the created prefix subflow. 4425 * 4426 * @return 4427 * The layers get from prefix subflow. 4428 */ 4429 static inline uint64_t 4430 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4431 { 4432 uint64_t layers = 0; 4433 4434 /* 4435 * Layers bits could be localization, but usually the compiler will 4436 * help to do the optimization work for source code. 4437 * If no decap actions, use the layers directly. 4438 */ 4439 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4440 return dev_flow->handle->layers; 4441 /* Convert L3 layers with decap action. */ 4442 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4443 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4444 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4445 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4446 /* Convert L4 layers with decap action. */ 4447 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4448 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4449 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4450 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4451 return layers; 4452 } 4453 4454 /** 4455 * Get metadata split action information. 4456 * 4457 * @param[in] actions 4458 * Pointer to the list of actions. 4459 * @param[out] qrss 4460 * Pointer to the return pointer. 4461 * @param[out] qrss_type 4462 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4463 * if no QUEUE/RSS is found. 4464 * @param[out] encap_idx 4465 * Pointer to the index of the encap action if exists, otherwise the last 4466 * action index. 4467 * 4468 * @return 4469 * Total number of actions. 4470 */ 4471 static int 4472 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4473 const struct rte_flow_action **qrss, 4474 int *encap_idx) 4475 { 4476 const struct rte_flow_action_raw_encap *raw_encap; 4477 int actions_n = 0; 4478 int raw_decap_idx = -1; 4479 4480 *encap_idx = -1; 4481 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4482 switch (actions->type) { 4483 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4484 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4485 *encap_idx = actions_n; 4486 break; 4487 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4488 raw_decap_idx = actions_n; 4489 break; 4490 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4491 raw_encap = actions->conf; 4492 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4493 *encap_idx = raw_decap_idx != -1 ? 4494 raw_decap_idx : actions_n; 4495 break; 4496 case RTE_FLOW_ACTION_TYPE_QUEUE: 4497 case RTE_FLOW_ACTION_TYPE_RSS: 4498 *qrss = actions; 4499 break; 4500 default: 4501 break; 4502 } 4503 actions_n++; 4504 } 4505 if (*encap_idx == -1) 4506 *encap_idx = actions_n; 4507 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4508 return actions_n + 1; 4509 } 4510 4511 /** 4512 * Check if the action will change packet. 4513 * 4514 * @param dev 4515 * Pointer to Ethernet device. 4516 * @param[in] type 4517 * action type. 4518 * 4519 * @return 4520 * true if action will change packet, false otherwise. 4521 */ 4522 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4523 enum rte_flow_action_type type) 4524 { 4525 struct mlx5_priv *priv = dev->data->dev_private; 4526 4527 switch (type) { 4528 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4529 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4530 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4531 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4532 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4533 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4534 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4535 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4536 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4537 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4538 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4539 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4540 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4541 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4542 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4543 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4544 case RTE_FLOW_ACTION_TYPE_SET_META: 4545 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4546 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4547 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4548 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4549 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4550 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4551 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4552 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4553 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4554 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4555 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4556 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4557 return true; 4558 case RTE_FLOW_ACTION_TYPE_FLAG: 4559 case RTE_FLOW_ACTION_TYPE_MARK: 4560 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4561 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4562 return true; 4563 else 4564 return false; 4565 default: 4566 return false; 4567 } 4568 } 4569 4570 /** 4571 * Check meter action from the action list. 4572 * 4573 * @param dev 4574 * Pointer to Ethernet device. 4575 * @param[in] actions 4576 * Pointer to the list of actions. 4577 * @param[out] has_mtr 4578 * Pointer to the meter exist flag. 4579 * @param[out] has_modify 4580 * Pointer to the flag showing there's packet change action. 4581 * @param[out] meter_id 4582 * Pointer to the meter id. 4583 * 4584 * @return 4585 * Total number of actions. 4586 */ 4587 static int 4588 flow_check_meter_action(struct rte_eth_dev *dev, 4589 const struct rte_flow_action actions[], 4590 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4591 { 4592 const struct rte_flow_action_meter *mtr = NULL; 4593 int actions_n = 0; 4594 4595 MLX5_ASSERT(has_mtr); 4596 *has_mtr = false; 4597 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4598 switch (actions->type) { 4599 case RTE_FLOW_ACTION_TYPE_METER: 4600 mtr = actions->conf; 4601 *meter_id = mtr->mtr_id; 4602 *has_mtr = true; 4603 break; 4604 default: 4605 break; 4606 } 4607 if (!*has_mtr) 4608 *has_modify |= flow_check_modify_action_type(dev, 4609 actions->type); 4610 actions_n++; 4611 } 4612 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4613 return actions_n + 1; 4614 } 4615 4616 /** 4617 * Check if the flow should be split due to hairpin. 4618 * The reason for the split is that in current HW we can't 4619 * support encap and push-vlan on Rx, so if a flow contains 4620 * these actions we move it to Tx. 4621 * 4622 * @param dev 4623 * Pointer to Ethernet device. 4624 * @param[in] attr 4625 * Flow rule attributes. 4626 * @param[in] actions 4627 * Associated actions (list terminated by the END action). 4628 * 4629 * @return 4630 * > 0 the number of actions and the flow should be split, 4631 * 0 when no split required. 4632 */ 4633 static int 4634 flow_check_hairpin_split(struct rte_eth_dev *dev, 4635 const struct rte_flow_attr *attr, 4636 const struct rte_flow_action actions[]) 4637 { 4638 int queue_action = 0; 4639 int action_n = 0; 4640 int split = 0; 4641 const struct rte_flow_action_queue *queue; 4642 const struct rte_flow_action_rss *rss; 4643 const struct rte_flow_action_raw_encap *raw_encap; 4644 const struct rte_eth_hairpin_conf *conf; 4645 4646 if (!attr->ingress) 4647 return 0; 4648 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4649 switch (actions->type) { 4650 case RTE_FLOW_ACTION_TYPE_QUEUE: 4651 queue = actions->conf; 4652 if (queue == NULL) 4653 return 0; 4654 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4655 if (conf == NULL || conf->tx_explicit != 0) 4656 return 0; 4657 queue_action = 1; 4658 action_n++; 4659 break; 4660 case RTE_FLOW_ACTION_TYPE_RSS: 4661 rss = actions->conf; 4662 if (rss == NULL || rss->queue_num == 0) 4663 return 0; 4664 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4665 if (conf == NULL || conf->tx_explicit != 0) 4666 return 0; 4667 queue_action = 1; 4668 action_n++; 4669 break; 4670 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4671 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4672 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4673 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4674 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4675 split++; 4676 action_n++; 4677 break; 4678 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4679 raw_encap = actions->conf; 4680 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4681 split++; 4682 action_n++; 4683 break; 4684 default: 4685 action_n++; 4686 break; 4687 } 4688 } 4689 if (split && queue_action) 4690 return action_n; 4691 return 0; 4692 } 4693 4694 /* Declare flow create/destroy prototype in advance. */ 4695 static uint32_t 4696 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4697 const struct rte_flow_attr *attr, 4698 const struct rte_flow_item items[], 4699 const struct rte_flow_action actions[], 4700 bool external, struct rte_flow_error *error); 4701 4702 static void 4703 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4704 uint32_t flow_idx); 4705 4706 int 4707 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4708 struct mlx5_list_entry *entry, void *cb_ctx) 4709 { 4710 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4711 struct mlx5_flow_mreg_copy_resource *mcp_res = 4712 container_of(entry, typeof(*mcp_res), hlist_ent); 4713 4714 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4715 } 4716 4717 struct mlx5_list_entry * 4718 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4719 { 4720 struct rte_eth_dev *dev = tool_ctx; 4721 struct mlx5_priv *priv = dev->data->dev_private; 4722 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4723 struct mlx5_flow_mreg_copy_resource *mcp_res; 4724 struct rte_flow_error *error = ctx->error; 4725 uint32_t idx = 0; 4726 int ret; 4727 uint32_t mark_id = *(uint32_t *)(ctx->data); 4728 struct rte_flow_attr attr = { 4729 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4730 .ingress = 1, 4731 }; 4732 struct mlx5_rte_flow_item_tag tag_spec = { 4733 .data = mark_id, 4734 }; 4735 struct rte_flow_item items[] = { 4736 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4737 }; 4738 struct rte_flow_action_mark ftag = { 4739 .id = mark_id, 4740 }; 4741 struct mlx5_flow_action_copy_mreg cp_mreg = { 4742 .dst = REG_B, 4743 .src = REG_NON, 4744 }; 4745 struct rte_flow_action_jump jump = { 4746 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4747 }; 4748 struct rte_flow_action actions[] = { 4749 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4750 }; 4751 4752 /* Fill the register fields in the flow. */ 4753 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4754 if (ret < 0) 4755 return NULL; 4756 tag_spec.id = ret; 4757 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4758 if (ret < 0) 4759 return NULL; 4760 cp_mreg.src = ret; 4761 /* Provide the full width of FLAG specific value. */ 4762 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4763 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4764 /* Build a new flow. */ 4765 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4766 items[0] = (struct rte_flow_item){ 4767 .type = (enum rte_flow_item_type) 4768 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4769 .spec = &tag_spec, 4770 }; 4771 items[1] = (struct rte_flow_item){ 4772 .type = RTE_FLOW_ITEM_TYPE_END, 4773 }; 4774 actions[0] = (struct rte_flow_action){ 4775 .type = (enum rte_flow_action_type) 4776 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4777 .conf = &ftag, 4778 }; 4779 actions[1] = (struct rte_flow_action){ 4780 .type = (enum rte_flow_action_type) 4781 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4782 .conf = &cp_mreg, 4783 }; 4784 actions[2] = (struct rte_flow_action){ 4785 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4786 .conf = &jump, 4787 }; 4788 actions[3] = (struct rte_flow_action){ 4789 .type = RTE_FLOW_ACTION_TYPE_END, 4790 }; 4791 } else { 4792 /* Default rule, wildcard match. */ 4793 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4794 items[0] = (struct rte_flow_item){ 4795 .type = RTE_FLOW_ITEM_TYPE_END, 4796 }; 4797 actions[0] = (struct rte_flow_action){ 4798 .type = (enum rte_flow_action_type) 4799 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4800 .conf = &cp_mreg, 4801 }; 4802 actions[1] = (struct rte_flow_action){ 4803 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4804 .conf = &jump, 4805 }; 4806 actions[2] = (struct rte_flow_action){ 4807 .type = RTE_FLOW_ACTION_TYPE_END, 4808 }; 4809 } 4810 /* Build a new entry. */ 4811 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4812 if (!mcp_res) { 4813 rte_errno = ENOMEM; 4814 return NULL; 4815 } 4816 mcp_res->idx = idx; 4817 mcp_res->mark_id = mark_id; 4818 /* 4819 * The copy Flows are not included in any list. There 4820 * ones are referenced from other Flows and can not 4821 * be applied, removed, deleted in arbitrary order 4822 * by list traversing. 4823 */ 4824 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4825 &attr, items, actions, false, error); 4826 if (!mcp_res->rix_flow) { 4827 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4828 return NULL; 4829 } 4830 return &mcp_res->hlist_ent; 4831 } 4832 4833 struct mlx5_list_entry * 4834 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4835 void *cb_ctx __rte_unused) 4836 { 4837 struct rte_eth_dev *dev = tool_ctx; 4838 struct mlx5_priv *priv = dev->data->dev_private; 4839 struct mlx5_flow_mreg_copy_resource *mcp_res; 4840 uint32_t idx = 0; 4841 4842 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4843 if (!mcp_res) { 4844 rte_errno = ENOMEM; 4845 return NULL; 4846 } 4847 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4848 mcp_res->idx = idx; 4849 return &mcp_res->hlist_ent; 4850 } 4851 4852 void 4853 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4854 { 4855 struct mlx5_flow_mreg_copy_resource *mcp_res = 4856 container_of(entry, typeof(*mcp_res), hlist_ent); 4857 struct rte_eth_dev *dev = tool_ctx; 4858 struct mlx5_priv *priv = dev->data->dev_private; 4859 4860 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4861 } 4862 4863 /** 4864 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4865 * 4866 * As mark_id is unique, if there's already a registered flow for the mark_id, 4867 * return by increasing the reference counter of the resource. Otherwise, create 4868 * the resource (mcp_res) and flow. 4869 * 4870 * Flow looks like, 4871 * - If ingress port is ANY and reg_c[1] is mark_id, 4872 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4873 * 4874 * For default flow (zero mark_id), flow is like, 4875 * - If ingress port is ANY, 4876 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4877 * 4878 * @param dev 4879 * Pointer to Ethernet device. 4880 * @param mark_id 4881 * ID of MARK action, zero means default flow for META. 4882 * @param[out] error 4883 * Perform verbose error reporting if not NULL. 4884 * 4885 * @return 4886 * Associated resource on success, NULL otherwise and rte_errno is set. 4887 */ 4888 static struct mlx5_flow_mreg_copy_resource * 4889 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4890 struct rte_flow_error *error) 4891 { 4892 struct mlx5_priv *priv = dev->data->dev_private; 4893 struct mlx5_list_entry *entry; 4894 struct mlx5_flow_cb_ctx ctx = { 4895 .dev = dev, 4896 .error = error, 4897 .data = &mark_id, 4898 }; 4899 4900 /* Check if already registered. */ 4901 MLX5_ASSERT(priv->mreg_cp_tbl); 4902 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4903 if (!entry) 4904 return NULL; 4905 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4906 hlist_ent); 4907 } 4908 4909 void 4910 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4911 { 4912 struct mlx5_flow_mreg_copy_resource *mcp_res = 4913 container_of(entry, typeof(*mcp_res), hlist_ent); 4914 struct rte_eth_dev *dev = tool_ctx; 4915 struct mlx5_priv *priv = dev->data->dev_private; 4916 4917 MLX5_ASSERT(mcp_res->rix_flow); 4918 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4919 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4920 } 4921 4922 /** 4923 * Release flow in RX_CP_TBL. 4924 * 4925 * @param dev 4926 * Pointer to Ethernet device. 4927 * @flow 4928 * Parent flow for wich copying is provided. 4929 */ 4930 static void 4931 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4932 struct rte_flow *flow) 4933 { 4934 struct mlx5_flow_mreg_copy_resource *mcp_res; 4935 struct mlx5_priv *priv = dev->data->dev_private; 4936 4937 if (!flow->rix_mreg_copy) 4938 return; 4939 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4940 flow->rix_mreg_copy); 4941 if (!mcp_res || !priv->mreg_cp_tbl) 4942 return; 4943 MLX5_ASSERT(mcp_res->rix_flow); 4944 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4945 flow->rix_mreg_copy = 0; 4946 } 4947 4948 /** 4949 * Remove the default copy action from RX_CP_TBL. 4950 * 4951 * This functions is called in the mlx5_dev_start(). No thread safe 4952 * is guaranteed. 4953 * 4954 * @param dev 4955 * Pointer to Ethernet device. 4956 */ 4957 static void 4958 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4959 { 4960 struct mlx5_list_entry *entry; 4961 struct mlx5_priv *priv = dev->data->dev_private; 4962 struct mlx5_flow_cb_ctx ctx; 4963 uint32_t mark_id; 4964 4965 /* Check if default flow is registered. */ 4966 if (!priv->mreg_cp_tbl) 4967 return; 4968 mark_id = MLX5_DEFAULT_COPY_ID; 4969 ctx.data = &mark_id; 4970 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4971 if (!entry) 4972 return; 4973 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4974 } 4975 4976 /** 4977 * Add the default copy action in in RX_CP_TBL. 4978 * 4979 * This functions is called in the mlx5_dev_start(). No thread safe 4980 * is guaranteed. 4981 * 4982 * @param dev 4983 * Pointer to Ethernet device. 4984 * @param[out] error 4985 * Perform verbose error reporting if not NULL. 4986 * 4987 * @return 4988 * 0 for success, negative value otherwise and rte_errno is set. 4989 */ 4990 static int 4991 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4992 struct rte_flow_error *error) 4993 { 4994 struct mlx5_priv *priv = dev->data->dev_private; 4995 struct mlx5_flow_mreg_copy_resource *mcp_res; 4996 struct mlx5_flow_cb_ctx ctx; 4997 uint32_t mark_id; 4998 4999 /* Check whether extensive metadata feature is engaged. */ 5000 if (!priv->sh->config.dv_flow_en || 5001 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5002 !mlx5_flow_ext_mreg_supported(dev) || 5003 !priv->sh->dv_regc0_mask) 5004 return 0; 5005 /* 5006 * Add default mreg copy flow may be called multiple time, but 5007 * only be called once in stop. Avoid register it twice. 5008 */ 5009 mark_id = MLX5_DEFAULT_COPY_ID; 5010 ctx.data = &mark_id; 5011 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5012 return 0; 5013 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5014 if (!mcp_res) 5015 return -rte_errno; 5016 return 0; 5017 } 5018 5019 /** 5020 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5021 * 5022 * All the flow having Q/RSS action should be split by 5023 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5024 * performs the following, 5025 * - CQE->flow_tag := reg_c[1] (MARK) 5026 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5027 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5028 * but there should be a flow per each MARK ID set by MARK action. 5029 * 5030 * For the aforementioned reason, if there's a MARK action in flow's action 5031 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5032 * the MARK ID to CQE's flow_tag like, 5033 * - If reg_c[1] is mark_id, 5034 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5035 * 5036 * For SET_META action which stores value in reg_c[0], as the destination is 5037 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5038 * MARK ID means the default flow. The default flow looks like, 5039 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5040 * 5041 * @param dev 5042 * Pointer to Ethernet device. 5043 * @param flow 5044 * Pointer to flow structure. 5045 * @param[in] actions 5046 * Pointer to the list of actions. 5047 * @param[out] error 5048 * Perform verbose error reporting if not NULL. 5049 * 5050 * @return 5051 * 0 on success, negative value otherwise and rte_errno is set. 5052 */ 5053 static int 5054 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5055 struct rte_flow *flow, 5056 const struct rte_flow_action *actions, 5057 struct rte_flow_error *error) 5058 { 5059 struct mlx5_priv *priv = dev->data->dev_private; 5060 struct mlx5_sh_config *config = &priv->sh->config; 5061 struct mlx5_flow_mreg_copy_resource *mcp_res; 5062 const struct rte_flow_action_mark *mark; 5063 5064 /* Check whether extensive metadata feature is engaged. */ 5065 if (!config->dv_flow_en || 5066 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5067 !mlx5_flow_ext_mreg_supported(dev) || 5068 !priv->sh->dv_regc0_mask) 5069 return 0; 5070 /* Find MARK action. */ 5071 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5072 switch (actions->type) { 5073 case RTE_FLOW_ACTION_TYPE_FLAG: 5074 mcp_res = flow_mreg_add_copy_action 5075 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5076 if (!mcp_res) 5077 return -rte_errno; 5078 flow->rix_mreg_copy = mcp_res->idx; 5079 return 0; 5080 case RTE_FLOW_ACTION_TYPE_MARK: 5081 mark = (const struct rte_flow_action_mark *) 5082 actions->conf; 5083 mcp_res = 5084 flow_mreg_add_copy_action(dev, mark->id, error); 5085 if (!mcp_res) 5086 return -rte_errno; 5087 flow->rix_mreg_copy = mcp_res->idx; 5088 return 0; 5089 default: 5090 break; 5091 } 5092 } 5093 return 0; 5094 } 5095 5096 #define MLX5_MAX_SPLIT_ACTIONS 24 5097 #define MLX5_MAX_SPLIT_ITEMS 24 5098 5099 /** 5100 * Split the hairpin flow. 5101 * Since HW can't support encap and push-vlan on Rx, we move these 5102 * actions to Tx. 5103 * If the count action is after the encap then we also 5104 * move the count action. in this case the count will also measure 5105 * the outer bytes. 5106 * 5107 * @param dev 5108 * Pointer to Ethernet device. 5109 * @param[in] actions 5110 * Associated actions (list terminated by the END action). 5111 * @param[out] actions_rx 5112 * Rx flow actions. 5113 * @param[out] actions_tx 5114 * Tx flow actions.. 5115 * @param[out] pattern_tx 5116 * The pattern items for the Tx flow. 5117 * @param[out] flow_id 5118 * The flow ID connected to this flow. 5119 * 5120 * @return 5121 * 0 on success. 5122 */ 5123 static int 5124 flow_hairpin_split(struct rte_eth_dev *dev, 5125 const struct rte_flow_action actions[], 5126 struct rte_flow_action actions_rx[], 5127 struct rte_flow_action actions_tx[], 5128 struct rte_flow_item pattern_tx[], 5129 uint32_t flow_id) 5130 { 5131 const struct rte_flow_action_raw_encap *raw_encap; 5132 const struct rte_flow_action_raw_decap *raw_decap; 5133 struct mlx5_rte_flow_action_set_tag *set_tag; 5134 struct rte_flow_action *tag_action; 5135 struct mlx5_rte_flow_item_tag *tag_item; 5136 struct rte_flow_item *item; 5137 char *addr; 5138 int encap = 0; 5139 5140 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5141 switch (actions->type) { 5142 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5143 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5144 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5145 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5146 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5147 rte_memcpy(actions_tx, actions, 5148 sizeof(struct rte_flow_action)); 5149 actions_tx++; 5150 break; 5151 case RTE_FLOW_ACTION_TYPE_COUNT: 5152 if (encap) { 5153 rte_memcpy(actions_tx, actions, 5154 sizeof(struct rte_flow_action)); 5155 actions_tx++; 5156 } else { 5157 rte_memcpy(actions_rx, actions, 5158 sizeof(struct rte_flow_action)); 5159 actions_rx++; 5160 } 5161 break; 5162 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5163 raw_encap = actions->conf; 5164 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5165 memcpy(actions_tx, actions, 5166 sizeof(struct rte_flow_action)); 5167 actions_tx++; 5168 encap = 1; 5169 } else { 5170 rte_memcpy(actions_rx, actions, 5171 sizeof(struct rte_flow_action)); 5172 actions_rx++; 5173 } 5174 break; 5175 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5176 raw_decap = actions->conf; 5177 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5178 memcpy(actions_tx, actions, 5179 sizeof(struct rte_flow_action)); 5180 actions_tx++; 5181 } else { 5182 rte_memcpy(actions_rx, actions, 5183 sizeof(struct rte_flow_action)); 5184 actions_rx++; 5185 } 5186 break; 5187 default: 5188 rte_memcpy(actions_rx, actions, 5189 sizeof(struct rte_flow_action)); 5190 actions_rx++; 5191 break; 5192 } 5193 } 5194 /* Add set meta action and end action for the Rx flow. */ 5195 tag_action = actions_rx; 5196 tag_action->type = (enum rte_flow_action_type) 5197 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5198 actions_rx++; 5199 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5200 actions_rx++; 5201 set_tag = (void *)actions_rx; 5202 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5203 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5204 .data = flow_id, 5205 }; 5206 MLX5_ASSERT(set_tag->id > REG_NON); 5207 tag_action->conf = set_tag; 5208 /* Create Tx item list. */ 5209 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5210 addr = (void *)&pattern_tx[2]; 5211 item = pattern_tx; 5212 item->type = (enum rte_flow_item_type) 5213 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5214 tag_item = (void *)addr; 5215 tag_item->data = flow_id; 5216 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5217 MLX5_ASSERT(set_tag->id > REG_NON); 5218 item->spec = tag_item; 5219 addr += sizeof(struct mlx5_rte_flow_item_tag); 5220 tag_item = (void *)addr; 5221 tag_item->data = UINT32_MAX; 5222 tag_item->id = UINT16_MAX; 5223 item->mask = tag_item; 5224 item->last = NULL; 5225 item++; 5226 item->type = RTE_FLOW_ITEM_TYPE_END; 5227 return 0; 5228 } 5229 5230 /** 5231 * The last stage of splitting chain, just creates the subflow 5232 * without any modification. 5233 * 5234 * @param[in] dev 5235 * Pointer to Ethernet device. 5236 * @param[in] flow 5237 * Parent flow structure pointer. 5238 * @param[in, out] sub_flow 5239 * Pointer to return the created subflow, may be NULL. 5240 * @param[in] attr 5241 * Flow rule attributes. 5242 * @param[in] items 5243 * Pattern specification (list terminated by the END pattern item). 5244 * @param[in] actions 5245 * Associated actions (list terminated by the END action). 5246 * @param[in] flow_split_info 5247 * Pointer to flow split info structure. 5248 * @param[out] error 5249 * Perform verbose error reporting if not NULL. 5250 * @return 5251 * 0 on success, negative value otherwise 5252 */ 5253 static int 5254 flow_create_split_inner(struct rte_eth_dev *dev, 5255 struct rte_flow *flow, 5256 struct mlx5_flow **sub_flow, 5257 const struct rte_flow_attr *attr, 5258 const struct rte_flow_item items[], 5259 const struct rte_flow_action actions[], 5260 struct mlx5_flow_split_info *flow_split_info, 5261 struct rte_flow_error *error) 5262 { 5263 struct mlx5_flow *dev_flow; 5264 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5265 5266 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5267 flow_split_info->flow_idx, error); 5268 if (!dev_flow) 5269 return -rte_errno; 5270 dev_flow->flow = flow; 5271 dev_flow->external = flow_split_info->external; 5272 dev_flow->skip_scale = flow_split_info->skip_scale; 5273 /* Subflow object was created, we must include one in the list. */ 5274 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5275 dev_flow->handle, next); 5276 /* 5277 * If dev_flow is as one of the suffix flow, some actions in suffix 5278 * flow may need some user defined item layer flags, and pass the 5279 * Metadata rxq mark flag to suffix flow as well. 5280 */ 5281 if (flow_split_info->prefix_layers) 5282 dev_flow->handle->layers = flow_split_info->prefix_layers; 5283 if (flow_split_info->prefix_mark) { 5284 MLX5_ASSERT(wks); 5285 wks->mark = 1; 5286 } 5287 if (sub_flow) 5288 *sub_flow = dev_flow; 5289 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5290 dev_flow->dv.table_id = flow_split_info->table_id; 5291 #endif 5292 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5293 } 5294 5295 /** 5296 * Get the sub policy of a meter. 5297 * 5298 * @param[in] dev 5299 * Pointer to Ethernet device. 5300 * @param[in] flow 5301 * Parent flow structure pointer. 5302 * @param wks 5303 * Pointer to thread flow work space. 5304 * @param[in] attr 5305 * Flow rule attributes. 5306 * @param[in] items 5307 * Pattern specification (list terminated by the END pattern item). 5308 * @param[out] error 5309 * Perform verbose error reporting if not NULL. 5310 * 5311 * @return 5312 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5313 */ 5314 static struct mlx5_flow_meter_sub_policy * 5315 get_meter_sub_policy(struct rte_eth_dev *dev, 5316 struct rte_flow *flow, 5317 struct mlx5_flow_workspace *wks, 5318 const struct rte_flow_attr *attr, 5319 const struct rte_flow_item items[], 5320 struct rte_flow_error *error) 5321 { 5322 struct mlx5_flow_meter_policy *policy; 5323 struct mlx5_flow_meter_policy *final_policy; 5324 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5325 5326 policy = wks->policy; 5327 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5328 if (final_policy->is_rss || final_policy->is_queue) { 5329 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5330 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5331 uint32_t i; 5332 5333 /* 5334 * This is a tmp dev_flow, 5335 * no need to register any matcher for it in translate. 5336 */ 5337 wks->skip_matcher_reg = 1; 5338 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5339 struct mlx5_flow dev_flow = {0}; 5340 struct mlx5_flow_handle dev_handle = { {0} }; 5341 uint8_t fate = final_policy->act_cnt[i].fate_action; 5342 5343 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5344 const struct rte_flow_action_rss *rss_act = 5345 final_policy->act_cnt[i].rss->conf; 5346 struct rte_flow_action rss_actions[2] = { 5347 [0] = { 5348 .type = RTE_FLOW_ACTION_TYPE_RSS, 5349 .conf = rss_act, 5350 }, 5351 [1] = { 5352 .type = RTE_FLOW_ACTION_TYPE_END, 5353 .conf = NULL, 5354 } 5355 }; 5356 5357 dev_flow.handle = &dev_handle; 5358 dev_flow.ingress = attr->ingress; 5359 dev_flow.flow = flow; 5360 dev_flow.external = 0; 5361 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5362 dev_flow.dv.transfer = attr->transfer; 5363 #endif 5364 /** 5365 * Translate RSS action to get rss hash fields. 5366 */ 5367 if (flow_drv_translate(dev, &dev_flow, attr, 5368 items, rss_actions, error)) 5369 goto exit; 5370 rss_desc_v[i] = wks->rss_desc; 5371 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5372 rss_desc_v[i].hash_fields = 5373 dev_flow.hash_fields; 5374 rss_desc_v[i].queue_num = 5375 rss_desc_v[i].hash_fields ? 5376 rss_desc_v[i].queue_num : 1; 5377 rss_desc_v[i].tunnel = 5378 !!(dev_flow.handle->layers & 5379 MLX5_FLOW_LAYER_TUNNEL); 5380 /* Use the RSS queues in the containers. */ 5381 rss_desc_v[i].queue = 5382 (uint16_t *)(uintptr_t)rss_act->queue; 5383 rss_desc[i] = &rss_desc_v[i]; 5384 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5385 /* This is queue action. */ 5386 rss_desc_v[i] = wks->rss_desc; 5387 rss_desc_v[i].key_len = 0; 5388 rss_desc_v[i].hash_fields = 0; 5389 rss_desc_v[i].queue = 5390 &final_policy->act_cnt[i].queue; 5391 rss_desc_v[i].queue_num = 1; 5392 rss_desc[i] = &rss_desc_v[i]; 5393 } else { 5394 rss_desc[i] = NULL; 5395 } 5396 } 5397 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5398 flow, policy, rss_desc); 5399 } else { 5400 enum mlx5_meter_domain mtr_domain = 5401 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5402 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5403 MLX5_MTR_DOMAIN_INGRESS); 5404 sub_policy = policy->sub_policys[mtr_domain][0]; 5405 } 5406 if (!sub_policy) 5407 rte_flow_error_set(error, EINVAL, 5408 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5409 "Failed to get meter sub-policy."); 5410 exit: 5411 return sub_policy; 5412 } 5413 5414 /** 5415 * Split the meter flow. 5416 * 5417 * As meter flow will split to three sub flow, other than meter 5418 * action, the other actions make sense to only meter accepts 5419 * the packet. If it need to be dropped, no other additional 5420 * actions should be take. 5421 * 5422 * One kind of special action which decapsulates the L3 tunnel 5423 * header will be in the prefix sub flow, as not to take the 5424 * L3 tunnel header into account. 5425 * 5426 * @param[in] dev 5427 * Pointer to Ethernet device. 5428 * @param[in] flow 5429 * Parent flow structure pointer. 5430 * @param wks 5431 * Pointer to thread flow work space. 5432 * @param[in] attr 5433 * Flow rule attributes. 5434 * @param[in] items 5435 * Pattern specification (list terminated by the END pattern item). 5436 * @param[out] sfx_items 5437 * Suffix flow match items (list terminated by the END pattern item). 5438 * @param[in] actions 5439 * Associated actions (list terminated by the END action). 5440 * @param[out] actions_sfx 5441 * Suffix flow actions. 5442 * @param[out] actions_pre 5443 * Prefix flow actions. 5444 * @param[out] mtr_flow_id 5445 * Pointer to meter flow id. 5446 * @param[out] error 5447 * Perform verbose error reporting if not NULL. 5448 * 5449 * @return 5450 * 0 on success, a negative errno value otherwise and rte_errno is set. 5451 */ 5452 static int 5453 flow_meter_split_prep(struct rte_eth_dev *dev, 5454 struct rte_flow *flow, 5455 struct mlx5_flow_workspace *wks, 5456 const struct rte_flow_attr *attr, 5457 const struct rte_flow_item items[], 5458 struct rte_flow_item sfx_items[], 5459 const struct rte_flow_action actions[], 5460 struct rte_flow_action actions_sfx[], 5461 struct rte_flow_action actions_pre[], 5462 uint32_t *mtr_flow_id, 5463 struct rte_flow_error *error) 5464 { 5465 struct mlx5_priv *priv = dev->data->dev_private; 5466 struct mlx5_flow_meter_info *fm = wks->fm; 5467 struct rte_flow_action *tag_action = NULL; 5468 struct rte_flow_item *tag_item; 5469 struct mlx5_rte_flow_action_set_tag *set_tag; 5470 const struct rte_flow_action_raw_encap *raw_encap; 5471 const struct rte_flow_action_raw_decap *raw_decap; 5472 struct mlx5_rte_flow_item_tag *tag_item_spec; 5473 struct mlx5_rte_flow_item_tag *tag_item_mask; 5474 uint32_t tag_id = 0; 5475 struct rte_flow_item *vlan_item_dst = NULL; 5476 const struct rte_flow_item *vlan_item_src = NULL; 5477 const struct rte_flow_item *orig_items = items; 5478 struct rte_flow_action *hw_mtr_action; 5479 struct rte_flow_action *action_pre_head = NULL; 5480 uint16_t flow_src_port = priv->representor_id; 5481 bool mtr_first; 5482 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5483 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5484 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5485 uint32_t flow_id = 0; 5486 uint32_t flow_id_reversed = 0; 5487 uint8_t flow_id_bits = 0; 5488 bool after_meter = false; 5489 int shift; 5490 5491 /* Prepare the suffix subflow items. */ 5492 tag_item = sfx_items++; 5493 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5494 int item_type = items->type; 5495 5496 switch (item_type) { 5497 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5498 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5499 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5500 return -rte_errno; 5501 if (!fm->def_policy && wks->policy->is_hierarchy && 5502 flow_src_port != priv->representor_id) { 5503 if (flow_drv_mtr_hierarchy_rule_create(dev, 5504 flow, fm, 5505 flow_src_port, 5506 items, 5507 error)) 5508 return -rte_errno; 5509 } 5510 memcpy(sfx_items, items, sizeof(*sfx_items)); 5511 sfx_items++; 5512 break; 5513 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5514 flow_src_port = 0; 5515 memcpy(sfx_items, items, sizeof(*sfx_items)); 5516 sfx_items++; 5517 break; 5518 case RTE_FLOW_ITEM_TYPE_VLAN: 5519 /* Determine if copy vlan item below. */ 5520 vlan_item_src = items; 5521 vlan_item_dst = sfx_items++; 5522 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5523 break; 5524 default: 5525 break; 5526 } 5527 } 5528 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5529 sfx_items++; 5530 mtr_first = priv->sh->meter_aso_en && 5531 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5532 /* For ASO meter, meter must be before tag in TX direction. */ 5533 if (mtr_first) { 5534 action_pre_head = actions_pre++; 5535 /* Leave space for tag action. */ 5536 tag_action = actions_pre++; 5537 } 5538 /* Prepare the actions for prefix and suffix flow. */ 5539 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5540 struct rte_flow_action *action_cur = NULL; 5541 5542 switch (actions->type) { 5543 case RTE_FLOW_ACTION_TYPE_METER: 5544 if (mtr_first) { 5545 action_cur = action_pre_head; 5546 } else { 5547 /* Leave space for tag action. */ 5548 tag_action = actions_pre++; 5549 action_cur = actions_pre++; 5550 } 5551 after_meter = true; 5552 break; 5553 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5554 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5555 action_cur = actions_pre++; 5556 break; 5557 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5558 raw_encap = actions->conf; 5559 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5560 action_cur = actions_pre++; 5561 break; 5562 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5563 raw_decap = actions->conf; 5564 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5565 action_cur = actions_pre++; 5566 break; 5567 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5568 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5569 if (vlan_item_dst && vlan_item_src) { 5570 memcpy(vlan_item_dst, vlan_item_src, 5571 sizeof(*vlan_item_dst)); 5572 /* 5573 * Convert to internal match item, it is used 5574 * for vlan push and set vid. 5575 */ 5576 vlan_item_dst->type = (enum rte_flow_item_type) 5577 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5578 } 5579 break; 5580 case RTE_FLOW_ACTION_TYPE_COUNT: 5581 if (fm->def_policy) 5582 action_cur = after_meter ? 5583 actions_sfx++ : actions_pre++; 5584 break; 5585 default: 5586 break; 5587 } 5588 if (!action_cur) 5589 action_cur = (fm->def_policy) ? 5590 actions_sfx++ : actions_pre++; 5591 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5592 } 5593 /* Add end action to the actions. */ 5594 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5595 if (priv->sh->meter_aso_en) { 5596 /** 5597 * For ASO meter, need to add an extra jump action explicitly, 5598 * to jump from meter to policer table. 5599 */ 5600 struct mlx5_flow_meter_sub_policy *sub_policy; 5601 struct mlx5_flow_tbl_data_entry *tbl_data; 5602 5603 if (!fm->def_policy) { 5604 sub_policy = get_meter_sub_policy(dev, flow, wks, 5605 attr, orig_items, 5606 error); 5607 if (!sub_policy) 5608 return -rte_errno; 5609 } else { 5610 enum mlx5_meter_domain mtr_domain = 5611 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5612 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5613 MLX5_MTR_DOMAIN_INGRESS); 5614 5615 sub_policy = 5616 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5617 } 5618 tbl_data = container_of(sub_policy->tbl_rsc, 5619 struct mlx5_flow_tbl_data_entry, tbl); 5620 hw_mtr_action = actions_pre++; 5621 hw_mtr_action->type = (enum rte_flow_action_type) 5622 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5623 hw_mtr_action->conf = tbl_data->jump.action; 5624 } 5625 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5626 actions_pre++; 5627 if (!tag_action) 5628 return rte_flow_error_set(error, ENOMEM, 5629 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5630 NULL, "No tag action space."); 5631 if (!mtr_flow_id) { 5632 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5633 goto exit; 5634 } 5635 /* Only default-policy Meter creates mtr flow id. */ 5636 if (fm->def_policy) { 5637 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5638 if (!tag_id) 5639 return rte_flow_error_set(error, ENOMEM, 5640 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5641 "Failed to allocate meter flow id."); 5642 flow_id = tag_id - 1; 5643 flow_id_bits = (!flow_id) ? 1 : 5644 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5645 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5646 mtr_reg_bits) { 5647 mlx5_ipool_free(fm->flow_ipool, tag_id); 5648 return rte_flow_error_set(error, EINVAL, 5649 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5650 "Meter flow id exceeds max limit."); 5651 } 5652 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5653 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5654 } 5655 /* Build tag actions and items for meter_id/meter flow_id. */ 5656 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5657 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5658 tag_item_mask = tag_item_spec + 1; 5659 /* Both flow_id and meter_id share the same register. */ 5660 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5661 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5662 0, error), 5663 .offset = mtr_id_offset, 5664 .length = mtr_reg_bits, 5665 .data = flow->meter, 5666 }; 5667 /* 5668 * The color Reg bits used by flow_id are growing from 5669 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5670 */ 5671 for (shift = 0; shift < flow_id_bits; shift++) 5672 flow_id_reversed = (flow_id_reversed << 1) | 5673 ((flow_id >> shift) & 0x1); 5674 set_tag->data |= 5675 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5676 tag_item_spec->id = set_tag->id; 5677 tag_item_spec->data = set_tag->data << mtr_id_offset; 5678 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5679 tag_action->type = (enum rte_flow_action_type) 5680 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5681 tag_action->conf = set_tag; 5682 tag_item->type = (enum rte_flow_item_type) 5683 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5684 tag_item->spec = tag_item_spec; 5685 tag_item->last = NULL; 5686 tag_item->mask = tag_item_mask; 5687 exit: 5688 if (mtr_flow_id) 5689 *mtr_flow_id = tag_id; 5690 return 0; 5691 } 5692 5693 /** 5694 * Split action list having QUEUE/RSS for metadata register copy. 5695 * 5696 * Once Q/RSS action is detected in user's action list, the flow action 5697 * should be split in order to copy metadata registers, which will happen in 5698 * RX_CP_TBL like, 5699 * - CQE->flow_tag := reg_c[1] (MARK) 5700 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5701 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5702 * This is because the last action of each flow must be a terminal action 5703 * (QUEUE, RSS or DROP). 5704 * 5705 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5706 * stored and kept in the mlx5_flow structure per each sub_flow. 5707 * 5708 * The Q/RSS action is replaced with, 5709 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5710 * And the following JUMP action is added at the end, 5711 * - JUMP, to RX_CP_TBL. 5712 * 5713 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5714 * flow_create_split_metadata() routine. The flow will look like, 5715 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5716 * 5717 * @param dev 5718 * Pointer to Ethernet device. 5719 * @param[out] split_actions 5720 * Pointer to store split actions to jump to CP_TBL. 5721 * @param[in] actions 5722 * Pointer to the list of original flow actions. 5723 * @param[in] qrss 5724 * Pointer to the Q/RSS action. 5725 * @param[in] actions_n 5726 * Number of original actions. 5727 * @param[in] mtr_sfx 5728 * Check if it is in meter suffix table. 5729 * @param[out] error 5730 * Perform verbose error reporting if not NULL. 5731 * 5732 * @return 5733 * non-zero unique flow_id on success, otherwise 0 and 5734 * error/rte_error are set. 5735 */ 5736 static uint32_t 5737 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5738 struct rte_flow_action *split_actions, 5739 const struct rte_flow_action *actions, 5740 const struct rte_flow_action *qrss, 5741 int actions_n, int mtr_sfx, 5742 struct rte_flow_error *error) 5743 { 5744 struct mlx5_priv *priv = dev->data->dev_private; 5745 struct mlx5_rte_flow_action_set_tag *set_tag; 5746 struct rte_flow_action_jump *jump; 5747 const int qrss_idx = qrss - actions; 5748 uint32_t flow_id = 0; 5749 int ret = 0; 5750 5751 /* 5752 * Given actions will be split 5753 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5754 * - Add jump to mreg CP_TBL. 5755 * As a result, there will be one more action. 5756 */ 5757 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5758 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5759 ++actions_n; 5760 set_tag = (void *)(split_actions + actions_n); 5761 /* 5762 * If we are not the meter suffix flow, add the tag action. 5763 * Since meter suffix flow already has the tag added. 5764 */ 5765 if (!mtr_sfx) { 5766 /* 5767 * Allocate the new subflow ID. This one is unique within 5768 * device and not shared with representors. Otherwise, 5769 * we would have to resolve multi-thread access synch 5770 * issue. Each flow on the shared device is appended 5771 * with source vport identifier, so the resulting 5772 * flows will be unique in the shared (by master and 5773 * representors) domain even if they have coinciding 5774 * IDs. 5775 */ 5776 mlx5_ipool_malloc(priv->sh->ipool 5777 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5778 if (!flow_id) 5779 return rte_flow_error_set(error, ENOMEM, 5780 RTE_FLOW_ERROR_TYPE_ACTION, 5781 NULL, "can't allocate id " 5782 "for split Q/RSS subflow"); 5783 /* Internal SET_TAG action to set flow ID. */ 5784 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5785 .data = flow_id, 5786 }; 5787 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5788 if (ret < 0) 5789 return ret; 5790 set_tag->id = ret; 5791 /* Construct new actions array. */ 5792 /* Replace QUEUE/RSS action. */ 5793 split_actions[qrss_idx] = (struct rte_flow_action){ 5794 .type = (enum rte_flow_action_type) 5795 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5796 .conf = set_tag, 5797 }; 5798 } else { 5799 /* 5800 * If we are the suffix flow of meter, tag already exist. 5801 * Set the QUEUE/RSS action to void. 5802 */ 5803 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5804 } 5805 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5806 jump = (void *)(set_tag + 1); 5807 *jump = (struct rte_flow_action_jump){ 5808 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5809 }; 5810 split_actions[actions_n - 2] = (struct rte_flow_action){ 5811 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5812 .conf = jump, 5813 }; 5814 split_actions[actions_n - 1] = (struct rte_flow_action){ 5815 .type = RTE_FLOW_ACTION_TYPE_END, 5816 }; 5817 return flow_id; 5818 } 5819 5820 /** 5821 * Extend the given action list for Tx metadata copy. 5822 * 5823 * Copy the given action list to the ext_actions and add flow metadata register 5824 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5825 * 5826 * @param[out] ext_actions 5827 * Pointer to the extended action list. 5828 * @param[in] actions 5829 * Pointer to the list of actions. 5830 * @param[in] actions_n 5831 * Number of actions in the list. 5832 * @param[out] error 5833 * Perform verbose error reporting if not NULL. 5834 * @param[in] encap_idx 5835 * The encap action index. 5836 * 5837 * @return 5838 * 0 on success, negative value otherwise 5839 */ 5840 static int 5841 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5842 struct rte_flow_action *ext_actions, 5843 const struct rte_flow_action *actions, 5844 int actions_n, struct rte_flow_error *error, 5845 int encap_idx) 5846 { 5847 struct mlx5_flow_action_copy_mreg *cp_mreg = 5848 (struct mlx5_flow_action_copy_mreg *) 5849 (ext_actions + actions_n + 1); 5850 int ret; 5851 5852 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5853 if (ret < 0) 5854 return ret; 5855 cp_mreg->dst = ret; 5856 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5857 if (ret < 0) 5858 return ret; 5859 cp_mreg->src = ret; 5860 if (encap_idx != 0) 5861 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5862 if (encap_idx == actions_n - 1) { 5863 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5864 .type = (enum rte_flow_action_type) 5865 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5866 .conf = cp_mreg, 5867 }; 5868 ext_actions[actions_n] = (struct rte_flow_action){ 5869 .type = RTE_FLOW_ACTION_TYPE_END, 5870 }; 5871 } else { 5872 ext_actions[encap_idx] = (struct rte_flow_action){ 5873 .type = (enum rte_flow_action_type) 5874 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5875 .conf = cp_mreg, 5876 }; 5877 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5878 sizeof(*ext_actions) * (actions_n - encap_idx)); 5879 } 5880 return 0; 5881 } 5882 5883 /** 5884 * Check the match action from the action list. 5885 * 5886 * @param[in] actions 5887 * Pointer to the list of actions. 5888 * @param[in] attr 5889 * Flow rule attributes. 5890 * @param[in] action 5891 * The action to be check if exist. 5892 * @param[out] match_action_pos 5893 * Pointer to the position of the matched action if exists, otherwise is -1. 5894 * @param[out] qrss_action_pos 5895 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5896 * @param[out] modify_after_mirror 5897 * Pointer to the flag of modify action after FDB mirroring. 5898 * 5899 * @return 5900 * > 0 the total number of actions. 5901 * 0 if not found match action in action list. 5902 */ 5903 static int 5904 flow_check_match_action(const struct rte_flow_action actions[], 5905 const struct rte_flow_attr *attr, 5906 enum rte_flow_action_type action, 5907 int *match_action_pos, int *qrss_action_pos, 5908 int *modify_after_mirror) 5909 { 5910 const struct rte_flow_action_sample *sample; 5911 const struct rte_flow_action_raw_decap *decap; 5912 int actions_n = 0; 5913 uint32_t ratio = 0; 5914 int sub_type = 0; 5915 int flag = 0; 5916 int fdb_mirror = 0; 5917 5918 *match_action_pos = -1; 5919 *qrss_action_pos = -1; 5920 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5921 if (actions->type == action) { 5922 flag = 1; 5923 *match_action_pos = actions_n; 5924 } 5925 switch (actions->type) { 5926 case RTE_FLOW_ACTION_TYPE_QUEUE: 5927 case RTE_FLOW_ACTION_TYPE_RSS: 5928 *qrss_action_pos = actions_n; 5929 break; 5930 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5931 sample = actions->conf; 5932 ratio = sample->ratio; 5933 sub_type = ((const struct rte_flow_action *) 5934 (sample->actions))->type; 5935 if (ratio == 1 && attr->transfer && 5936 sub_type != RTE_FLOW_ACTION_TYPE_END) 5937 fdb_mirror = 1; 5938 break; 5939 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5940 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5941 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5942 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5943 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5944 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5945 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5946 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5947 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5948 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5949 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5950 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5951 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5952 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5953 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5954 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5955 case RTE_FLOW_ACTION_TYPE_FLAG: 5956 case RTE_FLOW_ACTION_TYPE_MARK: 5957 case RTE_FLOW_ACTION_TYPE_SET_META: 5958 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5959 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5960 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5961 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5962 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5963 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5964 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5965 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5966 case RTE_FLOW_ACTION_TYPE_METER: 5967 if (fdb_mirror) 5968 *modify_after_mirror = 1; 5969 break; 5970 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5971 decap = actions->conf; 5972 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5973 ; 5974 actions_n++; 5975 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5976 const struct rte_flow_action_raw_encap *encap = 5977 actions->conf; 5978 if (decap->size <= 5979 MLX5_ENCAPSULATION_DECISION_SIZE && 5980 encap->size > 5981 MLX5_ENCAPSULATION_DECISION_SIZE) 5982 /* L3 encap. */ 5983 break; 5984 } 5985 if (fdb_mirror) 5986 *modify_after_mirror = 1; 5987 break; 5988 default: 5989 break; 5990 } 5991 actions_n++; 5992 } 5993 if (flag && fdb_mirror && !*modify_after_mirror) { 5994 /* FDB mirroring uses the destination array to implement 5995 * instead of FLOW_SAMPLER object. 5996 */ 5997 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5998 flag = 0; 5999 } 6000 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6001 return flag ? actions_n + 1 : 0; 6002 } 6003 6004 #define SAMPLE_SUFFIX_ITEM 3 6005 6006 /** 6007 * Split the sample flow. 6008 * 6009 * As sample flow will split to two sub flow, sample flow with 6010 * sample action, the other actions will move to new suffix flow. 6011 * 6012 * Also add unique tag id with tag action in the sample flow, 6013 * the same tag id will be as match in the suffix flow. 6014 * 6015 * @param dev 6016 * Pointer to Ethernet device. 6017 * @param[in] add_tag 6018 * Add extra tag action flag. 6019 * @param[out] sfx_items 6020 * Suffix flow match items (list terminated by the END pattern item). 6021 * @param[in] actions 6022 * Associated actions (list terminated by the END action). 6023 * @param[out] actions_sfx 6024 * Suffix flow actions. 6025 * @param[out] actions_pre 6026 * Prefix flow actions. 6027 * @param[in] actions_n 6028 * The total number of actions. 6029 * @param[in] sample_action_pos 6030 * The sample action position. 6031 * @param[in] qrss_action_pos 6032 * The Queue/RSS action position. 6033 * @param[in] jump_table 6034 * Add extra jump action flag. 6035 * @param[out] error 6036 * Perform verbose error reporting if not NULL. 6037 * 6038 * @return 6039 * 0 on success, or unique flow_id, a negative errno value 6040 * otherwise and rte_errno is set. 6041 */ 6042 static int 6043 flow_sample_split_prep(struct rte_eth_dev *dev, 6044 int add_tag, 6045 const struct rte_flow_item items[], 6046 struct rte_flow_item sfx_items[], 6047 const struct rte_flow_action actions[], 6048 struct rte_flow_action actions_sfx[], 6049 struct rte_flow_action actions_pre[], 6050 int actions_n, 6051 int sample_action_pos, 6052 int qrss_action_pos, 6053 int jump_table, 6054 struct rte_flow_error *error) 6055 { 6056 struct mlx5_priv *priv = dev->data->dev_private; 6057 struct mlx5_rte_flow_action_set_tag *set_tag; 6058 struct mlx5_rte_flow_item_tag *tag_spec; 6059 struct mlx5_rte_flow_item_tag *tag_mask; 6060 struct rte_flow_action_jump *jump_action; 6061 uint32_t tag_id = 0; 6062 int append_index = 0; 6063 int set_tag_idx = -1; 6064 int index; 6065 int ret; 6066 6067 if (sample_action_pos < 0) 6068 return rte_flow_error_set(error, EINVAL, 6069 RTE_FLOW_ERROR_TYPE_ACTION, 6070 NULL, "invalid position of sample " 6071 "action in list"); 6072 /* Prepare the actions for prefix and suffix flow. */ 6073 if (add_tag) { 6074 /* Update the new added tag action index preceding 6075 * the PUSH_VLAN or ENCAP action. 6076 */ 6077 const struct rte_flow_action_raw_encap *raw_encap; 6078 const struct rte_flow_action *action = actions; 6079 int encap_idx; 6080 int action_idx = 0; 6081 int raw_decap_idx = -1; 6082 int push_vlan_idx = -1; 6083 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6084 switch (action->type) { 6085 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6086 raw_decap_idx = action_idx; 6087 break; 6088 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6089 raw_encap = action->conf; 6090 if (raw_encap->size > 6091 MLX5_ENCAPSULATION_DECISION_SIZE) { 6092 encap_idx = raw_decap_idx != -1 ? 6093 raw_decap_idx : action_idx; 6094 if (encap_idx < sample_action_pos && 6095 push_vlan_idx == -1) 6096 set_tag_idx = encap_idx; 6097 } 6098 break; 6099 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6100 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6101 encap_idx = action_idx; 6102 if (encap_idx < sample_action_pos && 6103 push_vlan_idx == -1) 6104 set_tag_idx = encap_idx; 6105 break; 6106 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6107 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6108 if (action_idx < sample_action_pos && 6109 push_vlan_idx == -1) { 6110 set_tag_idx = action_idx; 6111 push_vlan_idx = action_idx; 6112 } 6113 break; 6114 default: 6115 break; 6116 } 6117 action_idx++; 6118 } 6119 } 6120 /* Prepare the actions for prefix and suffix flow. */ 6121 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6122 index = qrss_action_pos; 6123 /* Put the preceding the Queue/RSS action into prefix flow. */ 6124 if (index != 0) 6125 memcpy(actions_pre, actions, 6126 sizeof(struct rte_flow_action) * index); 6127 /* Put others preceding the sample action into prefix flow. */ 6128 if (sample_action_pos > index + 1) 6129 memcpy(actions_pre + index, actions + index + 1, 6130 sizeof(struct rte_flow_action) * 6131 (sample_action_pos - index - 1)); 6132 index = sample_action_pos - 1; 6133 /* Put Queue/RSS action into Suffix flow. */ 6134 memcpy(actions_sfx, actions + qrss_action_pos, 6135 sizeof(struct rte_flow_action)); 6136 actions_sfx++; 6137 } else if (add_tag && set_tag_idx >= 0) { 6138 if (set_tag_idx > 0) 6139 memcpy(actions_pre, actions, 6140 sizeof(struct rte_flow_action) * set_tag_idx); 6141 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6142 sizeof(struct rte_flow_action) * 6143 (sample_action_pos - set_tag_idx)); 6144 index = sample_action_pos; 6145 } else { 6146 index = sample_action_pos; 6147 if (index != 0) 6148 memcpy(actions_pre, actions, 6149 sizeof(struct rte_flow_action) * index); 6150 } 6151 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6152 * For CX6DX and above, metadata registers Cx preserve their value, 6153 * add an extra tag action for NIC-RX and E-Switch Domain. 6154 */ 6155 if (add_tag) { 6156 /* Prepare the prefix tag action. */ 6157 append_index++; 6158 set_tag = (void *)(actions_pre + actions_n + append_index); 6159 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6160 /* Trust VF/SF on CX5 not supported meter so that the reserved 6161 * metadata regC is REG_NON, back to use application tag 6162 * index 0. 6163 */ 6164 if (unlikely(ret == REG_NON)) 6165 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6166 if (ret < 0) 6167 return ret; 6168 mlx5_ipool_malloc(priv->sh->ipool 6169 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6170 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6171 .id = ret, 6172 .data = tag_id, 6173 }; 6174 /* Prepare the suffix subflow items. */ 6175 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6176 tag_spec->data = tag_id; 6177 tag_spec->id = set_tag->id; 6178 tag_mask = tag_spec + 1; 6179 tag_mask->data = UINT32_MAX; 6180 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6181 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6182 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6183 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6184 memcpy(sfx_items, items, sizeof(*sfx_items)); 6185 sfx_items++; 6186 break; 6187 } 6188 } 6189 sfx_items[0] = (struct rte_flow_item){ 6190 .type = (enum rte_flow_item_type) 6191 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6192 .spec = tag_spec, 6193 .last = NULL, 6194 .mask = tag_mask, 6195 }; 6196 sfx_items[1] = (struct rte_flow_item){ 6197 .type = (enum rte_flow_item_type) 6198 RTE_FLOW_ITEM_TYPE_END, 6199 }; 6200 /* Prepare the tag action in prefix subflow. */ 6201 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6202 actions_pre[set_tag_idx] = 6203 (struct rte_flow_action){ 6204 .type = (enum rte_flow_action_type) 6205 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6206 .conf = set_tag, 6207 }; 6208 /* Update next sample position due to add one tag action */ 6209 index += 1; 6210 } 6211 /* Copy the sample action into prefix flow. */ 6212 memcpy(actions_pre + index, actions + sample_action_pos, 6213 sizeof(struct rte_flow_action)); 6214 index += 1; 6215 /* For the modify action after the sample action in E-Switch mirroring, 6216 * Add the extra jump action in prefix subflow and jump into the next 6217 * table, then do the modify action in the new table. 6218 */ 6219 if (jump_table) { 6220 /* Prepare the prefix jump action. */ 6221 append_index++; 6222 jump_action = (void *)(actions_pre + actions_n + append_index); 6223 jump_action->group = jump_table; 6224 actions_pre[index++] = 6225 (struct rte_flow_action){ 6226 .type = (enum rte_flow_action_type) 6227 RTE_FLOW_ACTION_TYPE_JUMP, 6228 .conf = jump_action, 6229 }; 6230 } 6231 actions_pre[index] = (struct rte_flow_action){ 6232 .type = (enum rte_flow_action_type) 6233 RTE_FLOW_ACTION_TYPE_END, 6234 }; 6235 /* Put the actions after sample into Suffix flow. */ 6236 memcpy(actions_sfx, actions + sample_action_pos + 1, 6237 sizeof(struct rte_flow_action) * 6238 (actions_n - sample_action_pos - 1)); 6239 return tag_id; 6240 } 6241 6242 /** 6243 * The splitting for metadata feature. 6244 * 6245 * - Q/RSS action on NIC Rx should be split in order to pass by 6246 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6247 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6248 * 6249 * - All the actions on NIC Tx should have a mreg copy action to 6250 * copy reg_a from WQE to reg_c[0]. 6251 * 6252 * @param dev 6253 * Pointer to Ethernet device. 6254 * @param[in] flow 6255 * Parent flow structure pointer. 6256 * @param[in] attr 6257 * Flow rule attributes. 6258 * @param[in] items 6259 * Pattern specification (list terminated by the END pattern item). 6260 * @param[in] actions 6261 * Associated actions (list terminated by the END action). 6262 * @param[in] flow_split_info 6263 * Pointer to flow split info structure. 6264 * @param[out] error 6265 * Perform verbose error reporting if not NULL. 6266 * @return 6267 * 0 on success, negative value otherwise 6268 */ 6269 static int 6270 flow_create_split_metadata(struct rte_eth_dev *dev, 6271 struct rte_flow *flow, 6272 const struct rte_flow_attr *attr, 6273 const struct rte_flow_item items[], 6274 const struct rte_flow_action actions[], 6275 struct mlx5_flow_split_info *flow_split_info, 6276 struct rte_flow_error *error) 6277 { 6278 struct mlx5_priv *priv = dev->data->dev_private; 6279 struct mlx5_sh_config *config = &priv->sh->config; 6280 const struct rte_flow_action *qrss = NULL; 6281 struct rte_flow_action *ext_actions = NULL; 6282 struct mlx5_flow *dev_flow = NULL; 6283 uint32_t qrss_id = 0; 6284 int mtr_sfx = 0; 6285 size_t act_size; 6286 int actions_n; 6287 int encap_idx; 6288 int ret; 6289 6290 /* Check whether extensive metadata feature is engaged. */ 6291 if (!config->dv_flow_en || 6292 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6293 !mlx5_flow_ext_mreg_supported(dev)) 6294 return flow_create_split_inner(dev, flow, NULL, attr, items, 6295 actions, flow_split_info, error); 6296 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6297 &encap_idx); 6298 if (qrss) { 6299 /* Exclude hairpin flows from splitting. */ 6300 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6301 const struct rte_flow_action_queue *queue; 6302 6303 queue = qrss->conf; 6304 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6305 qrss = NULL; 6306 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6307 const struct rte_flow_action_rss *rss; 6308 6309 rss = qrss->conf; 6310 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6311 qrss = NULL; 6312 } 6313 } 6314 if (qrss) { 6315 /* Check if it is in meter suffix table. */ 6316 mtr_sfx = attr->group == 6317 ((attr->transfer && priv->fdb_def_rule) ? 6318 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6319 MLX5_FLOW_TABLE_LEVEL_METER); 6320 /* 6321 * Q/RSS action on NIC Rx should be split in order to pass by 6322 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6323 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6324 */ 6325 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6326 sizeof(struct rte_flow_action_set_tag) + 6327 sizeof(struct rte_flow_action_jump); 6328 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6329 SOCKET_ID_ANY); 6330 if (!ext_actions) 6331 return rte_flow_error_set(error, ENOMEM, 6332 RTE_FLOW_ERROR_TYPE_ACTION, 6333 NULL, "no memory to split " 6334 "metadata flow"); 6335 /* 6336 * Create the new actions list with removed Q/RSS action 6337 * and appended set tag and jump to register copy table 6338 * (RX_CP_TBL). We should preallocate unique tag ID here 6339 * in advance, because it is needed for set tag action. 6340 */ 6341 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6342 qrss, actions_n, 6343 mtr_sfx, error); 6344 if (!mtr_sfx && !qrss_id) { 6345 ret = -rte_errno; 6346 goto exit; 6347 } 6348 } else if (attr->egress && !attr->transfer) { 6349 /* 6350 * All the actions on NIC Tx should have a metadata register 6351 * copy action to copy reg_a from WQE to reg_c[meta] 6352 */ 6353 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6354 sizeof(struct mlx5_flow_action_copy_mreg); 6355 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6356 SOCKET_ID_ANY); 6357 if (!ext_actions) 6358 return rte_flow_error_set(error, ENOMEM, 6359 RTE_FLOW_ERROR_TYPE_ACTION, 6360 NULL, "no memory to split " 6361 "metadata flow"); 6362 /* Create the action list appended with copy register. */ 6363 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6364 actions_n, error, encap_idx); 6365 if (ret < 0) 6366 goto exit; 6367 } 6368 /* Add the unmodified original or prefix subflow. */ 6369 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6370 items, ext_actions ? ext_actions : 6371 actions, flow_split_info, error); 6372 if (ret < 0) 6373 goto exit; 6374 MLX5_ASSERT(dev_flow); 6375 if (qrss) { 6376 const struct rte_flow_attr q_attr = { 6377 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6378 .ingress = 1, 6379 }; 6380 /* Internal PMD action to set register. */ 6381 struct mlx5_rte_flow_item_tag q_tag_spec = { 6382 .data = qrss_id, 6383 .id = REG_NON, 6384 }; 6385 struct rte_flow_item q_items[] = { 6386 { 6387 .type = (enum rte_flow_item_type) 6388 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6389 .spec = &q_tag_spec, 6390 .last = NULL, 6391 .mask = NULL, 6392 }, 6393 { 6394 .type = RTE_FLOW_ITEM_TYPE_END, 6395 }, 6396 }; 6397 struct rte_flow_action q_actions[] = { 6398 { 6399 .type = qrss->type, 6400 .conf = qrss->conf, 6401 }, 6402 { 6403 .type = RTE_FLOW_ACTION_TYPE_END, 6404 }, 6405 }; 6406 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6407 6408 /* 6409 * Configure the tag item only if there is no meter subflow. 6410 * Since tag is already marked in the meter suffix subflow 6411 * we can just use the meter suffix items as is. 6412 */ 6413 if (qrss_id) { 6414 /* Not meter subflow. */ 6415 MLX5_ASSERT(!mtr_sfx); 6416 /* 6417 * Put unique id in prefix flow due to it is destroyed 6418 * after suffix flow and id will be freed after there 6419 * is no actual flows with this id and identifier 6420 * reallocation becomes possible (for example, for 6421 * other flows in other threads). 6422 */ 6423 dev_flow->handle->split_flow_id = qrss_id; 6424 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6425 error); 6426 if (ret < 0) 6427 goto exit; 6428 q_tag_spec.id = ret; 6429 } 6430 dev_flow = NULL; 6431 /* Add suffix subflow to execute Q/RSS. */ 6432 flow_split_info->prefix_layers = layers; 6433 flow_split_info->prefix_mark = 0; 6434 flow_split_info->table_id = 0; 6435 ret = flow_create_split_inner(dev, flow, &dev_flow, 6436 &q_attr, mtr_sfx ? items : 6437 q_items, q_actions, 6438 flow_split_info, error); 6439 if (ret < 0) 6440 goto exit; 6441 /* qrss ID should be freed if failed. */ 6442 qrss_id = 0; 6443 MLX5_ASSERT(dev_flow); 6444 } 6445 6446 exit: 6447 /* 6448 * We do not destroy the partially created sub_flows in case of error. 6449 * These ones are included into parent flow list and will be destroyed 6450 * by flow_drv_destroy. 6451 */ 6452 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6453 qrss_id); 6454 mlx5_free(ext_actions); 6455 return ret; 6456 } 6457 6458 /** 6459 * Create meter internal drop flow with the original pattern. 6460 * 6461 * @param dev 6462 * Pointer to Ethernet device. 6463 * @param[in] flow 6464 * Parent flow structure pointer. 6465 * @param[in] attr 6466 * Flow rule attributes. 6467 * @param[in] items 6468 * Pattern specification (list terminated by the END pattern item). 6469 * @param[in] flow_split_info 6470 * Pointer to flow split info structure. 6471 * @param[in] fm 6472 * Pointer to flow meter structure. 6473 * @param[out] error 6474 * Perform verbose error reporting if not NULL. 6475 * @return 6476 * 0 on success, negative value otherwise 6477 */ 6478 static uint32_t 6479 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6480 struct rte_flow *flow, 6481 const struct rte_flow_attr *attr, 6482 const struct rte_flow_item items[], 6483 struct mlx5_flow_split_info *flow_split_info, 6484 struct mlx5_flow_meter_info *fm, 6485 struct rte_flow_error *error) 6486 { 6487 struct mlx5_flow *dev_flow = NULL; 6488 struct rte_flow_attr drop_attr = *attr; 6489 struct rte_flow_action drop_actions[3]; 6490 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6491 6492 MLX5_ASSERT(fm->drop_cnt); 6493 drop_actions[0].type = 6494 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6495 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6496 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6497 drop_actions[1].conf = NULL; 6498 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6499 drop_actions[2].conf = NULL; 6500 drop_split_info.external = false; 6501 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6502 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6503 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6504 return flow_create_split_inner(dev, flow, &dev_flow, 6505 &drop_attr, items, drop_actions, 6506 &drop_split_info, error); 6507 } 6508 6509 /** 6510 * The splitting for meter feature. 6511 * 6512 * - The meter flow will be split to two flows as prefix and 6513 * suffix flow. The packets make sense only it pass the prefix 6514 * meter action. 6515 * 6516 * - Reg_C_5 is used for the packet to match betweend prefix and 6517 * suffix flow. 6518 * 6519 * @param dev 6520 * Pointer to Ethernet device. 6521 * @param[in] flow 6522 * Parent flow structure pointer. 6523 * @param[in] attr 6524 * Flow rule attributes. 6525 * @param[in] items 6526 * Pattern specification (list terminated by the END pattern item). 6527 * @param[in] actions 6528 * Associated actions (list terminated by the END action). 6529 * @param[in] flow_split_info 6530 * Pointer to flow split info structure. 6531 * @param[out] error 6532 * Perform verbose error reporting if not NULL. 6533 * @return 6534 * 0 on success, negative value otherwise 6535 */ 6536 static int 6537 flow_create_split_meter(struct rte_eth_dev *dev, 6538 struct rte_flow *flow, 6539 const struct rte_flow_attr *attr, 6540 const struct rte_flow_item items[], 6541 const struct rte_flow_action actions[], 6542 struct mlx5_flow_split_info *flow_split_info, 6543 struct rte_flow_error *error) 6544 { 6545 struct mlx5_priv *priv = dev->data->dev_private; 6546 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6547 struct rte_flow_action *sfx_actions = NULL; 6548 struct rte_flow_action *pre_actions = NULL; 6549 struct rte_flow_item *sfx_items = NULL; 6550 struct mlx5_flow *dev_flow = NULL; 6551 struct rte_flow_attr sfx_attr = *attr; 6552 struct mlx5_flow_meter_info *fm = NULL; 6553 uint8_t skip_scale_restore; 6554 bool has_mtr = false; 6555 bool has_modify = false; 6556 bool set_mtr_reg = true; 6557 bool is_mtr_hierarchy = false; 6558 uint32_t meter_id = 0; 6559 uint32_t mtr_idx = 0; 6560 uint32_t mtr_flow_id = 0; 6561 size_t act_size; 6562 size_t item_size; 6563 int actions_n = 0; 6564 int ret = 0; 6565 6566 if (priv->mtr_en) 6567 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6568 &has_modify, &meter_id); 6569 if (has_mtr) { 6570 if (flow->meter) { 6571 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6572 if (!fm) 6573 return rte_flow_error_set(error, EINVAL, 6574 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6575 NULL, "Meter not found."); 6576 } else { 6577 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6578 if (!fm) 6579 return rte_flow_error_set(error, EINVAL, 6580 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6581 NULL, "Meter not found."); 6582 ret = mlx5_flow_meter_attach(priv, fm, 6583 &sfx_attr, error); 6584 if (ret) 6585 return -rte_errno; 6586 flow->meter = mtr_idx; 6587 } 6588 MLX5_ASSERT(wks); 6589 wks->fm = fm; 6590 if (!fm->def_policy) { 6591 wks->policy = mlx5_flow_meter_policy_find(dev, 6592 fm->policy_id, 6593 NULL); 6594 MLX5_ASSERT(wks->policy); 6595 if (wks->policy->mark) 6596 wks->mark = 1; 6597 if (wks->policy->is_hierarchy) { 6598 wks->final_policy = 6599 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6600 wks->policy); 6601 if (!wks->final_policy) 6602 return rte_flow_error_set(error, 6603 EINVAL, 6604 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6605 "Failed to find terminal policy of hierarchy."); 6606 is_mtr_hierarchy = true; 6607 } 6608 } 6609 /* 6610 * If it isn't default-policy Meter, and 6611 * 1. Not meter hierarchy and there's no action in flow to change 6612 * packet (modify/encap/decap etc.), OR 6613 * 2. No drop count needed for this meter. 6614 * Then no need to use regC to save meter id anymore. 6615 */ 6616 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6617 set_mtr_reg = false; 6618 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6619 #define METER_PREFIX_ACTION 7 6620 act_size = (sizeof(struct rte_flow_action) * 6621 (actions_n + METER_PREFIX_ACTION)) + 6622 sizeof(struct mlx5_rte_flow_action_set_tag); 6623 /* Suffix items: tag, vlan, port id, end. */ 6624 #define METER_SUFFIX_ITEM 4 6625 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6626 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6627 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6628 0, SOCKET_ID_ANY); 6629 if (!sfx_actions) 6630 return rte_flow_error_set(error, ENOMEM, 6631 RTE_FLOW_ERROR_TYPE_ACTION, 6632 NULL, "no memory to split " 6633 "meter flow"); 6634 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6635 act_size); 6636 /* There's no suffix flow for meter of non-default policy. */ 6637 if (!fm->def_policy) 6638 pre_actions = sfx_actions + 1; 6639 else 6640 pre_actions = sfx_actions + actions_n; 6641 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6642 items, sfx_items, actions, 6643 sfx_actions, pre_actions, 6644 (set_mtr_reg ? &mtr_flow_id : NULL), 6645 error); 6646 if (ret) { 6647 ret = -rte_errno; 6648 goto exit; 6649 } 6650 /* Add the prefix subflow. */ 6651 skip_scale_restore = flow_split_info->skip_scale; 6652 flow_split_info->skip_scale |= 6653 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6654 ret = flow_create_split_inner(dev, flow, &dev_flow, 6655 attr, items, pre_actions, 6656 flow_split_info, error); 6657 flow_split_info->skip_scale = skip_scale_restore; 6658 if (ret) { 6659 if (mtr_flow_id) 6660 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6661 ret = -rte_errno; 6662 goto exit; 6663 } 6664 if (mtr_flow_id) { 6665 dev_flow->handle->split_flow_id = mtr_flow_id; 6666 dev_flow->handle->is_meter_flow_id = 1; 6667 } 6668 if (!fm->def_policy) { 6669 if (!set_mtr_reg && fm->drop_cnt) 6670 ret = 6671 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6672 &sfx_attr, items, 6673 flow_split_info, 6674 fm, error); 6675 goto exit; 6676 } 6677 /* Setting the sfx group atrr. */ 6678 sfx_attr.group = sfx_attr.transfer ? 6679 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6680 MLX5_FLOW_TABLE_LEVEL_METER; 6681 flow_split_info->prefix_layers = 6682 flow_get_prefix_layer_flags(dev_flow); 6683 flow_split_info->prefix_mark |= wks->mark; 6684 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6685 } 6686 /* Add the prefix subflow. */ 6687 ret = flow_create_split_metadata(dev, flow, 6688 &sfx_attr, sfx_items ? 6689 sfx_items : items, 6690 sfx_actions ? sfx_actions : actions, 6691 flow_split_info, error); 6692 exit: 6693 if (sfx_actions) 6694 mlx5_free(sfx_actions); 6695 return ret; 6696 } 6697 6698 /** 6699 * The splitting for sample feature. 6700 * 6701 * Once Sample action is detected in the action list, the flow actions should 6702 * be split into prefix sub flow and suffix sub flow. 6703 * 6704 * The original items remain in the prefix sub flow, all actions preceding the 6705 * sample action and the sample action itself will be copied to the prefix 6706 * sub flow, the actions following the sample action will be copied to the 6707 * suffix sub flow, Queue action always be located in the suffix sub flow. 6708 * 6709 * In order to make the packet from prefix sub flow matches with suffix sub 6710 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6711 * flow uses tag item with the unique flow id. 6712 * 6713 * @param dev 6714 * Pointer to Ethernet device. 6715 * @param[in] flow 6716 * Parent flow structure pointer. 6717 * @param[in] attr 6718 * Flow rule attributes. 6719 * @param[in] items 6720 * Pattern specification (list terminated by the END pattern item). 6721 * @param[in] actions 6722 * Associated actions (list terminated by the END action). 6723 * @param[in] flow_split_info 6724 * Pointer to flow split info structure. 6725 * @param[out] error 6726 * Perform verbose error reporting if not NULL. 6727 * @return 6728 * 0 on success, negative value otherwise 6729 */ 6730 static int 6731 flow_create_split_sample(struct rte_eth_dev *dev, 6732 struct rte_flow *flow, 6733 const struct rte_flow_attr *attr, 6734 const struct rte_flow_item items[], 6735 const struct rte_flow_action actions[], 6736 struct mlx5_flow_split_info *flow_split_info, 6737 struct rte_flow_error *error) 6738 { 6739 struct mlx5_priv *priv = dev->data->dev_private; 6740 struct rte_flow_action *sfx_actions = NULL; 6741 struct rte_flow_action *pre_actions = NULL; 6742 struct rte_flow_item *sfx_items = NULL; 6743 struct mlx5_flow *dev_flow = NULL; 6744 struct rte_flow_attr sfx_attr = *attr; 6745 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6746 struct mlx5_flow_dv_sample_resource *sample_res; 6747 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6748 struct mlx5_flow_tbl_resource *sfx_tbl; 6749 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6750 #endif 6751 size_t act_size; 6752 size_t item_size; 6753 uint32_t fdb_tx = 0; 6754 int32_t tag_id = 0; 6755 int actions_n = 0; 6756 int sample_action_pos; 6757 int qrss_action_pos; 6758 int add_tag = 0; 6759 int modify_after_mirror = 0; 6760 uint16_t jump_table = 0; 6761 const uint32_t next_ft_step = 1; 6762 int ret = 0; 6763 struct mlx5_priv *item_port_priv = NULL; 6764 const struct rte_flow_item *item; 6765 6766 if (priv->sampler_en) 6767 actions_n = flow_check_match_action(actions, attr, 6768 RTE_FLOW_ACTION_TYPE_SAMPLE, 6769 &sample_action_pos, &qrss_action_pos, 6770 &modify_after_mirror); 6771 if (actions_n) { 6772 /* The prefix actions must includes sample, tag, end. */ 6773 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6774 + sizeof(struct mlx5_rte_flow_action_set_tag); 6775 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6776 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6777 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6778 item_size), 0, SOCKET_ID_ANY); 6779 if (!sfx_actions) 6780 return rte_flow_error_set(error, ENOMEM, 6781 RTE_FLOW_ERROR_TYPE_ACTION, 6782 NULL, "no memory to split " 6783 "sample flow"); 6784 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 6785 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6786 const struct rte_flow_item_port_id *spec; 6787 6788 spec = (const struct rte_flow_item_port_id *)item->spec; 6789 if (spec) 6790 item_port_priv = 6791 mlx5_port_to_eswitch_info(spec->id, true); 6792 break; 6793 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6794 const struct rte_flow_item_ethdev *spec; 6795 6796 spec = (const struct rte_flow_item_ethdev *)item->spec; 6797 if (spec) 6798 item_port_priv = 6799 mlx5_port_to_eswitch_info(spec->port_id, true); 6800 break; 6801 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 6802 const struct rte_flow_item_ethdev *spec; 6803 6804 spec = (const struct rte_flow_item_ethdev *)item->spec; 6805 if (spec) 6806 item_port_priv = 6807 mlx5_port_to_eswitch_info(spec->port_id, true); 6808 break; 6809 } 6810 } 6811 /* The representor_id is UINT16_MAX for uplink. */ 6812 fdb_tx = (attr->transfer && 6813 flow_source_vport_representor(priv, item_port_priv)); 6814 /* 6815 * When reg_c_preserve is set, metadata registers Cx preserve 6816 * their value even through packet duplication. 6817 */ 6818 add_tag = (!fdb_tx || 6819 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6820 if (add_tag) 6821 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6822 + act_size); 6823 if (modify_after_mirror) 6824 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6825 next_ft_step; 6826 pre_actions = sfx_actions + actions_n; 6827 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6828 actions, sfx_actions, 6829 pre_actions, actions_n, 6830 sample_action_pos, 6831 qrss_action_pos, jump_table, 6832 error); 6833 if (tag_id < 0 || (add_tag && !tag_id)) { 6834 ret = -rte_errno; 6835 goto exit; 6836 } 6837 if (modify_after_mirror) 6838 flow_split_info->skip_scale = 6839 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6840 /* Add the prefix subflow. */ 6841 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6842 items, pre_actions, 6843 flow_split_info, error); 6844 if (ret) { 6845 ret = -rte_errno; 6846 goto exit; 6847 } 6848 dev_flow->handle->split_flow_id = tag_id; 6849 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6850 if (!modify_after_mirror) { 6851 /* Set the sfx group attr. */ 6852 sample_res = (struct mlx5_flow_dv_sample_resource *) 6853 dev_flow->dv.sample_res; 6854 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6855 sample_res->normal_path_tbl; 6856 sfx_tbl_data = container_of(sfx_tbl, 6857 struct mlx5_flow_tbl_data_entry, 6858 tbl); 6859 sfx_attr.group = sfx_attr.transfer ? 6860 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6861 } else { 6862 MLX5_ASSERT(attr->transfer); 6863 sfx_attr.group = jump_table; 6864 } 6865 flow_split_info->prefix_layers = 6866 flow_get_prefix_layer_flags(dev_flow); 6867 MLX5_ASSERT(wks); 6868 flow_split_info->prefix_mark |= wks->mark; 6869 /* Suffix group level already be scaled with factor, set 6870 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6871 * again in translation. 6872 */ 6873 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6874 #endif 6875 } 6876 /* Add the suffix subflow. */ 6877 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6878 sfx_items ? sfx_items : items, 6879 sfx_actions ? sfx_actions : actions, 6880 flow_split_info, error); 6881 exit: 6882 if (sfx_actions) 6883 mlx5_free(sfx_actions); 6884 return ret; 6885 } 6886 6887 /** 6888 * Split the flow to subflow set. The splitters might be linked 6889 * in the chain, like this: 6890 * flow_create_split_outer() calls: 6891 * flow_create_split_meter() calls: 6892 * flow_create_split_metadata(meter_subflow_0) calls: 6893 * flow_create_split_inner(metadata_subflow_0) 6894 * flow_create_split_inner(metadata_subflow_1) 6895 * flow_create_split_inner(metadata_subflow_2) 6896 * flow_create_split_metadata(meter_subflow_1) calls: 6897 * flow_create_split_inner(metadata_subflow_0) 6898 * flow_create_split_inner(metadata_subflow_1) 6899 * flow_create_split_inner(metadata_subflow_2) 6900 * 6901 * This provide flexible way to add new levels of flow splitting. 6902 * The all of successfully created subflows are included to the 6903 * parent flow dev_flow list. 6904 * 6905 * @param dev 6906 * Pointer to Ethernet device. 6907 * @param[in] flow 6908 * Parent flow structure pointer. 6909 * @param[in] attr 6910 * Flow rule attributes. 6911 * @param[in] items 6912 * Pattern specification (list terminated by the END pattern item). 6913 * @param[in] actions 6914 * Associated actions (list terminated by the END action). 6915 * @param[in] flow_split_info 6916 * Pointer to flow split info structure. 6917 * @param[out] error 6918 * Perform verbose error reporting if not NULL. 6919 * @return 6920 * 0 on success, negative value otherwise 6921 */ 6922 static int 6923 flow_create_split_outer(struct rte_eth_dev *dev, 6924 struct rte_flow *flow, 6925 const struct rte_flow_attr *attr, 6926 const struct rte_flow_item items[], 6927 const struct rte_flow_action actions[], 6928 struct mlx5_flow_split_info *flow_split_info, 6929 struct rte_flow_error *error) 6930 { 6931 int ret; 6932 6933 ret = flow_create_split_sample(dev, flow, attr, items, 6934 actions, flow_split_info, error); 6935 MLX5_ASSERT(ret <= 0); 6936 return ret; 6937 } 6938 6939 static inline struct mlx5_flow_tunnel * 6940 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6941 { 6942 struct mlx5_flow_tunnel *tunnel; 6943 6944 #pragma GCC diagnostic push 6945 #pragma GCC diagnostic ignored "-Wcast-qual" 6946 tunnel = (typeof(tunnel))flow->tunnel; 6947 #pragma GCC diagnostic pop 6948 6949 return tunnel; 6950 } 6951 6952 /** 6953 * Adjust flow RSS workspace if needed. 6954 * 6955 * @param wks 6956 * Pointer to thread flow work space. 6957 * @param rss_desc 6958 * Pointer to RSS descriptor. 6959 * @param[in] nrssq_num 6960 * New RSS queue number. 6961 * 6962 * @return 6963 * 0 on success, -1 otherwise and rte_errno is set. 6964 */ 6965 static int 6966 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6967 struct mlx5_flow_rss_desc *rss_desc, 6968 uint32_t nrssq_num) 6969 { 6970 if (likely(nrssq_num <= wks->rssq_num)) 6971 return 0; 6972 rss_desc->queue = realloc(rss_desc->queue, 6973 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6974 if (!rss_desc->queue) { 6975 rte_errno = ENOMEM; 6976 return -1; 6977 } 6978 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6979 return 0; 6980 } 6981 6982 /** 6983 * Create a flow and add it to @p list. 6984 * 6985 * @param dev 6986 * Pointer to Ethernet device. 6987 * @param list 6988 * Pointer to a TAILQ flow list. If this parameter NULL, 6989 * no list insertion occurred, flow is just created, 6990 * this is caller's responsibility to track the 6991 * created flow. 6992 * @param[in] attr 6993 * Flow rule attributes. 6994 * @param[in] items 6995 * Pattern specification (list terminated by the END pattern item). 6996 * @param[in] actions 6997 * Associated actions (list terminated by the END action). 6998 * @param[in] external 6999 * This flow rule is created by request external to PMD. 7000 * @param[out] error 7001 * Perform verbose error reporting if not NULL. 7002 * 7003 * @return 7004 * A flow index on success, 0 otherwise and rte_errno is set. 7005 */ 7006 static uint32_t 7007 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7008 const struct rte_flow_attr *attr, 7009 const struct rte_flow_item items[], 7010 const struct rte_flow_action original_actions[], 7011 bool external, struct rte_flow_error *error) 7012 { 7013 struct mlx5_priv *priv = dev->data->dev_private; 7014 struct rte_flow *flow = NULL; 7015 struct mlx5_flow *dev_flow; 7016 const struct rte_flow_action_rss *rss = NULL; 7017 struct mlx5_translated_action_handle 7018 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7019 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7020 union { 7021 struct mlx5_flow_expand_rss buf; 7022 uint8_t buffer[8192]; 7023 } expand_buffer; 7024 union { 7025 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7026 uint8_t buffer[2048]; 7027 } actions_rx; 7028 union { 7029 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7030 uint8_t buffer[2048]; 7031 } actions_hairpin_tx; 7032 union { 7033 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7034 uint8_t buffer[2048]; 7035 } items_tx; 7036 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7037 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7038 struct mlx5_flow_rss_desc *rss_desc; 7039 const struct rte_flow_action *p_actions_rx; 7040 uint32_t i; 7041 uint32_t idx = 0; 7042 int hairpin_flow; 7043 struct rte_flow_attr attr_tx = { .priority = 0 }; 7044 const struct rte_flow_action *actions; 7045 struct rte_flow_action *translated_actions = NULL; 7046 struct mlx5_flow_tunnel *tunnel; 7047 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7048 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7049 struct mlx5_flow_split_info flow_split_info = { 7050 .external = !!external, 7051 .skip_scale = 0, 7052 .flow_idx = 0, 7053 .prefix_mark = 0, 7054 .prefix_layers = 0, 7055 .table_id = 0 7056 }; 7057 int ret; 7058 7059 MLX5_ASSERT(wks); 7060 rss_desc = &wks->rss_desc; 7061 ret = flow_action_handles_translate(dev, original_actions, 7062 indir_actions, 7063 &indir_actions_n, 7064 &translated_actions, error); 7065 if (ret < 0) { 7066 MLX5_ASSERT(translated_actions == NULL); 7067 return 0; 7068 } 7069 actions = translated_actions ? translated_actions : original_actions; 7070 p_actions_rx = actions; 7071 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7072 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7073 external, hairpin_flow, error); 7074 if (ret < 0) 7075 goto error_before_hairpin_split; 7076 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7077 if (!flow) { 7078 rte_errno = ENOMEM; 7079 goto error_before_hairpin_split; 7080 } 7081 if (hairpin_flow > 0) { 7082 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7083 rte_errno = EINVAL; 7084 goto error_before_hairpin_split; 7085 } 7086 flow_hairpin_split(dev, actions, actions_rx.actions, 7087 actions_hairpin_tx.actions, items_tx.items, 7088 idx); 7089 p_actions_rx = actions_rx.actions; 7090 } 7091 flow_split_info.flow_idx = idx; 7092 flow->drv_type = flow_get_drv_type(dev, attr); 7093 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7094 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7095 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7096 /* RSS Action only works on NIC RX domain */ 7097 if (attr->ingress && !attr->transfer) 7098 rss = flow_get_rss_action(dev, p_actions_rx); 7099 if (rss) { 7100 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 7101 return 0; 7102 /* 7103 * The following information is required by 7104 * mlx5_flow_hashfields_adjust() in advance. 7105 */ 7106 rss_desc->level = rss->level; 7107 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7108 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7109 } 7110 flow->dev_handles = 0; 7111 if (rss && rss->types) { 7112 unsigned int graph_root; 7113 7114 graph_root = find_graph_root(rss->level); 7115 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7116 items, rss->types, 7117 mlx5_support_expansion, graph_root); 7118 MLX5_ASSERT(ret > 0 && 7119 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7120 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7121 for (i = 0; i < buf->entries; ++i) 7122 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7123 } 7124 } else { 7125 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7126 sizeof(expand_buffer.buffer), 7127 items, sq_specs); 7128 if (ret) { 7129 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7130 NULL, "not enough memory for rte_flow"); 7131 goto error; 7132 } 7133 if (buf->entries == 0) { 7134 buf->entries = 1; 7135 buf->entry[0].pattern = (void *)(uintptr_t)items; 7136 } 7137 } 7138 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7139 indir_actions_n); 7140 for (i = 0; i < buf->entries; ++i) { 7141 /* Initialize flow split data. */ 7142 flow_split_info.prefix_layers = 0; 7143 flow_split_info.prefix_mark = 0; 7144 flow_split_info.skip_scale = 0; 7145 /* 7146 * The splitter may create multiple dev_flows, 7147 * depending on configuration. In the simplest 7148 * case it just creates unmodified original flow. 7149 */ 7150 ret = flow_create_split_outer(dev, flow, attr, 7151 buf->entry[i].pattern, 7152 p_actions_rx, &flow_split_info, 7153 error); 7154 if (ret < 0) 7155 goto error; 7156 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7157 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7158 p_actions_rx, 7159 idx, 7160 wks->flows[0].tunnel, 7161 &default_miss_ctx, 7162 error); 7163 if (ret < 0) { 7164 mlx5_free(default_miss_ctx.queue); 7165 goto error; 7166 } 7167 } 7168 } 7169 /* Create the tx flow. */ 7170 if (hairpin_flow) { 7171 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7172 attr_tx.ingress = 0; 7173 attr_tx.egress = 1; 7174 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7175 actions_hairpin_tx.actions, 7176 idx, error); 7177 if (!dev_flow) 7178 goto error; 7179 dev_flow->flow = flow; 7180 dev_flow->external = 0; 7181 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7182 dev_flow->handle, next); 7183 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7184 items_tx.items, 7185 actions_hairpin_tx.actions, error); 7186 if (ret < 0) 7187 goto error; 7188 } 7189 /* 7190 * Update the metadata register copy table. If extensive 7191 * metadata feature is enabled and registers are supported 7192 * we might create the extra rte_flow for each unique 7193 * MARK/FLAG action ID. 7194 * 7195 * The table is updated for ingress Flows only, because 7196 * the egress Flows belong to the different device and 7197 * copy table should be updated in peer NIC Rx domain. 7198 */ 7199 if (attr->ingress && 7200 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7201 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7202 if (ret) 7203 goto error; 7204 } 7205 /* 7206 * If the flow is external (from application) OR device is started, 7207 * OR mreg discover, then apply immediately. 7208 */ 7209 if (external || dev->data->dev_started || 7210 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7211 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7212 ret = flow_drv_apply(dev, flow, error); 7213 if (ret < 0) 7214 goto error; 7215 } 7216 flow->type = type; 7217 flow_rxq_flags_set(dev, flow); 7218 rte_free(translated_actions); 7219 tunnel = flow_tunnel_from_rule(wks->flows); 7220 if (tunnel) { 7221 flow->tunnel = 1; 7222 flow->tunnel_id = tunnel->tunnel_id; 7223 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7224 mlx5_free(default_miss_ctx.queue); 7225 } 7226 mlx5_flow_pop_thread_workspace(); 7227 return idx; 7228 error: 7229 MLX5_ASSERT(flow); 7230 ret = rte_errno; /* Save rte_errno before cleanup. */ 7231 flow_mreg_del_copy_action(dev, flow); 7232 flow_drv_destroy(dev, flow); 7233 if (rss_desc->shared_rss) 7234 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7235 mlx5_ipool_get 7236 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7237 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7238 mlx5_ipool_free(priv->flows[type], idx); 7239 rte_errno = ret; /* Restore rte_errno. */ 7240 ret = rte_errno; 7241 rte_errno = ret; 7242 error_before_hairpin_split: 7243 mlx5_flow_pop_thread_workspace(); 7244 rte_free(translated_actions); 7245 return 0; 7246 } 7247 7248 /** 7249 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7250 * incoming packets to table 1. 7251 * 7252 * Other flow rules, requested for group n, will be created in 7253 * e-switch table n+1. 7254 * Jump action to e-switch group n will be created to group n+1. 7255 * 7256 * Used when working in switchdev mode, to utilise advantages of table 1 7257 * and above. 7258 * 7259 * @param dev 7260 * Pointer to Ethernet device. 7261 * 7262 * @return 7263 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7264 */ 7265 struct rte_flow * 7266 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7267 { 7268 const struct rte_flow_attr attr = { 7269 .group = 0, 7270 .priority = 0, 7271 .ingress = 1, 7272 .egress = 0, 7273 .transfer = 1, 7274 }; 7275 const struct rte_flow_item pattern = { 7276 .type = RTE_FLOW_ITEM_TYPE_END, 7277 }; 7278 struct rte_flow_action_jump jump = { 7279 .group = 1, 7280 }; 7281 const struct rte_flow_action actions[] = { 7282 { 7283 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7284 .conf = &jump, 7285 }, 7286 { 7287 .type = RTE_FLOW_ACTION_TYPE_END, 7288 }, 7289 }; 7290 struct rte_flow_error error; 7291 7292 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7293 &attr, &pattern, 7294 actions, false, &error); 7295 } 7296 7297 /** 7298 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7299 * and sq number, directs all packets to peer vport. 7300 * 7301 * @param dev 7302 * Pointer to Ethernet device. 7303 * @param sq_num 7304 * SQ number. 7305 * 7306 * @return 7307 * Flow ID on success, 0 otherwise and rte_errno is set. 7308 */ 7309 uint32_t 7310 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7311 { 7312 struct rte_flow_attr attr = { 7313 .group = 0, 7314 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7315 .ingress = 1, 7316 .egress = 0, 7317 .transfer = 1, 7318 }; 7319 struct rte_flow_item_port_id port_spec = { 7320 .id = MLX5_PORT_ESW_MGR, 7321 }; 7322 struct mlx5_rte_flow_item_sq sq_spec = { 7323 .queue = sq_num, 7324 }; 7325 struct rte_flow_item pattern[] = { 7326 { 7327 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7328 .spec = &port_spec, 7329 }, 7330 { 7331 .type = (enum rte_flow_item_type) 7332 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7333 .spec = &sq_spec, 7334 }, 7335 { 7336 .type = RTE_FLOW_ITEM_TYPE_END, 7337 }, 7338 }; 7339 struct rte_flow_action_jump jump = { 7340 .group = 1, 7341 }; 7342 struct rte_flow_action_port_id port = { 7343 .id = dev->data->port_id, 7344 }; 7345 struct rte_flow_action actions[] = { 7346 { 7347 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7348 .conf = &jump, 7349 }, 7350 { 7351 .type = RTE_FLOW_ACTION_TYPE_END, 7352 }, 7353 }; 7354 struct rte_flow_error error; 7355 7356 /* 7357 * Creates group 0, highest priority jump flow. 7358 * Matches txq to bypass kernel packets. 7359 */ 7360 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7361 false, &error) == 0) 7362 return 0; 7363 /* Create group 1, lowest priority redirect flow for txq. */ 7364 attr.group = 1; 7365 actions[0].conf = &port; 7366 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7367 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7368 actions, false, &error); 7369 } 7370 7371 /** 7372 * Validate a flow supported by the NIC. 7373 * 7374 * @see rte_flow_validate() 7375 * @see rte_flow_ops 7376 */ 7377 int 7378 mlx5_flow_validate(struct rte_eth_dev *dev, 7379 const struct rte_flow_attr *attr, 7380 const struct rte_flow_item items[], 7381 const struct rte_flow_action original_actions[], 7382 struct rte_flow_error *error) 7383 { 7384 int hairpin_flow; 7385 struct mlx5_translated_action_handle 7386 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7387 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7388 const struct rte_flow_action *actions; 7389 struct rte_flow_action *translated_actions = NULL; 7390 int ret = flow_action_handles_translate(dev, original_actions, 7391 indir_actions, 7392 &indir_actions_n, 7393 &translated_actions, error); 7394 7395 if (ret) 7396 return ret; 7397 actions = translated_actions ? translated_actions : original_actions; 7398 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7399 ret = flow_drv_validate(dev, attr, items, actions, 7400 true, hairpin_flow, error); 7401 rte_free(translated_actions); 7402 return ret; 7403 } 7404 7405 /** 7406 * Create a flow. 7407 * 7408 * @see rte_flow_create() 7409 * @see rte_flow_ops 7410 */ 7411 struct rte_flow * 7412 mlx5_flow_create(struct rte_eth_dev *dev, 7413 const struct rte_flow_attr *attr, 7414 const struct rte_flow_item items[], 7415 const struct rte_flow_action actions[], 7416 struct rte_flow_error *error) 7417 { 7418 struct mlx5_priv *priv = dev->data->dev_private; 7419 7420 if (priv->sh->config.dv_flow_en == 2) { 7421 rte_flow_error_set(error, ENOTSUP, 7422 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7423 NULL, 7424 "Flow non-Q creation not supported"); 7425 return NULL; 7426 } 7427 /* 7428 * If the device is not started yet, it is not allowed to created a 7429 * flow from application. PMD default flows and traffic control flows 7430 * are not affected. 7431 */ 7432 if (unlikely(!dev->data->dev_started)) { 7433 DRV_LOG(DEBUG, "port %u is not started when " 7434 "inserting a flow", dev->data->port_id); 7435 rte_flow_error_set(error, ENODEV, 7436 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7437 NULL, 7438 "port not started"); 7439 return NULL; 7440 } 7441 7442 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7443 attr, items, actions, 7444 true, error); 7445 } 7446 7447 /** 7448 * Destroy a flow in a list. 7449 * 7450 * @param dev 7451 * Pointer to Ethernet device. 7452 * @param[in] flow_idx 7453 * Index of flow to destroy. 7454 */ 7455 static void 7456 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7457 uint32_t flow_idx) 7458 { 7459 struct mlx5_priv *priv = dev->data->dev_private; 7460 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7461 7462 if (!flow) 7463 return; 7464 MLX5_ASSERT(flow->type == type); 7465 /* 7466 * Update RX queue flags only if port is started, otherwise it is 7467 * already clean. 7468 */ 7469 if (dev->data->dev_started) 7470 flow_rxq_flags_trim(dev, flow); 7471 flow_drv_destroy(dev, flow); 7472 if (flow->tunnel) { 7473 struct mlx5_flow_tunnel *tunnel; 7474 7475 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7476 RTE_VERIFY(tunnel); 7477 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7478 mlx5_flow_tunnel_free(dev, tunnel); 7479 } 7480 flow_mreg_del_copy_action(dev, flow); 7481 mlx5_ipool_free(priv->flows[type], flow_idx); 7482 } 7483 7484 /** 7485 * Destroy all flows. 7486 * 7487 * @param dev 7488 * Pointer to Ethernet device. 7489 * @param type 7490 * Flow type to be flushed. 7491 * @param active 7492 * If flushing is called actively. 7493 */ 7494 void 7495 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7496 bool active) 7497 { 7498 struct mlx5_priv *priv = dev->data->dev_private; 7499 uint32_t num_flushed = 0, fidx = 1; 7500 struct rte_flow *flow; 7501 7502 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7503 if (priv->sh->config.dv_flow_en == 2 && 7504 type == MLX5_FLOW_TYPE_GEN) { 7505 flow_hw_q_flow_flush(dev, NULL); 7506 return; 7507 } 7508 #endif 7509 7510 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7511 flow_list_destroy(dev, type, fidx); 7512 num_flushed++; 7513 } 7514 if (active) { 7515 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7516 dev->data->port_id, num_flushed); 7517 } 7518 } 7519 7520 /** 7521 * Stop all default actions for flows. 7522 * 7523 * @param dev 7524 * Pointer to Ethernet device. 7525 */ 7526 void 7527 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7528 { 7529 flow_mreg_del_default_copy_action(dev); 7530 flow_rxq_flags_clear(dev); 7531 } 7532 7533 /** 7534 * Start all default actions for flows. 7535 * 7536 * @param dev 7537 * Pointer to Ethernet device. 7538 * @return 7539 * 0 on success, a negative errno value otherwise and rte_errno is set. 7540 */ 7541 int 7542 mlx5_flow_start_default(struct rte_eth_dev *dev) 7543 { 7544 struct rte_flow_error error; 7545 7546 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7547 return flow_mreg_add_default_copy_action(dev, &error); 7548 } 7549 7550 /** 7551 * Release key of thread specific flow workspace data. 7552 */ 7553 void 7554 flow_release_workspace(void *data) 7555 { 7556 struct mlx5_flow_workspace *wks = data; 7557 struct mlx5_flow_workspace *next; 7558 7559 while (wks) { 7560 next = wks->next; 7561 free(wks->rss_desc.queue); 7562 free(wks); 7563 wks = next; 7564 } 7565 } 7566 7567 /** 7568 * Get thread specific current flow workspace. 7569 * 7570 * @return pointer to thread specific flow workspace data, NULL on error. 7571 */ 7572 struct mlx5_flow_workspace* 7573 mlx5_flow_get_thread_workspace(void) 7574 { 7575 struct mlx5_flow_workspace *data; 7576 7577 data = mlx5_flow_os_get_specific_workspace(); 7578 MLX5_ASSERT(data && data->inuse); 7579 if (!data || !data->inuse) 7580 DRV_LOG(ERR, "flow workspace not initialized."); 7581 return data; 7582 } 7583 7584 /** 7585 * Allocate and init new flow workspace. 7586 * 7587 * @return pointer to flow workspace data, NULL on error. 7588 */ 7589 static struct mlx5_flow_workspace* 7590 flow_alloc_thread_workspace(void) 7591 { 7592 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7593 7594 if (!data) { 7595 DRV_LOG(ERR, "Failed to allocate flow workspace " 7596 "memory."); 7597 return NULL; 7598 } 7599 data->rss_desc.queue = calloc(1, 7600 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7601 if (!data->rss_desc.queue) 7602 goto err; 7603 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7604 return data; 7605 err: 7606 free(data->rss_desc.queue); 7607 free(data); 7608 return NULL; 7609 } 7610 7611 /** 7612 * Get new thread specific flow workspace. 7613 * 7614 * If current workspace inuse, create new one and set as current. 7615 * 7616 * @return pointer to thread specific flow workspace data, NULL on error. 7617 */ 7618 struct mlx5_flow_workspace* 7619 mlx5_flow_push_thread_workspace(void) 7620 { 7621 struct mlx5_flow_workspace *curr; 7622 struct mlx5_flow_workspace *data; 7623 7624 curr = mlx5_flow_os_get_specific_workspace(); 7625 if (!curr) { 7626 data = flow_alloc_thread_workspace(); 7627 if (!data) 7628 return NULL; 7629 } else if (!curr->inuse) { 7630 data = curr; 7631 } else if (curr->next) { 7632 data = curr->next; 7633 } else { 7634 data = flow_alloc_thread_workspace(); 7635 if (!data) 7636 return NULL; 7637 curr->next = data; 7638 data->prev = curr; 7639 } 7640 data->inuse = 1; 7641 data->flow_idx = 0; 7642 /* Set as current workspace */ 7643 if (mlx5_flow_os_set_specific_workspace(data)) 7644 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7645 return data; 7646 } 7647 7648 /** 7649 * Close current thread specific flow workspace. 7650 * 7651 * If previous workspace available, set it as current. 7652 * 7653 * @return pointer to thread specific flow workspace data, NULL on error. 7654 */ 7655 void 7656 mlx5_flow_pop_thread_workspace(void) 7657 { 7658 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7659 7660 if (!data) 7661 return; 7662 if (!data->inuse) { 7663 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7664 return; 7665 } 7666 data->inuse = 0; 7667 if (!data->prev) 7668 return; 7669 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7670 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7671 } 7672 7673 /** 7674 * Verify the flow list is empty 7675 * 7676 * @param dev 7677 * Pointer to Ethernet device. 7678 * 7679 * @return the number of flows not released. 7680 */ 7681 int 7682 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7683 { 7684 struct mlx5_priv *priv = dev->data->dev_private; 7685 struct rte_flow *flow; 7686 uint32_t idx = 0; 7687 int ret = 0, i; 7688 7689 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7690 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7691 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7692 dev->data->port_id, (void *)flow); 7693 ret++; 7694 } 7695 } 7696 return ret; 7697 } 7698 7699 /** 7700 * Enable default hairpin egress flow. 7701 * 7702 * @param dev 7703 * Pointer to Ethernet device. 7704 * @param sq_num 7705 * The SQ hw number. 7706 * 7707 * @return 7708 * 0 on success, a negative errno value otherwise and rte_errno is set. 7709 */ 7710 int 7711 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7712 uint32_t sq_num) 7713 { 7714 const struct rte_flow_attr attr = { 7715 .egress = 1, 7716 .priority = 0, 7717 }; 7718 struct mlx5_rte_flow_item_sq queue_spec = { 7719 .queue = sq_num, 7720 }; 7721 struct mlx5_rte_flow_item_sq queue_mask = { 7722 .queue = UINT32_MAX, 7723 }; 7724 struct rte_flow_item items[] = { 7725 { 7726 .type = (enum rte_flow_item_type) 7727 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7728 .spec = &queue_spec, 7729 .last = NULL, 7730 .mask = &queue_mask, 7731 }, 7732 { 7733 .type = RTE_FLOW_ITEM_TYPE_END, 7734 }, 7735 }; 7736 struct rte_flow_action_jump jump = { 7737 .group = MLX5_HAIRPIN_TX_TABLE, 7738 }; 7739 struct rte_flow_action actions[2]; 7740 uint32_t flow_idx; 7741 struct rte_flow_error error; 7742 7743 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7744 actions[0].conf = &jump; 7745 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7746 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7747 &attr, items, actions, false, &error); 7748 if (!flow_idx) { 7749 DRV_LOG(DEBUG, 7750 "Failed to create ctrl flow: rte_errno(%d)," 7751 " type(%d), message(%s)", 7752 rte_errno, error.type, 7753 error.message ? error.message : " (no stated reason)"); 7754 return -rte_errno; 7755 } 7756 return 0; 7757 } 7758 7759 /** 7760 * Enable a control flow configured from the control plane. 7761 * 7762 * @param dev 7763 * Pointer to Ethernet device. 7764 * @param eth_spec 7765 * An Ethernet flow spec to apply. 7766 * @param eth_mask 7767 * An Ethernet flow mask to apply. 7768 * @param vlan_spec 7769 * A VLAN flow spec to apply. 7770 * @param vlan_mask 7771 * A VLAN flow mask to apply. 7772 * 7773 * @return 7774 * 0 on success, a negative errno value otherwise and rte_errno is set. 7775 */ 7776 int 7777 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7778 struct rte_flow_item_eth *eth_spec, 7779 struct rte_flow_item_eth *eth_mask, 7780 struct rte_flow_item_vlan *vlan_spec, 7781 struct rte_flow_item_vlan *vlan_mask) 7782 { 7783 struct mlx5_priv *priv = dev->data->dev_private; 7784 const struct rte_flow_attr attr = { 7785 .ingress = 1, 7786 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7787 }; 7788 struct rte_flow_item items[] = { 7789 { 7790 .type = RTE_FLOW_ITEM_TYPE_ETH, 7791 .spec = eth_spec, 7792 .last = NULL, 7793 .mask = eth_mask, 7794 }, 7795 { 7796 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7797 RTE_FLOW_ITEM_TYPE_END, 7798 .spec = vlan_spec, 7799 .last = NULL, 7800 .mask = vlan_mask, 7801 }, 7802 { 7803 .type = RTE_FLOW_ITEM_TYPE_END, 7804 }, 7805 }; 7806 uint16_t queue[priv->reta_idx_n]; 7807 struct rte_flow_action_rss action_rss = { 7808 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7809 .level = 0, 7810 .types = priv->rss_conf.rss_hf, 7811 .key_len = priv->rss_conf.rss_key_len, 7812 .queue_num = priv->reta_idx_n, 7813 .key = priv->rss_conf.rss_key, 7814 .queue = queue, 7815 }; 7816 struct rte_flow_action actions[] = { 7817 { 7818 .type = RTE_FLOW_ACTION_TYPE_RSS, 7819 .conf = &action_rss, 7820 }, 7821 { 7822 .type = RTE_FLOW_ACTION_TYPE_END, 7823 }, 7824 }; 7825 uint32_t flow_idx; 7826 struct rte_flow_error error; 7827 unsigned int i; 7828 7829 if (!priv->reta_idx_n || !priv->rxqs_n) { 7830 return 0; 7831 } 7832 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7833 action_rss.types = 0; 7834 for (i = 0; i != priv->reta_idx_n; ++i) 7835 queue[i] = (*priv->reta_idx)[i]; 7836 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7837 &attr, items, actions, false, &error); 7838 if (!flow_idx) 7839 return -rte_errno; 7840 return 0; 7841 } 7842 7843 /** 7844 * Enable a flow control configured from the control plane. 7845 * 7846 * @param dev 7847 * Pointer to Ethernet device. 7848 * @param eth_spec 7849 * An Ethernet flow spec to apply. 7850 * @param eth_mask 7851 * An Ethernet flow mask to apply. 7852 * 7853 * @return 7854 * 0 on success, a negative errno value otherwise and rte_errno is set. 7855 */ 7856 int 7857 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7858 struct rte_flow_item_eth *eth_spec, 7859 struct rte_flow_item_eth *eth_mask) 7860 { 7861 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7862 } 7863 7864 /** 7865 * Create default miss flow rule matching lacp traffic 7866 * 7867 * @param dev 7868 * Pointer to Ethernet device. 7869 * @param eth_spec 7870 * An Ethernet flow spec to apply. 7871 * 7872 * @return 7873 * 0 on success, a negative errno value otherwise and rte_errno is set. 7874 */ 7875 int 7876 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7877 { 7878 /* 7879 * The LACP matching is done by only using ether type since using 7880 * a multicast dst mac causes kernel to give low priority to this flow. 7881 */ 7882 static const struct rte_flow_item_eth lacp_spec = { 7883 .type = RTE_BE16(0x8809), 7884 }; 7885 static const struct rte_flow_item_eth lacp_mask = { 7886 .type = 0xffff, 7887 }; 7888 const struct rte_flow_attr attr = { 7889 .ingress = 1, 7890 }; 7891 struct rte_flow_item items[] = { 7892 { 7893 .type = RTE_FLOW_ITEM_TYPE_ETH, 7894 .spec = &lacp_spec, 7895 .mask = &lacp_mask, 7896 }, 7897 { 7898 .type = RTE_FLOW_ITEM_TYPE_END, 7899 }, 7900 }; 7901 struct rte_flow_action actions[] = { 7902 { 7903 .type = (enum rte_flow_action_type) 7904 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7905 }, 7906 { 7907 .type = RTE_FLOW_ACTION_TYPE_END, 7908 }, 7909 }; 7910 struct rte_flow_error error; 7911 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7912 &attr, items, actions, 7913 false, &error); 7914 7915 if (!flow_idx) 7916 return -rte_errno; 7917 return 0; 7918 } 7919 7920 /** 7921 * Destroy a flow. 7922 * 7923 * @see rte_flow_destroy() 7924 * @see rte_flow_ops 7925 */ 7926 int 7927 mlx5_flow_destroy(struct rte_eth_dev *dev, 7928 struct rte_flow *flow, 7929 struct rte_flow_error *error __rte_unused) 7930 { 7931 struct mlx5_priv *priv = dev->data->dev_private; 7932 7933 if (priv->sh->config.dv_flow_en == 2) 7934 return rte_flow_error_set(error, ENOTSUP, 7935 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7936 NULL, 7937 "Flow non-Q destruction not supported"); 7938 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7939 (uintptr_t)(void *)flow); 7940 return 0; 7941 } 7942 7943 /** 7944 * Destroy all flows. 7945 * 7946 * @see rte_flow_flush() 7947 * @see rte_flow_ops 7948 */ 7949 int 7950 mlx5_flow_flush(struct rte_eth_dev *dev, 7951 struct rte_flow_error *error __rte_unused) 7952 { 7953 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7954 return 0; 7955 } 7956 7957 /** 7958 * Isolated mode. 7959 * 7960 * @see rte_flow_isolate() 7961 * @see rte_flow_ops 7962 */ 7963 int 7964 mlx5_flow_isolate(struct rte_eth_dev *dev, 7965 int enable, 7966 struct rte_flow_error *error) 7967 { 7968 struct mlx5_priv *priv = dev->data->dev_private; 7969 7970 if (dev->data->dev_started) { 7971 rte_flow_error_set(error, EBUSY, 7972 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7973 NULL, 7974 "port must be stopped first"); 7975 return -rte_errno; 7976 } 7977 priv->isolated = !!enable; 7978 if (enable) 7979 dev->dev_ops = &mlx5_dev_ops_isolate; 7980 else 7981 dev->dev_ops = &mlx5_dev_ops; 7982 7983 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7984 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7985 7986 return 0; 7987 } 7988 7989 /** 7990 * Query a flow. 7991 * 7992 * @see rte_flow_query() 7993 * @see rte_flow_ops 7994 */ 7995 static int 7996 flow_drv_query(struct rte_eth_dev *dev, 7997 struct rte_flow *eflow, 7998 const struct rte_flow_action *actions, 7999 void *data, 8000 struct rte_flow_error *error) 8001 { 8002 struct mlx5_priv *priv = dev->data->dev_private; 8003 const struct mlx5_flow_driver_ops *fops; 8004 struct rte_flow *flow = NULL; 8005 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8006 8007 if (priv->sh->config.dv_flow_en == 2) { 8008 #ifdef HAVE_MLX5_HWS_SUPPORT 8009 flow = eflow; 8010 ftype = MLX5_FLOW_TYPE_HW; 8011 #endif 8012 } else { 8013 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8014 (uintptr_t)(void *)eflow); 8015 } 8016 if (!flow) { 8017 return rte_flow_error_set(error, ENOENT, 8018 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8019 NULL, 8020 "invalid flow handle"); 8021 } 8022 if (ftype == MLX5_FLOW_TYPE_MIN) 8023 ftype = flow->drv_type; 8024 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8025 fops = flow_get_drv_ops(ftype); 8026 8027 return fops->query(dev, flow, actions, data, error); 8028 } 8029 8030 /** 8031 * Query a flow. 8032 * 8033 * @see rte_flow_query() 8034 * @see rte_flow_ops 8035 */ 8036 int 8037 mlx5_flow_query(struct rte_eth_dev *dev, 8038 struct rte_flow *flow, 8039 const struct rte_flow_action *actions, 8040 void *data, 8041 struct rte_flow_error *error) 8042 { 8043 int ret; 8044 8045 ret = flow_drv_query(dev, flow, actions, data, 8046 error); 8047 if (ret < 0) 8048 return ret; 8049 return 0; 8050 } 8051 8052 /** 8053 * Get rte_flow callbacks. 8054 * 8055 * @param dev 8056 * Pointer to Ethernet device structure. 8057 * @param ops 8058 * Pointer to operation-specific structure. 8059 * 8060 * @return 0 8061 */ 8062 int 8063 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8064 const struct rte_flow_ops **ops) 8065 { 8066 *ops = &mlx5_flow_ops; 8067 return 0; 8068 } 8069 8070 /** 8071 * Validate meter policy actions. 8072 * Dispatcher for action type specific validation. 8073 * 8074 * @param[in] dev 8075 * Pointer to the Ethernet device structure. 8076 * @param[in] action 8077 * The meter policy action object to validate. 8078 * @param[in] attr 8079 * Attributes of flow to determine steering domain. 8080 * @param[out] is_rss 8081 * Is RSS or not. 8082 * @param[out] domain_bitmap 8083 * Domain bitmap. 8084 * @param[out] is_def_policy 8085 * Is default policy or not. 8086 * @param[out] error 8087 * Perform verbose error reporting if not NULL. Initialized in case of 8088 * error only. 8089 * 8090 * @return 8091 * 0 on success, otherwise negative errno value. 8092 */ 8093 int 8094 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8095 const struct rte_flow_action *actions[RTE_COLORS], 8096 struct rte_flow_attr *attr, 8097 bool *is_rss, 8098 uint8_t *domain_bitmap, 8099 uint8_t *policy_mode, 8100 struct rte_mtr_error *error) 8101 { 8102 const struct mlx5_flow_driver_ops *fops; 8103 8104 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8105 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8106 domain_bitmap, policy_mode, error); 8107 } 8108 8109 /** 8110 * Destroy the meter table set. 8111 * 8112 * @param[in] dev 8113 * Pointer to Ethernet device. 8114 * @param[in] mtr_policy 8115 * Meter policy struct. 8116 */ 8117 void 8118 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8119 struct mlx5_flow_meter_policy *mtr_policy) 8120 { 8121 const struct mlx5_flow_driver_ops *fops; 8122 8123 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8124 fops->destroy_mtr_acts(dev, mtr_policy); 8125 } 8126 8127 /** 8128 * Create policy action, lock free, 8129 * (mutex should be acquired by caller). 8130 * Dispatcher for action type specific call. 8131 * 8132 * @param[in] dev 8133 * Pointer to the Ethernet device structure. 8134 * @param[in] mtr_policy 8135 * Meter policy struct. 8136 * @param[in] action 8137 * Action specification used to create meter actions. 8138 * @param[in] attr 8139 * Flow rule attributes. 8140 * @param[out] error 8141 * Perform verbose error reporting if not NULL. Initialized in case of 8142 * error only. 8143 * 8144 * @return 8145 * 0 on success, otherwise negative errno value. 8146 */ 8147 int 8148 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8149 struct mlx5_flow_meter_policy *mtr_policy, 8150 const struct rte_flow_action *actions[RTE_COLORS], 8151 struct rte_flow_attr *attr, 8152 struct rte_mtr_error *error) 8153 { 8154 const struct mlx5_flow_driver_ops *fops; 8155 8156 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8157 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8158 } 8159 8160 /** 8161 * Create policy rules, lock free, 8162 * (mutex should be acquired by caller). 8163 * Dispatcher for action type specific call. 8164 * 8165 * @param[in] dev 8166 * Pointer to the Ethernet device structure. 8167 * @param[in] mtr_policy 8168 * Meter policy struct. 8169 * 8170 * @return 8171 * 0 on success, -1 otherwise. 8172 */ 8173 int 8174 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8175 struct mlx5_flow_meter_policy *mtr_policy) 8176 { 8177 const struct mlx5_flow_driver_ops *fops; 8178 8179 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8180 return fops->create_policy_rules(dev, mtr_policy); 8181 } 8182 8183 /** 8184 * Destroy policy rules, lock free, 8185 * (mutex should be acquired by caller). 8186 * Dispatcher for action type specific call. 8187 * 8188 * @param[in] dev 8189 * Pointer to the Ethernet device structure. 8190 * @param[in] mtr_policy 8191 * Meter policy struct. 8192 */ 8193 void 8194 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8195 struct mlx5_flow_meter_policy *mtr_policy) 8196 { 8197 const struct mlx5_flow_driver_ops *fops; 8198 8199 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8200 fops->destroy_policy_rules(dev, mtr_policy); 8201 } 8202 8203 /** 8204 * Destroy the default policy table set. 8205 * 8206 * @param[in] dev 8207 * Pointer to Ethernet device. 8208 */ 8209 void 8210 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8211 { 8212 const struct mlx5_flow_driver_ops *fops; 8213 8214 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8215 fops->destroy_def_policy(dev); 8216 } 8217 8218 /** 8219 * Destroy the default policy table set. 8220 * 8221 * @param[in] dev 8222 * Pointer to Ethernet device. 8223 * 8224 * @return 8225 * 0 on success, -1 otherwise. 8226 */ 8227 int 8228 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8229 { 8230 const struct mlx5_flow_driver_ops *fops; 8231 8232 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8233 return fops->create_def_policy(dev); 8234 } 8235 8236 /** 8237 * Create the needed meter and suffix tables. 8238 * 8239 * @param[in] dev 8240 * Pointer to Ethernet device. 8241 * 8242 * @return 8243 * 0 on success, -1 otherwise. 8244 */ 8245 int 8246 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8247 struct mlx5_flow_meter_info *fm, 8248 uint32_t mtr_idx, 8249 uint8_t domain_bitmap) 8250 { 8251 const struct mlx5_flow_driver_ops *fops; 8252 8253 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8254 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8255 } 8256 8257 /** 8258 * Destroy the meter table set. 8259 * 8260 * @param[in] dev 8261 * Pointer to Ethernet device. 8262 * @param[in] tbl 8263 * Pointer to the meter table set. 8264 */ 8265 void 8266 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8267 struct mlx5_flow_meter_info *fm) 8268 { 8269 const struct mlx5_flow_driver_ops *fops; 8270 8271 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8272 fops->destroy_mtr_tbls(dev, fm); 8273 } 8274 8275 /** 8276 * Destroy the global meter drop table. 8277 * 8278 * @param[in] dev 8279 * Pointer to Ethernet device. 8280 */ 8281 void 8282 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8283 { 8284 const struct mlx5_flow_driver_ops *fops; 8285 8286 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8287 fops->destroy_mtr_drop_tbls(dev); 8288 } 8289 8290 /** 8291 * Destroy the sub policy table with RX queue. 8292 * 8293 * @param[in] dev 8294 * Pointer to Ethernet device. 8295 * @param[in] mtr_policy 8296 * Pointer to meter policy table. 8297 */ 8298 void 8299 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8300 struct mlx5_flow_meter_policy *mtr_policy) 8301 { 8302 const struct mlx5_flow_driver_ops *fops; 8303 8304 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8305 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8306 } 8307 8308 /** 8309 * Allocate the needed aso flow meter id. 8310 * 8311 * @param[in] dev 8312 * Pointer to Ethernet device. 8313 * 8314 * @return 8315 * Index to aso flow meter on success, NULL otherwise. 8316 */ 8317 uint32_t 8318 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8319 { 8320 const struct mlx5_flow_driver_ops *fops; 8321 8322 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8323 return fops->create_meter(dev); 8324 } 8325 8326 /** 8327 * Free the aso flow meter id. 8328 * 8329 * @param[in] dev 8330 * Pointer to Ethernet device. 8331 * @param[in] mtr_idx 8332 * Index to aso flow meter to be free. 8333 * 8334 * @return 8335 * 0 on success. 8336 */ 8337 void 8338 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8339 { 8340 const struct mlx5_flow_driver_ops *fops; 8341 8342 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8343 fops->free_meter(dev, mtr_idx); 8344 } 8345 8346 /** 8347 * Allocate a counter. 8348 * 8349 * @param[in] dev 8350 * Pointer to Ethernet device structure. 8351 * 8352 * @return 8353 * Index to allocated counter on success, 0 otherwise. 8354 */ 8355 uint32_t 8356 mlx5_counter_alloc(struct rte_eth_dev *dev) 8357 { 8358 const struct mlx5_flow_driver_ops *fops; 8359 struct rte_flow_attr attr = { .transfer = 0 }; 8360 8361 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8362 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8363 return fops->counter_alloc(dev); 8364 } 8365 DRV_LOG(ERR, 8366 "port %u counter allocate is not supported.", 8367 dev->data->port_id); 8368 return 0; 8369 } 8370 8371 /** 8372 * Free a counter. 8373 * 8374 * @param[in] dev 8375 * Pointer to Ethernet device structure. 8376 * @param[in] cnt 8377 * Index to counter to be free. 8378 */ 8379 void 8380 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8381 { 8382 const struct mlx5_flow_driver_ops *fops; 8383 struct rte_flow_attr attr = { .transfer = 0 }; 8384 8385 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8386 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8387 fops->counter_free(dev, cnt); 8388 return; 8389 } 8390 DRV_LOG(ERR, 8391 "port %u counter free is not supported.", 8392 dev->data->port_id); 8393 } 8394 8395 /** 8396 * Query counter statistics. 8397 * 8398 * @param[in] dev 8399 * Pointer to Ethernet device structure. 8400 * @param[in] cnt 8401 * Index to counter to query. 8402 * @param[in] clear 8403 * Set to clear counter statistics. 8404 * @param[out] pkts 8405 * The counter hits packets number to save. 8406 * @param[out] bytes 8407 * The counter hits bytes number to save. 8408 * 8409 * @return 8410 * 0 on success, a negative errno value otherwise. 8411 */ 8412 int 8413 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8414 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8415 { 8416 const struct mlx5_flow_driver_ops *fops; 8417 struct rte_flow_attr attr = { .transfer = 0 }; 8418 8419 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8420 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8421 return fops->counter_query(dev, cnt, clear, pkts, 8422 bytes, action); 8423 } 8424 DRV_LOG(ERR, 8425 "port %u counter query is not supported.", 8426 dev->data->port_id); 8427 return -ENOTSUP; 8428 } 8429 8430 /** 8431 * Get information about HWS pre-configurable resources. 8432 * 8433 * @param[in] dev 8434 * Pointer to the rte_eth_dev structure. 8435 * @param[out] port_info 8436 * Pointer to port information. 8437 * @param[out] queue_info 8438 * Pointer to queue information. 8439 * @param[out] error 8440 * Pointer to error structure. 8441 * 8442 * @return 8443 * 0 on success, a negative errno value otherwise and rte_errno is set. 8444 */ 8445 static int 8446 mlx5_flow_info_get(struct rte_eth_dev *dev, 8447 struct rte_flow_port_info *port_info, 8448 struct rte_flow_queue_info *queue_info, 8449 struct rte_flow_error *error) 8450 { 8451 const struct mlx5_flow_driver_ops *fops; 8452 struct rte_flow_attr attr = {0}; 8453 8454 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8455 return rte_flow_error_set(error, ENOTSUP, 8456 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8457 NULL, 8458 "info get with incorrect steering mode"); 8459 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8460 return fops->info_get(dev, port_info, queue_info, error); 8461 } 8462 8463 /** 8464 * Configure port HWS resources. 8465 * 8466 * @param[in] dev 8467 * Pointer to the rte_eth_dev structure. 8468 * @param[in] port_attr 8469 * Port configuration attributes. 8470 * @param[in] nb_queue 8471 * Number of queue. 8472 * @param[in] queue_attr 8473 * Array that holds attributes for each flow queue. 8474 * @param[out] error 8475 * Pointer to error structure. 8476 * 8477 * @return 8478 * 0 on success, a negative errno value otherwise and rte_errno is set. 8479 */ 8480 static int 8481 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8482 const struct rte_flow_port_attr *port_attr, 8483 uint16_t nb_queue, 8484 const struct rte_flow_queue_attr *queue_attr[], 8485 struct rte_flow_error *error) 8486 { 8487 const struct mlx5_flow_driver_ops *fops; 8488 struct rte_flow_attr attr = {0}; 8489 8490 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8491 return rte_flow_error_set(error, ENOTSUP, 8492 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8493 NULL, 8494 "port configure with incorrect steering mode"); 8495 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8496 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8497 } 8498 8499 /** 8500 * Validate item template. 8501 * 8502 * @param[in] dev 8503 * Pointer to the rte_eth_dev structure. 8504 * @param[in] attr 8505 * Pointer to the item template attributes. 8506 * @param[in] items 8507 * The template item pattern. 8508 * @param[out] error 8509 * Pointer to error structure. 8510 * 8511 * @return 8512 * 0 on success, a negative errno value otherwise and rte_errno is set. 8513 */ 8514 int 8515 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8516 const struct rte_flow_pattern_template_attr *attr, 8517 const struct rte_flow_item items[], 8518 struct rte_flow_error *error) 8519 { 8520 const struct mlx5_flow_driver_ops *fops; 8521 struct rte_flow_attr fattr = {0}; 8522 8523 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8524 rte_flow_error_set(error, ENOTSUP, 8525 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8526 "pattern validate with incorrect steering mode"); 8527 return -ENOTSUP; 8528 } 8529 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8530 return fops->pattern_validate(dev, attr, items, error); 8531 } 8532 8533 /** 8534 * Create flow item template. 8535 * 8536 * @param[in] dev 8537 * Pointer to the rte_eth_dev structure. 8538 * @param[in] attr 8539 * Pointer to the item template attributes. 8540 * @param[in] items 8541 * The template item pattern. 8542 * @param[out] error 8543 * Pointer to error structure. 8544 * 8545 * @return 8546 * 0 on success, a negative errno value otherwise and rte_errno is set. 8547 */ 8548 static struct rte_flow_pattern_template * 8549 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8550 const struct rte_flow_pattern_template_attr *attr, 8551 const struct rte_flow_item items[], 8552 struct rte_flow_error *error) 8553 { 8554 const struct mlx5_flow_driver_ops *fops; 8555 struct rte_flow_attr fattr = {0}; 8556 8557 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8558 rte_flow_error_set(error, ENOTSUP, 8559 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8560 NULL, 8561 "pattern create with incorrect steering mode"); 8562 return NULL; 8563 } 8564 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8565 return fops->pattern_template_create(dev, attr, items, error); 8566 } 8567 8568 /** 8569 * Destroy flow item template. 8570 * 8571 * @param[in] dev 8572 * Pointer to the rte_eth_dev structure. 8573 * @param[in] template 8574 * Pointer to the item template to be destroyed. 8575 * @param[out] error 8576 * Pointer to error structure. 8577 * 8578 * @return 8579 * 0 on success, a negative errno value otherwise and rte_errno is set. 8580 */ 8581 static int 8582 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8583 struct rte_flow_pattern_template *template, 8584 struct rte_flow_error *error) 8585 { 8586 const struct mlx5_flow_driver_ops *fops; 8587 struct rte_flow_attr attr = {0}; 8588 8589 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8590 return rte_flow_error_set(error, ENOTSUP, 8591 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8592 NULL, 8593 "pattern destroy with incorrect steering mode"); 8594 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8595 return fops->pattern_template_destroy(dev, template, error); 8596 } 8597 8598 /** 8599 * Validate flow actions template. 8600 * 8601 * @param[in] dev 8602 * Pointer to the rte_eth_dev structure. 8603 * @param[in] attr 8604 * Pointer to the action template attributes. 8605 * @param[in] actions 8606 * Associated actions (list terminated by the END action). 8607 * @param[in] masks 8608 * List of actions that marks which of the action's member is constant. 8609 * @param[out] error 8610 * Pointer to error structure. 8611 * 8612 * @return 8613 * 0 on success, a negative errno value otherwise and rte_errno is set. 8614 */ 8615 int 8616 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8617 const struct rte_flow_actions_template_attr *attr, 8618 const struct rte_flow_action actions[], 8619 const struct rte_flow_action masks[], 8620 struct rte_flow_error *error) 8621 { 8622 const struct mlx5_flow_driver_ops *fops; 8623 struct rte_flow_attr fattr = {0}; 8624 8625 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8626 rte_flow_error_set(error, ENOTSUP, 8627 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8628 "actions validate with incorrect steering mode"); 8629 return -ENOTSUP; 8630 } 8631 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8632 return fops->actions_validate(dev, attr, actions, masks, error); 8633 } 8634 8635 /** 8636 * Create flow item template. 8637 * 8638 * @param[in] dev 8639 * Pointer to the rte_eth_dev structure. 8640 * @param[in] attr 8641 * Pointer to the action template attributes. 8642 * @param[in] actions 8643 * Associated actions (list terminated by the END action). 8644 * @param[in] masks 8645 * List of actions that marks which of the action's member is constant. 8646 * @param[out] error 8647 * Pointer to error structure. 8648 * 8649 * @return 8650 * 0 on success, a negative errno value otherwise and rte_errno is set. 8651 */ 8652 static struct rte_flow_actions_template * 8653 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8654 const struct rte_flow_actions_template_attr *attr, 8655 const struct rte_flow_action actions[], 8656 const struct rte_flow_action masks[], 8657 struct rte_flow_error *error) 8658 { 8659 const struct mlx5_flow_driver_ops *fops; 8660 struct rte_flow_attr fattr = {0}; 8661 8662 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8663 rte_flow_error_set(error, ENOTSUP, 8664 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8665 NULL, 8666 "action create with incorrect steering mode"); 8667 return NULL; 8668 } 8669 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8670 return fops->actions_template_create(dev, attr, actions, masks, error); 8671 } 8672 8673 /** 8674 * Destroy flow action template. 8675 * 8676 * @param[in] dev 8677 * Pointer to the rte_eth_dev structure. 8678 * @param[in] template 8679 * Pointer to the action template to be destroyed. 8680 * @param[out] error 8681 * Pointer to error structure. 8682 * 8683 * @return 8684 * 0 on success, a negative errno value otherwise and rte_errno is set. 8685 */ 8686 static int 8687 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8688 struct rte_flow_actions_template *template, 8689 struct rte_flow_error *error) 8690 { 8691 const struct mlx5_flow_driver_ops *fops; 8692 struct rte_flow_attr attr = {0}; 8693 8694 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8695 return rte_flow_error_set(error, ENOTSUP, 8696 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8697 NULL, 8698 "action destroy with incorrect steering mode"); 8699 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8700 return fops->actions_template_destroy(dev, template, error); 8701 } 8702 8703 /** 8704 * Create flow table. 8705 * 8706 * @param[in] dev 8707 * Pointer to the rte_eth_dev structure. 8708 * @param[in] attr 8709 * Pointer to the table attributes. 8710 * @param[in] item_templates 8711 * Item template array to be binded to the table. 8712 * @param[in] nb_item_templates 8713 * Number of item template. 8714 * @param[in] action_templates 8715 * Action template array to be binded to the table. 8716 * @param[in] nb_action_templates 8717 * Number of action template. 8718 * @param[out] error 8719 * Pointer to error structure. 8720 * 8721 * @return 8722 * Table on success, NULL otherwise and rte_errno is set. 8723 */ 8724 static struct rte_flow_template_table * 8725 mlx5_flow_table_create(struct rte_eth_dev *dev, 8726 const struct rte_flow_template_table_attr *attr, 8727 struct rte_flow_pattern_template *item_templates[], 8728 uint8_t nb_item_templates, 8729 struct rte_flow_actions_template *action_templates[], 8730 uint8_t nb_action_templates, 8731 struct rte_flow_error *error) 8732 { 8733 const struct mlx5_flow_driver_ops *fops; 8734 struct rte_flow_attr fattr = {0}; 8735 8736 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8737 rte_flow_error_set(error, ENOTSUP, 8738 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8739 NULL, 8740 "table create with incorrect steering mode"); 8741 return NULL; 8742 } 8743 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8744 return fops->template_table_create(dev, 8745 attr, 8746 item_templates, 8747 nb_item_templates, 8748 action_templates, 8749 nb_action_templates, 8750 error); 8751 } 8752 8753 /** 8754 * PMD destroy flow table. 8755 * 8756 * @param[in] dev 8757 * Pointer to the rte_eth_dev structure. 8758 * @param[in] table 8759 * Pointer to the table to be destroyed. 8760 * @param[out] error 8761 * Pointer to error structure. 8762 * 8763 * @return 8764 * 0 on success, a negative errno value otherwise and rte_errno is set. 8765 */ 8766 static int 8767 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8768 struct rte_flow_template_table *table, 8769 struct rte_flow_error *error) 8770 { 8771 const struct mlx5_flow_driver_ops *fops; 8772 struct rte_flow_attr attr = {0}; 8773 8774 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8775 return rte_flow_error_set(error, ENOTSUP, 8776 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8777 NULL, 8778 "table destroy with incorrect steering mode"); 8779 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8780 return fops->template_table_destroy(dev, table, error); 8781 } 8782 8783 /** 8784 * Enqueue flow creation. 8785 * 8786 * @param[in] dev 8787 * Pointer to the rte_eth_dev structure. 8788 * @param[in] queue_id 8789 * The queue to create the flow. 8790 * @param[in] attr 8791 * Pointer to the flow operation attributes. 8792 * @param[in] items 8793 * Items with flow spec value. 8794 * @param[in] pattern_template_index 8795 * The item pattern flow follows from the table. 8796 * @param[in] actions 8797 * Action with flow spec value. 8798 * @param[in] action_template_index 8799 * The action pattern flow follows from the table. 8800 * @param[in] user_data 8801 * Pointer to the user_data. 8802 * @param[out] error 8803 * Pointer to error structure. 8804 * 8805 * @return 8806 * Flow pointer on success, NULL otherwise and rte_errno is set. 8807 */ 8808 static struct rte_flow * 8809 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8810 uint32_t queue_id, 8811 const struct rte_flow_op_attr *attr, 8812 struct rte_flow_template_table *table, 8813 const struct rte_flow_item items[], 8814 uint8_t pattern_template_index, 8815 const struct rte_flow_action actions[], 8816 uint8_t action_template_index, 8817 void *user_data, 8818 struct rte_flow_error *error) 8819 { 8820 const struct mlx5_flow_driver_ops *fops; 8821 struct rte_flow_attr fattr = {0}; 8822 8823 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8824 rte_flow_error_set(error, ENOTSUP, 8825 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8826 NULL, 8827 "flow_q create with incorrect steering mode"); 8828 return NULL; 8829 } 8830 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8831 return fops->async_flow_create(dev, queue_id, attr, table, 8832 items, pattern_template_index, 8833 actions, action_template_index, 8834 user_data, error); 8835 } 8836 8837 /** 8838 * Enqueue flow destruction. 8839 * 8840 * @param[in] dev 8841 * Pointer to the rte_eth_dev structure. 8842 * @param[in] queue 8843 * The queue to destroy the flow. 8844 * @param[in] attr 8845 * Pointer to the flow operation attributes. 8846 * @param[in] flow 8847 * Pointer to the flow to be destroyed. 8848 * @param[in] user_data 8849 * Pointer to the user_data. 8850 * @param[out] error 8851 * Pointer to error structure. 8852 * 8853 * @return 8854 * 0 on success, negative value otherwise and rte_errno is set. 8855 */ 8856 static int 8857 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8858 uint32_t queue, 8859 const struct rte_flow_op_attr *attr, 8860 struct rte_flow *flow, 8861 void *user_data, 8862 struct rte_flow_error *error) 8863 { 8864 const struct mlx5_flow_driver_ops *fops; 8865 struct rte_flow_attr fattr = {0}; 8866 8867 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8868 return rte_flow_error_set(error, ENOTSUP, 8869 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8870 NULL, 8871 "flow_q destroy with incorrect steering mode"); 8872 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8873 return fops->async_flow_destroy(dev, queue, attr, flow, 8874 user_data, error); 8875 } 8876 8877 /** 8878 * Pull the enqueued flows. 8879 * 8880 * @param[in] dev 8881 * Pointer to the rte_eth_dev structure. 8882 * @param[in] queue 8883 * The queue to pull the result. 8884 * @param[in/out] res 8885 * Array to save the results. 8886 * @param[in] n_res 8887 * Available result with the array. 8888 * @param[out] error 8889 * Pointer to error structure. 8890 * 8891 * @return 8892 * Result number on success, negative value otherwise and rte_errno is set. 8893 */ 8894 static int 8895 mlx5_flow_pull(struct rte_eth_dev *dev, 8896 uint32_t queue, 8897 struct rte_flow_op_result res[], 8898 uint16_t n_res, 8899 struct rte_flow_error *error) 8900 { 8901 const struct mlx5_flow_driver_ops *fops; 8902 struct rte_flow_attr attr = {0}; 8903 8904 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8905 return rte_flow_error_set(error, ENOTSUP, 8906 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8907 NULL, 8908 "flow_q pull with incorrect steering mode"); 8909 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8910 return fops->pull(dev, queue, res, n_res, error); 8911 } 8912 8913 /** 8914 * Push the enqueued flows. 8915 * 8916 * @param[in] dev 8917 * Pointer to the rte_eth_dev structure. 8918 * @param[in] queue 8919 * The queue to push the flows. 8920 * @param[out] error 8921 * Pointer to error structure. 8922 * 8923 * @return 8924 * 0 on success, negative value otherwise and rte_errno is set. 8925 */ 8926 static int 8927 mlx5_flow_push(struct rte_eth_dev *dev, 8928 uint32_t queue, 8929 struct rte_flow_error *error) 8930 { 8931 const struct mlx5_flow_driver_ops *fops; 8932 struct rte_flow_attr attr = {0}; 8933 8934 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8935 return rte_flow_error_set(error, ENOTSUP, 8936 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8937 NULL, 8938 "flow_q push with incorrect steering mode"); 8939 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8940 return fops->push(dev, queue, error); 8941 } 8942 8943 /** 8944 * Create shared action. 8945 * 8946 * @param[in] dev 8947 * Pointer to the rte_eth_dev structure. 8948 * @param[in] queue 8949 * Which queue to be used.. 8950 * @param[in] attr 8951 * Operation attribute. 8952 * @param[in] conf 8953 * Indirect action configuration. 8954 * @param[in] action 8955 * rte_flow action detail. 8956 * @param[in] user_data 8957 * Pointer to the user_data. 8958 * @param[out] error 8959 * Pointer to error structure. 8960 * 8961 * @return 8962 * Action handle on success, NULL otherwise and rte_errno is set. 8963 */ 8964 static struct rte_flow_action_handle * 8965 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8966 const struct rte_flow_op_attr *attr, 8967 const struct rte_flow_indir_action_conf *conf, 8968 const struct rte_flow_action *action, 8969 void *user_data, 8970 struct rte_flow_error *error) 8971 { 8972 const struct mlx5_flow_driver_ops *fops = 8973 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8974 8975 return fops->async_action_create(dev, queue, attr, conf, action, 8976 user_data, error); 8977 } 8978 8979 /** 8980 * Update shared action. 8981 * 8982 * @param[in] dev 8983 * Pointer to the rte_eth_dev structure. 8984 * @param[in] queue 8985 * Which queue to be used.. 8986 * @param[in] attr 8987 * Operation attribute. 8988 * @param[in] handle 8989 * Action handle to be updated. 8990 * @param[in] update 8991 * Update value. 8992 * @param[in] user_data 8993 * Pointer to the user_data. 8994 * @param[out] error 8995 * Pointer to error structure. 8996 * 8997 * @return 8998 * 0 on success, negative value otherwise and rte_errno is set. 8999 */ 9000 static int 9001 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 9002 const struct rte_flow_op_attr *attr, 9003 struct rte_flow_action_handle *handle, 9004 const void *update, 9005 void *user_data, 9006 struct rte_flow_error *error) 9007 { 9008 const struct mlx5_flow_driver_ops *fops = 9009 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9010 9011 return fops->async_action_update(dev, queue, attr, handle, 9012 update, user_data, error); 9013 } 9014 9015 /** 9016 * Query shared action. 9017 * 9018 * @param[in] dev 9019 * Pointer to the rte_eth_dev structure. 9020 * @param[in] queue 9021 * Which queue to be used.. 9022 * @param[in] attr 9023 * Operation attribute. 9024 * @param[in] handle 9025 * Action handle to be updated. 9026 * @param[in] data 9027 * Pointer query result data. 9028 * @param[in] user_data 9029 * Pointer to the user_data. 9030 * @param[out] error 9031 * Pointer to error structure. 9032 * 9033 * @return 9034 * 0 on success, negative value otherwise and rte_errno is set. 9035 */ 9036 static int 9037 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9038 const struct rte_flow_op_attr *attr, 9039 const struct rte_flow_action_handle *handle, 9040 void *data, 9041 void *user_data, 9042 struct rte_flow_error *error) 9043 { 9044 const struct mlx5_flow_driver_ops *fops = 9045 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9046 9047 return fops->async_action_query(dev, queue, attr, handle, 9048 data, user_data, error); 9049 } 9050 9051 /** 9052 * Destroy shared action. 9053 * 9054 * @param[in] dev 9055 * Pointer to the rte_eth_dev structure. 9056 * @param[in] queue 9057 * Which queue to be used.. 9058 * @param[in] attr 9059 * Operation attribute. 9060 * @param[in] handle 9061 * Action handle to be destroyed. 9062 * @param[in] user_data 9063 * Pointer to the user_data. 9064 * @param[out] error 9065 * Pointer to error structure. 9066 * 9067 * @return 9068 * 0 on success, negative value otherwise and rte_errno is set. 9069 */ 9070 static int 9071 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9072 const struct rte_flow_op_attr *attr, 9073 struct rte_flow_action_handle *handle, 9074 void *user_data, 9075 struct rte_flow_error *error) 9076 { 9077 const struct mlx5_flow_driver_ops *fops = 9078 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9079 9080 return fops->async_action_destroy(dev, queue, attr, handle, 9081 user_data, error); 9082 } 9083 9084 /** 9085 * Allocate a new memory for the counter values wrapped by all the needed 9086 * management. 9087 * 9088 * @param[in] sh 9089 * Pointer to mlx5_dev_ctx_shared object. 9090 * 9091 * @return 9092 * 0 on success, a negative errno value otherwise. 9093 */ 9094 static int 9095 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9096 { 9097 struct mlx5_counter_stats_mem_mng *mem_mng; 9098 volatile struct flow_counter_stats *raw_data; 9099 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9100 int size = (sizeof(struct flow_counter_stats) * 9101 MLX5_COUNTERS_PER_POOL + 9102 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9103 sizeof(struct mlx5_counter_stats_mem_mng); 9104 size_t pgsize = rte_mem_page_size(); 9105 uint8_t *mem; 9106 int ret; 9107 int i; 9108 9109 if (pgsize == (size_t)-1) { 9110 DRV_LOG(ERR, "Failed to get mem page size"); 9111 rte_errno = ENOMEM; 9112 return -ENOMEM; 9113 } 9114 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9115 if (!mem) { 9116 rte_errno = ENOMEM; 9117 return -ENOMEM; 9118 } 9119 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9120 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9121 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9122 sh->cdev->pdn, mem, size, 9123 &mem_mng->wm); 9124 if (ret) { 9125 rte_errno = errno; 9126 mlx5_free(mem); 9127 return -rte_errno; 9128 } 9129 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9130 raw_data = (volatile struct flow_counter_stats *)mem; 9131 for (i = 0; i < raws_n; ++i) { 9132 mem_mng->raws[i].mem_mng = mem_mng; 9133 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9134 } 9135 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9136 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9137 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9138 next); 9139 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9140 sh->sws_cmng.mem_mng = mem_mng; 9141 return 0; 9142 } 9143 9144 /** 9145 * Set the statistic memory to the new counter pool. 9146 * 9147 * @param[in] sh 9148 * Pointer to mlx5_dev_ctx_shared object. 9149 * @param[in] pool 9150 * Pointer to the pool to set the statistic memory. 9151 * 9152 * @return 9153 * 0 on success, a negative errno value otherwise. 9154 */ 9155 static int 9156 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9157 struct mlx5_flow_counter_pool *pool) 9158 { 9159 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9160 /* Resize statistic memory once used out. */ 9161 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9162 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9163 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9164 return -1; 9165 } 9166 rte_spinlock_lock(&pool->sl); 9167 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9168 rte_spinlock_unlock(&pool->sl); 9169 pool->raw_hw = NULL; 9170 return 0; 9171 } 9172 9173 #define MLX5_POOL_QUERY_FREQ_US 1000000 9174 9175 /** 9176 * Set the periodic procedure for triggering asynchronous batch queries for all 9177 * the counter pools. 9178 * 9179 * @param[in] sh 9180 * Pointer to mlx5_dev_ctx_shared object. 9181 */ 9182 void 9183 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9184 { 9185 uint32_t pools_n, us; 9186 9187 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9188 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9189 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9190 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9191 sh->sws_cmng.query_thread_on = 0; 9192 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9193 } else { 9194 sh->sws_cmng.query_thread_on = 1; 9195 } 9196 } 9197 9198 /** 9199 * The periodic procedure for triggering asynchronous batch queries for all the 9200 * counter pools. This function is probably called by the host thread. 9201 * 9202 * @param[in] arg 9203 * The parameter for the alarm process. 9204 */ 9205 void 9206 mlx5_flow_query_alarm(void *arg) 9207 { 9208 struct mlx5_dev_ctx_shared *sh = arg; 9209 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9210 uint16_t pool_index = cmng->pool_index; 9211 struct mlx5_flow_counter_pool *pool; 9212 uint16_t n_valid; 9213 int ret; 9214 9215 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9216 goto set_alarm; 9217 rte_spinlock_lock(&cmng->pool_update_sl); 9218 pool = cmng->pools[pool_index]; 9219 n_valid = cmng->n_valid; 9220 rte_spinlock_unlock(&cmng->pool_update_sl); 9221 /* Set the statistic memory to the new created pool. */ 9222 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9223 goto set_alarm; 9224 if (pool->raw_hw) 9225 /* There is a pool query in progress. */ 9226 goto set_alarm; 9227 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9228 if (!pool->raw_hw) 9229 /* No free counter statistics raw memory. */ 9230 goto set_alarm; 9231 /* 9232 * Identify the counters released between query trigger and query 9233 * handle more efficiently. The counter released in this gap period 9234 * should wait for a new round of query as the new arrived packets 9235 * will not be taken into account. 9236 */ 9237 pool->query_gen++; 9238 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9239 MLX5_COUNTERS_PER_POOL, 9240 NULL, NULL, 9241 pool->raw_hw->mem_mng->wm.lkey, 9242 (void *)(uintptr_t) 9243 pool->raw_hw->data, 9244 sh->devx_comp, 9245 (uint64_t)(uintptr_t)pool); 9246 if (ret) { 9247 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9248 " %d", pool->min_dcs->id); 9249 pool->raw_hw = NULL; 9250 goto set_alarm; 9251 } 9252 LIST_REMOVE(pool->raw_hw, next); 9253 cmng->pending_queries++; 9254 pool_index++; 9255 if (pool_index >= n_valid) 9256 pool_index = 0; 9257 set_alarm: 9258 cmng->pool_index = pool_index; 9259 mlx5_set_query_alarm(sh); 9260 } 9261 9262 /** 9263 * Check and callback event for new aged flow in the counter pool 9264 * 9265 * @param[in] sh 9266 * Pointer to mlx5_dev_ctx_shared object. 9267 * @param[in] pool 9268 * Pointer to Current counter pool. 9269 */ 9270 static void 9271 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9272 struct mlx5_flow_counter_pool *pool) 9273 { 9274 struct mlx5_priv *priv; 9275 struct mlx5_flow_counter *cnt; 9276 struct mlx5_age_info *age_info; 9277 struct mlx5_age_param *age_param; 9278 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9279 struct mlx5_counter_stats_raw *prev = pool->raw; 9280 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9281 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9282 uint16_t expected = AGE_CANDIDATE; 9283 uint32_t i; 9284 9285 pool->time_of_last_age_check = curr_time; 9286 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9287 cnt = MLX5_POOL_GET_CNT(pool, i); 9288 age_param = MLX5_CNT_TO_AGE(cnt); 9289 if (__atomic_load_n(&age_param->state, 9290 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9291 continue; 9292 if (cur->data[i].hits != prev->data[i].hits) { 9293 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9294 __ATOMIC_RELAXED); 9295 continue; 9296 } 9297 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9298 time_delta, 9299 __ATOMIC_RELAXED) <= age_param->timeout) 9300 continue; 9301 /** 9302 * Hold the lock first, or if between the 9303 * state AGE_TMOUT and tailq operation the 9304 * release happened, the release procedure 9305 * may delete a non-existent tailq node. 9306 */ 9307 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9308 age_info = GET_PORT_AGE_INFO(priv); 9309 rte_spinlock_lock(&age_info->aged_sl); 9310 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9311 AGE_TMOUT, false, 9312 __ATOMIC_RELAXED, 9313 __ATOMIC_RELAXED)) { 9314 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9315 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9316 } 9317 rte_spinlock_unlock(&age_info->aged_sl); 9318 } 9319 mlx5_age_event_prepare(sh); 9320 } 9321 9322 /** 9323 * Handler for the HW respond about ready values from an asynchronous batch 9324 * query. This function is probably called by the host thread. 9325 * 9326 * @param[in] sh 9327 * The pointer to the shared device context. 9328 * @param[in] async_id 9329 * The Devx async ID. 9330 * @param[in] status 9331 * The status of the completion. 9332 */ 9333 void 9334 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9335 uint64_t async_id, int status) 9336 { 9337 struct mlx5_flow_counter_pool *pool = 9338 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9339 struct mlx5_counter_stats_raw *raw_to_free; 9340 uint8_t query_gen = pool->query_gen ^ 1; 9341 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9342 enum mlx5_counter_type cnt_type = 9343 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9344 MLX5_COUNTER_TYPE_ORIGIN; 9345 9346 if (unlikely(status)) { 9347 raw_to_free = pool->raw_hw; 9348 } else { 9349 raw_to_free = pool->raw; 9350 if (pool->is_aged) 9351 mlx5_flow_aging_check(sh, pool); 9352 rte_spinlock_lock(&pool->sl); 9353 pool->raw = pool->raw_hw; 9354 rte_spinlock_unlock(&pool->sl); 9355 /* Be sure the new raw counters data is updated in memory. */ 9356 rte_io_wmb(); 9357 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9358 rte_spinlock_lock(&cmng->csl[cnt_type]); 9359 TAILQ_CONCAT(&cmng->counters[cnt_type], 9360 &pool->counters[query_gen], next); 9361 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9362 } 9363 } 9364 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9365 pool->raw_hw = NULL; 9366 sh->sws_cmng.pending_queries--; 9367 } 9368 9369 static int 9370 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9371 const struct flow_grp_info *grp_info, 9372 struct rte_flow_error *error) 9373 { 9374 if (grp_info->transfer && grp_info->external && 9375 grp_info->fdb_def_rule) { 9376 if (group == UINT32_MAX) 9377 return rte_flow_error_set 9378 (error, EINVAL, 9379 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9380 NULL, 9381 "group index not supported"); 9382 *table = group + 1; 9383 } else { 9384 *table = group; 9385 } 9386 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9387 return 0; 9388 } 9389 9390 /** 9391 * Translate the rte_flow group index to HW table value. 9392 * 9393 * If tunnel offload is disabled, all group ids converted to flow table 9394 * id using the standard method. 9395 * If tunnel offload is enabled, group id can be converted using the 9396 * standard or tunnel conversion method. Group conversion method 9397 * selection depends on flags in `grp_info` parameter: 9398 * - Internal (grp_info.external == 0) groups conversion uses the 9399 * standard method. 9400 * - Group ids in JUMP action converted with the tunnel conversion. 9401 * - Group id in rule attribute conversion depends on a rule type and 9402 * group id value: 9403 * ** non zero group attributes converted with the tunnel method 9404 * ** zero group attribute in non-tunnel rule is converted using the 9405 * standard method - there's only one root table 9406 * ** zero group attribute in steer tunnel rule is converted with the 9407 * standard method - single root table 9408 * ** zero group attribute in match tunnel rule is a special OvS 9409 * case: that value is used for portability reasons. That group 9410 * id is converted with the tunnel conversion method. 9411 * 9412 * @param[in] dev 9413 * Port device 9414 * @param[in] tunnel 9415 * PMD tunnel offload object 9416 * @param[in] group 9417 * rte_flow group index value. 9418 * @param[out] table 9419 * HW table value. 9420 * @param[in] grp_info 9421 * flags used for conversion 9422 * @param[out] error 9423 * Pointer to error structure. 9424 * 9425 * @return 9426 * 0 on success, a negative errno value otherwise and rte_errno is set. 9427 */ 9428 int 9429 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9430 const struct mlx5_flow_tunnel *tunnel, 9431 uint32_t group, uint32_t *table, 9432 const struct flow_grp_info *grp_info, 9433 struct rte_flow_error *error) 9434 { 9435 int ret; 9436 bool standard_translation; 9437 9438 if (!grp_info->skip_scale && grp_info->external && 9439 group < MLX5_MAX_TABLES_EXTERNAL) 9440 group *= MLX5_FLOW_TABLE_FACTOR; 9441 if (is_tunnel_offload_active(dev)) { 9442 standard_translation = !grp_info->external || 9443 grp_info->std_tbl_fix; 9444 } else { 9445 standard_translation = true; 9446 } 9447 DRV_LOG(DEBUG, 9448 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9449 dev->data->port_id, group, grp_info->transfer, 9450 grp_info->external, grp_info->fdb_def_rule, 9451 standard_translation ? "STANDARD" : "TUNNEL"); 9452 if (standard_translation) 9453 ret = flow_group_to_table(dev->data->port_id, group, table, 9454 grp_info, error); 9455 else 9456 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9457 table, error); 9458 9459 return ret; 9460 } 9461 9462 /** 9463 * Discover availability of metadata reg_c's. 9464 * 9465 * Iteratively use test flows to check availability. 9466 * 9467 * @param[in] dev 9468 * Pointer to the Ethernet device structure. 9469 * 9470 * @return 9471 * 0 on success, a negative errno value otherwise and rte_errno is set. 9472 */ 9473 int 9474 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9475 { 9476 struct mlx5_priv *priv = dev->data->dev_private; 9477 enum modify_reg idx; 9478 int n = 0; 9479 9480 /* reg_c[0] and reg_c[1] are reserved. */ 9481 priv->sh->flow_mreg_c[n++] = REG_C_0; 9482 priv->sh->flow_mreg_c[n++] = REG_C_1; 9483 /* Discover availability of other reg_c's. */ 9484 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9485 struct rte_flow_attr attr = { 9486 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9487 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9488 .ingress = 1, 9489 }; 9490 struct rte_flow_item items[] = { 9491 [0] = { 9492 .type = RTE_FLOW_ITEM_TYPE_END, 9493 }, 9494 }; 9495 struct rte_flow_action actions[] = { 9496 [0] = { 9497 .type = (enum rte_flow_action_type) 9498 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9499 .conf = &(struct mlx5_flow_action_copy_mreg){ 9500 .src = REG_C_1, 9501 .dst = idx, 9502 }, 9503 }, 9504 [1] = { 9505 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9506 .conf = &(struct rte_flow_action_jump){ 9507 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9508 }, 9509 }, 9510 [2] = { 9511 .type = RTE_FLOW_ACTION_TYPE_END, 9512 }, 9513 }; 9514 uint32_t flow_idx; 9515 struct rte_flow *flow; 9516 struct rte_flow_error error; 9517 9518 if (!priv->sh->config.dv_flow_en) 9519 break; 9520 /* Create internal flow, validation skips copy action. */ 9521 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9522 items, actions, false, &error); 9523 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9524 flow_idx); 9525 if (!flow) 9526 continue; 9527 priv->sh->flow_mreg_c[n++] = idx; 9528 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9529 } 9530 for (; n < MLX5_MREG_C_NUM; ++n) 9531 priv->sh->flow_mreg_c[n] = REG_NON; 9532 priv->sh->metadata_regc_check_flag = 1; 9533 return 0; 9534 } 9535 9536 int 9537 save_dump_file(const uint8_t *data, uint32_t size, 9538 uint32_t type, uint64_t id, void *arg, FILE *file) 9539 { 9540 char line[BUF_SIZE]; 9541 uint32_t out = 0; 9542 uint32_t k; 9543 uint32_t actions_num; 9544 struct rte_flow_query_count *count; 9545 9546 memset(line, 0, BUF_SIZE); 9547 switch (type) { 9548 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9549 actions_num = *(uint32_t *)(arg); 9550 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9551 type, id, actions_num); 9552 break; 9553 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9554 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9555 type, id); 9556 break; 9557 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9558 count = (struct rte_flow_query_count *)arg; 9559 fprintf(file, 9560 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9561 type, id, count->hits, count->bytes); 9562 return 0; 9563 default: 9564 return -1; 9565 } 9566 9567 for (k = 0; k < size; k++) { 9568 /* Make sure we do not overrun the line buffer length. */ 9569 if (out >= BUF_SIZE - 4) { 9570 line[out] = '\0'; 9571 break; 9572 } 9573 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9574 (data[k]) & 0xff); 9575 } 9576 fprintf(file, "%s\n", line); 9577 return 0; 9578 } 9579 9580 int 9581 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9582 struct rte_flow_query_count *count, struct rte_flow_error *error) 9583 { 9584 struct rte_flow_action action[2]; 9585 enum mlx5_flow_drv_type ftype; 9586 const struct mlx5_flow_driver_ops *fops; 9587 9588 if (!flow) { 9589 return rte_flow_error_set(error, ENOENT, 9590 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9591 NULL, 9592 "invalid flow handle"); 9593 } 9594 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9595 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9596 if (flow->counter) { 9597 memset(count, 0, sizeof(struct rte_flow_query_count)); 9598 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9599 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9600 ftype < MLX5_FLOW_TYPE_MAX); 9601 fops = flow_get_drv_ops(ftype); 9602 return fops->query(dev, flow, action, count, error); 9603 } 9604 return -1; 9605 } 9606 9607 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9608 /** 9609 * Dump flow ipool data to file 9610 * 9611 * @param[in] dev 9612 * The pointer to Ethernet device. 9613 * @param[in] file 9614 * A pointer to a file for output. 9615 * @param[out] error 9616 * Perform verbose error reporting if not NULL. PMDs initialize this 9617 * structure in case of error only. 9618 * @return 9619 * 0 on success, a negative value otherwise. 9620 */ 9621 int 9622 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9623 struct rte_flow *flow, FILE *file, 9624 struct rte_flow_error *error) 9625 { 9626 struct mlx5_priv *priv = dev->data->dev_private; 9627 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9628 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9629 uint32_t handle_idx; 9630 struct mlx5_flow_handle *dh; 9631 struct rte_flow_query_count count; 9632 uint32_t actions_num; 9633 const uint8_t *data; 9634 size_t size; 9635 uint64_t id; 9636 uint32_t type; 9637 void *action = NULL; 9638 9639 if (!flow) { 9640 return rte_flow_error_set(error, ENOENT, 9641 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9642 NULL, 9643 "invalid flow handle"); 9644 } 9645 handle_idx = flow->dev_handles; 9646 /* query counter */ 9647 if (flow->counter && 9648 (!mlx5_counter_query(dev, flow->counter, false, 9649 &count.hits, &count.bytes, &action)) && action) { 9650 id = (uint64_t)(uintptr_t)action; 9651 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9652 save_dump_file(NULL, 0, type, 9653 id, (void *)&count, file); 9654 } 9655 9656 while (handle_idx) { 9657 dh = mlx5_ipool_get(priv->sh->ipool 9658 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9659 if (!dh) 9660 continue; 9661 handle_idx = dh->next.next; 9662 9663 /* Get modify_hdr and encap_decap buf from ipools. */ 9664 encap_decap = NULL; 9665 modify_hdr = dh->dvh.modify_hdr; 9666 9667 if (dh->dvh.rix_encap_decap) { 9668 encap_decap = mlx5_ipool_get(priv->sh->ipool 9669 [MLX5_IPOOL_DECAP_ENCAP], 9670 dh->dvh.rix_encap_decap); 9671 } 9672 if (modify_hdr) { 9673 data = (const uint8_t *)modify_hdr->actions; 9674 size = (size_t)(modify_hdr->actions_num) * 8; 9675 id = (uint64_t)(uintptr_t)modify_hdr->action; 9676 actions_num = modify_hdr->actions_num; 9677 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9678 save_dump_file(data, size, type, id, 9679 (void *)(&actions_num), file); 9680 } 9681 if (encap_decap) { 9682 data = encap_decap->buf; 9683 size = encap_decap->size; 9684 id = (uint64_t)(uintptr_t)encap_decap->action; 9685 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9686 save_dump_file(data, size, type, 9687 id, NULL, file); 9688 } 9689 } 9690 return 0; 9691 } 9692 9693 /** 9694 * Dump all flow's encap_decap/modify_hdr/counter data to file 9695 * 9696 * @param[in] dev 9697 * The pointer to Ethernet device. 9698 * @param[in] file 9699 * A pointer to a file for output. 9700 * @param[out] error 9701 * Perform verbose error reporting if not NULL. PMDs initialize this 9702 * structure in case of error only. 9703 * @return 9704 * 0 on success, a negative value otherwise. 9705 */ 9706 static int 9707 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9708 FILE *file, struct rte_flow_error *error __rte_unused) 9709 { 9710 struct mlx5_priv *priv = dev->data->dev_private; 9711 struct mlx5_dev_ctx_shared *sh = priv->sh; 9712 struct mlx5_hlist *h; 9713 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9714 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9715 struct rte_flow_query_count count; 9716 uint32_t actions_num; 9717 const uint8_t *data; 9718 size_t size; 9719 uint64_t id; 9720 uint32_t type; 9721 uint32_t i; 9722 uint32_t j; 9723 struct mlx5_list_inconst *l_inconst; 9724 struct mlx5_list_entry *e; 9725 int lcore_index; 9726 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9727 uint32_t max; 9728 void *action; 9729 9730 /* encap_decap hlist is lcore_share, get global core cache. */ 9731 i = MLX5_LIST_GLOBAL; 9732 h = sh->encaps_decaps; 9733 if (h) { 9734 for (j = 0; j <= h->mask; j++) { 9735 l_inconst = &h->buckets[j].l; 9736 if (!l_inconst || !l_inconst->cache[i]) 9737 continue; 9738 9739 e = LIST_FIRST(&l_inconst->cache[i]->h); 9740 while (e) { 9741 encap_decap = 9742 (struct mlx5_flow_dv_encap_decap_resource *)e; 9743 data = encap_decap->buf; 9744 size = encap_decap->size; 9745 id = (uint64_t)(uintptr_t)encap_decap->action; 9746 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9747 save_dump_file(data, size, type, 9748 id, NULL, file); 9749 e = LIST_NEXT(e, next); 9750 } 9751 } 9752 } 9753 9754 /* get modify_hdr */ 9755 h = sh->modify_cmds; 9756 if (h) { 9757 lcore_index = rte_lcore_index(rte_lcore_id()); 9758 if (unlikely(lcore_index == -1)) { 9759 lcore_index = MLX5_LIST_NLCORE; 9760 rte_spinlock_lock(&h->l_const.lcore_lock); 9761 } 9762 i = lcore_index; 9763 9764 for (j = 0; j <= h->mask; j++) { 9765 l_inconst = &h->buckets[j].l; 9766 if (!l_inconst || !l_inconst->cache[i]) 9767 continue; 9768 9769 e = LIST_FIRST(&l_inconst->cache[i]->h); 9770 while (e) { 9771 modify_hdr = 9772 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9773 data = (const uint8_t *)modify_hdr->actions; 9774 size = (size_t)(modify_hdr->actions_num) * 8; 9775 actions_num = modify_hdr->actions_num; 9776 id = (uint64_t)(uintptr_t)modify_hdr->action; 9777 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9778 save_dump_file(data, size, type, id, 9779 (void *)(&actions_num), file); 9780 e = LIST_NEXT(e, next); 9781 } 9782 } 9783 9784 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9785 rte_spinlock_unlock(&h->l_const.lcore_lock); 9786 } 9787 9788 /* get counter */ 9789 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 9790 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9791 for (j = 1; j <= max; j++) { 9792 action = NULL; 9793 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9794 &count.bytes, &action)) && action) { 9795 id = (uint64_t)(uintptr_t)action; 9796 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9797 save_dump_file(NULL, 0, type, 9798 id, (void *)&count, file); 9799 } 9800 } 9801 return 0; 9802 } 9803 #endif 9804 9805 /** 9806 * Dump flow raw hw data to file 9807 * 9808 * @param[in] dev 9809 * The pointer to Ethernet device. 9810 * @param[in] file 9811 * A pointer to a file for output. 9812 * @param[out] error 9813 * Perform verbose error reporting if not NULL. PMDs initialize this 9814 * structure in case of error only. 9815 * @return 9816 * 0 on success, a negative value otherwise. 9817 */ 9818 int 9819 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9820 FILE *file, 9821 struct rte_flow_error *error __rte_unused) 9822 { 9823 struct mlx5_priv *priv = dev->data->dev_private; 9824 struct mlx5_dev_ctx_shared *sh = priv->sh; 9825 uint32_t handle_idx; 9826 int ret; 9827 struct mlx5_flow_handle *dh; 9828 struct rte_flow *flow; 9829 9830 if (!sh->config.dv_flow_en) { 9831 if (fputs("device dv flow disabled\n", file) <= 0) 9832 return -errno; 9833 return -ENOTSUP; 9834 } 9835 9836 /* dump all */ 9837 if (!flow_idx) { 9838 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9839 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9840 return -EINVAL; 9841 #endif 9842 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9843 sh->rx_domain, 9844 sh->tx_domain, file); 9845 } 9846 /* dump one */ 9847 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9848 (uintptr_t)(void *)flow_idx); 9849 if (!flow) 9850 return -EINVAL; 9851 9852 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9853 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9854 #endif 9855 handle_idx = flow->dev_handles; 9856 while (handle_idx) { 9857 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9858 handle_idx); 9859 if (!dh) 9860 return -ENOENT; 9861 if (dh->drv_flow) { 9862 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9863 file); 9864 if (ret) 9865 return -ENOENT; 9866 } 9867 handle_idx = dh->next.next; 9868 } 9869 return 0; 9870 } 9871 9872 /** 9873 * Get aged-out flows. 9874 * 9875 * @param[in] dev 9876 * Pointer to the Ethernet device structure. 9877 * @param[in] context 9878 * The address of an array of pointers to the aged-out flows contexts. 9879 * @param[in] nb_countexts 9880 * The length of context array pointers. 9881 * @param[out] error 9882 * Perform verbose error reporting if not NULL. Initialized in case of 9883 * error only. 9884 * 9885 * @return 9886 * how many contexts get in success, otherwise negative errno value. 9887 * if nb_contexts is 0, return the amount of all aged contexts. 9888 * if nb_contexts is not 0 , return the amount of aged flows reported 9889 * in the context array. 9890 */ 9891 int 9892 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9893 uint32_t nb_contexts, struct rte_flow_error *error) 9894 { 9895 const struct mlx5_flow_driver_ops *fops; 9896 struct rte_flow_attr attr = { .transfer = 0 }; 9897 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, &attr); 9898 9899 if (type == MLX5_FLOW_TYPE_DV || type == MLX5_FLOW_TYPE_HW) { 9900 fops = flow_get_drv_ops(type); 9901 return fops->get_aged_flows(dev, contexts, nb_contexts, error); 9902 } 9903 DRV_LOG(ERR, "port %u get aged flows is not supported.", 9904 dev->data->port_id); 9905 return -ENOTSUP; 9906 } 9907 9908 /** 9909 * Get aged-out flows per HWS queue. 9910 * 9911 * @param[in] dev 9912 * Pointer to the Ethernet device structure. 9913 * @param[in] queue_id 9914 * Flow queue to query. 9915 * @param[in] context 9916 * The address of an array of pointers to the aged-out flows contexts. 9917 * @param[in] nb_countexts 9918 * The length of context array pointers. 9919 * @param[out] error 9920 * Perform verbose error reporting if not NULL. Initialized in case of 9921 * error only. 9922 * 9923 * @return 9924 * how many contexts get in success, otherwise negative errno value. 9925 * if nb_contexts is 0, return the amount of all aged contexts. 9926 * if nb_contexts is not 0 , return the amount of aged flows reported 9927 * in the context array. 9928 */ 9929 int 9930 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 9931 void **contexts, uint32_t nb_contexts, 9932 struct rte_flow_error *error) 9933 { 9934 const struct mlx5_flow_driver_ops *fops; 9935 struct rte_flow_attr attr = { 0 }; 9936 9937 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 9938 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9939 return fops->get_q_aged_flows(dev, queue_id, contexts, 9940 nb_contexts, error); 9941 } 9942 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 9943 dev->data->port_id, queue_id); 9944 return rte_flow_error_set(error, ENOTSUP, 9945 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9946 "get Q aged flows with incorrect steering mode"); 9947 } 9948 9949 /* Wrapper for driver action_validate op callback */ 9950 static int 9951 flow_drv_action_validate(struct rte_eth_dev *dev, 9952 const struct rte_flow_indir_action_conf *conf, 9953 const struct rte_flow_action *action, 9954 const struct mlx5_flow_driver_ops *fops, 9955 struct rte_flow_error *error) 9956 { 9957 static const char err_msg[] = "indirect action validation unsupported"; 9958 9959 if (!fops->action_validate) { 9960 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9961 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9962 NULL, err_msg); 9963 return -rte_errno; 9964 } 9965 return fops->action_validate(dev, conf, action, error); 9966 } 9967 9968 /** 9969 * Destroys the shared action by handle. 9970 * 9971 * @param dev 9972 * Pointer to Ethernet device structure. 9973 * @param[in] handle 9974 * Handle for the indirect action object to be destroyed. 9975 * @param[out] error 9976 * Perform verbose error reporting if not NULL. PMDs initialize this 9977 * structure in case of error only. 9978 * 9979 * @return 9980 * 0 on success, a negative errno value otherwise and rte_errno is set. 9981 * 9982 * @note: wrapper for driver action_create op callback. 9983 */ 9984 static int 9985 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9986 struct rte_flow_action_handle *handle, 9987 struct rte_flow_error *error) 9988 { 9989 static const char err_msg[] = "indirect action destruction unsupported"; 9990 struct rte_flow_attr attr = { .transfer = 0 }; 9991 const struct mlx5_flow_driver_ops *fops = 9992 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9993 9994 if (!fops->action_destroy) { 9995 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9996 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9997 NULL, err_msg); 9998 return -rte_errno; 9999 } 10000 return fops->action_destroy(dev, handle, error); 10001 } 10002 10003 /* Wrapper for driver action_destroy op callback */ 10004 static int 10005 flow_drv_action_update(struct rte_eth_dev *dev, 10006 struct rte_flow_action_handle *handle, 10007 const void *update, 10008 const struct mlx5_flow_driver_ops *fops, 10009 struct rte_flow_error *error) 10010 { 10011 static const char err_msg[] = "indirect action update unsupported"; 10012 10013 if (!fops->action_update) { 10014 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10015 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10016 NULL, err_msg); 10017 return -rte_errno; 10018 } 10019 return fops->action_update(dev, handle, update, error); 10020 } 10021 10022 /* Wrapper for driver action_destroy op callback */ 10023 static int 10024 flow_drv_action_query(struct rte_eth_dev *dev, 10025 const struct rte_flow_action_handle *handle, 10026 void *data, 10027 const struct mlx5_flow_driver_ops *fops, 10028 struct rte_flow_error *error) 10029 { 10030 static const char err_msg[] = "indirect action query unsupported"; 10031 10032 if (!fops->action_query) { 10033 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10034 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10035 NULL, err_msg); 10036 return -rte_errno; 10037 } 10038 return fops->action_query(dev, handle, data, error); 10039 } 10040 10041 /** 10042 * Create indirect action for reuse in multiple flow rules. 10043 * 10044 * @param dev 10045 * Pointer to Ethernet device structure. 10046 * @param conf 10047 * Pointer to indirect action object configuration. 10048 * @param[in] action 10049 * Action configuration for indirect action object creation. 10050 * @param[out] error 10051 * Perform verbose error reporting if not NULL. PMDs initialize this 10052 * structure in case of error only. 10053 * @return 10054 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10055 */ 10056 static struct rte_flow_action_handle * 10057 mlx5_action_handle_create(struct rte_eth_dev *dev, 10058 const struct rte_flow_indir_action_conf *conf, 10059 const struct rte_flow_action *action, 10060 struct rte_flow_error *error) 10061 { 10062 static const char err_msg[] = "indirect action creation unsupported"; 10063 struct rte_flow_attr attr = { .transfer = 0 }; 10064 const struct mlx5_flow_driver_ops *fops = 10065 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10066 10067 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10068 return NULL; 10069 if (!fops->action_create) { 10070 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10071 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10072 NULL, err_msg); 10073 return NULL; 10074 } 10075 return fops->action_create(dev, conf, action, error); 10076 } 10077 10078 /** 10079 * Updates inplace the indirect action configuration pointed by *handle* 10080 * with the configuration provided as *update* argument. 10081 * The update of the indirect action configuration effects all flow rules 10082 * reusing the action via handle. 10083 * 10084 * @param dev 10085 * Pointer to Ethernet device structure. 10086 * @param[in] handle 10087 * Handle for the indirect action to be updated. 10088 * @param[in] update 10089 * Action specification used to modify the action pointed by handle. 10090 * *update* could be of same type with the action pointed by the *handle* 10091 * handle argument, or some other structures like a wrapper, depending on 10092 * the indirect action type. 10093 * @param[out] error 10094 * Perform verbose error reporting if not NULL. PMDs initialize this 10095 * structure in case of error only. 10096 * 10097 * @return 10098 * 0 on success, a negative errno value otherwise and rte_errno is set. 10099 */ 10100 static int 10101 mlx5_action_handle_update(struct rte_eth_dev *dev, 10102 struct rte_flow_action_handle *handle, 10103 const void *update, 10104 struct rte_flow_error *error) 10105 { 10106 struct rte_flow_attr attr = { .transfer = 0 }; 10107 const struct mlx5_flow_driver_ops *fops = 10108 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10109 int ret; 10110 10111 ret = flow_drv_action_validate(dev, NULL, 10112 (const struct rte_flow_action *)update, fops, error); 10113 if (ret) 10114 return ret; 10115 return flow_drv_action_update(dev, handle, update, fops, 10116 error); 10117 } 10118 10119 /** 10120 * Query the indirect action by handle. 10121 * 10122 * This function allows retrieving action-specific data such as counters. 10123 * Data is gathered by special action which may be present/referenced in 10124 * more than one flow rule definition. 10125 * 10126 * see @RTE_FLOW_ACTION_TYPE_COUNT 10127 * 10128 * @param dev 10129 * Pointer to Ethernet device structure. 10130 * @param[in] handle 10131 * Handle for the indirect action to query. 10132 * @param[in, out] data 10133 * Pointer to storage for the associated query data type. 10134 * @param[out] error 10135 * Perform verbose error reporting if not NULL. PMDs initialize this 10136 * structure in case of error only. 10137 * 10138 * @return 10139 * 0 on success, a negative errno value otherwise and rte_errno is set. 10140 */ 10141 static int 10142 mlx5_action_handle_query(struct rte_eth_dev *dev, 10143 const struct rte_flow_action_handle *handle, 10144 void *data, 10145 struct rte_flow_error *error) 10146 { 10147 struct rte_flow_attr attr = { .transfer = 0 }; 10148 const struct mlx5_flow_driver_ops *fops = 10149 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10150 10151 return flow_drv_action_query(dev, handle, data, fops, error); 10152 } 10153 10154 /** 10155 * Destroy all indirect actions (shared RSS). 10156 * 10157 * @param dev 10158 * Pointer to Ethernet device. 10159 * 10160 * @return 10161 * 0 on success, a negative errno value otherwise and rte_errno is set. 10162 */ 10163 int 10164 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10165 { 10166 struct rte_flow_error error; 10167 struct mlx5_priv *priv = dev->data->dev_private; 10168 struct mlx5_shared_action_rss *shared_rss; 10169 int ret = 0; 10170 uint32_t idx; 10171 10172 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10173 priv->rss_shared_actions, idx, shared_rss, next) { 10174 ret |= mlx5_action_handle_destroy(dev, 10175 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10176 } 10177 return ret; 10178 } 10179 10180 /** 10181 * Validate existing indirect actions against current device configuration 10182 * and attach them to device resources. 10183 * 10184 * @param dev 10185 * Pointer to Ethernet device. 10186 * 10187 * @return 10188 * 0 on success, a negative errno value otherwise and rte_errno is set. 10189 */ 10190 int 10191 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10192 { 10193 struct mlx5_priv *priv = dev->data->dev_private; 10194 int ret = 0; 10195 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10196 10197 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10198 const char *message; 10199 uint32_t queue_idx; 10200 10201 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10202 ind_tbl->queues_n, 10203 &message, &queue_idx); 10204 if (ret != 0) { 10205 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10206 dev->data->port_id, ind_tbl->queues[queue_idx], 10207 message); 10208 break; 10209 } 10210 } 10211 if (ret != 0) 10212 return ret; 10213 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10214 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10215 if (ret != 0) { 10216 DRV_LOG(ERR, "Port %u could not attach " 10217 "indirection table obj %p", 10218 dev->data->port_id, (void *)ind_tbl); 10219 goto error; 10220 } 10221 } 10222 10223 return 0; 10224 error: 10225 ind_tbl_last = ind_tbl; 10226 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10227 if (ind_tbl == ind_tbl_last) 10228 break; 10229 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10230 DRV_LOG(CRIT, "Port %u could not detach " 10231 "indirection table obj %p on rollback", 10232 dev->data->port_id, (void *)ind_tbl); 10233 } 10234 return ret; 10235 } 10236 10237 /** 10238 * Detach indirect actions of the device from its resources. 10239 * 10240 * @param dev 10241 * Pointer to Ethernet device. 10242 * 10243 * @return 10244 * 0 on success, a negative errno value otherwise and rte_errno is set. 10245 */ 10246 int 10247 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10248 { 10249 struct mlx5_priv *priv = dev->data->dev_private; 10250 int ret = 0; 10251 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10252 10253 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10254 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10255 if (ret != 0) { 10256 DRV_LOG(ERR, "Port %u could not detach " 10257 "indirection table obj %p", 10258 dev->data->port_id, (void *)ind_tbl); 10259 goto error; 10260 } 10261 } 10262 return 0; 10263 error: 10264 ind_tbl_last = ind_tbl; 10265 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10266 if (ind_tbl == ind_tbl_last) 10267 break; 10268 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10269 DRV_LOG(CRIT, "Port %u could not attach " 10270 "indirection table obj %p on rollback", 10271 dev->data->port_id, (void *)ind_tbl); 10272 } 10273 return ret; 10274 } 10275 10276 #ifndef HAVE_MLX5DV_DR 10277 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10278 #else 10279 #define MLX5_DOMAIN_SYNC_FLOW \ 10280 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10281 #endif 10282 10283 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10284 { 10285 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10286 const struct mlx5_flow_driver_ops *fops; 10287 int ret; 10288 struct rte_flow_attr attr = { .transfer = 0 }; 10289 10290 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10291 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10292 if (ret > 0) 10293 ret = -ret; 10294 return ret; 10295 } 10296 10297 const struct mlx5_flow_tunnel * 10298 mlx5_get_tof(const struct rte_flow_item *item, 10299 const struct rte_flow_action *action, 10300 enum mlx5_tof_rule_type *rule_type) 10301 { 10302 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10303 if (item->type == (typeof(item->type)) 10304 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10305 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10306 return flow_items_to_tunnel(item); 10307 } 10308 } 10309 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10310 if (action->type == (typeof(action->type)) 10311 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10312 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10313 return flow_actions_to_tunnel(action); 10314 } 10315 } 10316 return NULL; 10317 } 10318 10319 /** 10320 * tunnel offload functionality is defined for DV environment only 10321 */ 10322 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10323 __extension__ 10324 union tunnel_offload_mark { 10325 uint32_t val; 10326 struct { 10327 uint32_t app_reserve:8; 10328 uint32_t table_id:15; 10329 uint32_t transfer:1; 10330 uint32_t _unused_:8; 10331 }; 10332 }; 10333 10334 static bool 10335 mlx5_access_tunnel_offload_db 10336 (struct rte_eth_dev *dev, 10337 bool (*match)(struct rte_eth_dev *, 10338 struct mlx5_flow_tunnel *, const void *), 10339 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10340 void (*miss)(struct rte_eth_dev *, void *), 10341 void *ctx, bool lock_op); 10342 10343 static int 10344 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10345 struct rte_flow *flow, 10346 const struct rte_flow_attr *attr, 10347 const struct rte_flow_action *app_actions, 10348 uint32_t flow_idx, 10349 const struct mlx5_flow_tunnel *tunnel, 10350 struct tunnel_default_miss_ctx *ctx, 10351 struct rte_flow_error *error) 10352 { 10353 struct mlx5_priv *priv = dev->data->dev_private; 10354 struct mlx5_flow *dev_flow; 10355 struct rte_flow_attr miss_attr = *attr; 10356 const struct rte_flow_item miss_items[2] = { 10357 { 10358 .type = RTE_FLOW_ITEM_TYPE_ETH, 10359 .spec = NULL, 10360 .last = NULL, 10361 .mask = NULL 10362 }, 10363 { 10364 .type = RTE_FLOW_ITEM_TYPE_END, 10365 .spec = NULL, 10366 .last = NULL, 10367 .mask = NULL 10368 } 10369 }; 10370 union tunnel_offload_mark mark_id; 10371 struct rte_flow_action_mark miss_mark; 10372 struct rte_flow_action miss_actions[3] = { 10373 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10374 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10375 }; 10376 const struct rte_flow_action_jump *jump_data; 10377 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10378 struct flow_grp_info grp_info = { 10379 .external = 1, 10380 .transfer = attr->transfer, 10381 .fdb_def_rule = !!priv->fdb_def_rule, 10382 .std_tbl_fix = 0, 10383 }; 10384 int ret; 10385 10386 if (!attr->transfer) { 10387 uint32_t q_size; 10388 10389 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10390 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10391 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10392 0, SOCKET_ID_ANY); 10393 if (!ctx->queue) 10394 return rte_flow_error_set 10395 (error, ENOMEM, 10396 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10397 NULL, "invalid default miss RSS"); 10398 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10399 ctx->action_rss.level = 0, 10400 ctx->action_rss.types = priv->rss_conf.rss_hf, 10401 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10402 ctx->action_rss.queue_num = priv->reta_idx_n, 10403 ctx->action_rss.key = priv->rss_conf.rss_key, 10404 ctx->action_rss.queue = ctx->queue; 10405 if (!priv->reta_idx_n || !priv->rxqs_n) 10406 return rte_flow_error_set 10407 (error, EINVAL, 10408 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10409 NULL, "invalid port configuration"); 10410 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10411 ctx->action_rss.types = 0; 10412 for (i = 0; i != priv->reta_idx_n; ++i) 10413 ctx->queue[i] = (*priv->reta_idx)[i]; 10414 } else { 10415 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10416 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10417 } 10418 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10419 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10420 jump_data = app_actions->conf; 10421 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10422 miss_attr.group = jump_data->group; 10423 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10424 &flow_table, &grp_info, error); 10425 if (ret) 10426 return rte_flow_error_set(error, EINVAL, 10427 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10428 NULL, "invalid tunnel id"); 10429 mark_id.app_reserve = 0; 10430 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10431 mark_id.transfer = !!attr->transfer; 10432 mark_id._unused_ = 0; 10433 miss_mark.id = mark_id.val; 10434 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10435 miss_items, miss_actions, flow_idx, error); 10436 if (!dev_flow) 10437 return -rte_errno; 10438 dev_flow->flow = flow; 10439 dev_flow->external = true; 10440 dev_flow->tunnel = tunnel; 10441 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10442 /* Subflow object was created, we must include one in the list. */ 10443 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10444 dev_flow->handle, next); 10445 DRV_LOG(DEBUG, 10446 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10447 dev->data->port_id, tunnel->app_tunnel.type, 10448 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10449 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10450 miss_actions, error); 10451 if (!ret) 10452 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10453 error); 10454 10455 return ret; 10456 } 10457 10458 static const struct mlx5_flow_tbl_data_entry * 10459 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10460 { 10461 struct mlx5_priv *priv = dev->data->dev_private; 10462 struct mlx5_dev_ctx_shared *sh = priv->sh; 10463 struct mlx5_list_entry *he; 10464 union tunnel_offload_mark mbits = { .val = mark }; 10465 union mlx5_flow_tbl_key table_key = { 10466 { 10467 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10468 .id = 0, 10469 .reserved = 0, 10470 .dummy = 0, 10471 .is_fdb = !!mbits.transfer, 10472 .is_egress = 0, 10473 } 10474 }; 10475 struct mlx5_flow_cb_ctx ctx = { 10476 .data = &table_key.v64, 10477 }; 10478 10479 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10480 return he ? 10481 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10482 } 10483 10484 static void 10485 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10486 struct mlx5_list_entry *entry) 10487 { 10488 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10489 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10490 10491 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10492 tunnel_flow_tbl_to_id(tte->flow_table)); 10493 mlx5_free(tte); 10494 } 10495 10496 static int 10497 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10498 struct mlx5_list_entry *entry, void *cb_ctx) 10499 { 10500 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10501 union tunnel_tbl_key tbl = { 10502 .val = *(uint64_t *)(ctx->data), 10503 }; 10504 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10505 10506 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10507 } 10508 10509 static struct mlx5_list_entry * 10510 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10511 { 10512 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10513 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10514 struct tunnel_tbl_entry *tte; 10515 union tunnel_tbl_key tbl = { 10516 .val = *(uint64_t *)(ctx->data), 10517 }; 10518 10519 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10520 sizeof(*tte), 0, 10521 SOCKET_ID_ANY); 10522 if (!tte) 10523 goto err; 10524 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10525 &tte->flow_table); 10526 if (tte->flow_table >= MLX5_MAX_TABLES) { 10527 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10528 tte->flow_table); 10529 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10530 tte->flow_table); 10531 goto err; 10532 } else if (!tte->flow_table) { 10533 goto err; 10534 } 10535 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10536 tte->tunnel_id = tbl.tunnel_id; 10537 tte->group = tbl.group; 10538 return &tte->hash; 10539 err: 10540 if (tte) 10541 mlx5_free(tte); 10542 return NULL; 10543 } 10544 10545 static struct mlx5_list_entry * 10546 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10547 struct mlx5_list_entry *oentry, 10548 void *cb_ctx __rte_unused) 10549 { 10550 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10551 0, SOCKET_ID_ANY); 10552 10553 if (!tte) 10554 return NULL; 10555 memcpy(tte, oentry, sizeof(*tte)); 10556 return &tte->hash; 10557 } 10558 10559 static void 10560 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10561 struct mlx5_list_entry *entry) 10562 { 10563 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10564 10565 mlx5_free(tte); 10566 } 10567 10568 static uint32_t 10569 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10570 const struct mlx5_flow_tunnel *tunnel, 10571 uint32_t group, uint32_t *table, 10572 struct rte_flow_error *error) 10573 { 10574 struct mlx5_list_entry *he; 10575 struct tunnel_tbl_entry *tte; 10576 union tunnel_tbl_key key = { 10577 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10578 .group = group 10579 }; 10580 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10581 struct mlx5_hlist *group_hash; 10582 struct mlx5_flow_cb_ctx ctx = { 10583 .data = &key.val, 10584 }; 10585 10586 group_hash = tunnel ? tunnel->groups : thub->groups; 10587 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10588 if (!he) 10589 return rte_flow_error_set(error, EINVAL, 10590 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10591 NULL, 10592 "tunnel group index not supported"); 10593 tte = container_of(he, typeof(*tte), hash); 10594 *table = tte->flow_table; 10595 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10596 dev->data->port_id, key.tunnel_id, group, *table); 10597 return 0; 10598 } 10599 10600 static void 10601 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10602 struct mlx5_flow_tunnel *tunnel) 10603 { 10604 struct mlx5_priv *priv = dev->data->dev_private; 10605 struct mlx5_indexed_pool *ipool; 10606 10607 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10608 dev->data->port_id, tunnel->tunnel_id); 10609 LIST_REMOVE(tunnel, chain); 10610 mlx5_hlist_destroy(tunnel->groups); 10611 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10612 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10613 } 10614 10615 static bool 10616 mlx5_access_tunnel_offload_db 10617 (struct rte_eth_dev *dev, 10618 bool (*match)(struct rte_eth_dev *, 10619 struct mlx5_flow_tunnel *, const void *), 10620 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10621 void (*miss)(struct rte_eth_dev *, void *), 10622 void *ctx, bool lock_op) 10623 { 10624 bool verdict = false; 10625 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10626 struct mlx5_flow_tunnel *tunnel; 10627 10628 rte_spinlock_lock(&thub->sl); 10629 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10630 verdict = match(dev, tunnel, (const void *)ctx); 10631 if (verdict) 10632 break; 10633 } 10634 if (!lock_op) 10635 rte_spinlock_unlock(&thub->sl); 10636 if (verdict && hit) 10637 hit(dev, tunnel, ctx); 10638 if (!verdict && miss) 10639 miss(dev, ctx); 10640 if (lock_op) 10641 rte_spinlock_unlock(&thub->sl); 10642 10643 return verdict; 10644 } 10645 10646 struct tunnel_db_find_tunnel_id_ctx { 10647 uint32_t tunnel_id; 10648 struct mlx5_flow_tunnel *tunnel; 10649 }; 10650 10651 static bool 10652 find_tunnel_id_match(struct rte_eth_dev *dev, 10653 struct mlx5_flow_tunnel *tunnel, const void *x) 10654 { 10655 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10656 10657 RTE_SET_USED(dev); 10658 return tunnel->tunnel_id == ctx->tunnel_id; 10659 } 10660 10661 static void 10662 find_tunnel_id_hit(struct rte_eth_dev *dev, 10663 struct mlx5_flow_tunnel *tunnel, void *x) 10664 { 10665 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10666 RTE_SET_USED(dev); 10667 ctx->tunnel = tunnel; 10668 } 10669 10670 static struct mlx5_flow_tunnel * 10671 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10672 { 10673 struct tunnel_db_find_tunnel_id_ctx ctx = { 10674 .tunnel_id = id, 10675 }; 10676 10677 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10678 find_tunnel_id_hit, NULL, &ctx, true); 10679 10680 return ctx.tunnel; 10681 } 10682 10683 static struct mlx5_flow_tunnel * 10684 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10685 const struct rte_flow_tunnel *app_tunnel) 10686 { 10687 struct mlx5_priv *priv = dev->data->dev_private; 10688 struct mlx5_indexed_pool *ipool; 10689 struct mlx5_flow_tunnel *tunnel; 10690 uint32_t id; 10691 10692 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10693 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10694 if (!tunnel) 10695 return NULL; 10696 if (id >= MLX5_MAX_TUNNELS) { 10697 mlx5_ipool_free(ipool, id); 10698 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10699 return NULL; 10700 } 10701 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10702 priv->sh, 10703 mlx5_flow_tunnel_grp2tbl_create_cb, 10704 mlx5_flow_tunnel_grp2tbl_match_cb, 10705 mlx5_flow_tunnel_grp2tbl_remove_cb, 10706 mlx5_flow_tunnel_grp2tbl_clone_cb, 10707 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10708 if (!tunnel->groups) { 10709 mlx5_ipool_free(ipool, id); 10710 return NULL; 10711 } 10712 /* initiate new PMD tunnel */ 10713 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10714 tunnel->tunnel_id = id; 10715 tunnel->action.type = (typeof(tunnel->action.type)) 10716 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10717 tunnel->action.conf = tunnel; 10718 tunnel->item.type = (typeof(tunnel->item.type)) 10719 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10720 tunnel->item.spec = tunnel; 10721 tunnel->item.last = NULL; 10722 tunnel->item.mask = NULL; 10723 10724 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10725 dev->data->port_id, tunnel->tunnel_id); 10726 10727 return tunnel; 10728 } 10729 10730 struct tunnel_db_get_tunnel_ctx { 10731 const struct rte_flow_tunnel *app_tunnel; 10732 struct mlx5_flow_tunnel *tunnel; 10733 }; 10734 10735 static bool get_tunnel_match(struct rte_eth_dev *dev, 10736 struct mlx5_flow_tunnel *tunnel, const void *x) 10737 { 10738 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10739 10740 RTE_SET_USED(dev); 10741 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10742 sizeof(*ctx->app_tunnel)); 10743 } 10744 10745 static void get_tunnel_hit(struct rte_eth_dev *dev, 10746 struct mlx5_flow_tunnel *tunnel, void *x) 10747 { 10748 /* called under tunnel spinlock protection */ 10749 struct tunnel_db_get_tunnel_ctx *ctx = x; 10750 10751 RTE_SET_USED(dev); 10752 tunnel->refctn++; 10753 ctx->tunnel = tunnel; 10754 } 10755 10756 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10757 { 10758 /* called under tunnel spinlock protection */ 10759 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10760 struct tunnel_db_get_tunnel_ctx *ctx = x; 10761 10762 rte_spinlock_unlock(&thub->sl); 10763 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10764 rte_spinlock_lock(&thub->sl); 10765 if (ctx->tunnel) { 10766 ctx->tunnel->refctn = 1; 10767 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10768 } 10769 } 10770 10771 10772 static int 10773 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10774 const struct rte_flow_tunnel *app_tunnel, 10775 struct mlx5_flow_tunnel **tunnel) 10776 { 10777 struct tunnel_db_get_tunnel_ctx ctx = { 10778 .app_tunnel = app_tunnel, 10779 }; 10780 10781 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10782 get_tunnel_miss, &ctx, true); 10783 *tunnel = ctx.tunnel; 10784 return ctx.tunnel ? 0 : -ENOMEM; 10785 } 10786 10787 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10788 { 10789 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10790 10791 if (!thub) 10792 return; 10793 if (!LIST_EMPTY(&thub->tunnels)) 10794 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10795 mlx5_hlist_destroy(thub->groups); 10796 mlx5_free(thub); 10797 } 10798 10799 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10800 { 10801 int err; 10802 struct mlx5_flow_tunnel_hub *thub; 10803 10804 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10805 0, SOCKET_ID_ANY); 10806 if (!thub) 10807 return -ENOMEM; 10808 LIST_INIT(&thub->tunnels); 10809 rte_spinlock_init(&thub->sl); 10810 thub->groups = mlx5_hlist_create("flow groups", 64, 10811 false, true, sh, 10812 mlx5_flow_tunnel_grp2tbl_create_cb, 10813 mlx5_flow_tunnel_grp2tbl_match_cb, 10814 mlx5_flow_tunnel_grp2tbl_remove_cb, 10815 mlx5_flow_tunnel_grp2tbl_clone_cb, 10816 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10817 if (!thub->groups) { 10818 err = -rte_errno; 10819 goto err; 10820 } 10821 sh->tunnel_hub = thub; 10822 10823 return 0; 10824 10825 err: 10826 if (thub->groups) 10827 mlx5_hlist_destroy(thub->groups); 10828 if (thub) 10829 mlx5_free(thub); 10830 return err; 10831 } 10832 10833 static inline int 10834 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10835 struct rte_flow_tunnel *tunnel, 10836 struct rte_flow_error *error) 10837 { 10838 struct mlx5_priv *priv = dev->data->dev_private; 10839 10840 if (!priv->sh->config.dv_flow_en) 10841 return rte_flow_error_set(error, ENOTSUP, 10842 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10843 "flow DV interface is off"); 10844 if (!is_tunnel_offload_active(dev)) 10845 return rte_flow_error_set(error, ENOTSUP, 10846 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10847 "tunnel offload was not activated"); 10848 if (!tunnel) 10849 return rte_flow_error_set(error, EINVAL, 10850 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10851 "no application tunnel"); 10852 switch (tunnel->type) { 10853 default: 10854 return rte_flow_error_set(error, EINVAL, 10855 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10856 "unsupported tunnel type"); 10857 case RTE_FLOW_ITEM_TYPE_VXLAN: 10858 case RTE_FLOW_ITEM_TYPE_GRE: 10859 case RTE_FLOW_ITEM_TYPE_NVGRE: 10860 case RTE_FLOW_ITEM_TYPE_GENEVE: 10861 break; 10862 } 10863 return 0; 10864 } 10865 10866 static int 10867 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10868 struct rte_flow_tunnel *app_tunnel, 10869 struct rte_flow_action **actions, 10870 uint32_t *num_of_actions, 10871 struct rte_flow_error *error) 10872 { 10873 struct mlx5_flow_tunnel *tunnel; 10874 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10875 10876 if (ret) 10877 return ret; 10878 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10879 if (ret < 0) { 10880 return rte_flow_error_set(error, ret, 10881 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10882 "failed to initialize pmd tunnel"); 10883 } 10884 *actions = &tunnel->action; 10885 *num_of_actions = 1; 10886 return 0; 10887 } 10888 10889 static int 10890 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10891 struct rte_flow_tunnel *app_tunnel, 10892 struct rte_flow_item **items, 10893 uint32_t *num_of_items, 10894 struct rte_flow_error *error) 10895 { 10896 struct mlx5_flow_tunnel *tunnel; 10897 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10898 10899 if (ret) 10900 return ret; 10901 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10902 if (ret < 0) { 10903 return rte_flow_error_set(error, ret, 10904 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10905 "failed to initialize pmd tunnel"); 10906 } 10907 *items = &tunnel->item; 10908 *num_of_items = 1; 10909 return 0; 10910 } 10911 10912 struct tunnel_db_element_release_ctx { 10913 struct rte_flow_item *items; 10914 struct rte_flow_action *actions; 10915 uint32_t num_elements; 10916 struct rte_flow_error *error; 10917 int ret; 10918 }; 10919 10920 static bool 10921 tunnel_element_release_match(struct rte_eth_dev *dev, 10922 struct mlx5_flow_tunnel *tunnel, const void *x) 10923 { 10924 const struct tunnel_db_element_release_ctx *ctx = x; 10925 10926 RTE_SET_USED(dev); 10927 if (ctx->num_elements != 1) 10928 return false; 10929 else if (ctx->items) 10930 return ctx->items == &tunnel->item; 10931 else if (ctx->actions) 10932 return ctx->actions == &tunnel->action; 10933 10934 return false; 10935 } 10936 10937 static void 10938 tunnel_element_release_hit(struct rte_eth_dev *dev, 10939 struct mlx5_flow_tunnel *tunnel, void *x) 10940 { 10941 struct tunnel_db_element_release_ctx *ctx = x; 10942 ctx->ret = 0; 10943 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10944 mlx5_flow_tunnel_free(dev, tunnel); 10945 } 10946 10947 static void 10948 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10949 { 10950 struct tunnel_db_element_release_ctx *ctx = x; 10951 RTE_SET_USED(dev); 10952 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10953 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10954 "invalid argument"); 10955 } 10956 10957 static int 10958 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10959 struct rte_flow_item *pmd_items, 10960 uint32_t num_items, struct rte_flow_error *err) 10961 { 10962 struct tunnel_db_element_release_ctx ctx = { 10963 .items = pmd_items, 10964 .actions = NULL, 10965 .num_elements = num_items, 10966 .error = err, 10967 }; 10968 10969 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10970 tunnel_element_release_hit, 10971 tunnel_element_release_miss, &ctx, false); 10972 10973 return ctx.ret; 10974 } 10975 10976 static int 10977 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10978 struct rte_flow_action *pmd_actions, 10979 uint32_t num_actions, struct rte_flow_error *err) 10980 { 10981 struct tunnel_db_element_release_ctx ctx = { 10982 .items = NULL, 10983 .actions = pmd_actions, 10984 .num_elements = num_actions, 10985 .error = err, 10986 }; 10987 10988 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10989 tunnel_element_release_hit, 10990 tunnel_element_release_miss, &ctx, false); 10991 10992 return ctx.ret; 10993 } 10994 10995 static int 10996 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10997 struct rte_mbuf *m, 10998 struct rte_flow_restore_info *info, 10999 struct rte_flow_error *err) 11000 { 11001 uint64_t ol_flags = m->ol_flags; 11002 const struct mlx5_flow_tbl_data_entry *tble; 11003 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11004 11005 if (!is_tunnel_offload_active(dev)) { 11006 info->flags = 0; 11007 return 0; 11008 } 11009 11010 if ((ol_flags & mask) != mask) 11011 goto err; 11012 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11013 if (!tble) { 11014 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11015 dev->data->port_id, m->hash.fdir.hi); 11016 goto err; 11017 } 11018 MLX5_ASSERT(tble->tunnel); 11019 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11020 info->group_id = tble->group_id; 11021 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11022 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11023 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11024 11025 return 0; 11026 11027 err: 11028 return rte_flow_error_set(err, EINVAL, 11029 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11030 "failed to get restore info"); 11031 } 11032 11033 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11034 static int 11035 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11036 __rte_unused struct rte_flow_tunnel *app_tunnel, 11037 __rte_unused struct rte_flow_action **actions, 11038 __rte_unused uint32_t *num_of_actions, 11039 __rte_unused struct rte_flow_error *error) 11040 { 11041 return -ENOTSUP; 11042 } 11043 11044 static int 11045 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11046 __rte_unused struct rte_flow_tunnel *app_tunnel, 11047 __rte_unused struct rte_flow_item **items, 11048 __rte_unused uint32_t *num_of_items, 11049 __rte_unused struct rte_flow_error *error) 11050 { 11051 return -ENOTSUP; 11052 } 11053 11054 static int 11055 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11056 __rte_unused struct rte_flow_item *pmd_items, 11057 __rte_unused uint32_t num_items, 11058 __rte_unused struct rte_flow_error *err) 11059 { 11060 return -ENOTSUP; 11061 } 11062 11063 static int 11064 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11065 __rte_unused struct rte_flow_action *pmd_action, 11066 __rte_unused uint32_t num_actions, 11067 __rte_unused struct rte_flow_error *err) 11068 { 11069 return -ENOTSUP; 11070 } 11071 11072 static int 11073 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11074 __rte_unused struct rte_mbuf *m, 11075 __rte_unused struct rte_flow_restore_info *i, 11076 __rte_unused struct rte_flow_error *err) 11077 { 11078 return -ENOTSUP; 11079 } 11080 11081 static int 11082 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11083 __rte_unused struct rte_flow *flow, 11084 __rte_unused const struct rte_flow_attr *attr, 11085 __rte_unused const struct rte_flow_action *actions, 11086 __rte_unused uint32_t flow_idx, 11087 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11088 __rte_unused struct tunnel_default_miss_ctx *ctx, 11089 __rte_unused struct rte_flow_error *error) 11090 { 11091 return -ENOTSUP; 11092 } 11093 11094 static struct mlx5_flow_tunnel * 11095 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11096 __rte_unused uint32_t id) 11097 { 11098 return NULL; 11099 } 11100 11101 static void 11102 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11103 __rte_unused struct mlx5_flow_tunnel *tunnel) 11104 { 11105 } 11106 11107 static uint32_t 11108 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11109 __rte_unused const struct mlx5_flow_tunnel *t, 11110 __rte_unused uint32_t group, 11111 __rte_unused uint32_t *table, 11112 struct rte_flow_error *error) 11113 { 11114 return rte_flow_error_set(error, ENOTSUP, 11115 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11116 "tunnel offload requires DV support"); 11117 } 11118 11119 void 11120 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11121 __rte_unused uint16_t port_id) 11122 { 11123 } 11124 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11125 11126 /* Flex flow item API */ 11127 static struct rte_flow_item_flex_handle * 11128 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11129 const struct rte_flow_item_flex_conf *conf, 11130 struct rte_flow_error *error) 11131 { 11132 static const char err_msg[] = "flex item creation unsupported"; 11133 struct mlx5_priv *priv = dev->data->dev_private; 11134 struct rte_flow_attr attr = { .transfer = 0 }; 11135 const struct mlx5_flow_driver_ops *fops = 11136 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11137 11138 if (!priv->pci_dev) { 11139 rte_flow_error_set(error, ENOTSUP, 11140 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11141 "create flex item on PF only"); 11142 return NULL; 11143 } 11144 switch (priv->pci_dev->id.device_id) { 11145 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11146 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11147 break; 11148 default: 11149 rte_flow_error_set(error, ENOTSUP, 11150 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11151 "flex item available on BlueField ports only"); 11152 return NULL; 11153 } 11154 if (!fops->item_create) { 11155 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11156 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11157 NULL, err_msg); 11158 return NULL; 11159 } 11160 return fops->item_create(dev, conf, error); 11161 } 11162 11163 static int 11164 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11165 const struct rte_flow_item_flex_handle *handle, 11166 struct rte_flow_error *error) 11167 { 11168 static const char err_msg[] = "flex item release unsupported"; 11169 struct rte_flow_attr attr = { .transfer = 0 }; 11170 const struct mlx5_flow_driver_ops *fops = 11171 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11172 11173 if (!fops->item_release) { 11174 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11175 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11176 NULL, err_msg); 11177 return -rte_errno; 11178 } 11179 return fops->item_release(dev, handle, error); 11180 } 11181 11182 static void 11183 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11184 { 11185 int ret; 11186 struct rte_flow_error error; 11187 11188 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11189 char *item_name; 11190 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11191 sizeof(item_name), 11192 (void *)(uintptr_t)item->type, &error); 11193 if (ret > 0) 11194 printf("%s ", item_name); 11195 else 11196 printf("%d\n", (int)item->type); 11197 } 11198 printf("END\n"); 11199 } 11200 11201 static int 11202 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11203 { 11204 const struct rte_flow_item_udp *spec = udp_item->spec; 11205 const struct rte_flow_item_udp *mask = udp_item->mask; 11206 uint16_t udp_dport = 0; 11207 11208 if (spec != NULL) { 11209 if (!mask) 11210 mask = &rte_flow_item_udp_mask; 11211 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11212 mask->hdr.dst_port); 11213 } 11214 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11215 } 11216 11217 static const struct mlx5_flow_expand_node * 11218 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11219 unsigned int item_idx, 11220 const struct mlx5_flow_expand_node graph[], 11221 const struct mlx5_flow_expand_node *node) 11222 { 11223 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11224 11225 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11226 node != NULL && 11227 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11228 /* 11229 * The expansion node is VXLAN and it is also the last 11230 * expandable item in the pattern, so need to continue 11231 * expansion of the inner tunnel. 11232 */ 11233 MLX5_ASSERT(item_idx > 0); 11234 prev_item = pattern + item_idx - 1; 11235 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11236 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11237 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11238 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11239 } 11240 return node; 11241 } 11242 11243 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11244 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11245 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11246 }; 11247 11248 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11249 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11250 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11251 { 9, 10, 11 }, { 12, 13, 14 }, 11252 }; 11253 11254 /** 11255 * Discover the number of available flow priorities. 11256 * 11257 * @param dev 11258 * Ethernet device. 11259 * 11260 * @return 11261 * On success, number of available flow priorities. 11262 * On failure, a negative errno-style code and rte_errno is set. 11263 */ 11264 int 11265 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11266 { 11267 static const uint16_t vprio[] = {8, 16}; 11268 const struct mlx5_priv *priv = dev->data->dev_private; 11269 const struct mlx5_flow_driver_ops *fops; 11270 enum mlx5_flow_drv_type type; 11271 int ret; 11272 11273 type = mlx5_flow_os_get_type(); 11274 if (type == MLX5_FLOW_TYPE_MAX) { 11275 type = MLX5_FLOW_TYPE_VERBS; 11276 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11277 type = MLX5_FLOW_TYPE_DV; 11278 } 11279 fops = flow_get_drv_ops(type); 11280 if (fops->discover_priorities == NULL) { 11281 DRV_LOG(ERR, "Priority discovery not supported"); 11282 rte_errno = ENOTSUP; 11283 return -rte_errno; 11284 } 11285 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11286 if (ret < 0) 11287 return ret; 11288 switch (ret) { 11289 case 8: 11290 ret = RTE_DIM(priority_map_3); 11291 break; 11292 case 16: 11293 ret = RTE_DIM(priority_map_5); 11294 break; 11295 default: 11296 rte_errno = ENOTSUP; 11297 DRV_LOG(ERR, 11298 "port %u maximum priority: %d expected 8/16", 11299 dev->data->port_id, ret); 11300 return -rte_errno; 11301 } 11302 DRV_LOG(INFO, "port %u supported flow priorities:" 11303 " 0-%d for ingress or egress root table," 11304 " 0-%d for non-root table or transfer root table.", 11305 dev->data->port_id, ret - 2, 11306 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11307 return ret; 11308 } 11309 11310 /** 11311 * Adjust flow priority based on the highest layer and the request priority. 11312 * 11313 * @param[in] dev 11314 * Pointer to the Ethernet device structure. 11315 * @param[in] priority 11316 * The rule base priority. 11317 * @param[in] subpriority 11318 * The priority based on the items. 11319 * 11320 * @return 11321 * The new priority. 11322 */ 11323 uint32_t 11324 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11325 uint32_t subpriority) 11326 { 11327 uint32_t res = 0; 11328 struct mlx5_priv *priv = dev->data->dev_private; 11329 11330 switch (priv->sh->flow_max_priority) { 11331 case RTE_DIM(priority_map_3): 11332 res = priority_map_3[priority][subpriority]; 11333 break; 11334 case RTE_DIM(priority_map_5): 11335 res = priority_map_5[priority][subpriority]; 11336 break; 11337 } 11338 return res; 11339 } 11340 11341 /** 11342 * Get the priority for sending traffic to kernel table. 11343 * 11344 * @param[in] dev 11345 * Pointer to the Ethernet device structure. 11346 * 11347 * @return 11348 * On success: the value of priority for sending traffic to kernel table 11349 * On failure: -1 11350 */ 11351 uint32_t 11352 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11353 { 11354 struct mlx5_priv *priv = dev->data->dev_private; 11355 uint32_t res; 11356 11357 switch (priv->sh->flow_max_priority) { 11358 case RTE_DIM(priority_map_5): 11359 res = 15; 11360 break; 11361 case RTE_DIM(priority_map_3): 11362 res = 7; 11363 break; 11364 default: 11365 DRV_LOG(ERR, 11366 "port %u maximum priority: %d expected 8/16", 11367 dev->data->port_id, priv->sh->flow_max_priority); 11368 res = (uint32_t)-1; 11369 } 11370 return res; 11371 } 11372 11373 /** 11374 * Get the E-Switch Manager vport id. 11375 * 11376 * @param[in] dev 11377 * Pointer to the Ethernet device structure. 11378 * 11379 * @return 11380 * The vport id. 11381 */ 11382 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11383 { 11384 struct mlx5_priv *priv = dev->data->dev_private; 11385 struct mlx5_common_device *cdev = priv->sh->cdev; 11386 11387 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11388 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11389 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11390 11391 if (priv->pci_dev == NULL) 11392 return 0; 11393 switch (priv->pci_dev->id.device_id) { 11394 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 11395 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11396 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11397 /* 11398 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11399 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11400 * ECPF vport ID is selected and not the host port (0) in any BF case. 11401 */ 11402 return (int16_t)MLX5_ECPF_VPORT_ID; 11403 default: 11404 return MLX5_PF_VPORT_ID; 11405 } 11406 } 11407 11408 /** 11409 * Parse item to get the vport id. 11410 * 11411 * @param[in] dev 11412 * Pointer to the Ethernet device structure. 11413 * @param[in] item 11414 * The src port id match item. 11415 * @param[out] vport_id 11416 * Pointer to put the vport id. 11417 * @param[out] all_ports 11418 * Indicate if the item matches all ports. 11419 * @param[out] error 11420 * Pointer to error structure. 11421 * 11422 * @return 11423 * 0 on success, a negative errno value otherwise and rte_errno is set. 11424 */ 11425 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11426 const struct rte_flow_item *item, 11427 uint16_t *vport_id, 11428 bool *all_ports, 11429 struct rte_flow_error *error) 11430 { 11431 struct mlx5_priv *port_priv; 11432 const struct rte_flow_item_port_id *pid_v; 11433 uint32_t esw_mgr_port; 11434 11435 if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID && 11436 item->type != RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) 11437 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11438 NULL, "Incorrect item type."); 11439 pid_v = item->spec; 11440 if (!pid_v) { 11441 if (all_ports) 11442 *all_ports = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT); 11443 return 0; 11444 } 11445 if (all_ports) 11446 *all_ports = false; 11447 esw_mgr_port = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) ? 11448 MLX5_REPRESENTED_PORT_ESW_MGR : MLX5_PORT_ESW_MGR; 11449 if (pid_v->id == esw_mgr_port) { 11450 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11451 } else { 11452 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 11453 if (!port_priv) 11454 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11455 NULL, "Failed to get port info."); 11456 *vport_id = port_priv->representor_id; 11457 } 11458 11459 return 0; 11460 } 11461 11462 int 11463 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11464 uint16_t *proxy_port_id, 11465 struct rte_flow_error *error) 11466 { 11467 const struct mlx5_priv *priv = dev->data->dev_private; 11468 uint16_t port_id; 11469 11470 if (!priv->sh->config.dv_esw_en) 11471 return rte_flow_error_set(error, EINVAL, 11472 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11473 NULL, 11474 "unable to provide a proxy port" 11475 " without E-Switch configured"); 11476 if (!priv->master && !priv->representor) 11477 return rte_flow_error_set(error, EINVAL, 11478 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11479 NULL, 11480 "unable to provide a proxy port" 11481 " for port which is not a master" 11482 " or a representor port"); 11483 if (priv->master) { 11484 *proxy_port_id = dev->data->port_id; 11485 return 0; 11486 } 11487 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11488 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11489 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11490 11491 if (port_priv->master && 11492 port_priv->domain_id == priv->domain_id) { 11493 *proxy_port_id = port_id; 11494 return 0; 11495 } 11496 } 11497 return rte_flow_error_set(error, ENODEV, 11498 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11499 NULL, "unable to find a proxy port"); 11500 } 11501