1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static bool 168 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 169 { 170 switch (item->type) { 171 case RTE_FLOW_ITEM_TYPE_ETH: 172 case RTE_FLOW_ITEM_TYPE_VLAN: 173 case RTE_FLOW_ITEM_TYPE_IPV4: 174 case RTE_FLOW_ITEM_TYPE_IPV6: 175 case RTE_FLOW_ITEM_TYPE_UDP: 176 case RTE_FLOW_ITEM_TYPE_TCP: 177 case RTE_FLOW_ITEM_TYPE_ESP: 178 case RTE_FLOW_ITEM_TYPE_ICMP: 179 case RTE_FLOW_ITEM_TYPE_ICMP6: 180 case RTE_FLOW_ITEM_TYPE_VXLAN: 181 case RTE_FLOW_ITEM_TYPE_NVGRE: 182 case RTE_FLOW_ITEM_TYPE_GRE: 183 case RTE_FLOW_ITEM_TYPE_GENEVE: 184 case RTE_FLOW_ITEM_TYPE_MPLS: 185 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 186 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 187 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 188 case RTE_FLOW_ITEM_TYPE_GTP: 189 return true; 190 default: 191 break; 192 } 193 return false; 194 } 195 196 /** 197 * Network Service Header (NSH) and its next protocol values 198 * are described in RFC-8393. 199 */ 200 static enum rte_flow_item_type 201 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 202 { 203 enum rte_flow_item_type type; 204 205 switch (proto_mask & proto_spec) { 206 case 0: 207 type = RTE_FLOW_ITEM_TYPE_VOID; 208 break; 209 case RTE_VXLAN_GPE_TYPE_IPV4: 210 type = RTE_FLOW_ITEM_TYPE_IPV4; 211 break; 212 case RTE_VXLAN_GPE_TYPE_IPV6: 213 type = RTE_VXLAN_GPE_TYPE_IPV6; 214 break; 215 case RTE_VXLAN_GPE_TYPE_ETH: 216 type = RTE_FLOW_ITEM_TYPE_ETH; 217 break; 218 default: 219 type = RTE_FLOW_ITEM_TYPE_END; 220 } 221 return type; 222 } 223 224 static enum rte_flow_item_type 225 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 226 { 227 enum rte_flow_item_type type; 228 229 switch (proto_mask & proto_spec) { 230 case 0: 231 type = RTE_FLOW_ITEM_TYPE_VOID; 232 break; 233 case IPPROTO_UDP: 234 type = RTE_FLOW_ITEM_TYPE_UDP; 235 break; 236 case IPPROTO_TCP: 237 type = RTE_FLOW_ITEM_TYPE_TCP; 238 break; 239 case IPPROTO_IPIP: 240 type = RTE_FLOW_ITEM_TYPE_IPV4; 241 break; 242 case IPPROTO_IPV6: 243 type = RTE_FLOW_ITEM_TYPE_IPV6; 244 break; 245 case IPPROTO_ESP: 246 type = RTE_FLOW_ITEM_TYPE_ESP; 247 break; 248 default: 249 type = RTE_FLOW_ITEM_TYPE_END; 250 } 251 return type; 252 } 253 254 static enum rte_flow_item_type 255 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 256 rte_be16_t type_mask, bool is_tunnel) 257 { 258 enum rte_flow_item_type type; 259 260 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 261 case 0: 262 type = RTE_FLOW_ITEM_TYPE_VOID; 263 break; 264 case RTE_ETHER_TYPE_TEB: 265 type = is_tunnel ? 266 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 267 break; 268 case RTE_ETHER_TYPE_VLAN: 269 type = !is_tunnel ? 270 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 271 break; 272 case RTE_ETHER_TYPE_IPV4: 273 type = RTE_FLOW_ITEM_TYPE_IPV4; 274 break; 275 case RTE_ETHER_TYPE_IPV6: 276 type = RTE_FLOW_ITEM_TYPE_IPV6; 277 break; 278 default: 279 type = RTE_FLOW_ITEM_TYPE_END; 280 } 281 return type; 282 } 283 284 static enum rte_flow_item_type 285 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 286 { 287 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 288 do { \ 289 const void *m = item->mask; \ 290 const void *s = item->spec; \ 291 mask = m ? \ 292 ((const struct rte_flow_item_##type *)m)->fld : \ 293 rte_flow_item_##type##_mask.fld; \ 294 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 295 } while (0) 296 297 enum rte_flow_item_type ret; 298 uint16_t spec, mask; 299 300 if (item == NULL || item->spec == NULL) 301 return RTE_FLOW_ITEM_TYPE_VOID; 302 switch (item->type) { 303 case RTE_FLOW_ITEM_TYPE_ETH: 304 MLX5_XSET_ITEM_MASK_SPEC(eth, type); 305 if (!mask) 306 return RTE_FLOW_ITEM_TYPE_VOID; 307 ret = mlx5_ethertype_to_item_type(spec, mask, false); 308 break; 309 case RTE_FLOW_ITEM_TYPE_VLAN: 310 MLX5_XSET_ITEM_MASK_SPEC(vlan, inner_type); 311 if (!mask) 312 return RTE_FLOW_ITEM_TYPE_VOID; 313 ret = mlx5_ethertype_to_item_type(spec, mask, false); 314 break; 315 case RTE_FLOW_ITEM_TYPE_IPV4: 316 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 317 if (!mask) 318 return RTE_FLOW_ITEM_TYPE_VOID; 319 ret = mlx5_inet_proto_to_item_type(spec, mask); 320 break; 321 case RTE_FLOW_ITEM_TYPE_IPV6: 322 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 323 if (!mask) 324 return RTE_FLOW_ITEM_TYPE_VOID; 325 ret = mlx5_inet_proto_to_item_type(spec, mask); 326 break; 327 case RTE_FLOW_ITEM_TYPE_GENEVE: 328 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 329 ret = mlx5_ethertype_to_item_type(spec, mask, true); 330 break; 331 case RTE_FLOW_ITEM_TYPE_GRE: 332 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 333 ret = mlx5_ethertype_to_item_type(spec, mask, true); 334 break; 335 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 336 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, protocol); 337 ret = mlx5_nsh_proto_to_item_type(spec, mask); 338 break; 339 default: 340 ret = RTE_FLOW_ITEM_TYPE_VOID; 341 break; 342 } 343 return ret; 344 #undef MLX5_XSET_ITEM_MASK_SPEC 345 } 346 347 static const int * 348 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 349 const int *next_node) 350 { 351 const struct mlx5_flow_expand_node *node = NULL; 352 const int *next = next_node; 353 354 while (next && *next) { 355 /* 356 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 357 * flag set, because they were not found in the flow pattern. 358 */ 359 node = &graph[*next]; 360 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 361 break; 362 next = node->next; 363 } 364 return next; 365 } 366 367 #define MLX5_RSS_EXP_ELT_N 16 368 369 /** 370 * Expand RSS flows into several possible flows according to the RSS hash 371 * fields requested and the driver capabilities. 372 * 373 * @param[out] buf 374 * Buffer to store the result expansion. 375 * @param[in] size 376 * Buffer size in bytes. If 0, @p buf can be NULL. 377 * @param[in] pattern 378 * User flow pattern. 379 * @param[in] types 380 * RSS types to expand (see RTE_ETH_RSS_* definitions). 381 * @param[in] graph 382 * Input graph to expand @p pattern according to @p types. 383 * @param[in] graph_root_index 384 * Index of root node in @p graph, typically 0. 385 * 386 * @return 387 * A positive value representing the size of @p buf in bytes regardless of 388 * @p size on success, a negative errno value otherwise and rte_errno is 389 * set, the following errors are defined: 390 * 391 * -E2BIG: graph-depth @p graph is too deep. 392 * -EINVAL: @p size has not enough space for expanded pattern. 393 */ 394 static int 395 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 396 const struct rte_flow_item *pattern, uint64_t types, 397 const struct mlx5_flow_expand_node graph[], 398 int graph_root_index) 399 { 400 const struct rte_flow_item *item; 401 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 402 const int *next_node; 403 const int *stack[MLX5_RSS_EXP_ELT_N]; 404 int stack_pos = 0; 405 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 406 unsigned int i, item_idx, last_expand_item_idx = 0; 407 size_t lsize; 408 size_t user_pattern_size = 0; 409 void *addr = NULL; 410 const struct mlx5_flow_expand_node *next = NULL; 411 struct rte_flow_item missed_item; 412 int missed = 0; 413 int elt = 0; 414 const struct rte_flow_item *last_expand_item = NULL; 415 416 memset(&missed_item, 0, sizeof(missed_item)); 417 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 418 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 419 if (lsize > size) 420 return -EINVAL; 421 buf->entry[0].priority = 0; 422 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 423 buf->entries = 0; 424 addr = buf->entry[0].pattern; 425 for (item = pattern, item_idx = 0; 426 item->type != RTE_FLOW_ITEM_TYPE_END; 427 item++, item_idx++) { 428 if (!mlx5_flow_is_rss_expandable_item(item)) { 429 user_pattern_size += sizeof(*item); 430 continue; 431 } 432 last_expand_item = item; 433 last_expand_item_idx = item_idx; 434 i = 0; 435 while (node->next && node->next[i]) { 436 next = &graph[node->next[i]]; 437 if (next->type == item->type) 438 break; 439 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 440 node = next; 441 i = 0; 442 } else { 443 ++i; 444 } 445 } 446 if (next) 447 node = next; 448 user_pattern_size += sizeof(*item); 449 } 450 user_pattern_size += sizeof(*item); /* Handle END item. */ 451 lsize += user_pattern_size; 452 if (lsize > size) 453 return -EINVAL; 454 /* Copy the user pattern in the first entry of the buffer. */ 455 rte_memcpy(addr, pattern, user_pattern_size); 456 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 457 buf->entries = 1; 458 /* Start expanding. */ 459 memset(flow_items, 0, sizeof(flow_items)); 460 user_pattern_size -= sizeof(*item); 461 /* 462 * Check if the last valid item has spec set, need complete pattern, 463 * and the pattern can be used for expansion. 464 */ 465 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 466 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 467 /* Item type END indicates expansion is not required. */ 468 return lsize; 469 } 470 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 471 next = NULL; 472 missed = 1; 473 i = 0; 474 while (node->next && node->next[i]) { 475 next = &graph[node->next[i]]; 476 if (next->type == missed_item.type) { 477 flow_items[0].type = missed_item.type; 478 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 479 break; 480 } 481 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 482 node = next; 483 i = 0; 484 } else { 485 ++i; 486 } 487 next = NULL; 488 } 489 } 490 if (next && missed) { 491 elt = 2; /* missed item + item end. */ 492 node = next; 493 lsize += elt * sizeof(*item) + user_pattern_size; 494 if (lsize > size) 495 return -EINVAL; 496 if (node->rss_types & types) { 497 buf->entry[buf->entries].priority = 1; 498 buf->entry[buf->entries].pattern = addr; 499 buf->entries++; 500 rte_memcpy(addr, buf->entry[0].pattern, 501 user_pattern_size); 502 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 503 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 504 addr = (void *)(((uintptr_t)addr) + 505 elt * sizeof(*item)); 506 } 507 } else if (last_expand_item != NULL) { 508 node = mlx5_flow_expand_rss_adjust_node(pattern, 509 last_expand_item_idx, graph, node); 510 } 511 memset(flow_items, 0, sizeof(flow_items)); 512 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 513 node->next); 514 stack[stack_pos] = next_node; 515 node = next_node ? &graph[*next_node] : NULL; 516 while (node) { 517 flow_items[stack_pos].type = node->type; 518 if (node->rss_types & types) { 519 size_t n; 520 /* 521 * compute the number of items to copy from the 522 * expansion and copy it. 523 * When the stack_pos is 0, there are 1 element in it, 524 * plus the addition END item. 525 */ 526 elt = stack_pos + 2; 527 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 528 lsize += elt * sizeof(*item) + user_pattern_size; 529 if (lsize > size) 530 return -EINVAL; 531 n = elt * sizeof(*item); 532 buf->entry[buf->entries].priority = 533 stack_pos + 1 + missed; 534 buf->entry[buf->entries].pattern = addr; 535 buf->entries++; 536 rte_memcpy(addr, buf->entry[0].pattern, 537 user_pattern_size); 538 addr = (void *)(((uintptr_t)addr) + 539 user_pattern_size); 540 rte_memcpy(addr, &missed_item, 541 missed * sizeof(*item)); 542 addr = (void *)(((uintptr_t)addr) + 543 missed * sizeof(*item)); 544 rte_memcpy(addr, flow_items, n); 545 addr = (void *)(((uintptr_t)addr) + n); 546 } 547 /* Go deeper. */ 548 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 549 node->next) { 550 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 551 node->next); 552 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 553 rte_errno = E2BIG; 554 return -rte_errno; 555 } 556 stack[stack_pos] = next_node; 557 } else if (*(next_node + 1)) { 558 /* Follow up with the next possibility. */ 559 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 560 ++next_node); 561 } else if (!stack_pos) { 562 /* 563 * Completing the traverse over the different paths. 564 * The next_node is advanced to the terminator. 565 */ 566 ++next_node; 567 } else { 568 /* Move to the next path. */ 569 while (stack_pos) { 570 next_node = stack[--stack_pos]; 571 next_node++; 572 if (*next_node) 573 break; 574 } 575 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 576 next_node); 577 stack[stack_pos] = next_node; 578 } 579 node = next_node && *next_node ? &graph[*next_node] : NULL; 580 }; 581 return lsize; 582 } 583 584 /** 585 * Expand SQN flows into several possible flows according to the Tx queue 586 * number 587 * 588 * @param[in] buf 589 * Buffer to store the result expansion. 590 * @param[in] size 591 * Buffer size in bytes. If 0, @p buf can be NULL. 592 * @param[in] pattern 593 * User flow pattern. 594 * @param[in] sq_specs 595 * Buffer to store sq spec. 596 * 597 * @return 598 * 0 for success and negative value for failure 599 * 600 */ 601 static int 602 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 603 const struct rte_flow_item *pattern, 604 struct mlx5_rte_flow_item_sq *sq_specs) 605 { 606 const struct rte_flow_item *item; 607 bool port_representor = false; 608 size_t user_pattern_size = 0; 609 struct rte_eth_dev *dev; 610 struct mlx5_priv *priv; 611 void *addr = NULL; 612 uint16_t port_id; 613 size_t lsize; 614 int elt = 2; 615 uint16_t i; 616 617 buf->entries = 0; 618 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 619 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 620 const struct rte_flow_item_ethdev *pid_v = item->spec; 621 622 if (!pid_v) 623 return 0; 624 port_id = pid_v->port_id; 625 port_representor = true; 626 } 627 user_pattern_size += sizeof(*item); 628 } 629 if (!port_representor) 630 return 0; 631 dev = &rte_eth_devices[port_id]; 632 priv = dev->data->dev_private; 633 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 634 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 635 sizeof(buf->entry[0]) * priv->txqs_n; 636 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 637 return -EINVAL; 638 addr = buf->entry[0].pattern; 639 for (i = 0; i != priv->txqs_n; ++i) { 640 struct rte_flow_item pattern_add[] = { 641 { 642 .type = (enum rte_flow_item_type) 643 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 644 .spec = &sq_specs[i], 645 }, 646 { 647 .type = RTE_FLOW_ITEM_TYPE_END, 648 }, 649 }; 650 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 651 652 if (txq == NULL) 653 return -EINVAL; 654 buf->entry[i].pattern = addr; 655 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 656 mlx5_txq_release(dev, i); 657 rte_memcpy(addr, pattern, user_pattern_size); 658 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 659 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 660 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 661 buf->entries++; 662 } 663 return 0; 664 } 665 666 enum mlx5_expansion { 667 MLX5_EXPANSION_ROOT, 668 MLX5_EXPANSION_ROOT_OUTER, 669 MLX5_EXPANSION_OUTER_ETH, 670 MLX5_EXPANSION_OUTER_VLAN, 671 MLX5_EXPANSION_OUTER_IPV4, 672 MLX5_EXPANSION_OUTER_IPV4_UDP, 673 MLX5_EXPANSION_OUTER_IPV4_TCP, 674 MLX5_EXPANSION_OUTER_IPV4_ESP, 675 MLX5_EXPANSION_OUTER_IPV4_ICMP, 676 MLX5_EXPANSION_OUTER_IPV6, 677 MLX5_EXPANSION_OUTER_IPV6_UDP, 678 MLX5_EXPANSION_OUTER_IPV6_TCP, 679 MLX5_EXPANSION_OUTER_IPV6_ESP, 680 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 681 MLX5_EXPANSION_VXLAN, 682 MLX5_EXPANSION_STD_VXLAN, 683 MLX5_EXPANSION_L3_VXLAN, 684 MLX5_EXPANSION_VXLAN_GPE, 685 MLX5_EXPANSION_GRE, 686 MLX5_EXPANSION_NVGRE, 687 MLX5_EXPANSION_GRE_KEY, 688 MLX5_EXPANSION_MPLS, 689 MLX5_EXPANSION_ETH, 690 MLX5_EXPANSION_VLAN, 691 MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV4_UDP, 693 MLX5_EXPANSION_IPV4_TCP, 694 MLX5_EXPANSION_IPV4_ESP, 695 MLX5_EXPANSION_IPV4_ICMP, 696 MLX5_EXPANSION_IPV6, 697 MLX5_EXPANSION_IPV6_UDP, 698 MLX5_EXPANSION_IPV6_TCP, 699 MLX5_EXPANSION_IPV6_ESP, 700 MLX5_EXPANSION_IPV6_ICMP6, 701 MLX5_EXPANSION_IPV6_FRAG_EXT, 702 MLX5_EXPANSION_GTP, 703 MLX5_EXPANSION_GENEVE, 704 }; 705 706 /** Supported expansion of items. */ 707 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 708 [MLX5_EXPANSION_ROOT] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 710 MLX5_EXPANSION_IPV4, 711 MLX5_EXPANSION_IPV6), 712 .type = RTE_FLOW_ITEM_TYPE_END, 713 }, 714 [MLX5_EXPANSION_ROOT_OUTER] = { 715 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 716 MLX5_EXPANSION_OUTER_IPV4, 717 MLX5_EXPANSION_OUTER_IPV6), 718 .type = RTE_FLOW_ITEM_TYPE_END, 719 }, 720 [MLX5_EXPANSION_OUTER_ETH] = { 721 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 722 .type = RTE_FLOW_ITEM_TYPE_ETH, 723 .rss_types = 0, 724 }, 725 [MLX5_EXPANSION_OUTER_VLAN] = { 726 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 727 MLX5_EXPANSION_OUTER_IPV6), 728 .type = RTE_FLOW_ITEM_TYPE_VLAN, 729 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 730 }, 731 [MLX5_EXPANSION_OUTER_IPV4] = { 732 .next = MLX5_FLOW_EXPAND_RSS_NEXT 733 (MLX5_EXPANSION_OUTER_IPV4_UDP, 734 MLX5_EXPANSION_OUTER_IPV4_TCP, 735 MLX5_EXPANSION_OUTER_IPV4_ESP, 736 MLX5_EXPANSION_OUTER_IPV4_ICMP, 737 MLX5_EXPANSION_GRE, 738 MLX5_EXPANSION_NVGRE, 739 MLX5_EXPANSION_IPV4, 740 MLX5_EXPANSION_IPV6), 741 .type = RTE_FLOW_ITEM_TYPE_IPV4, 742 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 743 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 744 }, 745 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 746 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 747 MLX5_EXPANSION_VXLAN_GPE, 748 MLX5_EXPANSION_MPLS, 749 MLX5_EXPANSION_GENEVE, 750 MLX5_EXPANSION_GTP), 751 .type = RTE_FLOW_ITEM_TYPE_UDP, 752 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 753 }, 754 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 755 .type = RTE_FLOW_ITEM_TYPE_TCP, 756 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 757 }, 758 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 759 .type = RTE_FLOW_ITEM_TYPE_ESP, 760 .rss_types = RTE_ETH_RSS_ESP, 761 }, 762 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 763 .type = RTE_FLOW_ITEM_TYPE_ICMP, 764 }, 765 [MLX5_EXPANSION_OUTER_IPV6] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT 767 (MLX5_EXPANSION_OUTER_IPV6_UDP, 768 MLX5_EXPANSION_OUTER_IPV6_TCP, 769 MLX5_EXPANSION_OUTER_IPV6_ESP, 770 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 771 MLX5_EXPANSION_IPV4, 772 MLX5_EXPANSION_IPV6, 773 MLX5_EXPANSION_GRE, 774 MLX5_EXPANSION_NVGRE), 775 .type = RTE_FLOW_ITEM_TYPE_IPV6, 776 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 777 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 778 }, 779 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 780 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 781 MLX5_EXPANSION_VXLAN_GPE, 782 MLX5_EXPANSION_MPLS, 783 MLX5_EXPANSION_GENEVE, 784 MLX5_EXPANSION_GTP), 785 .type = RTE_FLOW_ITEM_TYPE_UDP, 786 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 787 }, 788 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 789 .type = RTE_FLOW_ITEM_TYPE_TCP, 790 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 791 }, 792 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 793 .type = RTE_FLOW_ITEM_TYPE_ESP, 794 .rss_types = RTE_ETH_RSS_ESP, 795 }, 796 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 797 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 798 }, 799 [MLX5_EXPANSION_VXLAN] = { 800 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 801 MLX5_EXPANSION_IPV4, 802 MLX5_EXPANSION_IPV6), 803 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 804 }, 805 [MLX5_EXPANSION_STD_VXLAN] = { 806 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 807 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 808 }, 809 [MLX5_EXPANSION_L3_VXLAN] = { 810 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 811 MLX5_EXPANSION_IPV6), 812 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 813 }, 814 [MLX5_EXPANSION_VXLAN_GPE] = { 815 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 816 MLX5_EXPANSION_IPV4, 817 MLX5_EXPANSION_IPV6), 818 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 819 }, 820 [MLX5_EXPANSION_GRE] = { 821 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 822 MLX5_EXPANSION_IPV4, 823 MLX5_EXPANSION_IPV6, 824 MLX5_EXPANSION_GRE_KEY, 825 MLX5_EXPANSION_MPLS), 826 .type = RTE_FLOW_ITEM_TYPE_GRE, 827 }, 828 [MLX5_EXPANSION_GRE_KEY] = { 829 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 830 MLX5_EXPANSION_IPV6, 831 MLX5_EXPANSION_MPLS), 832 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 833 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 834 }, 835 [MLX5_EXPANSION_NVGRE] = { 836 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 837 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 838 }, 839 [MLX5_EXPANSION_MPLS] = { 840 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 841 MLX5_EXPANSION_IPV6, 842 MLX5_EXPANSION_ETH), 843 .type = RTE_FLOW_ITEM_TYPE_MPLS, 844 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 845 }, 846 [MLX5_EXPANSION_ETH] = { 847 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 848 .type = RTE_FLOW_ITEM_TYPE_ETH, 849 }, 850 [MLX5_EXPANSION_VLAN] = { 851 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 852 MLX5_EXPANSION_IPV6), 853 .type = RTE_FLOW_ITEM_TYPE_VLAN, 854 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 855 }, 856 [MLX5_EXPANSION_IPV4] = { 857 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 858 MLX5_EXPANSION_IPV4_TCP, 859 MLX5_EXPANSION_IPV4_ESP, 860 MLX5_EXPANSION_IPV4_ICMP), 861 .type = RTE_FLOW_ITEM_TYPE_IPV4, 862 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 863 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 864 }, 865 [MLX5_EXPANSION_IPV4_UDP] = { 866 .type = RTE_FLOW_ITEM_TYPE_UDP, 867 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 868 }, 869 [MLX5_EXPANSION_IPV4_TCP] = { 870 .type = RTE_FLOW_ITEM_TYPE_TCP, 871 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 872 }, 873 [MLX5_EXPANSION_IPV4_ESP] = { 874 .type = RTE_FLOW_ITEM_TYPE_ESP, 875 .rss_types = RTE_ETH_RSS_ESP, 876 }, 877 [MLX5_EXPANSION_IPV4_ICMP] = { 878 .type = RTE_FLOW_ITEM_TYPE_ICMP, 879 }, 880 [MLX5_EXPANSION_IPV6] = { 881 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 882 MLX5_EXPANSION_IPV6_TCP, 883 MLX5_EXPANSION_IPV6_ESP, 884 MLX5_EXPANSION_IPV6_ICMP6, 885 MLX5_EXPANSION_IPV6_FRAG_EXT), 886 .type = RTE_FLOW_ITEM_TYPE_IPV6, 887 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 888 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 889 }, 890 [MLX5_EXPANSION_IPV6_UDP] = { 891 .type = RTE_FLOW_ITEM_TYPE_UDP, 892 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 893 }, 894 [MLX5_EXPANSION_IPV6_TCP] = { 895 .type = RTE_FLOW_ITEM_TYPE_TCP, 896 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 897 }, 898 [MLX5_EXPANSION_IPV6_ESP] = { 899 .type = RTE_FLOW_ITEM_TYPE_ESP, 900 .rss_types = RTE_ETH_RSS_ESP, 901 }, 902 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 903 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 904 }, 905 [MLX5_EXPANSION_IPV6_ICMP6] = { 906 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 907 }, 908 [MLX5_EXPANSION_GTP] = { 909 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 910 MLX5_EXPANSION_IPV6), 911 .type = RTE_FLOW_ITEM_TYPE_GTP, 912 }, 913 [MLX5_EXPANSION_GENEVE] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 915 MLX5_EXPANSION_IPV4, 916 MLX5_EXPANSION_IPV6), 917 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 918 }, 919 }; 920 921 static struct rte_flow_action_handle * 922 mlx5_action_handle_create(struct rte_eth_dev *dev, 923 const struct rte_flow_indir_action_conf *conf, 924 const struct rte_flow_action *action, 925 struct rte_flow_error *error); 926 static int mlx5_action_handle_destroy 927 (struct rte_eth_dev *dev, 928 struct rte_flow_action_handle *handle, 929 struct rte_flow_error *error); 930 static int mlx5_action_handle_update 931 (struct rte_eth_dev *dev, 932 struct rte_flow_action_handle *handle, 933 const void *update, 934 struct rte_flow_error *error); 935 static int mlx5_action_handle_query 936 (struct rte_eth_dev *dev, 937 const struct rte_flow_action_handle *handle, 938 void *data, 939 struct rte_flow_error *error); 940 static int 941 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 942 struct rte_flow_tunnel *app_tunnel, 943 struct rte_flow_action **actions, 944 uint32_t *num_of_actions, 945 struct rte_flow_error *error); 946 static int 947 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 948 struct rte_flow_tunnel *app_tunnel, 949 struct rte_flow_item **items, 950 uint32_t *num_of_items, 951 struct rte_flow_error *error); 952 static int 953 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 954 struct rte_flow_item *pmd_items, 955 uint32_t num_items, struct rte_flow_error *err); 956 static int 957 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 958 struct rte_flow_action *pmd_actions, 959 uint32_t num_actions, 960 struct rte_flow_error *err); 961 static int 962 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 963 struct rte_mbuf *m, 964 struct rte_flow_restore_info *info, 965 struct rte_flow_error *err); 966 static struct rte_flow_item_flex_handle * 967 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 968 const struct rte_flow_item_flex_conf *conf, 969 struct rte_flow_error *error); 970 static int 971 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 972 const struct rte_flow_item_flex_handle *handle, 973 struct rte_flow_error *error); 974 static int 975 mlx5_flow_info_get(struct rte_eth_dev *dev, 976 struct rte_flow_port_info *port_info, 977 struct rte_flow_queue_info *queue_info, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_port_configure(struct rte_eth_dev *dev, 981 const struct rte_flow_port_attr *port_attr, 982 uint16_t nb_queue, 983 const struct rte_flow_queue_attr *queue_attr[], 984 struct rte_flow_error *err); 985 986 static struct rte_flow_pattern_template * 987 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 988 const struct rte_flow_pattern_template_attr *attr, 989 const struct rte_flow_item items[], 990 struct rte_flow_error *error); 991 992 static int 993 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 994 struct rte_flow_pattern_template *template, 995 struct rte_flow_error *error); 996 static struct rte_flow_actions_template * 997 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 998 const struct rte_flow_actions_template_attr *attr, 999 const struct rte_flow_action actions[], 1000 const struct rte_flow_action masks[], 1001 struct rte_flow_error *error); 1002 static int 1003 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1004 struct rte_flow_actions_template *template, 1005 struct rte_flow_error *error); 1006 1007 static struct rte_flow_template_table * 1008 mlx5_flow_table_create(struct rte_eth_dev *dev, 1009 const struct rte_flow_template_table_attr *attr, 1010 struct rte_flow_pattern_template *item_templates[], 1011 uint8_t nb_item_templates, 1012 struct rte_flow_actions_template *action_templates[], 1013 uint8_t nb_action_templates, 1014 struct rte_flow_error *error); 1015 static int 1016 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1017 struct rte_flow_template_table *table, 1018 struct rte_flow_error *error); 1019 static struct rte_flow * 1020 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1021 uint32_t queue, 1022 const struct rte_flow_op_attr *attr, 1023 struct rte_flow_template_table *table, 1024 const struct rte_flow_item items[], 1025 uint8_t pattern_template_index, 1026 const struct rte_flow_action actions[], 1027 uint8_t action_template_index, 1028 void *user_data, 1029 struct rte_flow_error *error); 1030 static int 1031 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1032 uint32_t queue, 1033 const struct rte_flow_op_attr *attr, 1034 struct rte_flow *flow, 1035 void *user_data, 1036 struct rte_flow_error *error); 1037 static int 1038 mlx5_flow_pull(struct rte_eth_dev *dev, 1039 uint32_t queue, 1040 struct rte_flow_op_result res[], 1041 uint16_t n_res, 1042 struct rte_flow_error *error); 1043 static int 1044 mlx5_flow_push(struct rte_eth_dev *dev, 1045 uint32_t queue, 1046 struct rte_flow_error *error); 1047 1048 static struct rte_flow_action_handle * 1049 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1050 const struct rte_flow_op_attr *attr, 1051 const struct rte_flow_indir_action_conf *conf, 1052 const struct rte_flow_action *action, 1053 void *user_data, 1054 struct rte_flow_error *error); 1055 1056 static int 1057 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1058 const struct rte_flow_op_attr *attr, 1059 struct rte_flow_action_handle *handle, 1060 const void *update, 1061 void *user_data, 1062 struct rte_flow_error *error); 1063 1064 static int 1065 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1066 const struct rte_flow_op_attr *attr, 1067 struct rte_flow_action_handle *handle, 1068 void *user_data, 1069 struct rte_flow_error *error); 1070 1071 static int 1072 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1073 const struct rte_flow_op_attr *attr, 1074 const struct rte_flow_action_handle *handle, 1075 void *data, 1076 void *user_data, 1077 struct rte_flow_error *error); 1078 1079 static const struct rte_flow_ops mlx5_flow_ops = { 1080 .validate = mlx5_flow_validate, 1081 .create = mlx5_flow_create, 1082 .destroy = mlx5_flow_destroy, 1083 .flush = mlx5_flow_flush, 1084 .isolate = mlx5_flow_isolate, 1085 .query = mlx5_flow_query, 1086 .dev_dump = mlx5_flow_dev_dump, 1087 #ifdef MLX5_HAVE_RTE_FLOW_Q_AGE 1088 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1089 #endif 1090 .get_aged_flows = mlx5_flow_get_aged_flows, 1091 .action_handle_create = mlx5_action_handle_create, 1092 .action_handle_destroy = mlx5_action_handle_destroy, 1093 .action_handle_update = mlx5_action_handle_update, 1094 .action_handle_query = mlx5_action_handle_query, 1095 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1096 .tunnel_match = mlx5_flow_tunnel_match, 1097 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1098 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1099 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1100 .flex_item_create = mlx5_flow_flex_item_create, 1101 .flex_item_release = mlx5_flow_flex_item_release, 1102 .info_get = mlx5_flow_info_get, 1103 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1104 .configure = mlx5_flow_port_configure, 1105 .pattern_template_create = mlx5_flow_pattern_template_create, 1106 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1107 .actions_template_create = mlx5_flow_actions_template_create, 1108 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1109 .template_table_create = mlx5_flow_table_create, 1110 .template_table_destroy = mlx5_flow_table_destroy, 1111 .async_create = mlx5_flow_async_flow_create, 1112 .async_destroy = mlx5_flow_async_flow_destroy, 1113 .pull = mlx5_flow_pull, 1114 .push = mlx5_flow_push, 1115 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1116 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1117 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1118 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1119 }; 1120 1121 /* Tunnel information. */ 1122 struct mlx5_flow_tunnel_info { 1123 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1124 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1125 }; 1126 1127 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1128 { 1129 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1130 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1131 }, 1132 { 1133 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1134 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1135 }, 1136 { 1137 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1138 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1139 }, 1140 { 1141 .tunnel = MLX5_FLOW_LAYER_GRE, 1142 .ptype = RTE_PTYPE_TUNNEL_GRE, 1143 }, 1144 { 1145 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1146 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1147 }, 1148 { 1149 .tunnel = MLX5_FLOW_LAYER_MPLS, 1150 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1151 }, 1152 { 1153 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1154 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1155 }, 1156 { 1157 .tunnel = MLX5_FLOW_LAYER_IPIP, 1158 .ptype = RTE_PTYPE_TUNNEL_IP, 1159 }, 1160 { 1161 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1162 .ptype = RTE_PTYPE_TUNNEL_IP, 1163 }, 1164 { 1165 .tunnel = MLX5_FLOW_LAYER_GTP, 1166 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1167 }, 1168 }; 1169 1170 1171 1172 /** 1173 * Translate tag ID to register. 1174 * 1175 * @param[in] dev 1176 * Pointer to the Ethernet device structure. 1177 * @param[in] feature 1178 * The feature that request the register. 1179 * @param[in] id 1180 * The request register ID. 1181 * @param[out] error 1182 * Error description in case of any. 1183 * 1184 * @return 1185 * The request register on success, a negative errno 1186 * value otherwise and rte_errno is set. 1187 */ 1188 int 1189 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1190 enum mlx5_feature_name feature, 1191 uint32_t id, 1192 struct rte_flow_error *error) 1193 { 1194 struct mlx5_priv *priv = dev->data->dev_private; 1195 struct mlx5_sh_config *config = &priv->sh->config; 1196 enum modify_reg start_reg; 1197 bool skip_mtr_reg = false; 1198 1199 switch (feature) { 1200 case MLX5_HAIRPIN_RX: 1201 return REG_B; 1202 case MLX5_HAIRPIN_TX: 1203 return REG_A; 1204 case MLX5_METADATA_RX: 1205 switch (config->dv_xmeta_en) { 1206 case MLX5_XMETA_MODE_LEGACY: 1207 return REG_B; 1208 case MLX5_XMETA_MODE_META16: 1209 return REG_C_0; 1210 case MLX5_XMETA_MODE_META32: 1211 return REG_C_1; 1212 case MLX5_XMETA_MODE_META32_HWS: 1213 return REG_C_1; 1214 } 1215 break; 1216 case MLX5_METADATA_TX: 1217 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1218 return REG_C_1; 1219 } else { 1220 return REG_A; 1221 } 1222 case MLX5_METADATA_FDB: 1223 switch (config->dv_xmeta_en) { 1224 case MLX5_XMETA_MODE_LEGACY: 1225 return REG_NON; 1226 case MLX5_XMETA_MODE_META16: 1227 return REG_C_0; 1228 case MLX5_XMETA_MODE_META32: 1229 return REG_C_1; 1230 case MLX5_XMETA_MODE_META32_HWS: 1231 return REG_C_1; 1232 } 1233 break; 1234 case MLX5_FLOW_MARK: 1235 switch (config->dv_xmeta_en) { 1236 case MLX5_XMETA_MODE_LEGACY: 1237 case MLX5_XMETA_MODE_META32_HWS: 1238 return REG_NON; 1239 case MLX5_XMETA_MODE_META16: 1240 return REG_C_1; 1241 case MLX5_XMETA_MODE_META32: 1242 return REG_C_0; 1243 } 1244 break; 1245 case MLX5_MTR_ID: 1246 /* 1247 * If meter color and meter id share one register, flow match 1248 * should use the meter color register for match. 1249 */ 1250 if (priv->mtr_reg_share) 1251 return priv->mtr_color_reg; 1252 else 1253 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1254 REG_C_3; 1255 case MLX5_MTR_COLOR: 1256 case MLX5_ASO_FLOW_HIT: 1257 case MLX5_ASO_CONNTRACK: 1258 case MLX5_SAMPLE_ID: 1259 /* All features use the same REG_C. */ 1260 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1261 return priv->mtr_color_reg; 1262 case MLX5_COPY_MARK: 1263 /* 1264 * Metadata COPY_MARK register using is in meter suffix sub 1265 * flow while with meter. It's safe to share the same register. 1266 */ 1267 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1268 case MLX5_APP_TAG: 1269 /* 1270 * If meter is enable, it will engage the register for color 1271 * match and flow match. If meter color match is not using the 1272 * REG_C_2, need to skip the REG_C_x be used by meter color 1273 * match. 1274 * If meter is disable, free to use all available registers. 1275 */ 1276 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1277 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1278 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1279 if (id > (uint32_t)(REG_C_7 - start_reg)) 1280 return rte_flow_error_set(error, EINVAL, 1281 RTE_FLOW_ERROR_TYPE_ITEM, 1282 NULL, "invalid tag id"); 1283 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1284 return rte_flow_error_set(error, ENOTSUP, 1285 RTE_FLOW_ERROR_TYPE_ITEM, 1286 NULL, "unsupported tag id"); 1287 /* 1288 * This case means meter is using the REG_C_x great than 2. 1289 * Take care not to conflict with meter color REG_C_x. 1290 * If the available index REG_C_y >= REG_C_x, skip the 1291 * color register. 1292 */ 1293 if (skip_mtr_reg && priv->sh->flow_mreg_c 1294 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1295 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1296 return rte_flow_error_set(error, EINVAL, 1297 RTE_FLOW_ERROR_TYPE_ITEM, 1298 NULL, "invalid tag id"); 1299 if (priv->sh->flow_mreg_c 1300 [id + 1 + start_reg - REG_C_0] != REG_NON) 1301 return priv->sh->flow_mreg_c 1302 [id + 1 + start_reg - REG_C_0]; 1303 return rte_flow_error_set(error, ENOTSUP, 1304 RTE_FLOW_ERROR_TYPE_ITEM, 1305 NULL, "unsupported tag id"); 1306 } 1307 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1308 } 1309 MLX5_ASSERT(false); 1310 return rte_flow_error_set(error, EINVAL, 1311 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1312 NULL, "invalid feature name"); 1313 } 1314 1315 /** 1316 * Check extensive flow metadata register support. 1317 * 1318 * @param dev 1319 * Pointer to rte_eth_dev structure. 1320 * 1321 * @return 1322 * True if device supports extensive flow metadata register, otherwise false. 1323 */ 1324 bool 1325 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1326 { 1327 struct mlx5_priv *priv = dev->data->dev_private; 1328 1329 /* 1330 * Having available reg_c can be regarded inclusively as supporting 1331 * extensive flow metadata register, which could mean, 1332 * - metadata register copy action by modify header. 1333 * - 16 modify header actions is supported. 1334 * - reg_c's are preserved across different domain (FDB and NIC) on 1335 * packet loopback by flow lookup miss. 1336 */ 1337 return priv->sh->flow_mreg_c[2] != REG_NON; 1338 } 1339 1340 /** 1341 * Get the lowest priority. 1342 * 1343 * @param[in] dev 1344 * Pointer to the Ethernet device structure. 1345 * @param[in] attributes 1346 * Pointer to device flow rule attributes. 1347 * 1348 * @return 1349 * The value of lowest priority of flow. 1350 */ 1351 uint32_t 1352 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1353 const struct rte_flow_attr *attr) 1354 { 1355 struct mlx5_priv *priv = dev->data->dev_private; 1356 1357 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1358 return priv->sh->flow_max_priority - 2; 1359 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1360 } 1361 1362 /** 1363 * Calculate matcher priority of the flow. 1364 * 1365 * @param[in] dev 1366 * Pointer to the Ethernet device structure. 1367 * @param[in] attr 1368 * Pointer to device flow rule attributes. 1369 * @param[in] subpriority 1370 * The priority based on the items. 1371 * @param[in] external 1372 * Flow is user flow. 1373 * @return 1374 * The matcher priority of the flow. 1375 */ 1376 uint16_t 1377 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1378 const struct rte_flow_attr *attr, 1379 uint32_t subpriority, bool external) 1380 { 1381 uint16_t priority = (uint16_t)attr->priority; 1382 struct mlx5_priv *priv = dev->data->dev_private; 1383 1384 /* NIC root rules */ 1385 if (!attr->group && !attr->transfer) { 1386 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1387 priority = priv->sh->flow_max_priority - 1; 1388 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1389 /* FDB root rules */ 1390 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1391 attr->group == 0 && 1392 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1393 return (priv->sh->flow_max_priority - 1) * 3; 1394 } 1395 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1396 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1397 return priority * 3 + subpriority; 1398 } 1399 1400 /** 1401 * Verify the @p item specifications (spec, last, mask) are compatible with the 1402 * NIC capabilities. 1403 * 1404 * @param[in] item 1405 * Item specification. 1406 * @param[in] mask 1407 * @p item->mask or flow default bit-masks. 1408 * @param[in] nic_mask 1409 * Bit-masks covering supported fields by the NIC to compare with user mask. 1410 * @param[in] size 1411 * Bit-masks size in bytes. 1412 * @param[in] range_accepted 1413 * True if range of values is accepted for specific fields, false otherwise. 1414 * @param[out] error 1415 * Pointer to error structure. 1416 * 1417 * @return 1418 * 0 on success, a negative errno value otherwise and rte_errno is set. 1419 */ 1420 int 1421 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1422 const uint8_t *mask, 1423 const uint8_t *nic_mask, 1424 unsigned int size, 1425 bool range_accepted, 1426 struct rte_flow_error *error) 1427 { 1428 unsigned int i; 1429 1430 MLX5_ASSERT(nic_mask); 1431 for (i = 0; i < size; ++i) 1432 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1433 return rte_flow_error_set(error, ENOTSUP, 1434 RTE_FLOW_ERROR_TYPE_ITEM, 1435 item, 1436 "mask enables non supported" 1437 " bits"); 1438 if (!item->spec && (item->mask || item->last)) 1439 return rte_flow_error_set(error, EINVAL, 1440 RTE_FLOW_ERROR_TYPE_ITEM, item, 1441 "mask/last without a spec is not" 1442 " supported"); 1443 if (item->spec && item->last && !range_accepted) { 1444 uint8_t spec[size]; 1445 uint8_t last[size]; 1446 unsigned int i; 1447 int ret; 1448 1449 for (i = 0; i < size; ++i) { 1450 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1451 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1452 } 1453 ret = memcmp(spec, last, size); 1454 if (ret != 0) 1455 return rte_flow_error_set(error, EINVAL, 1456 RTE_FLOW_ERROR_TYPE_ITEM, 1457 item, 1458 "range is not valid"); 1459 } 1460 return 0; 1461 } 1462 1463 /** 1464 * Adjust the hash fields according to the @p flow information. 1465 * 1466 * @param[in] dev_flow. 1467 * Pointer to the mlx5_flow. 1468 * @param[in] tunnel 1469 * 1 when the hash field is for a tunnel item. 1470 * @param[in] layer_types 1471 * RTE_ETH_RSS_* types. 1472 * @param[in] hash_fields 1473 * Item hash fields. 1474 * 1475 * @return 1476 * The hash fields that should be used. 1477 */ 1478 uint64_t 1479 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1480 int tunnel __rte_unused, uint64_t layer_types, 1481 uint64_t hash_fields) 1482 { 1483 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1484 int rss_request_inner = rss_desc->level >= 2; 1485 1486 /* Check RSS hash level for tunnel. */ 1487 if (tunnel && rss_request_inner) 1488 hash_fields |= IBV_RX_HASH_INNER; 1489 else if (tunnel || rss_request_inner) 1490 return 0; 1491 #endif 1492 /* Check if requested layer matches RSS hash fields. */ 1493 if (!(rss_desc->types & layer_types)) 1494 return 0; 1495 return hash_fields; 1496 } 1497 1498 /** 1499 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1500 * if several tunnel rules are used on this queue, the tunnel ptype will be 1501 * cleared. 1502 * 1503 * @param rxq_ctrl 1504 * Rx queue to update. 1505 */ 1506 static void 1507 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1508 { 1509 unsigned int i; 1510 uint32_t tunnel_ptype = 0; 1511 1512 /* Look up for the ptype to use. */ 1513 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1514 if (!rxq_ctrl->flow_tunnels_n[i]) 1515 continue; 1516 if (!tunnel_ptype) { 1517 tunnel_ptype = tunnels_info[i].ptype; 1518 } else { 1519 tunnel_ptype = 0; 1520 break; 1521 } 1522 } 1523 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1524 } 1525 1526 /** 1527 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1528 * flow. 1529 * 1530 * @param[in] dev 1531 * Pointer to the Ethernet device structure. 1532 * @param[in] dev_handle 1533 * Pointer to device flow handle structure. 1534 */ 1535 void 1536 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1537 struct mlx5_flow_handle *dev_handle) 1538 { 1539 struct mlx5_priv *priv = dev->data->dev_private; 1540 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1541 struct mlx5_ind_table_obj *ind_tbl = NULL; 1542 unsigned int i; 1543 1544 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1545 struct mlx5_hrxq *hrxq; 1546 1547 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1548 dev_handle->rix_hrxq); 1549 if (hrxq) 1550 ind_tbl = hrxq->ind_table; 1551 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1552 struct mlx5_shared_action_rss *shared_rss; 1553 1554 shared_rss = mlx5_ipool_get 1555 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1556 dev_handle->rix_srss); 1557 if (shared_rss) 1558 ind_tbl = shared_rss->ind_tbl; 1559 } 1560 if (!ind_tbl) 1561 return; 1562 for (i = 0; i != ind_tbl->queues_n; ++i) { 1563 int idx = ind_tbl->queues[i]; 1564 struct mlx5_rxq_ctrl *rxq_ctrl; 1565 1566 if (mlx5_is_external_rxq(dev, idx)) 1567 continue; 1568 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1569 MLX5_ASSERT(rxq_ctrl != NULL); 1570 if (rxq_ctrl == NULL) 1571 continue; 1572 /* 1573 * To support metadata register copy on Tx loopback, 1574 * this must be always enabled (metadata may arive 1575 * from other port - not from local flows only. 1576 */ 1577 if (tunnel) { 1578 unsigned int j; 1579 1580 /* Increase the counter matching the flow. */ 1581 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1582 if ((tunnels_info[j].tunnel & 1583 dev_handle->layers) == 1584 tunnels_info[j].tunnel) { 1585 rxq_ctrl->flow_tunnels_n[j]++; 1586 break; 1587 } 1588 } 1589 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1590 } 1591 } 1592 } 1593 1594 static void 1595 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1596 { 1597 struct mlx5_priv *priv = dev->data->dev_private; 1598 struct mlx5_rxq_ctrl *rxq_ctrl; 1599 uint16_t port_id; 1600 1601 if (priv->sh->shared_mark_enabled) 1602 return; 1603 if (priv->master || priv->representor) { 1604 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1605 struct mlx5_priv *opriv = 1606 rte_eth_devices[port_id].data->dev_private; 1607 1608 if (!opriv || 1609 opriv->sh != priv->sh || 1610 opriv->domain_id != priv->domain_id || 1611 opriv->mark_enabled) 1612 continue; 1613 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1614 rxq_ctrl->rxq.mark = 1; 1615 } 1616 opriv->mark_enabled = 1; 1617 } 1618 } else { 1619 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1620 rxq_ctrl->rxq.mark = 1; 1621 } 1622 priv->mark_enabled = 1; 1623 } 1624 priv->sh->shared_mark_enabled = 1; 1625 } 1626 1627 /** 1628 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1629 * 1630 * @param[in] dev 1631 * Pointer to the Ethernet device structure. 1632 * @param[in] flow 1633 * Pointer to flow structure. 1634 */ 1635 static void 1636 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1637 { 1638 struct mlx5_priv *priv = dev->data->dev_private; 1639 uint32_t handle_idx; 1640 struct mlx5_flow_handle *dev_handle; 1641 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1642 1643 MLX5_ASSERT(wks); 1644 if (wks->mark) 1645 flow_rxq_mark_flag_set(dev); 1646 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1647 handle_idx, dev_handle, next) 1648 flow_drv_rxq_flags_set(dev, dev_handle); 1649 } 1650 1651 /** 1652 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1653 * device flow if no other flow uses it with the same kind of request. 1654 * 1655 * @param dev 1656 * Pointer to Ethernet device. 1657 * @param[in] dev_handle 1658 * Pointer to the device flow handle structure. 1659 */ 1660 static void 1661 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1662 struct mlx5_flow_handle *dev_handle) 1663 { 1664 struct mlx5_priv *priv = dev->data->dev_private; 1665 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1666 struct mlx5_ind_table_obj *ind_tbl = NULL; 1667 unsigned int i; 1668 1669 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1670 struct mlx5_hrxq *hrxq; 1671 1672 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1673 dev_handle->rix_hrxq); 1674 if (hrxq) 1675 ind_tbl = hrxq->ind_table; 1676 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1677 struct mlx5_shared_action_rss *shared_rss; 1678 1679 shared_rss = mlx5_ipool_get 1680 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1681 dev_handle->rix_srss); 1682 if (shared_rss) 1683 ind_tbl = shared_rss->ind_tbl; 1684 } 1685 if (!ind_tbl) 1686 return; 1687 MLX5_ASSERT(dev->data->dev_started); 1688 for (i = 0; i != ind_tbl->queues_n; ++i) { 1689 int idx = ind_tbl->queues[i]; 1690 struct mlx5_rxq_ctrl *rxq_ctrl; 1691 1692 if (mlx5_is_external_rxq(dev, idx)) 1693 continue; 1694 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1695 MLX5_ASSERT(rxq_ctrl != NULL); 1696 if (rxq_ctrl == NULL) 1697 continue; 1698 if (tunnel) { 1699 unsigned int j; 1700 1701 /* Decrease the counter matching the flow. */ 1702 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1703 if ((tunnels_info[j].tunnel & 1704 dev_handle->layers) == 1705 tunnels_info[j].tunnel) { 1706 rxq_ctrl->flow_tunnels_n[j]--; 1707 break; 1708 } 1709 } 1710 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1711 } 1712 } 1713 } 1714 1715 /** 1716 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1717 * @p flow if no other flow uses it with the same kind of request. 1718 * 1719 * @param dev 1720 * Pointer to Ethernet device. 1721 * @param[in] flow 1722 * Pointer to the flow. 1723 */ 1724 static void 1725 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1726 { 1727 struct mlx5_priv *priv = dev->data->dev_private; 1728 uint32_t handle_idx; 1729 struct mlx5_flow_handle *dev_handle; 1730 1731 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1732 handle_idx, dev_handle, next) 1733 flow_drv_rxq_flags_trim(dev, dev_handle); 1734 } 1735 1736 /** 1737 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1738 * 1739 * @param dev 1740 * Pointer to Ethernet device. 1741 */ 1742 static void 1743 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1744 { 1745 struct mlx5_priv *priv = dev->data->dev_private; 1746 unsigned int i; 1747 1748 for (i = 0; i != priv->rxqs_n; ++i) { 1749 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1750 unsigned int j; 1751 1752 if (rxq == NULL || rxq->ctrl == NULL) 1753 continue; 1754 rxq->ctrl->rxq.mark = 0; 1755 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1756 rxq->ctrl->flow_tunnels_n[j] = 0; 1757 rxq->ctrl->rxq.tunnel = 0; 1758 } 1759 priv->mark_enabled = 0; 1760 priv->sh->shared_mark_enabled = 0; 1761 } 1762 1763 /** 1764 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1765 * 1766 * @param[in] dev 1767 * Pointer to the Ethernet device structure. 1768 */ 1769 void 1770 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1771 { 1772 struct mlx5_priv *priv = dev->data->dev_private; 1773 unsigned int i; 1774 1775 for (i = 0; i != priv->rxqs_n; ++i) { 1776 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1777 struct mlx5_rxq_data *data; 1778 1779 if (rxq == NULL || rxq->ctrl == NULL) 1780 continue; 1781 data = &rxq->ctrl->rxq; 1782 if (!rte_flow_dynf_metadata_avail()) { 1783 data->dynf_meta = 0; 1784 data->flow_meta_mask = 0; 1785 data->flow_meta_offset = -1; 1786 data->flow_meta_port_mask = 0; 1787 } else { 1788 data->dynf_meta = 1; 1789 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1790 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1791 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1792 } 1793 } 1794 } 1795 1796 /* 1797 * return a pointer to the desired action in the list of actions. 1798 * 1799 * @param[in] actions 1800 * The list of actions to search the action in. 1801 * @param[in] action 1802 * The action to find. 1803 * 1804 * @return 1805 * Pointer to the action in the list, if found. NULL otherwise. 1806 */ 1807 const struct rte_flow_action * 1808 mlx5_flow_find_action(const struct rte_flow_action *actions, 1809 enum rte_flow_action_type action) 1810 { 1811 if (actions == NULL) 1812 return NULL; 1813 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1814 if (actions->type == action) 1815 return actions; 1816 return NULL; 1817 } 1818 1819 /* 1820 * Validate the flag action. 1821 * 1822 * @param[in] action_flags 1823 * Bit-fields that holds the actions detected until now. 1824 * @param[in] attr 1825 * Attributes of flow that includes this action. 1826 * @param[out] error 1827 * Pointer to error structure. 1828 * 1829 * @return 1830 * 0 on success, a negative errno value otherwise and rte_errno is set. 1831 */ 1832 int 1833 mlx5_flow_validate_action_flag(uint64_t action_flags, 1834 const struct rte_flow_attr *attr, 1835 struct rte_flow_error *error) 1836 { 1837 if (action_flags & MLX5_FLOW_ACTION_MARK) 1838 return rte_flow_error_set(error, EINVAL, 1839 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1840 "can't mark and flag in same flow"); 1841 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1842 return rte_flow_error_set(error, EINVAL, 1843 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1844 "can't have 2 flag" 1845 " actions in same flow"); 1846 if (attr->egress) 1847 return rte_flow_error_set(error, ENOTSUP, 1848 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1849 "flag action not supported for " 1850 "egress"); 1851 return 0; 1852 } 1853 1854 /* 1855 * Validate the mark action. 1856 * 1857 * @param[in] action 1858 * Pointer to the queue action. 1859 * @param[in] action_flags 1860 * Bit-fields that holds the actions detected until now. 1861 * @param[in] attr 1862 * Attributes of flow that includes this action. 1863 * @param[out] error 1864 * Pointer to error structure. 1865 * 1866 * @return 1867 * 0 on success, a negative errno value otherwise and rte_errno is set. 1868 */ 1869 int 1870 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1871 uint64_t action_flags, 1872 const struct rte_flow_attr *attr, 1873 struct rte_flow_error *error) 1874 { 1875 const struct rte_flow_action_mark *mark = action->conf; 1876 1877 if (!mark) 1878 return rte_flow_error_set(error, EINVAL, 1879 RTE_FLOW_ERROR_TYPE_ACTION, 1880 action, 1881 "configuration cannot be null"); 1882 if (mark->id >= MLX5_FLOW_MARK_MAX) 1883 return rte_flow_error_set(error, EINVAL, 1884 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1885 &mark->id, 1886 "mark id must in 0 <= id < " 1887 RTE_STR(MLX5_FLOW_MARK_MAX)); 1888 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1889 return rte_flow_error_set(error, EINVAL, 1890 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1891 "can't flag and mark in same flow"); 1892 if (action_flags & MLX5_FLOW_ACTION_MARK) 1893 return rte_flow_error_set(error, EINVAL, 1894 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1895 "can't have 2 mark actions in same" 1896 " flow"); 1897 if (attr->egress) 1898 return rte_flow_error_set(error, ENOTSUP, 1899 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1900 "mark action not supported for " 1901 "egress"); 1902 return 0; 1903 } 1904 1905 /* 1906 * Validate the drop action. 1907 * 1908 * @param[in] action_flags 1909 * Bit-fields that holds the actions detected until now. 1910 * @param[in] attr 1911 * Attributes of flow that includes this action. 1912 * @param[out] error 1913 * Pointer to error structure. 1914 * 1915 * @return 1916 * 0 on success, a negative errno value otherwise and rte_errno is set. 1917 */ 1918 int 1919 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1920 const struct rte_flow_attr *attr, 1921 struct rte_flow_error *error) 1922 { 1923 if (attr->egress) 1924 return rte_flow_error_set(error, ENOTSUP, 1925 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1926 "drop action not supported for " 1927 "egress"); 1928 return 0; 1929 } 1930 1931 /* 1932 * Validate the queue action. 1933 * 1934 * @param[in] action 1935 * Pointer to the queue action. 1936 * @param[in] action_flags 1937 * Bit-fields that holds the actions detected until now. 1938 * @param[in] dev 1939 * Pointer to the Ethernet device structure. 1940 * @param[in] attr 1941 * Attributes of flow that includes this action. 1942 * @param[out] error 1943 * Pointer to error structure. 1944 * 1945 * @return 1946 * 0 on success, a negative errno value otherwise and rte_errno is set. 1947 */ 1948 int 1949 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1950 uint64_t action_flags, 1951 struct rte_eth_dev *dev, 1952 const struct rte_flow_attr *attr, 1953 struct rte_flow_error *error) 1954 { 1955 struct mlx5_priv *priv = dev->data->dev_private; 1956 const struct rte_flow_action_queue *queue = action->conf; 1957 1958 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1959 return rte_flow_error_set(error, EINVAL, 1960 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1961 "can't have 2 fate actions in" 1962 " same flow"); 1963 if (attr->egress) 1964 return rte_flow_error_set(error, ENOTSUP, 1965 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1966 "queue action not supported for egress."); 1967 if (mlx5_is_external_rxq(dev, queue->index)) 1968 return 0; 1969 if (!priv->rxqs_n) 1970 return rte_flow_error_set(error, EINVAL, 1971 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1972 NULL, "No Rx queues configured"); 1973 if (queue->index >= priv->rxqs_n) 1974 return rte_flow_error_set(error, EINVAL, 1975 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1976 &queue->index, 1977 "queue index out of range"); 1978 if (mlx5_rxq_get(dev, queue->index) == NULL) 1979 return rte_flow_error_set(error, EINVAL, 1980 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1981 &queue->index, 1982 "queue is not configured"); 1983 return 0; 1984 } 1985 1986 /** 1987 * Validate queue numbers for device RSS. 1988 * 1989 * @param[in] dev 1990 * Configured device. 1991 * @param[in] queues 1992 * Array of queue numbers. 1993 * @param[in] queues_n 1994 * Size of the @p queues array. 1995 * @param[out] error 1996 * On error, filled with a textual error description. 1997 * @param[out] queue_idx 1998 * On error, filled with an offending queue index in @p queues array. 1999 * 2000 * @return 2001 * 0 on success, a negative errno code on error. 2002 */ 2003 static int 2004 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2005 const uint16_t *queues, uint32_t queues_n, 2006 const char **error, uint32_t *queue_idx) 2007 { 2008 const struct mlx5_priv *priv = dev->data->dev_private; 2009 bool is_hairpin = false; 2010 bool is_ext_rss = false; 2011 uint32_t i; 2012 2013 for (i = 0; i != queues_n; ++i) { 2014 struct mlx5_rxq_ctrl *rxq_ctrl; 2015 2016 if (mlx5_is_external_rxq(dev, queues[0])) { 2017 is_ext_rss = true; 2018 continue; 2019 } 2020 if (is_ext_rss) { 2021 *error = "Combining external and regular RSS queues is not supported"; 2022 *queue_idx = i; 2023 return -ENOTSUP; 2024 } 2025 if (queues[i] >= priv->rxqs_n) { 2026 *error = "queue index out of range"; 2027 *queue_idx = i; 2028 return -EINVAL; 2029 } 2030 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2031 if (rxq_ctrl == NULL) { 2032 *error = "queue is not configured"; 2033 *queue_idx = i; 2034 return -EINVAL; 2035 } 2036 if (i == 0 && rxq_ctrl->is_hairpin) 2037 is_hairpin = true; 2038 if (is_hairpin != rxq_ctrl->is_hairpin) { 2039 *error = "combining hairpin and regular RSS queues is not supported"; 2040 *queue_idx = i; 2041 return -ENOTSUP; 2042 } 2043 } 2044 return 0; 2045 } 2046 2047 /* 2048 * Validate the rss action. 2049 * 2050 * @param[in] dev 2051 * Pointer to the Ethernet device structure. 2052 * @param[in] action 2053 * Pointer to the queue action. 2054 * @param[out] error 2055 * Pointer to error structure. 2056 * 2057 * @return 2058 * 0 on success, a negative errno value otherwise and rte_errno is set. 2059 */ 2060 int 2061 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2062 const struct rte_flow_action *action, 2063 struct rte_flow_error *error) 2064 { 2065 struct mlx5_priv *priv = dev->data->dev_private; 2066 const struct rte_flow_action_rss *rss = action->conf; 2067 int ret; 2068 const char *message; 2069 uint32_t queue_idx; 2070 2071 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2072 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2073 return rte_flow_error_set(error, ENOTSUP, 2074 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2075 &rss->func, 2076 "RSS hash function not supported"); 2077 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2078 if (rss->level > 2) 2079 #else 2080 if (rss->level > 1) 2081 #endif 2082 return rte_flow_error_set(error, ENOTSUP, 2083 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2084 &rss->level, 2085 "tunnel RSS is not supported"); 2086 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2087 if (rss->key_len == 0 && rss->key != NULL) 2088 return rte_flow_error_set(error, ENOTSUP, 2089 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2090 &rss->key_len, 2091 "RSS hash key length 0"); 2092 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2093 return rte_flow_error_set(error, ENOTSUP, 2094 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2095 &rss->key_len, 2096 "RSS hash key too small"); 2097 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2098 return rte_flow_error_set(error, ENOTSUP, 2099 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2100 &rss->key_len, 2101 "RSS hash key too large"); 2102 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2103 return rte_flow_error_set(error, ENOTSUP, 2104 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2105 &rss->queue_num, 2106 "number of queues too large"); 2107 if (rss->types & MLX5_RSS_HF_MASK) 2108 return rte_flow_error_set(error, ENOTSUP, 2109 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2110 &rss->types, 2111 "some RSS protocols are not" 2112 " supported"); 2113 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2114 !(rss->types & RTE_ETH_RSS_IP)) 2115 return rte_flow_error_set(error, EINVAL, 2116 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2117 "L3 partial RSS requested but L3 RSS" 2118 " type not specified"); 2119 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2120 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2121 return rte_flow_error_set(error, EINVAL, 2122 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2123 "L4 partial RSS requested but L4 RSS" 2124 " type not specified"); 2125 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2126 return rte_flow_error_set(error, EINVAL, 2127 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2128 NULL, "No Rx queues configured"); 2129 if (!rss->queue_num) 2130 return rte_flow_error_set(error, EINVAL, 2131 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2132 NULL, "No queues configured"); 2133 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2134 &message, &queue_idx); 2135 if (ret != 0) { 2136 return rte_flow_error_set(error, -ret, 2137 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2138 &rss->queue[queue_idx], message); 2139 } 2140 return 0; 2141 } 2142 2143 /* 2144 * Validate the rss action. 2145 * 2146 * @param[in] action 2147 * Pointer to the queue action. 2148 * @param[in] action_flags 2149 * Bit-fields that holds the actions detected until now. 2150 * @param[in] dev 2151 * Pointer to the Ethernet device structure. 2152 * @param[in] attr 2153 * Attributes of flow that includes this action. 2154 * @param[in] item_flags 2155 * Items that were detected. 2156 * @param[out] error 2157 * Pointer to error structure. 2158 * 2159 * @return 2160 * 0 on success, a negative errno value otherwise and rte_errno is set. 2161 */ 2162 int 2163 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2164 uint64_t action_flags, 2165 struct rte_eth_dev *dev, 2166 const struct rte_flow_attr *attr, 2167 uint64_t item_flags, 2168 struct rte_flow_error *error) 2169 { 2170 const struct rte_flow_action_rss *rss = action->conf; 2171 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2172 int ret; 2173 2174 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2175 return rte_flow_error_set(error, EINVAL, 2176 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2177 "can't have 2 fate actions" 2178 " in same flow"); 2179 ret = mlx5_validate_action_rss(dev, action, error); 2180 if (ret) 2181 return ret; 2182 if (attr->egress) 2183 return rte_flow_error_set(error, ENOTSUP, 2184 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2185 "rss action not supported for " 2186 "egress"); 2187 if (rss->level > 1 && !tunnel) 2188 return rte_flow_error_set(error, EINVAL, 2189 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2190 "inner RSS is not supported for " 2191 "non-tunnel flows"); 2192 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2193 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2194 return rte_flow_error_set(error, EINVAL, 2195 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2196 "RSS on eCPRI is not supported now"); 2197 } 2198 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2199 !(item_flags & 2200 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2201 rss->level > 1) 2202 return rte_flow_error_set(error, EINVAL, 2203 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2204 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2205 return 0; 2206 } 2207 2208 /* 2209 * Validate the default miss action. 2210 * 2211 * @param[in] action_flags 2212 * Bit-fields that holds the actions detected until now. 2213 * @param[out] error 2214 * Pointer to error structure. 2215 * 2216 * @return 2217 * 0 on success, a negative errno value otherwise and rte_errno is set. 2218 */ 2219 int 2220 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2221 const struct rte_flow_attr *attr, 2222 struct rte_flow_error *error) 2223 { 2224 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2225 return rte_flow_error_set(error, EINVAL, 2226 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2227 "can't have 2 fate actions in" 2228 " same flow"); 2229 if (attr->egress) 2230 return rte_flow_error_set(error, ENOTSUP, 2231 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2232 "default miss action not supported " 2233 "for egress"); 2234 if (attr->group) 2235 return rte_flow_error_set(error, ENOTSUP, 2236 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2237 "only group 0 is supported"); 2238 if (attr->transfer) 2239 return rte_flow_error_set(error, ENOTSUP, 2240 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2241 NULL, "transfer is not supported"); 2242 return 0; 2243 } 2244 2245 /* 2246 * Validate the count action. 2247 * 2248 * @param[in] dev 2249 * Pointer to the Ethernet device structure. 2250 * @param[in] attr 2251 * Attributes of flow that includes this action. 2252 * @param[out] error 2253 * Pointer to error structure. 2254 * 2255 * @return 2256 * 0 on success, a negative errno value otherwise and rte_errno is set. 2257 */ 2258 int 2259 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2260 const struct rte_flow_attr *attr, 2261 struct rte_flow_error *error) 2262 { 2263 if (attr->egress) 2264 return rte_flow_error_set(error, ENOTSUP, 2265 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2266 "count action not supported for " 2267 "egress"); 2268 return 0; 2269 } 2270 2271 /* 2272 * Validate the ASO CT action. 2273 * 2274 * @param[in] dev 2275 * Pointer to the Ethernet device structure. 2276 * @param[in] conntrack 2277 * Pointer to the CT action profile. 2278 * @param[out] error 2279 * Pointer to error structure. 2280 * 2281 * @return 2282 * 0 on success, a negative errno value otherwise and rte_errno is set. 2283 */ 2284 int 2285 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2286 const struct rte_flow_action_conntrack *conntrack, 2287 struct rte_flow_error *error) 2288 { 2289 RTE_SET_USED(dev); 2290 2291 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2292 return rte_flow_error_set(error, EINVAL, 2293 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2294 "Invalid CT state"); 2295 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2296 return rte_flow_error_set(error, EINVAL, 2297 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2298 "Invalid last TCP packet flag"); 2299 return 0; 2300 } 2301 2302 /** 2303 * Verify the @p attributes will be correctly understood by the NIC and store 2304 * them in the @p flow if everything is correct. 2305 * 2306 * @param[in] dev 2307 * Pointer to the Ethernet device structure. 2308 * @param[in] attributes 2309 * Pointer to flow attributes 2310 * @param[out] error 2311 * Pointer to error structure. 2312 * 2313 * @return 2314 * 0 on success, a negative errno value otherwise and rte_errno is set. 2315 */ 2316 int 2317 mlx5_flow_validate_attributes(struct rte_eth_dev *dev, 2318 const struct rte_flow_attr *attributes, 2319 struct rte_flow_error *error) 2320 { 2321 struct mlx5_priv *priv = dev->data->dev_private; 2322 uint32_t priority_max = priv->sh->flow_max_priority - 1; 2323 2324 if (attributes->group) 2325 return rte_flow_error_set(error, ENOTSUP, 2326 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 2327 NULL, "groups is not supported"); 2328 if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR && 2329 attributes->priority >= priority_max) 2330 return rte_flow_error_set(error, ENOTSUP, 2331 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY, 2332 NULL, "priority out of range"); 2333 if (attributes->egress) 2334 return rte_flow_error_set(error, ENOTSUP, 2335 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2336 "egress is not supported"); 2337 if (attributes->transfer && !priv->sh->config.dv_esw_en) 2338 return rte_flow_error_set(error, ENOTSUP, 2339 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2340 NULL, "transfer is not supported"); 2341 if (!attributes->ingress) 2342 return rte_flow_error_set(error, EINVAL, 2343 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, 2344 NULL, 2345 "ingress attribute is mandatory"); 2346 return 0; 2347 } 2348 2349 /** 2350 * Validate ICMP6 item. 2351 * 2352 * @param[in] item 2353 * Item specification. 2354 * @param[in] item_flags 2355 * Bit-fields that holds the items detected until now. 2356 * @param[in] ext_vlan_sup 2357 * Whether extended VLAN features are supported or not. 2358 * @param[out] error 2359 * Pointer to error structure. 2360 * 2361 * @return 2362 * 0 on success, a negative errno value otherwise and rte_errno is set. 2363 */ 2364 int 2365 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2366 uint64_t item_flags, 2367 uint8_t target_protocol, 2368 struct rte_flow_error *error) 2369 { 2370 const struct rte_flow_item_icmp6 *mask = item->mask; 2371 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2372 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2373 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2374 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2375 MLX5_FLOW_LAYER_OUTER_L4; 2376 int ret; 2377 2378 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2379 return rte_flow_error_set(error, EINVAL, 2380 RTE_FLOW_ERROR_TYPE_ITEM, item, 2381 "protocol filtering not compatible" 2382 " with ICMP6 layer"); 2383 if (!(item_flags & l3m)) 2384 return rte_flow_error_set(error, EINVAL, 2385 RTE_FLOW_ERROR_TYPE_ITEM, item, 2386 "IPv6 is mandatory to filter on" 2387 " ICMP6"); 2388 if (item_flags & l4m) 2389 return rte_flow_error_set(error, EINVAL, 2390 RTE_FLOW_ERROR_TYPE_ITEM, item, 2391 "multiple L4 layers not supported"); 2392 if (!mask) 2393 mask = &rte_flow_item_icmp6_mask; 2394 ret = mlx5_flow_item_acceptable 2395 (item, (const uint8_t *)mask, 2396 (const uint8_t *)&rte_flow_item_icmp6_mask, 2397 sizeof(struct rte_flow_item_icmp6), 2398 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2399 if (ret < 0) 2400 return ret; 2401 return 0; 2402 } 2403 2404 /** 2405 * Validate ICMP item. 2406 * 2407 * @param[in] item 2408 * Item specification. 2409 * @param[in] item_flags 2410 * Bit-fields that holds the items detected until now. 2411 * @param[out] error 2412 * Pointer to error structure. 2413 * 2414 * @return 2415 * 0 on success, a negative errno value otherwise and rte_errno is set. 2416 */ 2417 int 2418 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2419 uint64_t item_flags, 2420 uint8_t target_protocol, 2421 struct rte_flow_error *error) 2422 { 2423 const struct rte_flow_item_icmp *mask = item->mask; 2424 const struct rte_flow_item_icmp nic_mask = { 2425 .hdr.icmp_type = 0xff, 2426 .hdr.icmp_code = 0xff, 2427 .hdr.icmp_ident = RTE_BE16(0xffff), 2428 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2429 }; 2430 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2431 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2432 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2433 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2434 MLX5_FLOW_LAYER_OUTER_L4; 2435 int ret; 2436 2437 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2438 return rte_flow_error_set(error, EINVAL, 2439 RTE_FLOW_ERROR_TYPE_ITEM, item, 2440 "protocol filtering not compatible" 2441 " with ICMP layer"); 2442 if (!(item_flags & l3m)) 2443 return rte_flow_error_set(error, EINVAL, 2444 RTE_FLOW_ERROR_TYPE_ITEM, item, 2445 "IPv4 is mandatory to filter" 2446 " on ICMP"); 2447 if (item_flags & l4m) 2448 return rte_flow_error_set(error, EINVAL, 2449 RTE_FLOW_ERROR_TYPE_ITEM, item, 2450 "multiple L4 layers not supported"); 2451 if (!mask) 2452 mask = &nic_mask; 2453 ret = mlx5_flow_item_acceptable 2454 (item, (const uint8_t *)mask, 2455 (const uint8_t *)&nic_mask, 2456 sizeof(struct rte_flow_item_icmp), 2457 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2458 if (ret < 0) 2459 return ret; 2460 return 0; 2461 } 2462 2463 /** 2464 * Validate Ethernet item. 2465 * 2466 * @param[in] item 2467 * Item specification. 2468 * @param[in] item_flags 2469 * Bit-fields that holds the items detected until now. 2470 * @param[out] error 2471 * Pointer to error structure. 2472 * 2473 * @return 2474 * 0 on success, a negative errno value otherwise and rte_errno is set. 2475 */ 2476 int 2477 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2478 uint64_t item_flags, bool ext_vlan_sup, 2479 struct rte_flow_error *error) 2480 { 2481 const struct rte_flow_item_eth *mask = item->mask; 2482 const struct rte_flow_item_eth nic_mask = { 2483 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2484 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2485 .type = RTE_BE16(0xffff), 2486 .has_vlan = ext_vlan_sup ? 1 : 0, 2487 }; 2488 int ret; 2489 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2490 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2491 MLX5_FLOW_LAYER_OUTER_L2; 2492 2493 if (item_flags & ethm) 2494 return rte_flow_error_set(error, ENOTSUP, 2495 RTE_FLOW_ERROR_TYPE_ITEM, item, 2496 "multiple L2 layers not supported"); 2497 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2498 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2499 return rte_flow_error_set(error, EINVAL, 2500 RTE_FLOW_ERROR_TYPE_ITEM, item, 2501 "L2 layer should not follow " 2502 "L3 layers"); 2503 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2504 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2505 return rte_flow_error_set(error, EINVAL, 2506 RTE_FLOW_ERROR_TYPE_ITEM, item, 2507 "L2 layer should not follow VLAN"); 2508 if (item_flags & MLX5_FLOW_LAYER_GTP) 2509 return rte_flow_error_set(error, EINVAL, 2510 RTE_FLOW_ERROR_TYPE_ITEM, item, 2511 "L2 layer should not follow GTP"); 2512 if (!mask) 2513 mask = &rte_flow_item_eth_mask; 2514 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2515 (const uint8_t *)&nic_mask, 2516 sizeof(struct rte_flow_item_eth), 2517 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2518 return ret; 2519 } 2520 2521 /** 2522 * Validate VLAN item. 2523 * 2524 * @param[in] item 2525 * Item specification. 2526 * @param[in] item_flags 2527 * Bit-fields that holds the items detected until now. 2528 * @param[in] dev 2529 * Ethernet device flow is being created on. 2530 * @param[out] error 2531 * Pointer to error structure. 2532 * 2533 * @return 2534 * 0 on success, a negative errno value otherwise and rte_errno is set. 2535 */ 2536 int 2537 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2538 uint64_t item_flags, 2539 struct rte_eth_dev *dev, 2540 struct rte_flow_error *error) 2541 { 2542 const struct rte_flow_item_vlan *spec = item->spec; 2543 const struct rte_flow_item_vlan *mask = item->mask; 2544 const struct rte_flow_item_vlan nic_mask = { 2545 .tci = RTE_BE16(UINT16_MAX), 2546 .inner_type = RTE_BE16(UINT16_MAX), 2547 }; 2548 uint16_t vlan_tag = 0; 2549 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2550 int ret; 2551 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2552 MLX5_FLOW_LAYER_INNER_L4) : 2553 (MLX5_FLOW_LAYER_OUTER_L3 | 2554 MLX5_FLOW_LAYER_OUTER_L4); 2555 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2556 MLX5_FLOW_LAYER_OUTER_VLAN; 2557 2558 if (item_flags & vlanm) 2559 return rte_flow_error_set(error, EINVAL, 2560 RTE_FLOW_ERROR_TYPE_ITEM, item, 2561 "multiple VLAN layers not supported"); 2562 else if ((item_flags & l34m) != 0) 2563 return rte_flow_error_set(error, EINVAL, 2564 RTE_FLOW_ERROR_TYPE_ITEM, item, 2565 "VLAN cannot follow L3/L4 layer"); 2566 if (!mask) 2567 mask = &rte_flow_item_vlan_mask; 2568 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2569 (const uint8_t *)&nic_mask, 2570 sizeof(struct rte_flow_item_vlan), 2571 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2572 if (ret) 2573 return ret; 2574 if (!tunnel && mask->tci != RTE_BE16(0x0fff)) { 2575 struct mlx5_priv *priv = dev->data->dev_private; 2576 2577 if (priv->vmwa_context) { 2578 /* 2579 * Non-NULL context means we have a virtual machine 2580 * and SR-IOV enabled, we have to create VLAN interface 2581 * to make hypervisor to setup E-Switch vport 2582 * context correctly. We avoid creating the multiple 2583 * VLAN interfaces, so we cannot support VLAN tag mask. 2584 */ 2585 return rte_flow_error_set(error, EINVAL, 2586 RTE_FLOW_ERROR_TYPE_ITEM, 2587 item, 2588 "VLAN tag mask is not" 2589 " supported in virtual" 2590 " environment"); 2591 } 2592 } 2593 if (spec) { 2594 vlan_tag = spec->tci; 2595 vlan_tag &= mask->tci; 2596 } 2597 /* 2598 * From verbs perspective an empty VLAN is equivalent 2599 * to a packet without VLAN layer. 2600 */ 2601 if (!vlan_tag) 2602 return rte_flow_error_set(error, EINVAL, 2603 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2604 item->spec, 2605 "VLAN cannot be empty"); 2606 return 0; 2607 } 2608 2609 /** 2610 * Validate IPV4 item. 2611 * 2612 * @param[in] item 2613 * Item specification. 2614 * @param[in] item_flags 2615 * Bit-fields that holds the items detected until now. 2616 * @param[in] last_item 2617 * Previous validated item in the pattern items. 2618 * @param[in] ether_type 2619 * Type in the ethernet layer header (including dot1q). 2620 * @param[in] acc_mask 2621 * Acceptable mask, if NULL default internal default mask 2622 * will be used to check whether item fields are supported. 2623 * @param[in] range_accepted 2624 * True if range of values is accepted for specific fields, false otherwise. 2625 * @param[out] error 2626 * Pointer to error structure. 2627 * 2628 * @return 2629 * 0 on success, a negative errno value otherwise and rte_errno is set. 2630 */ 2631 int 2632 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2633 uint64_t item_flags, 2634 uint64_t last_item, 2635 uint16_t ether_type, 2636 const struct rte_flow_item_ipv4 *acc_mask, 2637 bool range_accepted, 2638 struct rte_flow_error *error) 2639 { 2640 const struct rte_flow_item_ipv4 *mask = item->mask; 2641 const struct rte_flow_item_ipv4 *spec = item->spec; 2642 const struct rte_flow_item_ipv4 nic_mask = { 2643 .hdr = { 2644 .src_addr = RTE_BE32(0xffffffff), 2645 .dst_addr = RTE_BE32(0xffffffff), 2646 .type_of_service = 0xff, 2647 .next_proto_id = 0xff, 2648 }, 2649 }; 2650 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2651 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2652 MLX5_FLOW_LAYER_OUTER_L3; 2653 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2654 MLX5_FLOW_LAYER_OUTER_L4; 2655 int ret; 2656 uint8_t next_proto = 0xFF; 2657 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2658 MLX5_FLOW_LAYER_OUTER_VLAN | 2659 MLX5_FLOW_LAYER_INNER_VLAN); 2660 2661 if ((last_item & l2_vlan) && ether_type && 2662 ether_type != RTE_ETHER_TYPE_IPV4) 2663 return rte_flow_error_set(error, EINVAL, 2664 RTE_FLOW_ERROR_TYPE_ITEM, item, 2665 "IPv4 cannot follow L2/VLAN layer " 2666 "which ether type is not IPv4"); 2667 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2668 if (mask && spec) 2669 next_proto = mask->hdr.next_proto_id & 2670 spec->hdr.next_proto_id; 2671 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2672 return rte_flow_error_set(error, EINVAL, 2673 RTE_FLOW_ERROR_TYPE_ITEM, 2674 item, 2675 "multiple tunnel " 2676 "not supported"); 2677 } 2678 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2679 return rte_flow_error_set(error, EINVAL, 2680 RTE_FLOW_ERROR_TYPE_ITEM, item, 2681 "wrong tunnel type - IPv6 specified " 2682 "but IPv4 item provided"); 2683 if (item_flags & l3m) 2684 return rte_flow_error_set(error, ENOTSUP, 2685 RTE_FLOW_ERROR_TYPE_ITEM, item, 2686 "multiple L3 layers not supported"); 2687 else if (item_flags & l4m) 2688 return rte_flow_error_set(error, EINVAL, 2689 RTE_FLOW_ERROR_TYPE_ITEM, item, 2690 "L3 cannot follow an L4 layer."); 2691 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2692 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2693 return rte_flow_error_set(error, EINVAL, 2694 RTE_FLOW_ERROR_TYPE_ITEM, item, 2695 "L3 cannot follow an NVGRE layer."); 2696 if (!mask) 2697 mask = &rte_flow_item_ipv4_mask; 2698 else if (mask->hdr.next_proto_id != 0 && 2699 mask->hdr.next_proto_id != 0xff) 2700 return rte_flow_error_set(error, EINVAL, 2701 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2702 "partial mask is not supported" 2703 " for protocol"); 2704 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2705 acc_mask ? (const uint8_t *)acc_mask 2706 : (const uint8_t *)&nic_mask, 2707 sizeof(struct rte_flow_item_ipv4), 2708 range_accepted, error); 2709 if (ret < 0) 2710 return ret; 2711 return 0; 2712 } 2713 2714 /** 2715 * Validate IPV6 item. 2716 * 2717 * @param[in] item 2718 * Item specification. 2719 * @param[in] item_flags 2720 * Bit-fields that holds the items detected until now. 2721 * @param[in] last_item 2722 * Previous validated item in the pattern items. 2723 * @param[in] ether_type 2724 * Type in the ethernet layer header (including dot1q). 2725 * @param[in] acc_mask 2726 * Acceptable mask, if NULL default internal default mask 2727 * will be used to check whether item fields are supported. 2728 * @param[out] error 2729 * Pointer to error structure. 2730 * 2731 * @return 2732 * 0 on success, a negative errno value otherwise and rte_errno is set. 2733 */ 2734 int 2735 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2736 uint64_t item_flags, 2737 uint64_t last_item, 2738 uint16_t ether_type, 2739 const struct rte_flow_item_ipv6 *acc_mask, 2740 struct rte_flow_error *error) 2741 { 2742 const struct rte_flow_item_ipv6 *mask = item->mask; 2743 const struct rte_flow_item_ipv6 *spec = item->spec; 2744 const struct rte_flow_item_ipv6 nic_mask = { 2745 .hdr = { 2746 .src_addr = 2747 "\xff\xff\xff\xff\xff\xff\xff\xff" 2748 "\xff\xff\xff\xff\xff\xff\xff\xff", 2749 .dst_addr = 2750 "\xff\xff\xff\xff\xff\xff\xff\xff" 2751 "\xff\xff\xff\xff\xff\xff\xff\xff", 2752 .vtc_flow = RTE_BE32(0xffffffff), 2753 .proto = 0xff, 2754 }, 2755 }; 2756 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2757 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2758 MLX5_FLOW_LAYER_OUTER_L3; 2759 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2760 MLX5_FLOW_LAYER_OUTER_L4; 2761 int ret; 2762 uint8_t next_proto = 0xFF; 2763 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2764 MLX5_FLOW_LAYER_OUTER_VLAN | 2765 MLX5_FLOW_LAYER_INNER_VLAN); 2766 2767 if ((last_item & l2_vlan) && ether_type && 2768 ether_type != RTE_ETHER_TYPE_IPV6) 2769 return rte_flow_error_set(error, EINVAL, 2770 RTE_FLOW_ERROR_TYPE_ITEM, item, 2771 "IPv6 cannot follow L2/VLAN layer " 2772 "which ether type is not IPv6"); 2773 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2774 next_proto = spec->hdr.proto; 2775 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2776 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2777 return rte_flow_error_set(error, EINVAL, 2778 RTE_FLOW_ERROR_TYPE_ITEM, 2779 item, 2780 "multiple tunnel " 2781 "not supported"); 2782 } 2783 if (next_proto == IPPROTO_HOPOPTS || 2784 next_proto == IPPROTO_ROUTING || 2785 next_proto == IPPROTO_FRAGMENT || 2786 next_proto == IPPROTO_ESP || 2787 next_proto == IPPROTO_AH || 2788 next_proto == IPPROTO_DSTOPTS) 2789 return rte_flow_error_set(error, EINVAL, 2790 RTE_FLOW_ERROR_TYPE_ITEM, item, 2791 "IPv6 proto (next header) should " 2792 "not be set as extension header"); 2793 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2794 return rte_flow_error_set(error, EINVAL, 2795 RTE_FLOW_ERROR_TYPE_ITEM, item, 2796 "wrong tunnel type - IPv4 specified " 2797 "but IPv6 item provided"); 2798 if (item_flags & l3m) 2799 return rte_flow_error_set(error, ENOTSUP, 2800 RTE_FLOW_ERROR_TYPE_ITEM, item, 2801 "multiple L3 layers not supported"); 2802 else if (item_flags & l4m) 2803 return rte_flow_error_set(error, EINVAL, 2804 RTE_FLOW_ERROR_TYPE_ITEM, item, 2805 "L3 cannot follow an L4 layer."); 2806 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2807 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2808 return rte_flow_error_set(error, EINVAL, 2809 RTE_FLOW_ERROR_TYPE_ITEM, item, 2810 "L3 cannot follow an NVGRE layer."); 2811 if (!mask) 2812 mask = &rte_flow_item_ipv6_mask; 2813 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2814 acc_mask ? (const uint8_t *)acc_mask 2815 : (const uint8_t *)&nic_mask, 2816 sizeof(struct rte_flow_item_ipv6), 2817 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2818 if (ret < 0) 2819 return ret; 2820 return 0; 2821 } 2822 2823 /** 2824 * Validate UDP item. 2825 * 2826 * @param[in] item 2827 * Item specification. 2828 * @param[in] item_flags 2829 * Bit-fields that holds the items detected until now. 2830 * @param[in] target_protocol 2831 * The next protocol in the previous item. 2832 * @param[in] flow_mask 2833 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2834 * @param[out] error 2835 * Pointer to error structure. 2836 * 2837 * @return 2838 * 0 on success, a negative errno value otherwise and rte_errno is set. 2839 */ 2840 int 2841 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2842 uint64_t item_flags, 2843 uint8_t target_protocol, 2844 struct rte_flow_error *error) 2845 { 2846 const struct rte_flow_item_udp *mask = item->mask; 2847 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2848 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2849 MLX5_FLOW_LAYER_OUTER_L3; 2850 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2851 MLX5_FLOW_LAYER_OUTER_L4; 2852 int ret; 2853 2854 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2855 return rte_flow_error_set(error, EINVAL, 2856 RTE_FLOW_ERROR_TYPE_ITEM, item, 2857 "protocol filtering not compatible" 2858 " with UDP layer"); 2859 if (!(item_flags & l3m)) 2860 return rte_flow_error_set(error, EINVAL, 2861 RTE_FLOW_ERROR_TYPE_ITEM, item, 2862 "L3 is mandatory to filter on L4"); 2863 if (item_flags & l4m) 2864 return rte_flow_error_set(error, EINVAL, 2865 RTE_FLOW_ERROR_TYPE_ITEM, item, 2866 "multiple L4 layers not supported"); 2867 if (!mask) 2868 mask = &rte_flow_item_udp_mask; 2869 ret = mlx5_flow_item_acceptable 2870 (item, (const uint8_t *)mask, 2871 (const uint8_t *)&rte_flow_item_udp_mask, 2872 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2873 error); 2874 if (ret < 0) 2875 return ret; 2876 return 0; 2877 } 2878 2879 /** 2880 * Validate TCP item. 2881 * 2882 * @param[in] item 2883 * Item specification. 2884 * @param[in] item_flags 2885 * Bit-fields that holds the items detected until now. 2886 * @param[in] target_protocol 2887 * The next protocol in the previous item. 2888 * @param[out] error 2889 * Pointer to error structure. 2890 * 2891 * @return 2892 * 0 on success, a negative errno value otherwise and rte_errno is set. 2893 */ 2894 int 2895 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2896 uint64_t item_flags, 2897 uint8_t target_protocol, 2898 const struct rte_flow_item_tcp *flow_mask, 2899 struct rte_flow_error *error) 2900 { 2901 const struct rte_flow_item_tcp *mask = item->mask; 2902 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2903 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2904 MLX5_FLOW_LAYER_OUTER_L3; 2905 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2906 MLX5_FLOW_LAYER_OUTER_L4; 2907 int ret; 2908 2909 MLX5_ASSERT(flow_mask); 2910 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2911 return rte_flow_error_set(error, EINVAL, 2912 RTE_FLOW_ERROR_TYPE_ITEM, item, 2913 "protocol filtering not compatible" 2914 " with TCP layer"); 2915 if (!(item_flags & l3m)) 2916 return rte_flow_error_set(error, EINVAL, 2917 RTE_FLOW_ERROR_TYPE_ITEM, item, 2918 "L3 is mandatory to filter on L4"); 2919 if (item_flags & l4m) 2920 return rte_flow_error_set(error, EINVAL, 2921 RTE_FLOW_ERROR_TYPE_ITEM, item, 2922 "multiple L4 layers not supported"); 2923 if (!mask) 2924 mask = &rte_flow_item_tcp_mask; 2925 ret = mlx5_flow_item_acceptable 2926 (item, (const uint8_t *)mask, 2927 (const uint8_t *)flow_mask, 2928 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2929 error); 2930 if (ret < 0) 2931 return ret; 2932 return 0; 2933 } 2934 2935 /** 2936 * Validate VXLAN item. 2937 * 2938 * @param[in] dev 2939 * Pointer to the Ethernet device structure. 2940 * @param[in] udp_dport 2941 * UDP destination port 2942 * @param[in] item 2943 * Item specification. 2944 * @param[in] item_flags 2945 * Bit-fields that holds the items detected until now. 2946 * @param root 2947 * Whether action is on root table. 2948 * @param[out] error 2949 * Pointer to error structure. 2950 * 2951 * @return 2952 * 0 on success, a negative errno value otherwise and rte_errno is set. 2953 */ 2954 int 2955 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2956 uint16_t udp_dport, 2957 const struct rte_flow_item *item, 2958 uint64_t item_flags, 2959 bool root, 2960 struct rte_flow_error *error) 2961 { 2962 const struct rte_flow_item_vxlan *spec = item->spec; 2963 const struct rte_flow_item_vxlan *mask = item->mask; 2964 int ret; 2965 struct mlx5_priv *priv = dev->data->dev_private; 2966 union vni { 2967 uint32_t vlan_id; 2968 uint8_t vni[4]; 2969 } id = { .vlan_id = 0, }; 2970 const struct rte_flow_item_vxlan nic_mask = { 2971 .vni = "\xff\xff\xff", 2972 .rsvd1 = 0xff, 2973 }; 2974 const struct rte_flow_item_vxlan *valid_mask; 2975 2976 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2977 return rte_flow_error_set(error, ENOTSUP, 2978 RTE_FLOW_ERROR_TYPE_ITEM, item, 2979 "multiple tunnel layers not" 2980 " supported"); 2981 valid_mask = &rte_flow_item_vxlan_mask; 2982 /* 2983 * Verify only UDPv4 is present as defined in 2984 * https://tools.ietf.org/html/rfc7348 2985 */ 2986 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2987 return rte_flow_error_set(error, EINVAL, 2988 RTE_FLOW_ERROR_TYPE_ITEM, item, 2989 "no outer UDP layer found"); 2990 if (!mask) 2991 mask = &rte_flow_item_vxlan_mask; 2992 2993 if (priv->sh->steering_format_version != 2994 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2995 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2996 /* non-root table */ 2997 if (!root && priv->sh->misc5_cap) 2998 valid_mask = &nic_mask; 2999 /* Group zero in NIC domain */ 3000 if (!root && priv->sh->tunnel_header_0_1) 3001 valid_mask = &nic_mask; 3002 } 3003 ret = mlx5_flow_item_acceptable 3004 (item, (const uint8_t *)mask, 3005 (const uint8_t *)valid_mask, 3006 sizeof(struct rte_flow_item_vxlan), 3007 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3008 if (ret < 0) 3009 return ret; 3010 if (spec) { 3011 memcpy(&id.vni[1], spec->vni, 3); 3012 memcpy(&id.vni[1], mask->vni, 3); 3013 } 3014 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3015 return rte_flow_error_set(error, ENOTSUP, 3016 RTE_FLOW_ERROR_TYPE_ITEM, item, 3017 "VXLAN tunnel must be fully defined"); 3018 return 0; 3019 } 3020 3021 /** 3022 * Validate VXLAN_GPE item. 3023 * 3024 * @param[in] item 3025 * Item specification. 3026 * @param[in] item_flags 3027 * Bit-fields that holds the items detected until now. 3028 * @param[in] priv 3029 * Pointer to the private data structure. 3030 * @param[in] target_protocol 3031 * The next protocol in the previous item. 3032 * @param[out] error 3033 * Pointer to error structure. 3034 * 3035 * @return 3036 * 0 on success, a negative errno value otherwise and rte_errno is set. 3037 */ 3038 int 3039 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3040 uint64_t item_flags, 3041 struct rte_eth_dev *dev, 3042 struct rte_flow_error *error) 3043 { 3044 struct mlx5_priv *priv = dev->data->dev_private; 3045 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3046 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3047 int ret; 3048 union vni { 3049 uint32_t vlan_id; 3050 uint8_t vni[4]; 3051 } id = { .vlan_id = 0, }; 3052 3053 if (!priv->sh->config.l3_vxlan_en) 3054 return rte_flow_error_set(error, ENOTSUP, 3055 RTE_FLOW_ERROR_TYPE_ITEM, item, 3056 "L3 VXLAN is not enabled by device" 3057 " parameter and/or not configured in" 3058 " firmware"); 3059 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3060 return rte_flow_error_set(error, ENOTSUP, 3061 RTE_FLOW_ERROR_TYPE_ITEM, item, 3062 "multiple tunnel layers not" 3063 " supported"); 3064 /* 3065 * Verify only UDPv4 is present as defined in 3066 * https://tools.ietf.org/html/rfc7348 3067 */ 3068 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3069 return rte_flow_error_set(error, EINVAL, 3070 RTE_FLOW_ERROR_TYPE_ITEM, item, 3071 "no outer UDP layer found"); 3072 if (!mask) 3073 mask = &rte_flow_item_vxlan_gpe_mask; 3074 ret = mlx5_flow_item_acceptable 3075 (item, (const uint8_t *)mask, 3076 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 3077 sizeof(struct rte_flow_item_vxlan_gpe), 3078 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3079 if (ret < 0) 3080 return ret; 3081 if (spec) { 3082 if (spec->protocol) 3083 return rte_flow_error_set(error, ENOTSUP, 3084 RTE_FLOW_ERROR_TYPE_ITEM, 3085 item, 3086 "VxLAN-GPE protocol" 3087 " not supported"); 3088 memcpy(&id.vni[1], spec->vni, 3); 3089 memcpy(&id.vni[1], mask->vni, 3); 3090 } 3091 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3092 return rte_flow_error_set(error, ENOTSUP, 3093 RTE_FLOW_ERROR_TYPE_ITEM, item, 3094 "VXLAN-GPE tunnel must be fully" 3095 " defined"); 3096 return 0; 3097 } 3098 /** 3099 * Validate GRE Key item. 3100 * 3101 * @param[in] item 3102 * Item specification. 3103 * @param[in] item_flags 3104 * Bit flags to mark detected items. 3105 * @param[in] gre_item 3106 * Pointer to gre_item 3107 * @param[out] error 3108 * Pointer to error structure. 3109 * 3110 * @return 3111 * 0 on success, a negative errno value otherwise and rte_errno is set. 3112 */ 3113 int 3114 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3115 uint64_t item_flags, 3116 const struct rte_flow_item *gre_item, 3117 struct rte_flow_error *error) 3118 { 3119 const rte_be32_t *mask = item->mask; 3120 int ret = 0; 3121 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3122 const struct rte_flow_item_gre *gre_spec; 3123 const struct rte_flow_item_gre *gre_mask; 3124 3125 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3126 return rte_flow_error_set(error, ENOTSUP, 3127 RTE_FLOW_ERROR_TYPE_ITEM, item, 3128 "Multiple GRE key not support"); 3129 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3130 return rte_flow_error_set(error, ENOTSUP, 3131 RTE_FLOW_ERROR_TYPE_ITEM, item, 3132 "No preceding GRE header"); 3133 if (item_flags & MLX5_FLOW_LAYER_INNER) 3134 return rte_flow_error_set(error, ENOTSUP, 3135 RTE_FLOW_ERROR_TYPE_ITEM, item, 3136 "GRE key following a wrong item"); 3137 gre_mask = gre_item->mask; 3138 if (!gre_mask) 3139 gre_mask = &rte_flow_item_gre_mask; 3140 gre_spec = gre_item->spec; 3141 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3142 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3143 return rte_flow_error_set(error, EINVAL, 3144 RTE_FLOW_ERROR_TYPE_ITEM, item, 3145 "Key bit must be on"); 3146 3147 if (!mask) 3148 mask = &gre_key_default_mask; 3149 ret = mlx5_flow_item_acceptable 3150 (item, (const uint8_t *)mask, 3151 (const uint8_t *)&gre_key_default_mask, 3152 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3153 return ret; 3154 } 3155 3156 /** 3157 * Validate GRE optional item. 3158 * 3159 * @param[in] dev 3160 * Pointer to the Ethernet device structure. 3161 * @param[in] item 3162 * Item specification. 3163 * @param[in] item_flags 3164 * Bit flags to mark detected items. 3165 * @param[in] attr 3166 * Flow rule attributes. 3167 * @param[in] gre_item 3168 * Pointer to gre_item 3169 * @param[out] error 3170 * Pointer to error structure. 3171 * 3172 * @return 3173 * 0 on success, a negative errno value otherwise and rte_errno is set. 3174 */ 3175 int 3176 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3177 const struct rte_flow_item *item, 3178 uint64_t item_flags, 3179 const struct rte_flow_attr *attr, 3180 const struct rte_flow_item *gre_item, 3181 struct rte_flow_error *error) 3182 { 3183 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3184 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3185 const struct rte_flow_item_gre_opt *spec = item->spec; 3186 const struct rte_flow_item_gre_opt *mask = item->mask; 3187 struct mlx5_priv *priv = dev->data->dev_private; 3188 int ret = 0; 3189 struct rte_flow_item_gre_opt nic_mask = { 3190 .checksum_rsvd = { 3191 .checksum = RTE_BE16(UINT16_MAX), 3192 .reserved1 = 0x0, 3193 }, 3194 .key = { 3195 .key = RTE_BE32(UINT32_MAX), 3196 }, 3197 .sequence = { 3198 .sequence = RTE_BE32(UINT32_MAX), 3199 }, 3200 }; 3201 3202 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3203 return rte_flow_error_set(error, ENOTSUP, 3204 RTE_FLOW_ERROR_TYPE_ITEM, item, 3205 "No preceding GRE header"); 3206 if (item_flags & MLX5_FLOW_LAYER_INNER) 3207 return rte_flow_error_set(error, ENOTSUP, 3208 RTE_FLOW_ERROR_TYPE_ITEM, item, 3209 "GRE option following a wrong item"); 3210 if (!spec || !mask) 3211 return rte_flow_error_set(error, EINVAL, 3212 RTE_FLOW_ERROR_TYPE_ITEM, item, 3213 "At least one field gre_option(checksum/key/sequence) must be specified"); 3214 if (!gre_mask) 3215 gre_mask = &rte_flow_item_gre_mask; 3216 if (mask->checksum_rsvd.checksum) 3217 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3218 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3219 return rte_flow_error_set(error, EINVAL, 3220 RTE_FLOW_ERROR_TYPE_ITEM, 3221 item, 3222 "Checksum bit must be on"); 3223 if (mask->key.key) 3224 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3225 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3226 return rte_flow_error_set(error, EINVAL, 3227 RTE_FLOW_ERROR_TYPE_ITEM, 3228 item, "Key bit must be on"); 3229 if (mask->sequence.sequence) 3230 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3231 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3232 return rte_flow_error_set(error, EINVAL, 3233 RTE_FLOW_ERROR_TYPE_ITEM, 3234 item, 3235 "Sequence bit must be on"); 3236 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3237 if (priv->sh->steering_format_version == 3238 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3239 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3240 !priv->sh->misc5_cap) || 3241 (!(priv->sh->tunnel_header_0_1 && 3242 priv->sh->tunnel_header_2_3) && 3243 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3244 return rte_flow_error_set(error, EINVAL, 3245 RTE_FLOW_ERROR_TYPE_ITEM, 3246 item, 3247 "Checksum/Sequence not supported"); 3248 } 3249 ret = mlx5_flow_item_acceptable 3250 (item, (const uint8_t *)mask, 3251 (const uint8_t *)&nic_mask, 3252 sizeof(struct rte_flow_item_gre_opt), 3253 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3254 return ret; 3255 } 3256 3257 /** 3258 * Validate GRE item. 3259 * 3260 * @param[in] item 3261 * Item specification. 3262 * @param[in] item_flags 3263 * Bit flags to mark detected items. 3264 * @param[in] target_protocol 3265 * The next protocol in the previous item. 3266 * @param[out] error 3267 * Pointer to error structure. 3268 * 3269 * @return 3270 * 0 on success, a negative errno value otherwise and rte_errno is set. 3271 */ 3272 int 3273 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3274 uint64_t item_flags, 3275 uint8_t target_protocol, 3276 struct rte_flow_error *error) 3277 { 3278 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3279 const struct rte_flow_item_gre *mask = item->mask; 3280 int ret; 3281 const struct rte_flow_item_gre nic_mask = { 3282 .c_rsvd0_ver = RTE_BE16(0xB000), 3283 .protocol = RTE_BE16(UINT16_MAX), 3284 }; 3285 3286 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3287 return rte_flow_error_set(error, EINVAL, 3288 RTE_FLOW_ERROR_TYPE_ITEM, item, 3289 "protocol filtering not compatible" 3290 " with this GRE layer"); 3291 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3292 return rte_flow_error_set(error, ENOTSUP, 3293 RTE_FLOW_ERROR_TYPE_ITEM, item, 3294 "multiple tunnel layers not" 3295 " supported"); 3296 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3297 return rte_flow_error_set(error, ENOTSUP, 3298 RTE_FLOW_ERROR_TYPE_ITEM, item, 3299 "L3 Layer is missing"); 3300 if (!mask) 3301 mask = &rte_flow_item_gre_mask; 3302 ret = mlx5_flow_item_acceptable 3303 (item, (const uint8_t *)mask, 3304 (const uint8_t *)&nic_mask, 3305 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3306 error); 3307 if (ret < 0) 3308 return ret; 3309 #ifndef HAVE_MLX5DV_DR 3310 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3311 if (spec && (spec->protocol & mask->protocol)) 3312 return rte_flow_error_set(error, ENOTSUP, 3313 RTE_FLOW_ERROR_TYPE_ITEM, item, 3314 "without MPLS support the" 3315 " specification cannot be used for" 3316 " filtering"); 3317 #endif 3318 #endif 3319 return 0; 3320 } 3321 3322 /** 3323 * Validate Geneve item. 3324 * 3325 * @param[in] item 3326 * Item specification. 3327 * @param[in] itemFlags 3328 * Bit-fields that holds the items detected until now. 3329 * @param[in] enPriv 3330 * Pointer to the private data structure. 3331 * @param[out] error 3332 * Pointer to error structure. 3333 * 3334 * @return 3335 * 0 on success, a negative errno value otherwise and rte_errno is set. 3336 */ 3337 3338 int 3339 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3340 uint64_t item_flags, 3341 struct rte_eth_dev *dev, 3342 struct rte_flow_error *error) 3343 { 3344 struct mlx5_priv *priv = dev->data->dev_private; 3345 const struct rte_flow_item_geneve *spec = item->spec; 3346 const struct rte_flow_item_geneve *mask = item->mask; 3347 int ret; 3348 uint16_t gbhdr; 3349 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3350 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3351 const struct rte_flow_item_geneve nic_mask = { 3352 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3353 .vni = "\xff\xff\xff", 3354 .protocol = RTE_BE16(UINT16_MAX), 3355 }; 3356 3357 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3358 return rte_flow_error_set(error, ENOTSUP, 3359 RTE_FLOW_ERROR_TYPE_ITEM, item, 3360 "L3 Geneve is not enabled by device" 3361 " parameter and/or not configured in" 3362 " firmware"); 3363 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3364 return rte_flow_error_set(error, ENOTSUP, 3365 RTE_FLOW_ERROR_TYPE_ITEM, item, 3366 "multiple tunnel layers not" 3367 " supported"); 3368 /* 3369 * Verify only UDPv4 is present as defined in 3370 * https://tools.ietf.org/html/rfc7348 3371 */ 3372 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3373 return rte_flow_error_set(error, EINVAL, 3374 RTE_FLOW_ERROR_TYPE_ITEM, item, 3375 "no outer UDP layer found"); 3376 if (!mask) 3377 mask = &rte_flow_item_geneve_mask; 3378 ret = mlx5_flow_item_acceptable 3379 (item, (const uint8_t *)mask, 3380 (const uint8_t *)&nic_mask, 3381 sizeof(struct rte_flow_item_geneve), 3382 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3383 if (ret) 3384 return ret; 3385 if (spec) { 3386 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3387 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3388 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3389 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3390 return rte_flow_error_set(error, ENOTSUP, 3391 RTE_FLOW_ERROR_TYPE_ITEM, 3392 item, 3393 "Geneve protocol unsupported" 3394 " fields are being used"); 3395 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3396 return rte_flow_error_set 3397 (error, ENOTSUP, 3398 RTE_FLOW_ERROR_TYPE_ITEM, 3399 item, 3400 "Unsupported Geneve options length"); 3401 } 3402 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3403 return rte_flow_error_set 3404 (error, ENOTSUP, 3405 RTE_FLOW_ERROR_TYPE_ITEM, item, 3406 "Geneve tunnel must be fully defined"); 3407 return 0; 3408 } 3409 3410 /** 3411 * Validate Geneve TLV option item. 3412 * 3413 * @param[in] item 3414 * Item specification. 3415 * @param[in] last_item 3416 * Previous validated item in the pattern items. 3417 * @param[in] geneve_item 3418 * Previous GENEVE item specification. 3419 * @param[in] dev 3420 * Pointer to the rte_eth_dev structure. 3421 * @param[out] error 3422 * Pointer to error structure. 3423 * 3424 * @return 3425 * 0 on success, a negative errno value otherwise and rte_errno is set. 3426 */ 3427 int 3428 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3429 uint64_t last_item, 3430 const struct rte_flow_item *geneve_item, 3431 struct rte_eth_dev *dev, 3432 struct rte_flow_error *error) 3433 { 3434 struct mlx5_priv *priv = dev->data->dev_private; 3435 struct mlx5_dev_ctx_shared *sh = priv->sh; 3436 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3437 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3438 uint8_t data_max_supported = 3439 hca_attr->max_geneve_tlv_option_data_len * 4; 3440 const struct rte_flow_item_geneve *geneve_spec; 3441 const struct rte_flow_item_geneve *geneve_mask; 3442 const struct rte_flow_item_geneve_opt *spec = item->spec; 3443 const struct rte_flow_item_geneve_opt *mask = item->mask; 3444 unsigned int i; 3445 unsigned int data_len; 3446 uint8_t tlv_option_len; 3447 uint16_t optlen_m, optlen_v; 3448 const struct rte_flow_item_geneve_opt full_mask = { 3449 .option_class = RTE_BE16(0xffff), 3450 .option_type = 0xff, 3451 .option_len = 0x1f, 3452 }; 3453 3454 if (!mask) 3455 mask = &rte_flow_item_geneve_opt_mask; 3456 if (!spec) 3457 return rte_flow_error_set 3458 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3459 "Geneve TLV opt class/type/length must be specified"); 3460 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3461 return rte_flow_error_set 3462 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3463 "Geneve TLV opt length exceeds the limit (31)"); 3464 /* Check if class type and length masks are full. */ 3465 if (full_mask.option_class != mask->option_class || 3466 full_mask.option_type != mask->option_type || 3467 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3468 return rte_flow_error_set 3469 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3470 "Geneve TLV opt class/type/length masks must be full"); 3471 /* Check if length is supported */ 3472 if ((uint32_t)spec->option_len > 3473 hca_attr->max_geneve_tlv_option_data_len) 3474 return rte_flow_error_set 3475 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3476 "Geneve TLV opt length not supported"); 3477 if (hca_attr->max_geneve_tlv_options > 1) 3478 DRV_LOG(DEBUG, 3479 "max_geneve_tlv_options supports more than 1 option"); 3480 /* Check GENEVE item preceding. */ 3481 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3482 return rte_flow_error_set 3483 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3484 "Geneve opt item must be preceded with Geneve item"); 3485 geneve_spec = geneve_item->spec; 3486 geneve_mask = geneve_item->mask ? geneve_item->mask : 3487 &rte_flow_item_geneve_mask; 3488 /* Check if GENEVE TLV option size doesn't exceed option length */ 3489 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3490 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3491 tlv_option_len = spec->option_len & mask->option_len; 3492 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3493 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3494 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3495 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3496 if ((optlen_v & optlen_m) <= tlv_option_len) 3497 return rte_flow_error_set 3498 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3499 "GENEVE TLV option length exceeds optlen"); 3500 } 3501 /* Check if length is 0 or data is 0. */ 3502 if (spec->data == NULL || spec->option_len == 0) 3503 return rte_flow_error_set 3504 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3505 "Geneve TLV opt with zero data/length not supported"); 3506 /* Check not all data & mask are 0. */ 3507 data_len = spec->option_len * 4; 3508 if (mask->data == NULL) { 3509 for (i = 0; i < data_len; i++) 3510 if (spec->data[i]) 3511 break; 3512 if (i == data_len) 3513 return rte_flow_error_set(error, ENOTSUP, 3514 RTE_FLOW_ERROR_TYPE_ITEM, item, 3515 "Can't match on Geneve option data 0"); 3516 } else { 3517 for (i = 0; i < data_len; i++) 3518 if (spec->data[i] & mask->data[i]) 3519 break; 3520 if (i == data_len) 3521 return rte_flow_error_set(error, ENOTSUP, 3522 RTE_FLOW_ERROR_TYPE_ITEM, item, 3523 "Can't match on Geneve option data and mask 0"); 3524 /* Check data mask supported. */ 3525 for (i = data_max_supported; i < data_len ; i++) 3526 if (mask->data[i]) 3527 return rte_flow_error_set(error, ENOTSUP, 3528 RTE_FLOW_ERROR_TYPE_ITEM, item, 3529 "Data mask is of unsupported size"); 3530 } 3531 /* Check GENEVE option is supported in NIC. */ 3532 if (!hca_attr->geneve_tlv_opt) 3533 return rte_flow_error_set 3534 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3535 "Geneve TLV opt not supported"); 3536 /* Check if we already have geneve option with different type/class. */ 3537 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3538 geneve_opt_resource = sh->geneve_tlv_option_resource; 3539 if (geneve_opt_resource != NULL) 3540 if (geneve_opt_resource->option_class != spec->option_class || 3541 geneve_opt_resource->option_type != spec->option_type || 3542 geneve_opt_resource->length != spec->option_len) { 3543 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3544 return rte_flow_error_set(error, ENOTSUP, 3545 RTE_FLOW_ERROR_TYPE_ITEM, item, 3546 "Only one Geneve TLV option supported"); 3547 } 3548 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3549 return 0; 3550 } 3551 3552 /** 3553 * Validate MPLS item. 3554 * 3555 * @param[in] dev 3556 * Pointer to the rte_eth_dev structure. 3557 * @param[in] item 3558 * Item specification. 3559 * @param[in] item_flags 3560 * Bit-fields that holds the items detected until now. 3561 * @param[in] prev_layer 3562 * The protocol layer indicated in previous item. 3563 * @param[out] error 3564 * Pointer to error structure. 3565 * 3566 * @return 3567 * 0 on success, a negative errno value otherwise and rte_errno is set. 3568 */ 3569 int 3570 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3571 const struct rte_flow_item *item __rte_unused, 3572 uint64_t item_flags __rte_unused, 3573 uint64_t prev_layer __rte_unused, 3574 struct rte_flow_error *error) 3575 { 3576 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3577 const struct rte_flow_item_mpls *mask = item->mask; 3578 struct mlx5_priv *priv = dev->data->dev_private; 3579 int ret; 3580 3581 if (!priv->sh->dev_cap.mpls_en) 3582 return rte_flow_error_set(error, ENOTSUP, 3583 RTE_FLOW_ERROR_TYPE_ITEM, item, 3584 "MPLS not supported or" 3585 " disabled in firmware" 3586 " configuration."); 3587 /* MPLS over UDP, GRE is allowed */ 3588 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3589 MLX5_FLOW_LAYER_GRE | 3590 MLX5_FLOW_LAYER_GRE_KEY))) 3591 return rte_flow_error_set(error, EINVAL, 3592 RTE_FLOW_ERROR_TYPE_ITEM, item, 3593 "protocol filtering not compatible" 3594 " with MPLS layer"); 3595 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3596 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3597 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3598 return rte_flow_error_set(error, ENOTSUP, 3599 RTE_FLOW_ERROR_TYPE_ITEM, item, 3600 "multiple tunnel layers not" 3601 " supported"); 3602 if (!mask) 3603 mask = &rte_flow_item_mpls_mask; 3604 ret = mlx5_flow_item_acceptable 3605 (item, (const uint8_t *)mask, 3606 (const uint8_t *)&rte_flow_item_mpls_mask, 3607 sizeof(struct rte_flow_item_mpls), 3608 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3609 if (ret < 0) 3610 return ret; 3611 return 0; 3612 #else 3613 return rte_flow_error_set(error, ENOTSUP, 3614 RTE_FLOW_ERROR_TYPE_ITEM, item, 3615 "MPLS is not supported by Verbs, please" 3616 " update."); 3617 #endif 3618 } 3619 3620 /** 3621 * Validate NVGRE item. 3622 * 3623 * @param[in] item 3624 * Item specification. 3625 * @param[in] item_flags 3626 * Bit flags to mark detected items. 3627 * @param[in] target_protocol 3628 * The next protocol in the previous item. 3629 * @param[out] error 3630 * Pointer to error structure. 3631 * 3632 * @return 3633 * 0 on success, a negative errno value otherwise and rte_errno is set. 3634 */ 3635 int 3636 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3637 uint64_t item_flags, 3638 uint8_t target_protocol, 3639 struct rte_flow_error *error) 3640 { 3641 const struct rte_flow_item_nvgre *mask = item->mask; 3642 int ret; 3643 3644 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3645 return rte_flow_error_set(error, EINVAL, 3646 RTE_FLOW_ERROR_TYPE_ITEM, item, 3647 "protocol filtering not compatible" 3648 " with this GRE layer"); 3649 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3650 return rte_flow_error_set(error, ENOTSUP, 3651 RTE_FLOW_ERROR_TYPE_ITEM, item, 3652 "multiple tunnel layers not" 3653 " supported"); 3654 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3655 return rte_flow_error_set(error, ENOTSUP, 3656 RTE_FLOW_ERROR_TYPE_ITEM, item, 3657 "L3 Layer is missing"); 3658 if (!mask) 3659 mask = &rte_flow_item_nvgre_mask; 3660 ret = mlx5_flow_item_acceptable 3661 (item, (const uint8_t *)mask, 3662 (const uint8_t *)&rte_flow_item_nvgre_mask, 3663 sizeof(struct rte_flow_item_nvgre), 3664 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3665 if (ret < 0) 3666 return ret; 3667 return 0; 3668 } 3669 3670 /** 3671 * Validate eCPRI item. 3672 * 3673 * @param[in] item 3674 * Item specification. 3675 * @param[in] item_flags 3676 * Bit-fields that holds the items detected until now. 3677 * @param[in] last_item 3678 * Previous validated item in the pattern items. 3679 * @param[in] ether_type 3680 * Type in the ethernet layer header (including dot1q). 3681 * @param[in] acc_mask 3682 * Acceptable mask, if NULL default internal default mask 3683 * will be used to check whether item fields are supported. 3684 * @param[out] error 3685 * Pointer to error structure. 3686 * 3687 * @return 3688 * 0 on success, a negative errno value otherwise and rte_errno is set. 3689 */ 3690 int 3691 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3692 uint64_t item_flags, 3693 uint64_t last_item, 3694 uint16_t ether_type, 3695 const struct rte_flow_item_ecpri *acc_mask, 3696 struct rte_flow_error *error) 3697 { 3698 const struct rte_flow_item_ecpri *mask = item->mask; 3699 const struct rte_flow_item_ecpri nic_mask = { 3700 .hdr = { 3701 .common = { 3702 .u32 = 3703 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3704 .type = 0xFF, 3705 }).u32), 3706 }, 3707 .dummy[0] = 0xFFFFFFFF, 3708 }, 3709 }; 3710 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3711 MLX5_FLOW_LAYER_OUTER_VLAN); 3712 struct rte_flow_item_ecpri mask_lo; 3713 3714 if (!(last_item & outer_l2_vlan) && 3715 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3716 return rte_flow_error_set(error, EINVAL, 3717 RTE_FLOW_ERROR_TYPE_ITEM, item, 3718 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3719 if ((last_item & outer_l2_vlan) && ether_type && 3720 ether_type != RTE_ETHER_TYPE_ECPRI) 3721 return rte_flow_error_set(error, EINVAL, 3722 RTE_FLOW_ERROR_TYPE_ITEM, item, 3723 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3724 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3725 return rte_flow_error_set(error, EINVAL, 3726 RTE_FLOW_ERROR_TYPE_ITEM, item, 3727 "eCPRI with tunnel is not supported right now"); 3728 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3729 return rte_flow_error_set(error, ENOTSUP, 3730 RTE_FLOW_ERROR_TYPE_ITEM, item, 3731 "multiple L3 layers not supported"); 3732 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3733 return rte_flow_error_set(error, EINVAL, 3734 RTE_FLOW_ERROR_TYPE_ITEM, item, 3735 "eCPRI cannot coexist with a TCP layer"); 3736 /* In specification, eCPRI could be over UDP layer. */ 3737 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3738 return rte_flow_error_set(error, EINVAL, 3739 RTE_FLOW_ERROR_TYPE_ITEM, item, 3740 "eCPRI over UDP layer is not yet supported right now"); 3741 /* Mask for type field in common header could be zero. */ 3742 if (!mask) 3743 mask = &rte_flow_item_ecpri_mask; 3744 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3745 /* Input mask is in big-endian format. */ 3746 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3747 return rte_flow_error_set(error, EINVAL, 3748 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3749 "partial mask is not supported for protocol"); 3750 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3751 return rte_flow_error_set(error, EINVAL, 3752 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3753 "message header mask must be after a type mask"); 3754 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3755 acc_mask ? (const uint8_t *)acc_mask 3756 : (const uint8_t *)&nic_mask, 3757 sizeof(struct rte_flow_item_ecpri), 3758 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3759 } 3760 3761 static int 3762 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3763 const struct rte_flow_attr *attr __rte_unused, 3764 const struct rte_flow_item items[] __rte_unused, 3765 const struct rte_flow_action actions[] __rte_unused, 3766 bool external __rte_unused, 3767 int hairpin __rte_unused, 3768 struct rte_flow_error *error) 3769 { 3770 return rte_flow_error_set(error, ENOTSUP, 3771 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3772 } 3773 3774 static struct mlx5_flow * 3775 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3776 const struct rte_flow_attr *attr __rte_unused, 3777 const struct rte_flow_item items[] __rte_unused, 3778 const struct rte_flow_action actions[] __rte_unused, 3779 struct rte_flow_error *error) 3780 { 3781 rte_flow_error_set(error, ENOTSUP, 3782 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3783 return NULL; 3784 } 3785 3786 static int 3787 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3788 struct mlx5_flow *dev_flow __rte_unused, 3789 const struct rte_flow_attr *attr __rte_unused, 3790 const struct rte_flow_item items[] __rte_unused, 3791 const struct rte_flow_action actions[] __rte_unused, 3792 struct rte_flow_error *error) 3793 { 3794 return rte_flow_error_set(error, ENOTSUP, 3795 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3796 } 3797 3798 static int 3799 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3800 struct rte_flow *flow __rte_unused, 3801 struct rte_flow_error *error) 3802 { 3803 return rte_flow_error_set(error, ENOTSUP, 3804 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3805 } 3806 3807 static void 3808 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3809 struct rte_flow *flow __rte_unused) 3810 { 3811 } 3812 3813 static void 3814 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3815 struct rte_flow *flow __rte_unused) 3816 { 3817 } 3818 3819 static int 3820 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3821 struct rte_flow *flow __rte_unused, 3822 const struct rte_flow_action *actions __rte_unused, 3823 void *data __rte_unused, 3824 struct rte_flow_error *error) 3825 { 3826 return rte_flow_error_set(error, ENOTSUP, 3827 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3828 } 3829 3830 static int 3831 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3832 uint32_t domains __rte_unused, 3833 uint32_t flags __rte_unused) 3834 { 3835 return 0; 3836 } 3837 3838 /* Void driver to protect from null pointer reference. */ 3839 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3840 .validate = flow_null_validate, 3841 .prepare = flow_null_prepare, 3842 .translate = flow_null_translate, 3843 .apply = flow_null_apply, 3844 .remove = flow_null_remove, 3845 .destroy = flow_null_destroy, 3846 .query = flow_null_query, 3847 .sync_domain = flow_null_sync_domain, 3848 }; 3849 3850 /** 3851 * Select flow driver type according to flow attributes and device 3852 * configuration. 3853 * 3854 * @param[in] dev 3855 * Pointer to the dev structure. 3856 * @param[in] attr 3857 * Pointer to the flow attributes. 3858 * 3859 * @return 3860 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3861 */ 3862 static enum mlx5_flow_drv_type 3863 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3864 { 3865 struct mlx5_priv *priv = dev->data->dev_private; 3866 /* The OS can determine first a specific flow type (DV, VERBS) */ 3867 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3868 3869 if (type != MLX5_FLOW_TYPE_MAX) 3870 return type; 3871 /* 3872 * Currently when dv_flow_en == 2, only HW steering engine is 3873 * supported. New engines can also be chosen here if ready. 3874 */ 3875 if (priv->sh->config.dv_flow_en == 2) 3876 return MLX5_FLOW_TYPE_HW; 3877 if (!attr) 3878 return MLX5_FLOW_TYPE_MIN; 3879 /* If no OS specific type - continue with DV/VERBS selection */ 3880 if (attr->transfer && priv->sh->config.dv_esw_en) 3881 type = MLX5_FLOW_TYPE_DV; 3882 if (!attr->transfer) 3883 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3884 MLX5_FLOW_TYPE_VERBS; 3885 return type; 3886 } 3887 3888 #define flow_get_drv_ops(type) flow_drv_ops[type] 3889 3890 /** 3891 * Flow driver validation API. This abstracts calling driver specific functions. 3892 * The type of flow driver is determined according to flow attributes. 3893 * 3894 * @param[in] dev 3895 * Pointer to the dev structure. 3896 * @param[in] attr 3897 * Pointer to the flow attributes. 3898 * @param[in] items 3899 * Pointer to the list of items. 3900 * @param[in] actions 3901 * Pointer to the list of actions. 3902 * @param[in] external 3903 * This flow rule is created by request external to PMD. 3904 * @param[in] hairpin 3905 * Number of hairpin TX actions, 0 means classic flow. 3906 * @param[out] error 3907 * Pointer to the error structure. 3908 * 3909 * @return 3910 * 0 on success, a negative errno value otherwise and rte_errno is set. 3911 */ 3912 static inline int 3913 flow_drv_validate(struct rte_eth_dev *dev, 3914 const struct rte_flow_attr *attr, 3915 const struct rte_flow_item items[], 3916 const struct rte_flow_action actions[], 3917 bool external, int hairpin, struct rte_flow_error *error) 3918 { 3919 const struct mlx5_flow_driver_ops *fops; 3920 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3921 3922 fops = flow_get_drv_ops(type); 3923 return fops->validate(dev, attr, items, actions, external, 3924 hairpin, error); 3925 } 3926 3927 /** 3928 * Flow driver preparation API. This abstracts calling driver specific 3929 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3930 * calculates the size of memory required for device flow, allocates the memory, 3931 * initializes the device flow and returns the pointer. 3932 * 3933 * @note 3934 * This function initializes device flow structure such as dv or verbs in 3935 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3936 * rest. For example, adding returning device flow to flow->dev_flow list and 3937 * setting backward reference to the flow should be done out of this function. 3938 * layers field is not filled either. 3939 * 3940 * @param[in] dev 3941 * Pointer to the dev structure. 3942 * @param[in] attr 3943 * Pointer to the flow attributes. 3944 * @param[in] items 3945 * Pointer to the list of items. 3946 * @param[in] actions 3947 * Pointer to the list of actions. 3948 * @param[in] flow_idx 3949 * This memory pool index to the flow. 3950 * @param[out] error 3951 * Pointer to the error structure. 3952 * 3953 * @return 3954 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3955 */ 3956 static inline struct mlx5_flow * 3957 flow_drv_prepare(struct rte_eth_dev *dev, 3958 const struct rte_flow *flow, 3959 const struct rte_flow_attr *attr, 3960 const struct rte_flow_item items[], 3961 const struct rte_flow_action actions[], 3962 uint32_t flow_idx, 3963 struct rte_flow_error *error) 3964 { 3965 const struct mlx5_flow_driver_ops *fops; 3966 enum mlx5_flow_drv_type type = flow->drv_type; 3967 struct mlx5_flow *mlx5_flow = NULL; 3968 3969 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3970 fops = flow_get_drv_ops(type); 3971 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3972 if (mlx5_flow) 3973 mlx5_flow->flow_idx = flow_idx; 3974 return mlx5_flow; 3975 } 3976 3977 /** 3978 * Flow driver translation API. This abstracts calling driver specific 3979 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3980 * translates a generic flow into a driver flow. flow_drv_prepare() must 3981 * precede. 3982 * 3983 * @note 3984 * dev_flow->layers could be filled as a result of parsing during translation 3985 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3986 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3987 * flow->actions could be overwritten even though all the expanded dev_flows 3988 * have the same actions. 3989 * 3990 * @param[in] dev 3991 * Pointer to the rte dev structure. 3992 * @param[in, out] dev_flow 3993 * Pointer to the mlx5 flow. 3994 * @param[in] attr 3995 * Pointer to the flow attributes. 3996 * @param[in] items 3997 * Pointer to the list of items. 3998 * @param[in] actions 3999 * Pointer to the list of actions. 4000 * @param[out] error 4001 * Pointer to the error structure. 4002 * 4003 * @return 4004 * 0 on success, a negative errno value otherwise and rte_errno is set. 4005 */ 4006 static inline int 4007 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4008 const struct rte_flow_attr *attr, 4009 const struct rte_flow_item items[], 4010 const struct rte_flow_action actions[], 4011 struct rte_flow_error *error) 4012 { 4013 const struct mlx5_flow_driver_ops *fops; 4014 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4015 4016 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4017 fops = flow_get_drv_ops(type); 4018 return fops->translate(dev, dev_flow, attr, items, actions, error); 4019 } 4020 4021 /** 4022 * Flow driver apply API. This abstracts calling driver specific functions. 4023 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4024 * translated driver flows on to device. flow_drv_translate() must precede. 4025 * 4026 * @param[in] dev 4027 * Pointer to Ethernet device structure. 4028 * @param[in, out] flow 4029 * Pointer to flow structure. 4030 * @param[out] error 4031 * Pointer to error structure. 4032 * 4033 * @return 4034 * 0 on success, a negative errno value otherwise and rte_errno is set. 4035 */ 4036 static inline int 4037 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4038 struct rte_flow_error *error) 4039 { 4040 const struct mlx5_flow_driver_ops *fops; 4041 enum mlx5_flow_drv_type type = flow->drv_type; 4042 4043 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4044 fops = flow_get_drv_ops(type); 4045 return fops->apply(dev, flow, error); 4046 } 4047 4048 /** 4049 * Flow driver destroy API. This abstracts calling driver specific functions. 4050 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4051 * on device and releases resources of the flow. 4052 * 4053 * @param[in] dev 4054 * Pointer to Ethernet device. 4055 * @param[in, out] flow 4056 * Pointer to flow structure. 4057 */ 4058 static inline void 4059 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4060 { 4061 const struct mlx5_flow_driver_ops *fops; 4062 enum mlx5_flow_drv_type type = flow->drv_type; 4063 4064 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4065 fops = flow_get_drv_ops(type); 4066 fops->destroy(dev, flow); 4067 } 4068 4069 /** 4070 * Flow driver find RSS policy tbl API. This abstracts calling driver 4071 * specific functions. Parent flow (rte_flow) should have driver 4072 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4073 * 4074 * @param[in] dev 4075 * Pointer to Ethernet device. 4076 * @param[in, out] flow 4077 * Pointer to flow structure. 4078 * @param[in] policy 4079 * Pointer to meter policy table. 4080 * @param[in] rss_desc 4081 * Pointer to rss_desc 4082 */ 4083 static struct mlx5_flow_meter_sub_policy * 4084 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4085 struct rte_flow *flow, 4086 struct mlx5_flow_meter_policy *policy, 4087 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4088 { 4089 const struct mlx5_flow_driver_ops *fops; 4090 enum mlx5_flow_drv_type type = flow->drv_type; 4091 4092 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4093 fops = flow_get_drv_ops(type); 4094 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4095 } 4096 4097 /** 4098 * Flow driver color tag rule API. This abstracts calling driver 4099 * specific functions. Parent flow (rte_flow) should have driver 4100 * type (drv_type). It will create the color tag rules in hierarchy meter. 4101 * 4102 * @param[in] dev 4103 * Pointer to Ethernet device. 4104 * @param[in, out] flow 4105 * Pointer to flow structure. 4106 * @param[in] fm 4107 * Pointer to flow meter structure. 4108 * @param[in] src_port 4109 * The src port this extra rule should use. 4110 * @param[in] item 4111 * The src port id match item. 4112 * @param[out] error 4113 * Pointer to error structure. 4114 */ 4115 static int 4116 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4117 struct rte_flow *flow, 4118 struct mlx5_flow_meter_info *fm, 4119 int32_t src_port, 4120 const struct rte_flow_item *item, 4121 struct rte_flow_error *error) 4122 { 4123 const struct mlx5_flow_driver_ops *fops; 4124 enum mlx5_flow_drv_type type = flow->drv_type; 4125 4126 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4127 fops = flow_get_drv_ops(type); 4128 return fops->meter_hierarchy_rule_create(dev, fm, 4129 src_port, item, error); 4130 } 4131 4132 /** 4133 * Get RSS action from the action list. 4134 * 4135 * @param[in] dev 4136 * Pointer to Ethernet device. 4137 * @param[in] actions 4138 * Pointer to the list of actions. 4139 * @param[in] flow 4140 * Parent flow structure pointer. 4141 * 4142 * @return 4143 * Pointer to the RSS action if exist, else return NULL. 4144 */ 4145 static const struct rte_flow_action_rss* 4146 flow_get_rss_action(struct rte_eth_dev *dev, 4147 const struct rte_flow_action actions[]) 4148 { 4149 struct mlx5_priv *priv = dev->data->dev_private; 4150 const struct rte_flow_action_rss *rss = NULL; 4151 struct mlx5_meter_policy_action_container *acg; 4152 struct mlx5_meter_policy_action_container *acy; 4153 4154 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4155 switch (actions->type) { 4156 case RTE_FLOW_ACTION_TYPE_RSS: 4157 rss = actions->conf; 4158 break; 4159 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4160 { 4161 const struct rte_flow_action_sample *sample = 4162 actions->conf; 4163 const struct rte_flow_action *act = sample->actions; 4164 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4165 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4166 rss = act->conf; 4167 break; 4168 } 4169 case RTE_FLOW_ACTION_TYPE_METER: 4170 { 4171 uint32_t mtr_idx; 4172 struct mlx5_flow_meter_info *fm; 4173 struct mlx5_flow_meter_policy *policy; 4174 const struct rte_flow_action_meter *mtr = actions->conf; 4175 4176 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4177 if (fm && !fm->def_policy) { 4178 policy = mlx5_flow_meter_policy_find(dev, 4179 fm->policy_id, NULL); 4180 MLX5_ASSERT(policy); 4181 if (policy->is_hierarchy) { 4182 policy = 4183 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4184 policy); 4185 if (!policy) 4186 return NULL; 4187 } 4188 if (policy->is_rss) { 4189 acg = 4190 &policy->act_cnt[RTE_COLOR_GREEN]; 4191 acy = 4192 &policy->act_cnt[RTE_COLOR_YELLOW]; 4193 if (acg->fate_action == 4194 MLX5_FLOW_FATE_SHARED_RSS) 4195 rss = acg->rss->conf; 4196 else if (acy->fate_action == 4197 MLX5_FLOW_FATE_SHARED_RSS) 4198 rss = acy->rss->conf; 4199 } 4200 } 4201 break; 4202 } 4203 default: 4204 break; 4205 } 4206 } 4207 return rss; 4208 } 4209 4210 /** 4211 * Get ASO age action by index. 4212 * 4213 * @param[in] dev 4214 * Pointer to the Ethernet device structure. 4215 * @param[in] age_idx 4216 * Index to the ASO age action. 4217 * 4218 * @return 4219 * The specified ASO age action. 4220 */ 4221 struct mlx5_aso_age_action* 4222 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4223 { 4224 uint16_t pool_idx = age_idx & UINT16_MAX; 4225 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4226 struct mlx5_priv *priv = dev->data->dev_private; 4227 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4228 struct mlx5_aso_age_pool *pool; 4229 4230 rte_rwlock_read_lock(&mng->resize_rwl); 4231 pool = mng->pools[pool_idx]; 4232 rte_rwlock_read_unlock(&mng->resize_rwl); 4233 return &pool->actions[offset - 1]; 4234 } 4235 4236 /* maps indirect action to translated direct in some actions array */ 4237 struct mlx5_translated_action_handle { 4238 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4239 int index; /**< Index in related array of rte_flow_action. */ 4240 }; 4241 4242 /** 4243 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4244 * direct action if translation possible. 4245 * This functionality used to run same execution path for both direct and 4246 * indirect actions on flow create. All necessary preparations for indirect 4247 * action handling should be performed on *handle* actions list returned 4248 * from this call. 4249 * 4250 * @param[in] dev 4251 * Pointer to Ethernet device. 4252 * @param[in] actions 4253 * List of actions to translate. 4254 * @param[out] handle 4255 * List to store translated indirect action object handles. 4256 * @param[in, out] indir_n 4257 * Size of *handle* array. On return should be updated with number of 4258 * indirect actions retrieved from the *actions* list. 4259 * @param[out] translated_actions 4260 * List of actions where all indirect actions were translated to direct 4261 * if possible. NULL if no translation took place. 4262 * @param[out] error 4263 * Pointer to the error structure. 4264 * 4265 * @return 4266 * 0 on success, a negative errno value otherwise and rte_errno is set. 4267 */ 4268 static int 4269 flow_action_handles_translate(struct rte_eth_dev *dev, 4270 const struct rte_flow_action actions[], 4271 struct mlx5_translated_action_handle *handle, 4272 int *indir_n, 4273 struct rte_flow_action **translated_actions, 4274 struct rte_flow_error *error) 4275 { 4276 struct mlx5_priv *priv = dev->data->dev_private; 4277 struct rte_flow_action *translated = NULL; 4278 size_t actions_size; 4279 int n; 4280 int copied_n = 0; 4281 struct mlx5_translated_action_handle *handle_end = NULL; 4282 4283 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4284 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4285 continue; 4286 if (copied_n == *indir_n) { 4287 return rte_flow_error_set 4288 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4289 NULL, "too many shared actions"); 4290 } 4291 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4292 sizeof(actions[n].conf)); 4293 handle[copied_n].index = n; 4294 copied_n++; 4295 } 4296 n++; 4297 *indir_n = copied_n; 4298 if (!copied_n) 4299 return 0; 4300 actions_size = sizeof(struct rte_flow_action) * n; 4301 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4302 if (!translated) { 4303 rte_errno = ENOMEM; 4304 return -ENOMEM; 4305 } 4306 memcpy(translated, actions, actions_size); 4307 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4308 struct mlx5_shared_action_rss *shared_rss; 4309 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4310 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4311 uint32_t idx = act_idx & 4312 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4313 4314 switch (type) { 4315 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4316 shared_rss = mlx5_ipool_get 4317 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4318 translated[handle->index].type = 4319 RTE_FLOW_ACTION_TYPE_RSS; 4320 translated[handle->index].conf = 4321 &shared_rss->origin; 4322 break; 4323 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4324 translated[handle->index].type = 4325 (enum rte_flow_action_type) 4326 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4327 translated[handle->index].conf = (void *)(uintptr_t)idx; 4328 break; 4329 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4330 translated[handle->index].type = 4331 (enum rte_flow_action_type) 4332 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4333 translated[handle->index].conf = (void *)(uintptr_t)idx; 4334 break; 4335 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4336 if (priv->sh->flow_hit_aso_en) { 4337 translated[handle->index].type = 4338 (enum rte_flow_action_type) 4339 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4340 translated[handle->index].conf = 4341 (void *)(uintptr_t)idx; 4342 break; 4343 } 4344 /* Fall-through */ 4345 case MLX5_INDIRECT_ACTION_TYPE_CT: 4346 if (priv->sh->ct_aso_en) { 4347 translated[handle->index].type = 4348 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4349 translated[handle->index].conf = 4350 (void *)(uintptr_t)idx; 4351 break; 4352 } 4353 /* Fall-through */ 4354 default: 4355 mlx5_free(translated); 4356 return rte_flow_error_set 4357 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4358 NULL, "invalid indirect action type"); 4359 } 4360 } 4361 *translated_actions = translated; 4362 return 0; 4363 } 4364 4365 /** 4366 * Get Shared RSS action from the action list. 4367 * 4368 * @param[in] dev 4369 * Pointer to Ethernet device. 4370 * @param[in] shared 4371 * Pointer to the list of actions. 4372 * @param[in] shared_n 4373 * Actions list length. 4374 * 4375 * @return 4376 * The MLX5 RSS action ID if exists, otherwise return 0. 4377 */ 4378 static uint32_t 4379 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4380 struct mlx5_translated_action_handle *handle, 4381 int shared_n) 4382 { 4383 struct mlx5_translated_action_handle *handle_end; 4384 struct mlx5_priv *priv = dev->data->dev_private; 4385 struct mlx5_shared_action_rss *shared_rss; 4386 4387 4388 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4389 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4390 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4391 uint32_t idx = act_idx & 4392 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4393 switch (type) { 4394 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4395 shared_rss = mlx5_ipool_get 4396 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4397 idx); 4398 __atomic_add_fetch(&shared_rss->refcnt, 1, 4399 __ATOMIC_RELAXED); 4400 return idx; 4401 default: 4402 break; 4403 } 4404 } 4405 return 0; 4406 } 4407 4408 static unsigned int 4409 find_graph_root(uint32_t rss_level) 4410 { 4411 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4412 MLX5_EXPANSION_ROOT_OUTER; 4413 } 4414 4415 /** 4416 * Get layer flags from the prefix flow. 4417 * 4418 * Some flows may be split to several subflows, the prefix subflow gets the 4419 * match items and the suffix sub flow gets the actions. 4420 * Some actions need the user defined match item flags to get the detail for 4421 * the action. 4422 * This function helps the suffix flow to get the item layer flags from prefix 4423 * subflow. 4424 * 4425 * @param[in] dev_flow 4426 * Pointer the created prefix subflow. 4427 * 4428 * @return 4429 * The layers get from prefix subflow. 4430 */ 4431 static inline uint64_t 4432 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4433 { 4434 uint64_t layers = 0; 4435 4436 /* 4437 * Layers bits could be localization, but usually the compiler will 4438 * help to do the optimization work for source code. 4439 * If no decap actions, use the layers directly. 4440 */ 4441 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4442 return dev_flow->handle->layers; 4443 /* Convert L3 layers with decap action. */ 4444 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4445 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4446 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4447 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4448 /* Convert L4 layers with decap action. */ 4449 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4450 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4451 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4452 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4453 return layers; 4454 } 4455 4456 /** 4457 * Get metadata split action information. 4458 * 4459 * @param[in] actions 4460 * Pointer to the list of actions. 4461 * @param[out] qrss 4462 * Pointer to the return pointer. 4463 * @param[out] qrss_type 4464 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4465 * if no QUEUE/RSS is found. 4466 * @param[out] encap_idx 4467 * Pointer to the index of the encap action if exists, otherwise the last 4468 * action index. 4469 * 4470 * @return 4471 * Total number of actions. 4472 */ 4473 static int 4474 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4475 const struct rte_flow_action **qrss, 4476 int *encap_idx) 4477 { 4478 const struct rte_flow_action_raw_encap *raw_encap; 4479 int actions_n = 0; 4480 int raw_decap_idx = -1; 4481 4482 *encap_idx = -1; 4483 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4484 switch (actions->type) { 4485 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4486 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4487 *encap_idx = actions_n; 4488 break; 4489 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4490 raw_decap_idx = actions_n; 4491 break; 4492 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4493 raw_encap = actions->conf; 4494 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4495 *encap_idx = raw_decap_idx != -1 ? 4496 raw_decap_idx : actions_n; 4497 break; 4498 case RTE_FLOW_ACTION_TYPE_QUEUE: 4499 case RTE_FLOW_ACTION_TYPE_RSS: 4500 *qrss = actions; 4501 break; 4502 default: 4503 break; 4504 } 4505 actions_n++; 4506 } 4507 if (*encap_idx == -1) 4508 *encap_idx = actions_n; 4509 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4510 return actions_n + 1; 4511 } 4512 4513 /** 4514 * Check if the action will change packet. 4515 * 4516 * @param dev 4517 * Pointer to Ethernet device. 4518 * @param[in] type 4519 * action type. 4520 * 4521 * @return 4522 * true if action will change packet, false otherwise. 4523 */ 4524 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4525 enum rte_flow_action_type type) 4526 { 4527 struct mlx5_priv *priv = dev->data->dev_private; 4528 4529 switch (type) { 4530 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4531 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4532 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4533 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4534 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4535 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4536 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4537 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4538 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4539 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4540 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4541 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4542 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4543 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4544 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4545 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4546 case RTE_FLOW_ACTION_TYPE_SET_META: 4547 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4548 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4549 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4550 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4551 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4552 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4553 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4554 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4555 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4556 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4557 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4558 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4559 return true; 4560 case RTE_FLOW_ACTION_TYPE_FLAG: 4561 case RTE_FLOW_ACTION_TYPE_MARK: 4562 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4563 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4564 return true; 4565 else 4566 return false; 4567 default: 4568 return false; 4569 } 4570 } 4571 4572 /** 4573 * Check meter action from the action list. 4574 * 4575 * @param dev 4576 * Pointer to Ethernet device. 4577 * @param[in] actions 4578 * Pointer to the list of actions. 4579 * @param[out] has_mtr 4580 * Pointer to the meter exist flag. 4581 * @param[out] has_modify 4582 * Pointer to the flag showing there's packet change action. 4583 * @param[out] meter_id 4584 * Pointer to the meter id. 4585 * 4586 * @return 4587 * Total number of actions. 4588 */ 4589 static int 4590 flow_check_meter_action(struct rte_eth_dev *dev, 4591 const struct rte_flow_action actions[], 4592 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4593 { 4594 const struct rte_flow_action_meter *mtr = NULL; 4595 int actions_n = 0; 4596 4597 MLX5_ASSERT(has_mtr); 4598 *has_mtr = false; 4599 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4600 switch (actions->type) { 4601 case RTE_FLOW_ACTION_TYPE_METER: 4602 mtr = actions->conf; 4603 *meter_id = mtr->mtr_id; 4604 *has_mtr = true; 4605 break; 4606 default: 4607 break; 4608 } 4609 if (!*has_mtr) 4610 *has_modify |= flow_check_modify_action_type(dev, 4611 actions->type); 4612 actions_n++; 4613 } 4614 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4615 return actions_n + 1; 4616 } 4617 4618 /** 4619 * Check if the flow should be split due to hairpin. 4620 * The reason for the split is that in current HW we can't 4621 * support encap and push-vlan on Rx, so if a flow contains 4622 * these actions we move it to Tx. 4623 * 4624 * @param dev 4625 * Pointer to Ethernet device. 4626 * @param[in] attr 4627 * Flow rule attributes. 4628 * @param[in] actions 4629 * Associated actions (list terminated by the END action). 4630 * 4631 * @return 4632 * > 0 the number of actions and the flow should be split, 4633 * 0 when no split required. 4634 */ 4635 static int 4636 flow_check_hairpin_split(struct rte_eth_dev *dev, 4637 const struct rte_flow_attr *attr, 4638 const struct rte_flow_action actions[]) 4639 { 4640 int queue_action = 0; 4641 int action_n = 0; 4642 int split = 0; 4643 const struct rte_flow_action_queue *queue; 4644 const struct rte_flow_action_rss *rss; 4645 const struct rte_flow_action_raw_encap *raw_encap; 4646 const struct rte_eth_hairpin_conf *conf; 4647 4648 if (!attr->ingress) 4649 return 0; 4650 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4651 switch (actions->type) { 4652 case RTE_FLOW_ACTION_TYPE_QUEUE: 4653 queue = actions->conf; 4654 if (queue == NULL) 4655 return 0; 4656 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4657 if (conf == NULL || conf->tx_explicit != 0) 4658 return 0; 4659 queue_action = 1; 4660 action_n++; 4661 break; 4662 case RTE_FLOW_ACTION_TYPE_RSS: 4663 rss = actions->conf; 4664 if (rss == NULL || rss->queue_num == 0) 4665 return 0; 4666 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4667 if (conf == NULL || conf->tx_explicit != 0) 4668 return 0; 4669 queue_action = 1; 4670 action_n++; 4671 break; 4672 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4673 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4674 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4675 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4676 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4677 split++; 4678 action_n++; 4679 break; 4680 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4681 raw_encap = actions->conf; 4682 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4683 split++; 4684 action_n++; 4685 break; 4686 default: 4687 action_n++; 4688 break; 4689 } 4690 } 4691 if (split && queue_action) 4692 return action_n; 4693 return 0; 4694 } 4695 4696 /* Declare flow create/destroy prototype in advance. */ 4697 static uint32_t 4698 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4699 const struct rte_flow_attr *attr, 4700 const struct rte_flow_item items[], 4701 const struct rte_flow_action actions[], 4702 bool external, struct rte_flow_error *error); 4703 4704 static void 4705 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4706 uint32_t flow_idx); 4707 4708 int 4709 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4710 struct mlx5_list_entry *entry, void *cb_ctx) 4711 { 4712 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4713 struct mlx5_flow_mreg_copy_resource *mcp_res = 4714 container_of(entry, typeof(*mcp_res), hlist_ent); 4715 4716 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4717 } 4718 4719 struct mlx5_list_entry * 4720 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4721 { 4722 struct rte_eth_dev *dev = tool_ctx; 4723 struct mlx5_priv *priv = dev->data->dev_private; 4724 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4725 struct mlx5_flow_mreg_copy_resource *mcp_res; 4726 struct rte_flow_error *error = ctx->error; 4727 uint32_t idx = 0; 4728 int ret; 4729 uint32_t mark_id = *(uint32_t *)(ctx->data); 4730 struct rte_flow_attr attr = { 4731 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4732 .ingress = 1, 4733 }; 4734 struct mlx5_rte_flow_item_tag tag_spec = { 4735 .data = mark_id, 4736 }; 4737 struct rte_flow_item items[] = { 4738 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4739 }; 4740 struct rte_flow_action_mark ftag = { 4741 .id = mark_id, 4742 }; 4743 struct mlx5_flow_action_copy_mreg cp_mreg = { 4744 .dst = REG_B, 4745 .src = REG_NON, 4746 }; 4747 struct rte_flow_action_jump jump = { 4748 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4749 }; 4750 struct rte_flow_action actions[] = { 4751 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4752 }; 4753 4754 /* Fill the register fields in the flow. */ 4755 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4756 if (ret < 0) 4757 return NULL; 4758 tag_spec.id = ret; 4759 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4760 if (ret < 0) 4761 return NULL; 4762 cp_mreg.src = ret; 4763 /* Provide the full width of FLAG specific value. */ 4764 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4765 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4766 /* Build a new flow. */ 4767 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4768 items[0] = (struct rte_flow_item){ 4769 .type = (enum rte_flow_item_type) 4770 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4771 .spec = &tag_spec, 4772 }; 4773 items[1] = (struct rte_flow_item){ 4774 .type = RTE_FLOW_ITEM_TYPE_END, 4775 }; 4776 actions[0] = (struct rte_flow_action){ 4777 .type = (enum rte_flow_action_type) 4778 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4779 .conf = &ftag, 4780 }; 4781 actions[1] = (struct rte_flow_action){ 4782 .type = (enum rte_flow_action_type) 4783 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4784 .conf = &cp_mreg, 4785 }; 4786 actions[2] = (struct rte_flow_action){ 4787 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4788 .conf = &jump, 4789 }; 4790 actions[3] = (struct rte_flow_action){ 4791 .type = RTE_FLOW_ACTION_TYPE_END, 4792 }; 4793 } else { 4794 /* Default rule, wildcard match. */ 4795 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4796 items[0] = (struct rte_flow_item){ 4797 .type = RTE_FLOW_ITEM_TYPE_END, 4798 }; 4799 actions[0] = (struct rte_flow_action){ 4800 .type = (enum rte_flow_action_type) 4801 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4802 .conf = &cp_mreg, 4803 }; 4804 actions[1] = (struct rte_flow_action){ 4805 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4806 .conf = &jump, 4807 }; 4808 actions[2] = (struct rte_flow_action){ 4809 .type = RTE_FLOW_ACTION_TYPE_END, 4810 }; 4811 } 4812 /* Build a new entry. */ 4813 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4814 if (!mcp_res) { 4815 rte_errno = ENOMEM; 4816 return NULL; 4817 } 4818 mcp_res->idx = idx; 4819 mcp_res->mark_id = mark_id; 4820 /* 4821 * The copy Flows are not included in any list. There 4822 * ones are referenced from other Flows and can not 4823 * be applied, removed, deleted in arbitrary order 4824 * by list traversing. 4825 */ 4826 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4827 &attr, items, actions, false, error); 4828 if (!mcp_res->rix_flow) { 4829 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4830 return NULL; 4831 } 4832 return &mcp_res->hlist_ent; 4833 } 4834 4835 struct mlx5_list_entry * 4836 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4837 void *cb_ctx __rte_unused) 4838 { 4839 struct rte_eth_dev *dev = tool_ctx; 4840 struct mlx5_priv *priv = dev->data->dev_private; 4841 struct mlx5_flow_mreg_copy_resource *mcp_res; 4842 uint32_t idx = 0; 4843 4844 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4845 if (!mcp_res) { 4846 rte_errno = ENOMEM; 4847 return NULL; 4848 } 4849 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4850 mcp_res->idx = idx; 4851 return &mcp_res->hlist_ent; 4852 } 4853 4854 void 4855 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4856 { 4857 struct mlx5_flow_mreg_copy_resource *mcp_res = 4858 container_of(entry, typeof(*mcp_res), hlist_ent); 4859 struct rte_eth_dev *dev = tool_ctx; 4860 struct mlx5_priv *priv = dev->data->dev_private; 4861 4862 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4863 } 4864 4865 /** 4866 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4867 * 4868 * As mark_id is unique, if there's already a registered flow for the mark_id, 4869 * return by increasing the reference counter of the resource. Otherwise, create 4870 * the resource (mcp_res) and flow. 4871 * 4872 * Flow looks like, 4873 * - If ingress port is ANY and reg_c[1] is mark_id, 4874 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4875 * 4876 * For default flow (zero mark_id), flow is like, 4877 * - If ingress port is ANY, 4878 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4879 * 4880 * @param dev 4881 * Pointer to Ethernet device. 4882 * @param mark_id 4883 * ID of MARK action, zero means default flow for META. 4884 * @param[out] error 4885 * Perform verbose error reporting if not NULL. 4886 * 4887 * @return 4888 * Associated resource on success, NULL otherwise and rte_errno is set. 4889 */ 4890 static struct mlx5_flow_mreg_copy_resource * 4891 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4892 struct rte_flow_error *error) 4893 { 4894 struct mlx5_priv *priv = dev->data->dev_private; 4895 struct mlx5_list_entry *entry; 4896 struct mlx5_flow_cb_ctx ctx = { 4897 .dev = dev, 4898 .error = error, 4899 .data = &mark_id, 4900 }; 4901 4902 /* Check if already registered. */ 4903 MLX5_ASSERT(priv->mreg_cp_tbl); 4904 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4905 if (!entry) 4906 return NULL; 4907 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4908 hlist_ent); 4909 } 4910 4911 void 4912 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4913 { 4914 struct mlx5_flow_mreg_copy_resource *mcp_res = 4915 container_of(entry, typeof(*mcp_res), hlist_ent); 4916 struct rte_eth_dev *dev = tool_ctx; 4917 struct mlx5_priv *priv = dev->data->dev_private; 4918 4919 MLX5_ASSERT(mcp_res->rix_flow); 4920 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4921 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4922 } 4923 4924 /** 4925 * Release flow in RX_CP_TBL. 4926 * 4927 * @param dev 4928 * Pointer to Ethernet device. 4929 * @flow 4930 * Parent flow for wich copying is provided. 4931 */ 4932 static void 4933 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4934 struct rte_flow *flow) 4935 { 4936 struct mlx5_flow_mreg_copy_resource *mcp_res; 4937 struct mlx5_priv *priv = dev->data->dev_private; 4938 4939 if (!flow->rix_mreg_copy) 4940 return; 4941 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4942 flow->rix_mreg_copy); 4943 if (!mcp_res || !priv->mreg_cp_tbl) 4944 return; 4945 MLX5_ASSERT(mcp_res->rix_flow); 4946 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4947 flow->rix_mreg_copy = 0; 4948 } 4949 4950 /** 4951 * Remove the default copy action from RX_CP_TBL. 4952 * 4953 * This functions is called in the mlx5_dev_start(). No thread safe 4954 * is guaranteed. 4955 * 4956 * @param dev 4957 * Pointer to Ethernet device. 4958 */ 4959 static void 4960 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4961 { 4962 struct mlx5_list_entry *entry; 4963 struct mlx5_priv *priv = dev->data->dev_private; 4964 struct mlx5_flow_cb_ctx ctx; 4965 uint32_t mark_id; 4966 4967 /* Check if default flow is registered. */ 4968 if (!priv->mreg_cp_tbl) 4969 return; 4970 mark_id = MLX5_DEFAULT_COPY_ID; 4971 ctx.data = &mark_id; 4972 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4973 if (!entry) 4974 return; 4975 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4976 } 4977 4978 /** 4979 * Add the default copy action in in RX_CP_TBL. 4980 * 4981 * This functions is called in the mlx5_dev_start(). No thread safe 4982 * is guaranteed. 4983 * 4984 * @param dev 4985 * Pointer to Ethernet device. 4986 * @param[out] error 4987 * Perform verbose error reporting if not NULL. 4988 * 4989 * @return 4990 * 0 for success, negative value otherwise and rte_errno is set. 4991 */ 4992 static int 4993 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4994 struct rte_flow_error *error) 4995 { 4996 struct mlx5_priv *priv = dev->data->dev_private; 4997 struct mlx5_flow_mreg_copy_resource *mcp_res; 4998 struct mlx5_flow_cb_ctx ctx; 4999 uint32_t mark_id; 5000 5001 /* Check whether extensive metadata feature is engaged. */ 5002 if (!priv->sh->config.dv_flow_en || 5003 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5004 !mlx5_flow_ext_mreg_supported(dev) || 5005 !priv->sh->dv_regc0_mask) 5006 return 0; 5007 /* 5008 * Add default mreg copy flow may be called multiple time, but 5009 * only be called once in stop. Avoid register it twice. 5010 */ 5011 mark_id = MLX5_DEFAULT_COPY_ID; 5012 ctx.data = &mark_id; 5013 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5014 return 0; 5015 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5016 if (!mcp_res) 5017 return -rte_errno; 5018 return 0; 5019 } 5020 5021 /** 5022 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5023 * 5024 * All the flow having Q/RSS action should be split by 5025 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5026 * performs the following, 5027 * - CQE->flow_tag := reg_c[1] (MARK) 5028 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5029 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5030 * but there should be a flow per each MARK ID set by MARK action. 5031 * 5032 * For the aforementioned reason, if there's a MARK action in flow's action 5033 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5034 * the MARK ID to CQE's flow_tag like, 5035 * - If reg_c[1] is mark_id, 5036 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5037 * 5038 * For SET_META action which stores value in reg_c[0], as the destination is 5039 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5040 * MARK ID means the default flow. The default flow looks like, 5041 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5042 * 5043 * @param dev 5044 * Pointer to Ethernet device. 5045 * @param flow 5046 * Pointer to flow structure. 5047 * @param[in] actions 5048 * Pointer to the list of actions. 5049 * @param[out] error 5050 * Perform verbose error reporting if not NULL. 5051 * 5052 * @return 5053 * 0 on success, negative value otherwise and rte_errno is set. 5054 */ 5055 static int 5056 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5057 struct rte_flow *flow, 5058 const struct rte_flow_action *actions, 5059 struct rte_flow_error *error) 5060 { 5061 struct mlx5_priv *priv = dev->data->dev_private; 5062 struct mlx5_sh_config *config = &priv->sh->config; 5063 struct mlx5_flow_mreg_copy_resource *mcp_res; 5064 const struct rte_flow_action_mark *mark; 5065 5066 /* Check whether extensive metadata feature is engaged. */ 5067 if (!config->dv_flow_en || 5068 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5069 !mlx5_flow_ext_mreg_supported(dev) || 5070 !priv->sh->dv_regc0_mask) 5071 return 0; 5072 /* Find MARK action. */ 5073 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5074 switch (actions->type) { 5075 case RTE_FLOW_ACTION_TYPE_FLAG: 5076 mcp_res = flow_mreg_add_copy_action 5077 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5078 if (!mcp_res) 5079 return -rte_errno; 5080 flow->rix_mreg_copy = mcp_res->idx; 5081 return 0; 5082 case RTE_FLOW_ACTION_TYPE_MARK: 5083 mark = (const struct rte_flow_action_mark *) 5084 actions->conf; 5085 mcp_res = 5086 flow_mreg_add_copy_action(dev, mark->id, error); 5087 if (!mcp_res) 5088 return -rte_errno; 5089 flow->rix_mreg_copy = mcp_res->idx; 5090 return 0; 5091 default: 5092 break; 5093 } 5094 } 5095 return 0; 5096 } 5097 5098 #define MLX5_MAX_SPLIT_ACTIONS 24 5099 #define MLX5_MAX_SPLIT_ITEMS 24 5100 5101 /** 5102 * Split the hairpin flow. 5103 * Since HW can't support encap and push-vlan on Rx, we move these 5104 * actions to Tx. 5105 * If the count action is after the encap then we also 5106 * move the count action. in this case the count will also measure 5107 * the outer bytes. 5108 * 5109 * @param dev 5110 * Pointer to Ethernet device. 5111 * @param[in] actions 5112 * Associated actions (list terminated by the END action). 5113 * @param[out] actions_rx 5114 * Rx flow actions. 5115 * @param[out] actions_tx 5116 * Tx flow actions.. 5117 * @param[out] pattern_tx 5118 * The pattern items for the Tx flow. 5119 * @param[out] flow_id 5120 * The flow ID connected to this flow. 5121 * 5122 * @return 5123 * 0 on success. 5124 */ 5125 static int 5126 flow_hairpin_split(struct rte_eth_dev *dev, 5127 const struct rte_flow_action actions[], 5128 struct rte_flow_action actions_rx[], 5129 struct rte_flow_action actions_tx[], 5130 struct rte_flow_item pattern_tx[], 5131 uint32_t flow_id) 5132 { 5133 const struct rte_flow_action_raw_encap *raw_encap; 5134 const struct rte_flow_action_raw_decap *raw_decap; 5135 struct mlx5_rte_flow_action_set_tag *set_tag; 5136 struct rte_flow_action *tag_action; 5137 struct mlx5_rte_flow_item_tag *tag_item; 5138 struct rte_flow_item *item; 5139 char *addr; 5140 int encap = 0; 5141 5142 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5143 switch (actions->type) { 5144 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5145 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5146 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5147 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5148 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5149 rte_memcpy(actions_tx, actions, 5150 sizeof(struct rte_flow_action)); 5151 actions_tx++; 5152 break; 5153 case RTE_FLOW_ACTION_TYPE_COUNT: 5154 if (encap) { 5155 rte_memcpy(actions_tx, actions, 5156 sizeof(struct rte_flow_action)); 5157 actions_tx++; 5158 } else { 5159 rte_memcpy(actions_rx, actions, 5160 sizeof(struct rte_flow_action)); 5161 actions_rx++; 5162 } 5163 break; 5164 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5165 raw_encap = actions->conf; 5166 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5167 memcpy(actions_tx, actions, 5168 sizeof(struct rte_flow_action)); 5169 actions_tx++; 5170 encap = 1; 5171 } else { 5172 rte_memcpy(actions_rx, actions, 5173 sizeof(struct rte_flow_action)); 5174 actions_rx++; 5175 } 5176 break; 5177 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5178 raw_decap = actions->conf; 5179 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5180 memcpy(actions_tx, actions, 5181 sizeof(struct rte_flow_action)); 5182 actions_tx++; 5183 } else { 5184 rte_memcpy(actions_rx, actions, 5185 sizeof(struct rte_flow_action)); 5186 actions_rx++; 5187 } 5188 break; 5189 default: 5190 rte_memcpy(actions_rx, actions, 5191 sizeof(struct rte_flow_action)); 5192 actions_rx++; 5193 break; 5194 } 5195 } 5196 /* Add set meta action and end action for the Rx flow. */ 5197 tag_action = actions_rx; 5198 tag_action->type = (enum rte_flow_action_type) 5199 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5200 actions_rx++; 5201 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5202 actions_rx++; 5203 set_tag = (void *)actions_rx; 5204 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5205 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5206 .data = flow_id, 5207 }; 5208 MLX5_ASSERT(set_tag->id > REG_NON); 5209 tag_action->conf = set_tag; 5210 /* Create Tx item list. */ 5211 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5212 addr = (void *)&pattern_tx[2]; 5213 item = pattern_tx; 5214 item->type = (enum rte_flow_item_type) 5215 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5216 tag_item = (void *)addr; 5217 tag_item->data = flow_id; 5218 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5219 MLX5_ASSERT(set_tag->id > REG_NON); 5220 item->spec = tag_item; 5221 addr += sizeof(struct mlx5_rte_flow_item_tag); 5222 tag_item = (void *)addr; 5223 tag_item->data = UINT32_MAX; 5224 tag_item->id = UINT16_MAX; 5225 item->mask = tag_item; 5226 item->last = NULL; 5227 item++; 5228 item->type = RTE_FLOW_ITEM_TYPE_END; 5229 return 0; 5230 } 5231 5232 /** 5233 * The last stage of splitting chain, just creates the subflow 5234 * without any modification. 5235 * 5236 * @param[in] dev 5237 * Pointer to Ethernet device. 5238 * @param[in] flow 5239 * Parent flow structure pointer. 5240 * @param[in, out] sub_flow 5241 * Pointer to return the created subflow, may be NULL. 5242 * @param[in] attr 5243 * Flow rule attributes. 5244 * @param[in] items 5245 * Pattern specification (list terminated by the END pattern item). 5246 * @param[in] actions 5247 * Associated actions (list terminated by the END action). 5248 * @param[in] flow_split_info 5249 * Pointer to flow split info structure. 5250 * @param[out] error 5251 * Perform verbose error reporting if not NULL. 5252 * @return 5253 * 0 on success, negative value otherwise 5254 */ 5255 static int 5256 flow_create_split_inner(struct rte_eth_dev *dev, 5257 struct rte_flow *flow, 5258 struct mlx5_flow **sub_flow, 5259 const struct rte_flow_attr *attr, 5260 const struct rte_flow_item items[], 5261 const struct rte_flow_action actions[], 5262 struct mlx5_flow_split_info *flow_split_info, 5263 struct rte_flow_error *error) 5264 { 5265 struct mlx5_flow *dev_flow; 5266 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5267 5268 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5269 flow_split_info->flow_idx, error); 5270 if (!dev_flow) 5271 return -rte_errno; 5272 dev_flow->flow = flow; 5273 dev_flow->external = flow_split_info->external; 5274 dev_flow->skip_scale = flow_split_info->skip_scale; 5275 /* Subflow object was created, we must include one in the list. */ 5276 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5277 dev_flow->handle, next); 5278 /* 5279 * If dev_flow is as one of the suffix flow, some actions in suffix 5280 * flow may need some user defined item layer flags, and pass the 5281 * Metadata rxq mark flag to suffix flow as well. 5282 */ 5283 if (flow_split_info->prefix_layers) 5284 dev_flow->handle->layers = flow_split_info->prefix_layers; 5285 if (flow_split_info->prefix_mark) { 5286 MLX5_ASSERT(wks); 5287 wks->mark = 1; 5288 } 5289 if (sub_flow) 5290 *sub_flow = dev_flow; 5291 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5292 dev_flow->dv.table_id = flow_split_info->table_id; 5293 #endif 5294 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5295 } 5296 5297 /** 5298 * Get the sub policy of a meter. 5299 * 5300 * @param[in] dev 5301 * Pointer to Ethernet device. 5302 * @param[in] flow 5303 * Parent flow structure pointer. 5304 * @param wks 5305 * Pointer to thread flow work space. 5306 * @param[in] attr 5307 * Flow rule attributes. 5308 * @param[in] items 5309 * Pattern specification (list terminated by the END pattern item). 5310 * @param[out] error 5311 * Perform verbose error reporting if not NULL. 5312 * 5313 * @return 5314 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5315 */ 5316 static struct mlx5_flow_meter_sub_policy * 5317 get_meter_sub_policy(struct rte_eth_dev *dev, 5318 struct rte_flow *flow, 5319 struct mlx5_flow_workspace *wks, 5320 const struct rte_flow_attr *attr, 5321 const struct rte_flow_item items[], 5322 struct rte_flow_error *error) 5323 { 5324 struct mlx5_flow_meter_policy *policy; 5325 struct mlx5_flow_meter_policy *final_policy; 5326 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5327 5328 policy = wks->policy; 5329 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5330 if (final_policy->is_rss || final_policy->is_queue) { 5331 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5332 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5333 uint32_t i; 5334 5335 /* 5336 * This is a tmp dev_flow, 5337 * no need to register any matcher for it in translate. 5338 */ 5339 wks->skip_matcher_reg = 1; 5340 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5341 struct mlx5_flow dev_flow = {0}; 5342 struct mlx5_flow_handle dev_handle = { {0} }; 5343 uint8_t fate = final_policy->act_cnt[i].fate_action; 5344 5345 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5346 const struct rte_flow_action_rss *rss_act = 5347 final_policy->act_cnt[i].rss->conf; 5348 struct rte_flow_action rss_actions[2] = { 5349 [0] = { 5350 .type = RTE_FLOW_ACTION_TYPE_RSS, 5351 .conf = rss_act, 5352 }, 5353 [1] = { 5354 .type = RTE_FLOW_ACTION_TYPE_END, 5355 .conf = NULL, 5356 } 5357 }; 5358 5359 dev_flow.handle = &dev_handle; 5360 dev_flow.ingress = attr->ingress; 5361 dev_flow.flow = flow; 5362 dev_flow.external = 0; 5363 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5364 dev_flow.dv.transfer = attr->transfer; 5365 #endif 5366 /** 5367 * Translate RSS action to get rss hash fields. 5368 */ 5369 if (flow_drv_translate(dev, &dev_flow, attr, 5370 items, rss_actions, error)) 5371 goto exit; 5372 rss_desc_v[i] = wks->rss_desc; 5373 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5374 rss_desc_v[i].hash_fields = 5375 dev_flow.hash_fields; 5376 rss_desc_v[i].queue_num = 5377 rss_desc_v[i].hash_fields ? 5378 rss_desc_v[i].queue_num : 1; 5379 rss_desc_v[i].tunnel = 5380 !!(dev_flow.handle->layers & 5381 MLX5_FLOW_LAYER_TUNNEL); 5382 /* Use the RSS queues in the containers. */ 5383 rss_desc_v[i].queue = 5384 (uint16_t *)(uintptr_t)rss_act->queue; 5385 rss_desc[i] = &rss_desc_v[i]; 5386 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5387 /* This is queue action. */ 5388 rss_desc_v[i] = wks->rss_desc; 5389 rss_desc_v[i].key_len = 0; 5390 rss_desc_v[i].hash_fields = 0; 5391 rss_desc_v[i].queue = 5392 &final_policy->act_cnt[i].queue; 5393 rss_desc_v[i].queue_num = 1; 5394 rss_desc[i] = &rss_desc_v[i]; 5395 } else { 5396 rss_desc[i] = NULL; 5397 } 5398 } 5399 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5400 flow, policy, rss_desc); 5401 } else { 5402 enum mlx5_meter_domain mtr_domain = 5403 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5404 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5405 MLX5_MTR_DOMAIN_INGRESS); 5406 sub_policy = policy->sub_policys[mtr_domain][0]; 5407 } 5408 if (!sub_policy) 5409 rte_flow_error_set(error, EINVAL, 5410 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5411 "Failed to get meter sub-policy."); 5412 exit: 5413 return sub_policy; 5414 } 5415 5416 /** 5417 * Split the meter flow. 5418 * 5419 * As meter flow will split to three sub flow, other than meter 5420 * action, the other actions make sense to only meter accepts 5421 * the packet. If it need to be dropped, no other additional 5422 * actions should be take. 5423 * 5424 * One kind of special action which decapsulates the L3 tunnel 5425 * header will be in the prefix sub flow, as not to take the 5426 * L3 tunnel header into account. 5427 * 5428 * @param[in] dev 5429 * Pointer to Ethernet device. 5430 * @param[in] flow 5431 * Parent flow structure pointer. 5432 * @param wks 5433 * Pointer to thread flow work space. 5434 * @param[in] attr 5435 * Flow rule attributes. 5436 * @param[in] items 5437 * Pattern specification (list terminated by the END pattern item). 5438 * @param[out] sfx_items 5439 * Suffix flow match items (list terminated by the END pattern item). 5440 * @param[in] actions 5441 * Associated actions (list terminated by the END action). 5442 * @param[out] actions_sfx 5443 * Suffix flow actions. 5444 * @param[out] actions_pre 5445 * Prefix flow actions. 5446 * @param[out] mtr_flow_id 5447 * Pointer to meter flow id. 5448 * @param[out] error 5449 * Perform verbose error reporting if not NULL. 5450 * 5451 * @return 5452 * 0 on success, a negative errno value otherwise and rte_errno is set. 5453 */ 5454 static int 5455 flow_meter_split_prep(struct rte_eth_dev *dev, 5456 struct rte_flow *flow, 5457 struct mlx5_flow_workspace *wks, 5458 const struct rte_flow_attr *attr, 5459 const struct rte_flow_item items[], 5460 struct rte_flow_item sfx_items[], 5461 const struct rte_flow_action actions[], 5462 struct rte_flow_action actions_sfx[], 5463 struct rte_flow_action actions_pre[], 5464 uint32_t *mtr_flow_id, 5465 struct rte_flow_error *error) 5466 { 5467 struct mlx5_priv *priv = dev->data->dev_private; 5468 struct mlx5_flow_meter_info *fm = wks->fm; 5469 struct rte_flow_action *tag_action = NULL; 5470 struct rte_flow_item *tag_item; 5471 struct mlx5_rte_flow_action_set_tag *set_tag; 5472 const struct rte_flow_action_raw_encap *raw_encap; 5473 const struct rte_flow_action_raw_decap *raw_decap; 5474 struct mlx5_rte_flow_item_tag *tag_item_spec; 5475 struct mlx5_rte_flow_item_tag *tag_item_mask; 5476 uint32_t tag_id = 0; 5477 struct rte_flow_item *vlan_item_dst = NULL; 5478 const struct rte_flow_item *vlan_item_src = NULL; 5479 const struct rte_flow_item *orig_items = items; 5480 struct rte_flow_action *hw_mtr_action; 5481 struct rte_flow_action *action_pre_head = NULL; 5482 uint16_t flow_src_port = priv->representor_id; 5483 bool mtr_first; 5484 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5485 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5486 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5487 uint32_t flow_id = 0; 5488 uint32_t flow_id_reversed = 0; 5489 uint8_t flow_id_bits = 0; 5490 bool after_meter = false; 5491 int shift; 5492 5493 /* Prepare the suffix subflow items. */ 5494 tag_item = sfx_items++; 5495 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5496 int item_type = items->type; 5497 5498 switch (item_type) { 5499 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5500 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5501 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5502 return -rte_errno; 5503 if (!fm->def_policy && wks->policy->is_hierarchy && 5504 flow_src_port != priv->representor_id) { 5505 if (flow_drv_mtr_hierarchy_rule_create(dev, 5506 flow, fm, 5507 flow_src_port, 5508 items, 5509 error)) 5510 return -rte_errno; 5511 } 5512 memcpy(sfx_items, items, sizeof(*sfx_items)); 5513 sfx_items++; 5514 break; 5515 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5516 flow_src_port = 0; 5517 memcpy(sfx_items, items, sizeof(*sfx_items)); 5518 sfx_items++; 5519 break; 5520 case RTE_FLOW_ITEM_TYPE_VLAN: 5521 /* Determine if copy vlan item below. */ 5522 vlan_item_src = items; 5523 vlan_item_dst = sfx_items++; 5524 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5525 break; 5526 default: 5527 break; 5528 } 5529 } 5530 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5531 sfx_items++; 5532 mtr_first = priv->sh->meter_aso_en && 5533 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5534 /* For ASO meter, meter must be before tag in TX direction. */ 5535 if (mtr_first) { 5536 action_pre_head = actions_pre++; 5537 /* Leave space for tag action. */ 5538 tag_action = actions_pre++; 5539 } 5540 /* Prepare the actions for prefix and suffix flow. */ 5541 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5542 struct rte_flow_action *action_cur = NULL; 5543 5544 switch (actions->type) { 5545 case RTE_FLOW_ACTION_TYPE_METER: 5546 if (mtr_first) { 5547 action_cur = action_pre_head; 5548 } else { 5549 /* Leave space for tag action. */ 5550 tag_action = actions_pre++; 5551 action_cur = actions_pre++; 5552 } 5553 after_meter = true; 5554 break; 5555 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5556 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5557 action_cur = actions_pre++; 5558 break; 5559 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5560 raw_encap = actions->conf; 5561 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5562 action_cur = actions_pre++; 5563 break; 5564 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5565 raw_decap = actions->conf; 5566 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5567 action_cur = actions_pre++; 5568 break; 5569 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5570 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5571 if (vlan_item_dst && vlan_item_src) { 5572 memcpy(vlan_item_dst, vlan_item_src, 5573 sizeof(*vlan_item_dst)); 5574 /* 5575 * Convert to internal match item, it is used 5576 * for vlan push and set vid. 5577 */ 5578 vlan_item_dst->type = (enum rte_flow_item_type) 5579 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5580 } 5581 break; 5582 case RTE_FLOW_ACTION_TYPE_COUNT: 5583 if (fm->def_policy) 5584 action_cur = after_meter ? 5585 actions_sfx++ : actions_pre++; 5586 break; 5587 default: 5588 break; 5589 } 5590 if (!action_cur) 5591 action_cur = (fm->def_policy) ? 5592 actions_sfx++ : actions_pre++; 5593 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5594 } 5595 /* Add end action to the actions. */ 5596 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5597 if (priv->sh->meter_aso_en) { 5598 /** 5599 * For ASO meter, need to add an extra jump action explicitly, 5600 * to jump from meter to policer table. 5601 */ 5602 struct mlx5_flow_meter_sub_policy *sub_policy; 5603 struct mlx5_flow_tbl_data_entry *tbl_data; 5604 5605 if (!fm->def_policy) { 5606 sub_policy = get_meter_sub_policy(dev, flow, wks, 5607 attr, orig_items, 5608 error); 5609 if (!sub_policy) 5610 return -rte_errno; 5611 } else { 5612 enum mlx5_meter_domain mtr_domain = 5613 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5614 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5615 MLX5_MTR_DOMAIN_INGRESS); 5616 5617 sub_policy = 5618 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5619 } 5620 tbl_data = container_of(sub_policy->tbl_rsc, 5621 struct mlx5_flow_tbl_data_entry, tbl); 5622 hw_mtr_action = actions_pre++; 5623 hw_mtr_action->type = (enum rte_flow_action_type) 5624 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5625 hw_mtr_action->conf = tbl_data->jump.action; 5626 } 5627 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5628 actions_pre++; 5629 if (!tag_action) 5630 return rte_flow_error_set(error, ENOMEM, 5631 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5632 NULL, "No tag action space."); 5633 if (!mtr_flow_id) { 5634 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5635 goto exit; 5636 } 5637 /* Only default-policy Meter creates mtr flow id. */ 5638 if (fm->def_policy) { 5639 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5640 if (!tag_id) 5641 return rte_flow_error_set(error, ENOMEM, 5642 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5643 "Failed to allocate meter flow id."); 5644 flow_id = tag_id - 1; 5645 flow_id_bits = (!flow_id) ? 1 : 5646 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5647 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5648 mtr_reg_bits) { 5649 mlx5_ipool_free(fm->flow_ipool, tag_id); 5650 return rte_flow_error_set(error, EINVAL, 5651 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5652 "Meter flow id exceeds max limit."); 5653 } 5654 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5655 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5656 } 5657 /* Build tag actions and items for meter_id/meter flow_id. */ 5658 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5659 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5660 tag_item_mask = tag_item_spec + 1; 5661 /* Both flow_id and meter_id share the same register. */ 5662 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5663 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5664 0, error), 5665 .offset = mtr_id_offset, 5666 .length = mtr_reg_bits, 5667 .data = flow->meter, 5668 }; 5669 /* 5670 * The color Reg bits used by flow_id are growing from 5671 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5672 */ 5673 for (shift = 0; shift < flow_id_bits; shift++) 5674 flow_id_reversed = (flow_id_reversed << 1) | 5675 ((flow_id >> shift) & 0x1); 5676 set_tag->data |= 5677 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5678 tag_item_spec->id = set_tag->id; 5679 tag_item_spec->data = set_tag->data << mtr_id_offset; 5680 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5681 tag_action->type = (enum rte_flow_action_type) 5682 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5683 tag_action->conf = set_tag; 5684 tag_item->type = (enum rte_flow_item_type) 5685 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5686 tag_item->spec = tag_item_spec; 5687 tag_item->last = NULL; 5688 tag_item->mask = tag_item_mask; 5689 exit: 5690 if (mtr_flow_id) 5691 *mtr_flow_id = tag_id; 5692 return 0; 5693 } 5694 5695 /** 5696 * Split action list having QUEUE/RSS for metadata register copy. 5697 * 5698 * Once Q/RSS action is detected in user's action list, the flow action 5699 * should be split in order to copy metadata registers, which will happen in 5700 * RX_CP_TBL like, 5701 * - CQE->flow_tag := reg_c[1] (MARK) 5702 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5703 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5704 * This is because the last action of each flow must be a terminal action 5705 * (QUEUE, RSS or DROP). 5706 * 5707 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5708 * stored and kept in the mlx5_flow structure per each sub_flow. 5709 * 5710 * The Q/RSS action is replaced with, 5711 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5712 * And the following JUMP action is added at the end, 5713 * - JUMP, to RX_CP_TBL. 5714 * 5715 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5716 * flow_create_split_metadata() routine. The flow will look like, 5717 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5718 * 5719 * @param dev 5720 * Pointer to Ethernet device. 5721 * @param[out] split_actions 5722 * Pointer to store split actions to jump to CP_TBL. 5723 * @param[in] actions 5724 * Pointer to the list of original flow actions. 5725 * @param[in] qrss 5726 * Pointer to the Q/RSS action. 5727 * @param[in] actions_n 5728 * Number of original actions. 5729 * @param[in] mtr_sfx 5730 * Check if it is in meter suffix table. 5731 * @param[out] error 5732 * Perform verbose error reporting if not NULL. 5733 * 5734 * @return 5735 * non-zero unique flow_id on success, otherwise 0 and 5736 * error/rte_error are set. 5737 */ 5738 static uint32_t 5739 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5740 struct rte_flow_action *split_actions, 5741 const struct rte_flow_action *actions, 5742 const struct rte_flow_action *qrss, 5743 int actions_n, int mtr_sfx, 5744 struct rte_flow_error *error) 5745 { 5746 struct mlx5_priv *priv = dev->data->dev_private; 5747 struct mlx5_rte_flow_action_set_tag *set_tag; 5748 struct rte_flow_action_jump *jump; 5749 const int qrss_idx = qrss - actions; 5750 uint32_t flow_id = 0; 5751 int ret = 0; 5752 5753 /* 5754 * Given actions will be split 5755 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5756 * - Add jump to mreg CP_TBL. 5757 * As a result, there will be one more action. 5758 */ 5759 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5760 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5761 ++actions_n; 5762 set_tag = (void *)(split_actions + actions_n); 5763 /* 5764 * If we are not the meter suffix flow, add the tag action. 5765 * Since meter suffix flow already has the tag added. 5766 */ 5767 if (!mtr_sfx) { 5768 /* 5769 * Allocate the new subflow ID. This one is unique within 5770 * device and not shared with representors. Otherwise, 5771 * we would have to resolve multi-thread access synch 5772 * issue. Each flow on the shared device is appended 5773 * with source vport identifier, so the resulting 5774 * flows will be unique in the shared (by master and 5775 * representors) domain even if they have coinciding 5776 * IDs. 5777 */ 5778 mlx5_ipool_malloc(priv->sh->ipool 5779 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5780 if (!flow_id) 5781 return rte_flow_error_set(error, ENOMEM, 5782 RTE_FLOW_ERROR_TYPE_ACTION, 5783 NULL, "can't allocate id " 5784 "for split Q/RSS subflow"); 5785 /* Internal SET_TAG action to set flow ID. */ 5786 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5787 .data = flow_id, 5788 }; 5789 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5790 if (ret < 0) 5791 return ret; 5792 set_tag->id = ret; 5793 /* Construct new actions array. */ 5794 /* Replace QUEUE/RSS action. */ 5795 split_actions[qrss_idx] = (struct rte_flow_action){ 5796 .type = (enum rte_flow_action_type) 5797 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5798 .conf = set_tag, 5799 }; 5800 } else { 5801 /* 5802 * If we are the suffix flow of meter, tag already exist. 5803 * Set the QUEUE/RSS action to void. 5804 */ 5805 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5806 } 5807 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5808 jump = (void *)(set_tag + 1); 5809 *jump = (struct rte_flow_action_jump){ 5810 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5811 }; 5812 split_actions[actions_n - 2] = (struct rte_flow_action){ 5813 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5814 .conf = jump, 5815 }; 5816 split_actions[actions_n - 1] = (struct rte_flow_action){ 5817 .type = RTE_FLOW_ACTION_TYPE_END, 5818 }; 5819 return flow_id; 5820 } 5821 5822 /** 5823 * Extend the given action list for Tx metadata copy. 5824 * 5825 * Copy the given action list to the ext_actions and add flow metadata register 5826 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5827 * 5828 * @param[out] ext_actions 5829 * Pointer to the extended action list. 5830 * @param[in] actions 5831 * Pointer to the list of actions. 5832 * @param[in] actions_n 5833 * Number of actions in the list. 5834 * @param[out] error 5835 * Perform verbose error reporting if not NULL. 5836 * @param[in] encap_idx 5837 * The encap action index. 5838 * 5839 * @return 5840 * 0 on success, negative value otherwise 5841 */ 5842 static int 5843 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5844 struct rte_flow_action *ext_actions, 5845 const struct rte_flow_action *actions, 5846 int actions_n, struct rte_flow_error *error, 5847 int encap_idx) 5848 { 5849 struct mlx5_flow_action_copy_mreg *cp_mreg = 5850 (struct mlx5_flow_action_copy_mreg *) 5851 (ext_actions + actions_n + 1); 5852 int ret; 5853 5854 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5855 if (ret < 0) 5856 return ret; 5857 cp_mreg->dst = ret; 5858 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5859 if (ret < 0) 5860 return ret; 5861 cp_mreg->src = ret; 5862 if (encap_idx != 0) 5863 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5864 if (encap_idx == actions_n - 1) { 5865 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5866 .type = (enum rte_flow_action_type) 5867 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5868 .conf = cp_mreg, 5869 }; 5870 ext_actions[actions_n] = (struct rte_flow_action){ 5871 .type = RTE_FLOW_ACTION_TYPE_END, 5872 }; 5873 } else { 5874 ext_actions[encap_idx] = (struct rte_flow_action){ 5875 .type = (enum rte_flow_action_type) 5876 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5877 .conf = cp_mreg, 5878 }; 5879 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5880 sizeof(*ext_actions) * (actions_n - encap_idx)); 5881 } 5882 return 0; 5883 } 5884 5885 /** 5886 * Check the match action from the action list. 5887 * 5888 * @param[in] actions 5889 * Pointer to the list of actions. 5890 * @param[in] attr 5891 * Flow rule attributes. 5892 * @param[in] action 5893 * The action to be check if exist. 5894 * @param[out] match_action_pos 5895 * Pointer to the position of the matched action if exists, otherwise is -1. 5896 * @param[out] qrss_action_pos 5897 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5898 * @param[out] modify_after_mirror 5899 * Pointer to the flag of modify action after FDB mirroring. 5900 * 5901 * @return 5902 * > 0 the total number of actions. 5903 * 0 if not found match action in action list. 5904 */ 5905 static int 5906 flow_check_match_action(const struct rte_flow_action actions[], 5907 const struct rte_flow_attr *attr, 5908 enum rte_flow_action_type action, 5909 int *match_action_pos, int *qrss_action_pos, 5910 int *modify_after_mirror) 5911 { 5912 const struct rte_flow_action_sample *sample; 5913 const struct rte_flow_action_raw_decap *decap; 5914 int actions_n = 0; 5915 uint32_t ratio = 0; 5916 int sub_type = 0; 5917 int flag = 0; 5918 int fdb_mirror = 0; 5919 5920 *match_action_pos = -1; 5921 *qrss_action_pos = -1; 5922 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5923 if (actions->type == action) { 5924 flag = 1; 5925 *match_action_pos = actions_n; 5926 } 5927 switch (actions->type) { 5928 case RTE_FLOW_ACTION_TYPE_QUEUE: 5929 case RTE_FLOW_ACTION_TYPE_RSS: 5930 *qrss_action_pos = actions_n; 5931 break; 5932 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5933 sample = actions->conf; 5934 ratio = sample->ratio; 5935 sub_type = ((const struct rte_flow_action *) 5936 (sample->actions))->type; 5937 if (ratio == 1 && attr->transfer) 5938 fdb_mirror = 1; 5939 break; 5940 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5941 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5942 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5943 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5944 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5945 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5946 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5947 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5948 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5949 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5950 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5951 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5952 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5953 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5954 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5955 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5956 case RTE_FLOW_ACTION_TYPE_FLAG: 5957 case RTE_FLOW_ACTION_TYPE_MARK: 5958 case RTE_FLOW_ACTION_TYPE_SET_META: 5959 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5960 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5961 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5962 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5963 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5964 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5965 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5966 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5967 case RTE_FLOW_ACTION_TYPE_METER: 5968 if (fdb_mirror) 5969 *modify_after_mirror = 1; 5970 break; 5971 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5972 decap = actions->conf; 5973 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5974 ; 5975 actions_n++; 5976 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5977 const struct rte_flow_action_raw_encap *encap = 5978 actions->conf; 5979 if (decap->size <= 5980 MLX5_ENCAPSULATION_DECISION_SIZE && 5981 encap->size > 5982 MLX5_ENCAPSULATION_DECISION_SIZE) 5983 /* L3 encap. */ 5984 break; 5985 } 5986 if (fdb_mirror) 5987 *modify_after_mirror = 1; 5988 break; 5989 default: 5990 break; 5991 } 5992 actions_n++; 5993 } 5994 if (flag && fdb_mirror && !*modify_after_mirror) { 5995 /* FDB mirroring uses the destination array to implement 5996 * instead of FLOW_SAMPLER object. 5997 */ 5998 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5999 flag = 0; 6000 } 6001 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6002 return flag ? actions_n + 1 : 0; 6003 } 6004 6005 #define SAMPLE_SUFFIX_ITEM 3 6006 6007 /** 6008 * Split the sample flow. 6009 * 6010 * As sample flow will split to two sub flow, sample flow with 6011 * sample action, the other actions will move to new suffix flow. 6012 * 6013 * Also add unique tag id with tag action in the sample flow, 6014 * the same tag id will be as match in the suffix flow. 6015 * 6016 * @param dev 6017 * Pointer to Ethernet device. 6018 * @param[in] add_tag 6019 * Add extra tag action flag. 6020 * @param[out] sfx_items 6021 * Suffix flow match items (list terminated by the END pattern item). 6022 * @param[in] actions 6023 * Associated actions (list terminated by the END action). 6024 * @param[out] actions_sfx 6025 * Suffix flow actions. 6026 * @param[out] actions_pre 6027 * Prefix flow actions. 6028 * @param[in] actions_n 6029 * The total number of actions. 6030 * @param[in] sample_action_pos 6031 * The sample action position. 6032 * @param[in] qrss_action_pos 6033 * The Queue/RSS action position. 6034 * @param[in] jump_table 6035 * Add extra jump action flag. 6036 * @param[out] error 6037 * Perform verbose error reporting if not NULL. 6038 * 6039 * @return 6040 * 0 on success, or unique flow_id, a negative errno value 6041 * otherwise and rte_errno is set. 6042 */ 6043 static int 6044 flow_sample_split_prep(struct rte_eth_dev *dev, 6045 int add_tag, 6046 const struct rte_flow_item items[], 6047 struct rte_flow_item sfx_items[], 6048 const struct rte_flow_action actions[], 6049 struct rte_flow_action actions_sfx[], 6050 struct rte_flow_action actions_pre[], 6051 int actions_n, 6052 int sample_action_pos, 6053 int qrss_action_pos, 6054 int jump_table, 6055 struct rte_flow_error *error) 6056 { 6057 struct mlx5_priv *priv = dev->data->dev_private; 6058 struct mlx5_rte_flow_action_set_tag *set_tag; 6059 struct mlx5_rte_flow_item_tag *tag_spec; 6060 struct mlx5_rte_flow_item_tag *tag_mask; 6061 struct rte_flow_action_jump *jump_action; 6062 uint32_t tag_id = 0; 6063 int append_index = 0; 6064 int set_tag_idx = -1; 6065 int index; 6066 int ret; 6067 6068 if (sample_action_pos < 0) 6069 return rte_flow_error_set(error, EINVAL, 6070 RTE_FLOW_ERROR_TYPE_ACTION, 6071 NULL, "invalid position of sample " 6072 "action in list"); 6073 /* Prepare the actions for prefix and suffix flow. */ 6074 if (add_tag) { 6075 /* Update the new added tag action index preceding 6076 * the PUSH_VLAN or ENCAP action. 6077 */ 6078 const struct rte_flow_action_raw_encap *raw_encap; 6079 const struct rte_flow_action *action = actions; 6080 int encap_idx; 6081 int action_idx = 0; 6082 int raw_decap_idx = -1; 6083 int push_vlan_idx = -1; 6084 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6085 switch (action->type) { 6086 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6087 raw_decap_idx = action_idx; 6088 break; 6089 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6090 raw_encap = action->conf; 6091 if (raw_encap->size > 6092 MLX5_ENCAPSULATION_DECISION_SIZE) { 6093 encap_idx = raw_decap_idx != -1 ? 6094 raw_decap_idx : action_idx; 6095 if (encap_idx < sample_action_pos && 6096 push_vlan_idx == -1) 6097 set_tag_idx = encap_idx; 6098 } 6099 break; 6100 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6101 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6102 encap_idx = action_idx; 6103 if (encap_idx < sample_action_pos && 6104 push_vlan_idx == -1) 6105 set_tag_idx = encap_idx; 6106 break; 6107 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6108 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6109 push_vlan_idx = action_idx; 6110 if (push_vlan_idx < sample_action_pos) 6111 set_tag_idx = action_idx; 6112 break; 6113 default: 6114 break; 6115 } 6116 action_idx++; 6117 } 6118 } 6119 /* Prepare the actions for prefix and suffix flow. */ 6120 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6121 index = qrss_action_pos; 6122 /* Put the preceding the Queue/RSS action into prefix flow. */ 6123 if (index != 0) 6124 memcpy(actions_pre, actions, 6125 sizeof(struct rte_flow_action) * index); 6126 /* Put others preceding the sample action into prefix flow. */ 6127 if (sample_action_pos > index + 1) 6128 memcpy(actions_pre + index, actions + index + 1, 6129 sizeof(struct rte_flow_action) * 6130 (sample_action_pos - index - 1)); 6131 index = sample_action_pos - 1; 6132 /* Put Queue/RSS action into Suffix flow. */ 6133 memcpy(actions_sfx, actions + qrss_action_pos, 6134 sizeof(struct rte_flow_action)); 6135 actions_sfx++; 6136 } else if (add_tag && set_tag_idx >= 0) { 6137 if (set_tag_idx > 0) 6138 memcpy(actions_pre, actions, 6139 sizeof(struct rte_flow_action) * set_tag_idx); 6140 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6141 sizeof(struct rte_flow_action) * 6142 (sample_action_pos - set_tag_idx)); 6143 index = sample_action_pos; 6144 } else { 6145 index = sample_action_pos; 6146 if (index != 0) 6147 memcpy(actions_pre, actions, 6148 sizeof(struct rte_flow_action) * index); 6149 } 6150 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6151 * For CX6DX and above, metadata registers Cx preserve their value, 6152 * add an extra tag action for NIC-RX and E-Switch Domain. 6153 */ 6154 if (add_tag) { 6155 /* Prepare the prefix tag action. */ 6156 append_index++; 6157 set_tag = (void *)(actions_pre + actions_n + append_index); 6158 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6159 /* Trust VF/SF on CX5 not supported meter so that the reserved 6160 * metadata regC is REG_NON, back to use application tag 6161 * index 0. 6162 */ 6163 if (unlikely(ret == REG_NON)) 6164 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6165 if (ret < 0) 6166 return ret; 6167 mlx5_ipool_malloc(priv->sh->ipool 6168 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6169 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6170 .id = ret, 6171 .data = tag_id, 6172 }; 6173 /* Prepare the suffix subflow items. */ 6174 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6175 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6176 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 6177 memcpy(sfx_items, items, sizeof(*sfx_items)); 6178 sfx_items++; 6179 } 6180 } 6181 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6182 tag_spec->data = tag_id; 6183 tag_spec->id = set_tag->id; 6184 tag_mask = tag_spec + 1; 6185 tag_mask->data = UINT32_MAX; 6186 sfx_items[0] = (struct rte_flow_item){ 6187 .type = (enum rte_flow_item_type) 6188 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6189 .spec = tag_spec, 6190 .last = NULL, 6191 .mask = tag_mask, 6192 }; 6193 sfx_items[1] = (struct rte_flow_item){ 6194 .type = (enum rte_flow_item_type) 6195 RTE_FLOW_ITEM_TYPE_END, 6196 }; 6197 /* Prepare the tag action in prefix subflow. */ 6198 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6199 actions_pre[set_tag_idx] = 6200 (struct rte_flow_action){ 6201 .type = (enum rte_flow_action_type) 6202 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6203 .conf = set_tag, 6204 }; 6205 /* Update next sample position due to add one tag action */ 6206 index += 1; 6207 } 6208 /* Copy the sample action into prefix flow. */ 6209 memcpy(actions_pre + index, actions + sample_action_pos, 6210 sizeof(struct rte_flow_action)); 6211 index += 1; 6212 /* For the modify action after the sample action in E-Switch mirroring, 6213 * Add the extra jump action in prefix subflow and jump into the next 6214 * table, then do the modify action in the new table. 6215 */ 6216 if (jump_table) { 6217 /* Prepare the prefix jump action. */ 6218 append_index++; 6219 jump_action = (void *)(actions_pre + actions_n + append_index); 6220 jump_action->group = jump_table; 6221 actions_pre[index++] = 6222 (struct rte_flow_action){ 6223 .type = (enum rte_flow_action_type) 6224 RTE_FLOW_ACTION_TYPE_JUMP, 6225 .conf = jump_action, 6226 }; 6227 } 6228 actions_pre[index] = (struct rte_flow_action){ 6229 .type = (enum rte_flow_action_type) 6230 RTE_FLOW_ACTION_TYPE_END, 6231 }; 6232 /* Put the actions after sample into Suffix flow. */ 6233 memcpy(actions_sfx, actions + sample_action_pos + 1, 6234 sizeof(struct rte_flow_action) * 6235 (actions_n - sample_action_pos - 1)); 6236 return tag_id; 6237 } 6238 6239 /** 6240 * The splitting for metadata feature. 6241 * 6242 * - Q/RSS action on NIC Rx should be split in order to pass by 6243 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6244 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6245 * 6246 * - All the actions on NIC Tx should have a mreg copy action to 6247 * copy reg_a from WQE to reg_c[0]. 6248 * 6249 * @param dev 6250 * Pointer to Ethernet device. 6251 * @param[in] flow 6252 * Parent flow structure pointer. 6253 * @param[in] attr 6254 * Flow rule attributes. 6255 * @param[in] items 6256 * Pattern specification (list terminated by the END pattern item). 6257 * @param[in] actions 6258 * Associated actions (list terminated by the END action). 6259 * @param[in] flow_split_info 6260 * Pointer to flow split info structure. 6261 * @param[out] error 6262 * Perform verbose error reporting if not NULL. 6263 * @return 6264 * 0 on success, negative value otherwise 6265 */ 6266 static int 6267 flow_create_split_metadata(struct rte_eth_dev *dev, 6268 struct rte_flow *flow, 6269 const struct rte_flow_attr *attr, 6270 const struct rte_flow_item items[], 6271 const struct rte_flow_action actions[], 6272 struct mlx5_flow_split_info *flow_split_info, 6273 struct rte_flow_error *error) 6274 { 6275 struct mlx5_priv *priv = dev->data->dev_private; 6276 struct mlx5_sh_config *config = &priv->sh->config; 6277 const struct rte_flow_action *qrss = NULL; 6278 struct rte_flow_action *ext_actions = NULL; 6279 struct mlx5_flow *dev_flow = NULL; 6280 uint32_t qrss_id = 0; 6281 int mtr_sfx = 0; 6282 size_t act_size; 6283 int actions_n; 6284 int encap_idx; 6285 int ret; 6286 6287 /* Check whether extensive metadata feature is engaged. */ 6288 if (!config->dv_flow_en || 6289 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6290 !mlx5_flow_ext_mreg_supported(dev)) 6291 return flow_create_split_inner(dev, flow, NULL, attr, items, 6292 actions, flow_split_info, error); 6293 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6294 &encap_idx); 6295 if (qrss) { 6296 /* Exclude hairpin flows from splitting. */ 6297 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6298 const struct rte_flow_action_queue *queue; 6299 6300 queue = qrss->conf; 6301 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6302 qrss = NULL; 6303 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6304 const struct rte_flow_action_rss *rss; 6305 6306 rss = qrss->conf; 6307 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6308 qrss = NULL; 6309 } 6310 } 6311 if (qrss) { 6312 /* Check if it is in meter suffix table. */ 6313 mtr_sfx = attr->group == 6314 ((attr->transfer && priv->fdb_def_rule) ? 6315 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6316 MLX5_FLOW_TABLE_LEVEL_METER); 6317 /* 6318 * Q/RSS action on NIC Rx should be split in order to pass by 6319 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6320 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6321 */ 6322 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6323 sizeof(struct rte_flow_action_set_tag) + 6324 sizeof(struct rte_flow_action_jump); 6325 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6326 SOCKET_ID_ANY); 6327 if (!ext_actions) 6328 return rte_flow_error_set(error, ENOMEM, 6329 RTE_FLOW_ERROR_TYPE_ACTION, 6330 NULL, "no memory to split " 6331 "metadata flow"); 6332 /* 6333 * Create the new actions list with removed Q/RSS action 6334 * and appended set tag and jump to register copy table 6335 * (RX_CP_TBL). We should preallocate unique tag ID here 6336 * in advance, because it is needed for set tag action. 6337 */ 6338 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6339 qrss, actions_n, 6340 mtr_sfx, error); 6341 if (!mtr_sfx && !qrss_id) { 6342 ret = -rte_errno; 6343 goto exit; 6344 } 6345 } else if (attr->egress && !attr->transfer) { 6346 /* 6347 * All the actions on NIC Tx should have a metadata register 6348 * copy action to copy reg_a from WQE to reg_c[meta] 6349 */ 6350 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6351 sizeof(struct mlx5_flow_action_copy_mreg); 6352 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6353 SOCKET_ID_ANY); 6354 if (!ext_actions) 6355 return rte_flow_error_set(error, ENOMEM, 6356 RTE_FLOW_ERROR_TYPE_ACTION, 6357 NULL, "no memory to split " 6358 "metadata flow"); 6359 /* Create the action list appended with copy register. */ 6360 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6361 actions_n, error, encap_idx); 6362 if (ret < 0) 6363 goto exit; 6364 } 6365 /* Add the unmodified original or prefix subflow. */ 6366 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6367 items, ext_actions ? ext_actions : 6368 actions, flow_split_info, error); 6369 if (ret < 0) 6370 goto exit; 6371 MLX5_ASSERT(dev_flow); 6372 if (qrss) { 6373 const struct rte_flow_attr q_attr = { 6374 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6375 .ingress = 1, 6376 }; 6377 /* Internal PMD action to set register. */ 6378 struct mlx5_rte_flow_item_tag q_tag_spec = { 6379 .data = qrss_id, 6380 .id = REG_NON, 6381 }; 6382 struct rte_flow_item q_items[] = { 6383 { 6384 .type = (enum rte_flow_item_type) 6385 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6386 .spec = &q_tag_spec, 6387 .last = NULL, 6388 .mask = NULL, 6389 }, 6390 { 6391 .type = RTE_FLOW_ITEM_TYPE_END, 6392 }, 6393 }; 6394 struct rte_flow_action q_actions[] = { 6395 { 6396 .type = qrss->type, 6397 .conf = qrss->conf, 6398 }, 6399 { 6400 .type = RTE_FLOW_ACTION_TYPE_END, 6401 }, 6402 }; 6403 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6404 6405 /* 6406 * Configure the tag item only if there is no meter subflow. 6407 * Since tag is already marked in the meter suffix subflow 6408 * we can just use the meter suffix items as is. 6409 */ 6410 if (qrss_id) { 6411 /* Not meter subflow. */ 6412 MLX5_ASSERT(!mtr_sfx); 6413 /* 6414 * Put unique id in prefix flow due to it is destroyed 6415 * after suffix flow and id will be freed after there 6416 * is no actual flows with this id and identifier 6417 * reallocation becomes possible (for example, for 6418 * other flows in other threads). 6419 */ 6420 dev_flow->handle->split_flow_id = qrss_id; 6421 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6422 error); 6423 if (ret < 0) 6424 goto exit; 6425 q_tag_spec.id = ret; 6426 } 6427 dev_flow = NULL; 6428 /* Add suffix subflow to execute Q/RSS. */ 6429 flow_split_info->prefix_layers = layers; 6430 flow_split_info->prefix_mark = 0; 6431 flow_split_info->table_id = 0; 6432 ret = flow_create_split_inner(dev, flow, &dev_flow, 6433 &q_attr, mtr_sfx ? items : 6434 q_items, q_actions, 6435 flow_split_info, error); 6436 if (ret < 0) 6437 goto exit; 6438 /* qrss ID should be freed if failed. */ 6439 qrss_id = 0; 6440 MLX5_ASSERT(dev_flow); 6441 } 6442 6443 exit: 6444 /* 6445 * We do not destroy the partially created sub_flows in case of error. 6446 * These ones are included into parent flow list and will be destroyed 6447 * by flow_drv_destroy. 6448 */ 6449 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6450 qrss_id); 6451 mlx5_free(ext_actions); 6452 return ret; 6453 } 6454 6455 /** 6456 * Create meter internal drop flow with the original pattern. 6457 * 6458 * @param dev 6459 * Pointer to Ethernet device. 6460 * @param[in] flow 6461 * Parent flow structure pointer. 6462 * @param[in] attr 6463 * Flow rule attributes. 6464 * @param[in] items 6465 * Pattern specification (list terminated by the END pattern item). 6466 * @param[in] flow_split_info 6467 * Pointer to flow split info structure. 6468 * @param[in] fm 6469 * Pointer to flow meter structure. 6470 * @param[out] error 6471 * Perform verbose error reporting if not NULL. 6472 * @return 6473 * 0 on success, negative value otherwise 6474 */ 6475 static uint32_t 6476 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6477 struct rte_flow *flow, 6478 const struct rte_flow_attr *attr, 6479 const struct rte_flow_item items[], 6480 struct mlx5_flow_split_info *flow_split_info, 6481 struct mlx5_flow_meter_info *fm, 6482 struct rte_flow_error *error) 6483 { 6484 struct mlx5_flow *dev_flow = NULL; 6485 struct rte_flow_attr drop_attr = *attr; 6486 struct rte_flow_action drop_actions[3]; 6487 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6488 6489 MLX5_ASSERT(fm->drop_cnt); 6490 drop_actions[0].type = 6491 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6492 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6493 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6494 drop_actions[1].conf = NULL; 6495 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6496 drop_actions[2].conf = NULL; 6497 drop_split_info.external = false; 6498 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6499 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6500 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6501 return flow_create_split_inner(dev, flow, &dev_flow, 6502 &drop_attr, items, drop_actions, 6503 &drop_split_info, error); 6504 } 6505 6506 /** 6507 * The splitting for meter feature. 6508 * 6509 * - The meter flow will be split to two flows as prefix and 6510 * suffix flow. The packets make sense only it pass the prefix 6511 * meter action. 6512 * 6513 * - Reg_C_5 is used for the packet to match betweend prefix and 6514 * suffix flow. 6515 * 6516 * @param dev 6517 * Pointer to Ethernet device. 6518 * @param[in] flow 6519 * Parent flow structure pointer. 6520 * @param[in] attr 6521 * Flow rule attributes. 6522 * @param[in] items 6523 * Pattern specification (list terminated by the END pattern item). 6524 * @param[in] actions 6525 * Associated actions (list terminated by the END action). 6526 * @param[in] flow_split_info 6527 * Pointer to flow split info structure. 6528 * @param[out] error 6529 * Perform verbose error reporting if not NULL. 6530 * @return 6531 * 0 on success, negative value otherwise 6532 */ 6533 static int 6534 flow_create_split_meter(struct rte_eth_dev *dev, 6535 struct rte_flow *flow, 6536 const struct rte_flow_attr *attr, 6537 const struct rte_flow_item items[], 6538 const struct rte_flow_action actions[], 6539 struct mlx5_flow_split_info *flow_split_info, 6540 struct rte_flow_error *error) 6541 { 6542 struct mlx5_priv *priv = dev->data->dev_private; 6543 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6544 struct rte_flow_action *sfx_actions = NULL; 6545 struct rte_flow_action *pre_actions = NULL; 6546 struct rte_flow_item *sfx_items = NULL; 6547 struct mlx5_flow *dev_flow = NULL; 6548 struct rte_flow_attr sfx_attr = *attr; 6549 struct mlx5_flow_meter_info *fm = NULL; 6550 uint8_t skip_scale_restore; 6551 bool has_mtr = false; 6552 bool has_modify = false; 6553 bool set_mtr_reg = true; 6554 bool is_mtr_hierarchy = false; 6555 uint32_t meter_id = 0; 6556 uint32_t mtr_idx = 0; 6557 uint32_t mtr_flow_id = 0; 6558 size_t act_size; 6559 size_t item_size; 6560 int actions_n = 0; 6561 int ret = 0; 6562 6563 if (priv->mtr_en) 6564 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6565 &has_modify, &meter_id); 6566 if (has_mtr) { 6567 if (flow->meter) { 6568 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6569 if (!fm) 6570 return rte_flow_error_set(error, EINVAL, 6571 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6572 NULL, "Meter not found."); 6573 } else { 6574 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6575 if (!fm) 6576 return rte_flow_error_set(error, EINVAL, 6577 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6578 NULL, "Meter not found."); 6579 ret = mlx5_flow_meter_attach(priv, fm, 6580 &sfx_attr, error); 6581 if (ret) 6582 return -rte_errno; 6583 flow->meter = mtr_idx; 6584 } 6585 MLX5_ASSERT(wks); 6586 wks->fm = fm; 6587 if (!fm->def_policy) { 6588 wks->policy = mlx5_flow_meter_policy_find(dev, 6589 fm->policy_id, 6590 NULL); 6591 MLX5_ASSERT(wks->policy); 6592 if (wks->policy->mark) 6593 wks->mark = 1; 6594 if (wks->policy->is_hierarchy) { 6595 wks->final_policy = 6596 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6597 wks->policy); 6598 if (!wks->final_policy) 6599 return rte_flow_error_set(error, 6600 EINVAL, 6601 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6602 "Failed to find terminal policy of hierarchy."); 6603 is_mtr_hierarchy = true; 6604 } 6605 } 6606 /* 6607 * If it isn't default-policy Meter, and 6608 * 1. Not meter hierarchy and there's no action in flow to change 6609 * packet (modify/encap/decap etc.), OR 6610 * 2. No drop count needed for this meter. 6611 * Then no need to use regC to save meter id anymore. 6612 */ 6613 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6614 set_mtr_reg = false; 6615 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6616 #define METER_PREFIX_ACTION 7 6617 act_size = (sizeof(struct rte_flow_action) * 6618 (actions_n + METER_PREFIX_ACTION)) + 6619 sizeof(struct mlx5_rte_flow_action_set_tag); 6620 /* Suffix items: tag, vlan, port id, end. */ 6621 #define METER_SUFFIX_ITEM 4 6622 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6623 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6624 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6625 0, SOCKET_ID_ANY); 6626 if (!sfx_actions) 6627 return rte_flow_error_set(error, ENOMEM, 6628 RTE_FLOW_ERROR_TYPE_ACTION, 6629 NULL, "no memory to split " 6630 "meter flow"); 6631 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6632 act_size); 6633 /* There's no suffix flow for meter of non-default policy. */ 6634 if (!fm->def_policy) 6635 pre_actions = sfx_actions + 1; 6636 else 6637 pre_actions = sfx_actions + actions_n; 6638 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6639 items, sfx_items, actions, 6640 sfx_actions, pre_actions, 6641 (set_mtr_reg ? &mtr_flow_id : NULL), 6642 error); 6643 if (ret) { 6644 ret = -rte_errno; 6645 goto exit; 6646 } 6647 /* Add the prefix subflow. */ 6648 skip_scale_restore = flow_split_info->skip_scale; 6649 flow_split_info->skip_scale |= 6650 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6651 ret = flow_create_split_inner(dev, flow, &dev_flow, 6652 attr, items, pre_actions, 6653 flow_split_info, error); 6654 flow_split_info->skip_scale = skip_scale_restore; 6655 if (ret) { 6656 if (mtr_flow_id) 6657 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6658 ret = -rte_errno; 6659 goto exit; 6660 } 6661 if (mtr_flow_id) { 6662 dev_flow->handle->split_flow_id = mtr_flow_id; 6663 dev_flow->handle->is_meter_flow_id = 1; 6664 } 6665 if (!fm->def_policy) { 6666 if (!set_mtr_reg && fm->drop_cnt) 6667 ret = 6668 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6669 &sfx_attr, items, 6670 flow_split_info, 6671 fm, error); 6672 goto exit; 6673 } 6674 /* Setting the sfx group atrr. */ 6675 sfx_attr.group = sfx_attr.transfer ? 6676 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6677 MLX5_FLOW_TABLE_LEVEL_METER; 6678 flow_split_info->prefix_layers = 6679 flow_get_prefix_layer_flags(dev_flow); 6680 flow_split_info->prefix_mark |= wks->mark; 6681 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6682 } 6683 /* Add the prefix subflow. */ 6684 ret = flow_create_split_metadata(dev, flow, 6685 &sfx_attr, sfx_items ? 6686 sfx_items : items, 6687 sfx_actions ? sfx_actions : actions, 6688 flow_split_info, error); 6689 exit: 6690 if (sfx_actions) 6691 mlx5_free(sfx_actions); 6692 return ret; 6693 } 6694 6695 /** 6696 * The splitting for sample feature. 6697 * 6698 * Once Sample action is detected in the action list, the flow actions should 6699 * be split into prefix sub flow and suffix sub flow. 6700 * 6701 * The original items remain in the prefix sub flow, all actions preceding the 6702 * sample action and the sample action itself will be copied to the prefix 6703 * sub flow, the actions following the sample action will be copied to the 6704 * suffix sub flow, Queue action always be located in the suffix sub flow. 6705 * 6706 * In order to make the packet from prefix sub flow matches with suffix sub 6707 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6708 * flow uses tag item with the unique flow id. 6709 * 6710 * @param dev 6711 * Pointer to Ethernet device. 6712 * @param[in] flow 6713 * Parent flow structure pointer. 6714 * @param[in] attr 6715 * Flow rule attributes. 6716 * @param[in] items 6717 * Pattern specification (list terminated by the END pattern item). 6718 * @param[in] actions 6719 * Associated actions (list terminated by the END action). 6720 * @param[in] flow_split_info 6721 * Pointer to flow split info structure. 6722 * @param[out] error 6723 * Perform verbose error reporting if not NULL. 6724 * @return 6725 * 0 on success, negative value otherwise 6726 */ 6727 static int 6728 flow_create_split_sample(struct rte_eth_dev *dev, 6729 struct rte_flow *flow, 6730 const struct rte_flow_attr *attr, 6731 const struct rte_flow_item items[], 6732 const struct rte_flow_action actions[], 6733 struct mlx5_flow_split_info *flow_split_info, 6734 struct rte_flow_error *error) 6735 { 6736 struct mlx5_priv *priv = dev->data->dev_private; 6737 struct rte_flow_action *sfx_actions = NULL; 6738 struct rte_flow_action *pre_actions = NULL; 6739 struct rte_flow_item *sfx_items = NULL; 6740 struct mlx5_flow *dev_flow = NULL; 6741 struct rte_flow_attr sfx_attr = *attr; 6742 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6743 struct mlx5_flow_dv_sample_resource *sample_res; 6744 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6745 struct mlx5_flow_tbl_resource *sfx_tbl; 6746 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6747 #endif 6748 size_t act_size; 6749 size_t item_size; 6750 uint32_t fdb_tx = 0; 6751 int32_t tag_id = 0; 6752 int actions_n = 0; 6753 int sample_action_pos; 6754 int qrss_action_pos; 6755 int add_tag = 0; 6756 int modify_after_mirror = 0; 6757 uint16_t jump_table = 0; 6758 const uint32_t next_ft_step = 1; 6759 int ret = 0; 6760 6761 if (priv->sampler_en) 6762 actions_n = flow_check_match_action(actions, attr, 6763 RTE_FLOW_ACTION_TYPE_SAMPLE, 6764 &sample_action_pos, &qrss_action_pos, 6765 &modify_after_mirror); 6766 if (actions_n) { 6767 /* The prefix actions must includes sample, tag, end. */ 6768 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6769 + sizeof(struct mlx5_rte_flow_action_set_tag); 6770 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6771 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6772 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6773 item_size), 0, SOCKET_ID_ANY); 6774 if (!sfx_actions) 6775 return rte_flow_error_set(error, ENOMEM, 6776 RTE_FLOW_ERROR_TYPE_ACTION, 6777 NULL, "no memory to split " 6778 "sample flow"); 6779 /* The representor_id is UINT16_MAX for uplink. */ 6780 fdb_tx = (attr->transfer && priv->representor_id != UINT16_MAX); 6781 /* 6782 * When reg_c_preserve is set, metadata registers Cx preserve 6783 * their value even through packet duplication. 6784 */ 6785 add_tag = (!fdb_tx || 6786 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6787 if (add_tag) 6788 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6789 + act_size); 6790 if (modify_after_mirror) 6791 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6792 next_ft_step; 6793 pre_actions = sfx_actions + actions_n; 6794 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6795 actions, sfx_actions, 6796 pre_actions, actions_n, 6797 sample_action_pos, 6798 qrss_action_pos, jump_table, 6799 error); 6800 if (tag_id < 0 || (add_tag && !tag_id)) { 6801 ret = -rte_errno; 6802 goto exit; 6803 } 6804 if (modify_after_mirror) 6805 flow_split_info->skip_scale = 6806 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6807 /* Add the prefix subflow. */ 6808 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6809 items, pre_actions, 6810 flow_split_info, error); 6811 if (ret) { 6812 ret = -rte_errno; 6813 goto exit; 6814 } 6815 dev_flow->handle->split_flow_id = tag_id; 6816 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6817 if (!modify_after_mirror) { 6818 /* Set the sfx group attr. */ 6819 sample_res = (struct mlx5_flow_dv_sample_resource *) 6820 dev_flow->dv.sample_res; 6821 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6822 sample_res->normal_path_tbl; 6823 sfx_tbl_data = container_of(sfx_tbl, 6824 struct mlx5_flow_tbl_data_entry, 6825 tbl); 6826 sfx_attr.group = sfx_attr.transfer ? 6827 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6828 } else { 6829 MLX5_ASSERT(attr->transfer); 6830 sfx_attr.group = jump_table; 6831 } 6832 flow_split_info->prefix_layers = 6833 flow_get_prefix_layer_flags(dev_flow); 6834 MLX5_ASSERT(wks); 6835 flow_split_info->prefix_mark |= wks->mark; 6836 /* Suffix group level already be scaled with factor, set 6837 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6838 * again in translation. 6839 */ 6840 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6841 #endif 6842 } 6843 /* Add the suffix subflow. */ 6844 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6845 sfx_items ? sfx_items : items, 6846 sfx_actions ? sfx_actions : actions, 6847 flow_split_info, error); 6848 exit: 6849 if (sfx_actions) 6850 mlx5_free(sfx_actions); 6851 return ret; 6852 } 6853 6854 /** 6855 * Split the flow to subflow set. The splitters might be linked 6856 * in the chain, like this: 6857 * flow_create_split_outer() calls: 6858 * flow_create_split_meter() calls: 6859 * flow_create_split_metadata(meter_subflow_0) calls: 6860 * flow_create_split_inner(metadata_subflow_0) 6861 * flow_create_split_inner(metadata_subflow_1) 6862 * flow_create_split_inner(metadata_subflow_2) 6863 * flow_create_split_metadata(meter_subflow_1) calls: 6864 * flow_create_split_inner(metadata_subflow_0) 6865 * flow_create_split_inner(metadata_subflow_1) 6866 * flow_create_split_inner(metadata_subflow_2) 6867 * 6868 * This provide flexible way to add new levels of flow splitting. 6869 * The all of successfully created subflows are included to the 6870 * parent flow dev_flow list. 6871 * 6872 * @param dev 6873 * Pointer to Ethernet device. 6874 * @param[in] flow 6875 * Parent flow structure pointer. 6876 * @param[in] attr 6877 * Flow rule attributes. 6878 * @param[in] items 6879 * Pattern specification (list terminated by the END pattern item). 6880 * @param[in] actions 6881 * Associated actions (list terminated by the END action). 6882 * @param[in] flow_split_info 6883 * Pointer to flow split info structure. 6884 * @param[out] error 6885 * Perform verbose error reporting if not NULL. 6886 * @return 6887 * 0 on success, negative value otherwise 6888 */ 6889 static int 6890 flow_create_split_outer(struct rte_eth_dev *dev, 6891 struct rte_flow *flow, 6892 const struct rte_flow_attr *attr, 6893 const struct rte_flow_item items[], 6894 const struct rte_flow_action actions[], 6895 struct mlx5_flow_split_info *flow_split_info, 6896 struct rte_flow_error *error) 6897 { 6898 int ret; 6899 6900 ret = flow_create_split_sample(dev, flow, attr, items, 6901 actions, flow_split_info, error); 6902 MLX5_ASSERT(ret <= 0); 6903 return ret; 6904 } 6905 6906 static inline struct mlx5_flow_tunnel * 6907 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6908 { 6909 struct mlx5_flow_tunnel *tunnel; 6910 6911 #pragma GCC diagnostic push 6912 #pragma GCC diagnostic ignored "-Wcast-qual" 6913 tunnel = (typeof(tunnel))flow->tunnel; 6914 #pragma GCC diagnostic pop 6915 6916 return tunnel; 6917 } 6918 6919 /** 6920 * Adjust flow RSS workspace if needed. 6921 * 6922 * @param wks 6923 * Pointer to thread flow work space. 6924 * @param rss_desc 6925 * Pointer to RSS descriptor. 6926 * @param[in] nrssq_num 6927 * New RSS queue number. 6928 * 6929 * @return 6930 * 0 on success, -1 otherwise and rte_errno is set. 6931 */ 6932 static int 6933 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6934 struct mlx5_flow_rss_desc *rss_desc, 6935 uint32_t nrssq_num) 6936 { 6937 if (likely(nrssq_num <= wks->rssq_num)) 6938 return 0; 6939 rss_desc->queue = realloc(rss_desc->queue, 6940 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6941 if (!rss_desc->queue) { 6942 rte_errno = ENOMEM; 6943 return -1; 6944 } 6945 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6946 return 0; 6947 } 6948 6949 /** 6950 * Create a flow and add it to @p list. 6951 * 6952 * @param dev 6953 * Pointer to Ethernet device. 6954 * @param list 6955 * Pointer to a TAILQ flow list. If this parameter NULL, 6956 * no list insertion occurred, flow is just created, 6957 * this is caller's responsibility to track the 6958 * created flow. 6959 * @param[in] attr 6960 * Flow rule attributes. 6961 * @param[in] items 6962 * Pattern specification (list terminated by the END pattern item). 6963 * @param[in] actions 6964 * Associated actions (list terminated by the END action). 6965 * @param[in] external 6966 * This flow rule is created by request external to PMD. 6967 * @param[out] error 6968 * Perform verbose error reporting if not NULL. 6969 * 6970 * @return 6971 * A flow index on success, 0 otherwise and rte_errno is set. 6972 */ 6973 static uint32_t 6974 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6975 const struct rte_flow_attr *attr, 6976 const struct rte_flow_item items[], 6977 const struct rte_flow_action original_actions[], 6978 bool external, struct rte_flow_error *error) 6979 { 6980 struct mlx5_priv *priv = dev->data->dev_private; 6981 struct rte_flow *flow = NULL; 6982 struct mlx5_flow *dev_flow; 6983 const struct rte_flow_action_rss *rss = NULL; 6984 struct mlx5_translated_action_handle 6985 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6986 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6987 union { 6988 struct mlx5_flow_expand_rss buf; 6989 uint8_t buffer[8192]; 6990 } expand_buffer; 6991 union { 6992 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6993 uint8_t buffer[2048]; 6994 } actions_rx; 6995 union { 6996 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6997 uint8_t buffer[2048]; 6998 } actions_hairpin_tx; 6999 union { 7000 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7001 uint8_t buffer[2048]; 7002 } items_tx; 7003 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7004 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7005 struct mlx5_flow_rss_desc *rss_desc; 7006 const struct rte_flow_action *p_actions_rx; 7007 uint32_t i; 7008 uint32_t idx = 0; 7009 int hairpin_flow; 7010 struct rte_flow_attr attr_tx = { .priority = 0 }; 7011 const struct rte_flow_action *actions; 7012 struct rte_flow_action *translated_actions = NULL; 7013 struct mlx5_flow_tunnel *tunnel; 7014 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7015 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7016 struct mlx5_flow_split_info flow_split_info = { 7017 .external = !!external, 7018 .skip_scale = 0, 7019 .flow_idx = 0, 7020 .prefix_mark = 0, 7021 .prefix_layers = 0, 7022 .table_id = 0 7023 }; 7024 int ret; 7025 7026 MLX5_ASSERT(wks); 7027 rss_desc = &wks->rss_desc; 7028 ret = flow_action_handles_translate(dev, original_actions, 7029 indir_actions, 7030 &indir_actions_n, 7031 &translated_actions, error); 7032 if (ret < 0) { 7033 MLX5_ASSERT(translated_actions == NULL); 7034 return 0; 7035 } 7036 actions = translated_actions ? translated_actions : original_actions; 7037 p_actions_rx = actions; 7038 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7039 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7040 external, hairpin_flow, error); 7041 if (ret < 0) 7042 goto error_before_hairpin_split; 7043 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7044 if (!flow) { 7045 rte_errno = ENOMEM; 7046 goto error_before_hairpin_split; 7047 } 7048 if (hairpin_flow > 0) { 7049 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7050 rte_errno = EINVAL; 7051 goto error_before_hairpin_split; 7052 } 7053 flow_hairpin_split(dev, actions, actions_rx.actions, 7054 actions_hairpin_tx.actions, items_tx.items, 7055 idx); 7056 p_actions_rx = actions_rx.actions; 7057 } 7058 flow_split_info.flow_idx = idx; 7059 flow->drv_type = flow_get_drv_type(dev, attr); 7060 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7061 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7062 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7063 /* RSS Action only works on NIC RX domain */ 7064 if (attr->ingress && !attr->transfer) 7065 rss = flow_get_rss_action(dev, p_actions_rx); 7066 if (rss) { 7067 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 7068 return 0; 7069 /* 7070 * The following information is required by 7071 * mlx5_flow_hashfields_adjust() in advance. 7072 */ 7073 rss_desc->level = rss->level; 7074 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7075 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7076 } 7077 flow->dev_handles = 0; 7078 if (rss && rss->types) { 7079 unsigned int graph_root; 7080 7081 graph_root = find_graph_root(rss->level); 7082 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7083 items, rss->types, 7084 mlx5_support_expansion, graph_root); 7085 MLX5_ASSERT(ret > 0 && 7086 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7087 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7088 for (i = 0; i < buf->entries; ++i) 7089 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7090 } 7091 } else { 7092 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7093 sizeof(expand_buffer.buffer), 7094 items, sq_specs); 7095 if (ret) { 7096 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7097 NULL, "not enough memory for rte_flow"); 7098 goto error; 7099 } 7100 if (buf->entries == 0) { 7101 buf->entries = 1; 7102 buf->entry[0].pattern = (void *)(uintptr_t)items; 7103 } 7104 } 7105 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7106 indir_actions_n); 7107 for (i = 0; i < buf->entries; ++i) { 7108 /* Initialize flow split data. */ 7109 flow_split_info.prefix_layers = 0; 7110 flow_split_info.prefix_mark = 0; 7111 flow_split_info.skip_scale = 0; 7112 /* 7113 * The splitter may create multiple dev_flows, 7114 * depending on configuration. In the simplest 7115 * case it just creates unmodified original flow. 7116 */ 7117 ret = flow_create_split_outer(dev, flow, attr, 7118 buf->entry[i].pattern, 7119 p_actions_rx, &flow_split_info, 7120 error); 7121 if (ret < 0) 7122 goto error; 7123 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7124 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7125 p_actions_rx, 7126 idx, 7127 wks->flows[0].tunnel, 7128 &default_miss_ctx, 7129 error); 7130 if (ret < 0) { 7131 mlx5_free(default_miss_ctx.queue); 7132 goto error; 7133 } 7134 } 7135 } 7136 /* Create the tx flow. */ 7137 if (hairpin_flow) { 7138 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7139 attr_tx.ingress = 0; 7140 attr_tx.egress = 1; 7141 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7142 actions_hairpin_tx.actions, 7143 idx, error); 7144 if (!dev_flow) 7145 goto error; 7146 dev_flow->flow = flow; 7147 dev_flow->external = 0; 7148 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7149 dev_flow->handle, next); 7150 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7151 items_tx.items, 7152 actions_hairpin_tx.actions, error); 7153 if (ret < 0) 7154 goto error; 7155 } 7156 /* 7157 * Update the metadata register copy table. If extensive 7158 * metadata feature is enabled and registers are supported 7159 * we might create the extra rte_flow for each unique 7160 * MARK/FLAG action ID. 7161 * 7162 * The table is updated for ingress Flows only, because 7163 * the egress Flows belong to the different device and 7164 * copy table should be updated in peer NIC Rx domain. 7165 */ 7166 if (attr->ingress && 7167 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7168 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7169 if (ret) 7170 goto error; 7171 } 7172 /* 7173 * If the flow is external (from application) OR device is started, 7174 * OR mreg discover, then apply immediately. 7175 */ 7176 if (external || dev->data->dev_started || 7177 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7178 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7179 ret = flow_drv_apply(dev, flow, error); 7180 if (ret < 0) 7181 goto error; 7182 } 7183 flow->type = type; 7184 flow_rxq_flags_set(dev, flow); 7185 rte_free(translated_actions); 7186 tunnel = flow_tunnel_from_rule(wks->flows); 7187 if (tunnel) { 7188 flow->tunnel = 1; 7189 flow->tunnel_id = tunnel->tunnel_id; 7190 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7191 mlx5_free(default_miss_ctx.queue); 7192 } 7193 mlx5_flow_pop_thread_workspace(); 7194 return idx; 7195 error: 7196 MLX5_ASSERT(flow); 7197 ret = rte_errno; /* Save rte_errno before cleanup. */ 7198 flow_mreg_del_copy_action(dev, flow); 7199 flow_drv_destroy(dev, flow); 7200 if (rss_desc->shared_rss) 7201 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7202 mlx5_ipool_get 7203 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7204 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7205 mlx5_ipool_free(priv->flows[type], idx); 7206 rte_errno = ret; /* Restore rte_errno. */ 7207 ret = rte_errno; 7208 rte_errno = ret; 7209 error_before_hairpin_split: 7210 mlx5_flow_pop_thread_workspace(); 7211 rte_free(translated_actions); 7212 return 0; 7213 } 7214 7215 /** 7216 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7217 * incoming packets to table 1. 7218 * 7219 * Other flow rules, requested for group n, will be created in 7220 * e-switch table n+1. 7221 * Jump action to e-switch group n will be created to group n+1. 7222 * 7223 * Used when working in switchdev mode, to utilise advantages of table 1 7224 * and above. 7225 * 7226 * @param dev 7227 * Pointer to Ethernet device. 7228 * 7229 * @return 7230 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7231 */ 7232 struct rte_flow * 7233 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7234 { 7235 const struct rte_flow_attr attr = { 7236 .group = 0, 7237 .priority = 0, 7238 .ingress = 1, 7239 .egress = 0, 7240 .transfer = 1, 7241 }; 7242 const struct rte_flow_item pattern = { 7243 .type = RTE_FLOW_ITEM_TYPE_END, 7244 }; 7245 struct rte_flow_action_jump jump = { 7246 .group = 1, 7247 }; 7248 const struct rte_flow_action actions[] = { 7249 { 7250 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7251 .conf = &jump, 7252 }, 7253 { 7254 .type = RTE_FLOW_ACTION_TYPE_END, 7255 }, 7256 }; 7257 struct rte_flow_error error; 7258 7259 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7260 &attr, &pattern, 7261 actions, false, &error); 7262 } 7263 7264 /** 7265 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7266 * and sq number, directs all packets to peer vport. 7267 * 7268 * @param dev 7269 * Pointer to Ethernet device. 7270 * @param sq_num 7271 * SQ number. 7272 * 7273 * @return 7274 * Flow ID on success, 0 otherwise and rte_errno is set. 7275 */ 7276 uint32_t 7277 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7278 { 7279 struct rte_flow_attr attr = { 7280 .group = 0, 7281 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7282 .ingress = 1, 7283 .egress = 0, 7284 .transfer = 1, 7285 }; 7286 struct rte_flow_item_port_id port_spec = { 7287 .id = MLX5_PORT_ESW_MGR, 7288 }; 7289 struct mlx5_rte_flow_item_sq sq_spec = { 7290 .queue = sq_num, 7291 }; 7292 struct rte_flow_item pattern[] = { 7293 { 7294 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7295 .spec = &port_spec, 7296 }, 7297 { 7298 .type = (enum rte_flow_item_type) 7299 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7300 .spec = &sq_spec, 7301 }, 7302 { 7303 .type = RTE_FLOW_ITEM_TYPE_END, 7304 }, 7305 }; 7306 struct rte_flow_action_jump jump = { 7307 .group = 1, 7308 }; 7309 struct rte_flow_action_port_id port = { 7310 .id = dev->data->port_id, 7311 }; 7312 struct rte_flow_action actions[] = { 7313 { 7314 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7315 .conf = &jump, 7316 }, 7317 { 7318 .type = RTE_FLOW_ACTION_TYPE_END, 7319 }, 7320 }; 7321 struct rte_flow_error error; 7322 7323 /* 7324 * Creates group 0, highest priority jump flow. 7325 * Matches txq to bypass kernel packets. 7326 */ 7327 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7328 false, &error) == 0) 7329 return 0; 7330 /* Create group 1, lowest priority redirect flow for txq. */ 7331 attr.group = 1; 7332 actions[0].conf = &port; 7333 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7334 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7335 actions, false, &error); 7336 } 7337 7338 /** 7339 * Validate a flow supported by the NIC. 7340 * 7341 * @see rte_flow_validate() 7342 * @see rte_flow_ops 7343 */ 7344 int 7345 mlx5_flow_validate(struct rte_eth_dev *dev, 7346 const struct rte_flow_attr *attr, 7347 const struct rte_flow_item items[], 7348 const struct rte_flow_action original_actions[], 7349 struct rte_flow_error *error) 7350 { 7351 int hairpin_flow; 7352 struct mlx5_translated_action_handle 7353 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7354 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7355 const struct rte_flow_action *actions; 7356 struct rte_flow_action *translated_actions = NULL; 7357 int ret = flow_action_handles_translate(dev, original_actions, 7358 indir_actions, 7359 &indir_actions_n, 7360 &translated_actions, error); 7361 7362 if (ret) 7363 return ret; 7364 actions = translated_actions ? translated_actions : original_actions; 7365 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7366 ret = flow_drv_validate(dev, attr, items, actions, 7367 true, hairpin_flow, error); 7368 rte_free(translated_actions); 7369 return ret; 7370 } 7371 7372 /** 7373 * Create a flow. 7374 * 7375 * @see rte_flow_create() 7376 * @see rte_flow_ops 7377 */ 7378 struct rte_flow * 7379 mlx5_flow_create(struct rte_eth_dev *dev, 7380 const struct rte_flow_attr *attr, 7381 const struct rte_flow_item items[], 7382 const struct rte_flow_action actions[], 7383 struct rte_flow_error *error) 7384 { 7385 struct mlx5_priv *priv = dev->data->dev_private; 7386 7387 if (priv->sh->config.dv_flow_en == 2) { 7388 rte_flow_error_set(error, ENOTSUP, 7389 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7390 NULL, 7391 "Flow non-Q creation not supported"); 7392 return NULL; 7393 } 7394 /* 7395 * If the device is not started yet, it is not allowed to created a 7396 * flow from application. PMD default flows and traffic control flows 7397 * are not affected. 7398 */ 7399 if (unlikely(!dev->data->dev_started)) { 7400 DRV_LOG(DEBUG, "port %u is not started when " 7401 "inserting a flow", dev->data->port_id); 7402 rte_flow_error_set(error, ENODEV, 7403 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7404 NULL, 7405 "port not started"); 7406 return NULL; 7407 } 7408 7409 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7410 attr, items, actions, 7411 true, error); 7412 } 7413 7414 /** 7415 * Destroy a flow in a list. 7416 * 7417 * @param dev 7418 * Pointer to Ethernet device. 7419 * @param[in] flow_idx 7420 * Index of flow to destroy. 7421 */ 7422 static void 7423 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7424 uint32_t flow_idx) 7425 { 7426 struct mlx5_priv *priv = dev->data->dev_private; 7427 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7428 7429 if (!flow) 7430 return; 7431 MLX5_ASSERT(flow->type == type); 7432 /* 7433 * Update RX queue flags only if port is started, otherwise it is 7434 * already clean. 7435 */ 7436 if (dev->data->dev_started) 7437 flow_rxq_flags_trim(dev, flow); 7438 flow_drv_destroy(dev, flow); 7439 if (flow->tunnel) { 7440 struct mlx5_flow_tunnel *tunnel; 7441 7442 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7443 RTE_VERIFY(tunnel); 7444 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7445 mlx5_flow_tunnel_free(dev, tunnel); 7446 } 7447 flow_mreg_del_copy_action(dev, flow); 7448 mlx5_ipool_free(priv->flows[type], flow_idx); 7449 } 7450 7451 /** 7452 * Destroy all flows. 7453 * 7454 * @param dev 7455 * Pointer to Ethernet device. 7456 * @param type 7457 * Flow type to be flushed. 7458 * @param active 7459 * If flushing is called actively. 7460 */ 7461 void 7462 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7463 bool active) 7464 { 7465 struct mlx5_priv *priv = dev->data->dev_private; 7466 uint32_t num_flushed = 0, fidx = 1; 7467 struct rte_flow *flow; 7468 7469 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7470 if (priv->sh->config.dv_flow_en == 2 && 7471 type == MLX5_FLOW_TYPE_GEN) { 7472 flow_hw_q_flow_flush(dev, NULL); 7473 return; 7474 } 7475 #endif 7476 7477 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7478 flow_list_destroy(dev, type, fidx); 7479 num_flushed++; 7480 } 7481 if (active) { 7482 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7483 dev->data->port_id, num_flushed); 7484 } 7485 } 7486 7487 /** 7488 * Stop all default actions for flows. 7489 * 7490 * @param dev 7491 * Pointer to Ethernet device. 7492 */ 7493 void 7494 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7495 { 7496 flow_mreg_del_default_copy_action(dev); 7497 flow_rxq_flags_clear(dev); 7498 } 7499 7500 /** 7501 * Start all default actions for flows. 7502 * 7503 * @param dev 7504 * Pointer to Ethernet device. 7505 * @return 7506 * 0 on success, a negative errno value otherwise and rte_errno is set. 7507 */ 7508 int 7509 mlx5_flow_start_default(struct rte_eth_dev *dev) 7510 { 7511 struct rte_flow_error error; 7512 7513 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7514 return flow_mreg_add_default_copy_action(dev, &error); 7515 } 7516 7517 /** 7518 * Release key of thread specific flow workspace data. 7519 */ 7520 void 7521 flow_release_workspace(void *data) 7522 { 7523 struct mlx5_flow_workspace *wks = data; 7524 struct mlx5_flow_workspace *next; 7525 7526 while (wks) { 7527 next = wks->next; 7528 free(wks->rss_desc.queue); 7529 free(wks); 7530 wks = next; 7531 } 7532 } 7533 7534 /** 7535 * Get thread specific current flow workspace. 7536 * 7537 * @return pointer to thread specific flow workspace data, NULL on error. 7538 */ 7539 struct mlx5_flow_workspace* 7540 mlx5_flow_get_thread_workspace(void) 7541 { 7542 struct mlx5_flow_workspace *data; 7543 7544 data = mlx5_flow_os_get_specific_workspace(); 7545 MLX5_ASSERT(data && data->inuse); 7546 if (!data || !data->inuse) 7547 DRV_LOG(ERR, "flow workspace not initialized."); 7548 return data; 7549 } 7550 7551 /** 7552 * Allocate and init new flow workspace. 7553 * 7554 * @return pointer to flow workspace data, NULL on error. 7555 */ 7556 static struct mlx5_flow_workspace* 7557 flow_alloc_thread_workspace(void) 7558 { 7559 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7560 7561 if (!data) { 7562 DRV_LOG(ERR, "Failed to allocate flow workspace " 7563 "memory."); 7564 return NULL; 7565 } 7566 data->rss_desc.queue = calloc(1, 7567 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7568 if (!data->rss_desc.queue) 7569 goto err; 7570 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7571 return data; 7572 err: 7573 free(data->rss_desc.queue); 7574 free(data); 7575 return NULL; 7576 } 7577 7578 /** 7579 * Get new thread specific flow workspace. 7580 * 7581 * If current workspace inuse, create new one and set as current. 7582 * 7583 * @return pointer to thread specific flow workspace data, NULL on error. 7584 */ 7585 struct mlx5_flow_workspace* 7586 mlx5_flow_push_thread_workspace(void) 7587 { 7588 struct mlx5_flow_workspace *curr; 7589 struct mlx5_flow_workspace *data; 7590 7591 curr = mlx5_flow_os_get_specific_workspace(); 7592 if (!curr) { 7593 data = flow_alloc_thread_workspace(); 7594 if (!data) 7595 return NULL; 7596 } else if (!curr->inuse) { 7597 data = curr; 7598 } else if (curr->next) { 7599 data = curr->next; 7600 } else { 7601 data = flow_alloc_thread_workspace(); 7602 if (!data) 7603 return NULL; 7604 curr->next = data; 7605 data->prev = curr; 7606 } 7607 data->inuse = 1; 7608 data->flow_idx = 0; 7609 /* Set as current workspace */ 7610 if (mlx5_flow_os_set_specific_workspace(data)) 7611 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7612 return data; 7613 } 7614 7615 /** 7616 * Close current thread specific flow workspace. 7617 * 7618 * If previous workspace available, set it as current. 7619 * 7620 * @return pointer to thread specific flow workspace data, NULL on error. 7621 */ 7622 void 7623 mlx5_flow_pop_thread_workspace(void) 7624 { 7625 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7626 7627 if (!data) 7628 return; 7629 if (!data->inuse) { 7630 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7631 return; 7632 } 7633 data->inuse = 0; 7634 if (!data->prev) 7635 return; 7636 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7637 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7638 } 7639 7640 /** 7641 * Verify the flow list is empty 7642 * 7643 * @param dev 7644 * Pointer to Ethernet device. 7645 * 7646 * @return the number of flows not released. 7647 */ 7648 int 7649 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7650 { 7651 struct mlx5_priv *priv = dev->data->dev_private; 7652 struct rte_flow *flow; 7653 uint32_t idx = 0; 7654 int ret = 0, i; 7655 7656 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7657 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7658 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7659 dev->data->port_id, (void *)flow); 7660 ret++; 7661 } 7662 } 7663 return ret; 7664 } 7665 7666 /** 7667 * Enable default hairpin egress flow. 7668 * 7669 * @param dev 7670 * Pointer to Ethernet device. 7671 * @param sq_num 7672 * The SQ hw number. 7673 * 7674 * @return 7675 * 0 on success, a negative errno value otherwise and rte_errno is set. 7676 */ 7677 int 7678 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7679 uint32_t sq_num) 7680 { 7681 const struct rte_flow_attr attr = { 7682 .egress = 1, 7683 .priority = 0, 7684 }; 7685 struct mlx5_rte_flow_item_sq queue_spec = { 7686 .queue = sq_num, 7687 }; 7688 struct mlx5_rte_flow_item_sq queue_mask = { 7689 .queue = UINT32_MAX, 7690 }; 7691 struct rte_flow_item items[] = { 7692 { 7693 .type = (enum rte_flow_item_type) 7694 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7695 .spec = &queue_spec, 7696 .last = NULL, 7697 .mask = &queue_mask, 7698 }, 7699 { 7700 .type = RTE_FLOW_ITEM_TYPE_END, 7701 }, 7702 }; 7703 struct rte_flow_action_jump jump = { 7704 .group = MLX5_HAIRPIN_TX_TABLE, 7705 }; 7706 struct rte_flow_action actions[2]; 7707 uint32_t flow_idx; 7708 struct rte_flow_error error; 7709 7710 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7711 actions[0].conf = &jump; 7712 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7713 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7714 &attr, items, actions, false, &error); 7715 if (!flow_idx) { 7716 DRV_LOG(DEBUG, 7717 "Failed to create ctrl flow: rte_errno(%d)," 7718 " type(%d), message(%s)", 7719 rte_errno, error.type, 7720 error.message ? error.message : " (no stated reason)"); 7721 return -rte_errno; 7722 } 7723 return 0; 7724 } 7725 7726 /** 7727 * Enable a control flow configured from the control plane. 7728 * 7729 * @param dev 7730 * Pointer to Ethernet device. 7731 * @param eth_spec 7732 * An Ethernet flow spec to apply. 7733 * @param eth_mask 7734 * An Ethernet flow mask to apply. 7735 * @param vlan_spec 7736 * A VLAN flow spec to apply. 7737 * @param vlan_mask 7738 * A VLAN flow mask to apply. 7739 * 7740 * @return 7741 * 0 on success, a negative errno value otherwise and rte_errno is set. 7742 */ 7743 int 7744 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7745 struct rte_flow_item_eth *eth_spec, 7746 struct rte_flow_item_eth *eth_mask, 7747 struct rte_flow_item_vlan *vlan_spec, 7748 struct rte_flow_item_vlan *vlan_mask) 7749 { 7750 struct mlx5_priv *priv = dev->data->dev_private; 7751 const struct rte_flow_attr attr = { 7752 .ingress = 1, 7753 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7754 }; 7755 struct rte_flow_item items[] = { 7756 { 7757 .type = RTE_FLOW_ITEM_TYPE_ETH, 7758 .spec = eth_spec, 7759 .last = NULL, 7760 .mask = eth_mask, 7761 }, 7762 { 7763 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7764 RTE_FLOW_ITEM_TYPE_END, 7765 .spec = vlan_spec, 7766 .last = NULL, 7767 .mask = vlan_mask, 7768 }, 7769 { 7770 .type = RTE_FLOW_ITEM_TYPE_END, 7771 }, 7772 }; 7773 uint16_t queue[priv->reta_idx_n]; 7774 struct rte_flow_action_rss action_rss = { 7775 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7776 .level = 0, 7777 .types = priv->rss_conf.rss_hf, 7778 .key_len = priv->rss_conf.rss_key_len, 7779 .queue_num = priv->reta_idx_n, 7780 .key = priv->rss_conf.rss_key, 7781 .queue = queue, 7782 }; 7783 struct rte_flow_action actions[] = { 7784 { 7785 .type = RTE_FLOW_ACTION_TYPE_RSS, 7786 .conf = &action_rss, 7787 }, 7788 { 7789 .type = RTE_FLOW_ACTION_TYPE_END, 7790 }, 7791 }; 7792 uint32_t flow_idx; 7793 struct rte_flow_error error; 7794 unsigned int i; 7795 7796 if (!priv->reta_idx_n || !priv->rxqs_n) { 7797 return 0; 7798 } 7799 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7800 action_rss.types = 0; 7801 for (i = 0; i != priv->reta_idx_n; ++i) 7802 queue[i] = (*priv->reta_idx)[i]; 7803 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7804 &attr, items, actions, false, &error); 7805 if (!flow_idx) 7806 return -rte_errno; 7807 return 0; 7808 } 7809 7810 /** 7811 * Enable a flow control configured from the control plane. 7812 * 7813 * @param dev 7814 * Pointer to Ethernet device. 7815 * @param eth_spec 7816 * An Ethernet flow spec to apply. 7817 * @param eth_mask 7818 * An Ethernet flow mask to apply. 7819 * 7820 * @return 7821 * 0 on success, a negative errno value otherwise and rte_errno is set. 7822 */ 7823 int 7824 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7825 struct rte_flow_item_eth *eth_spec, 7826 struct rte_flow_item_eth *eth_mask) 7827 { 7828 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7829 } 7830 7831 /** 7832 * Create default miss flow rule matching lacp traffic 7833 * 7834 * @param dev 7835 * Pointer to Ethernet device. 7836 * @param eth_spec 7837 * An Ethernet flow spec to apply. 7838 * 7839 * @return 7840 * 0 on success, a negative errno value otherwise and rte_errno is set. 7841 */ 7842 int 7843 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7844 { 7845 /* 7846 * The LACP matching is done by only using ether type since using 7847 * a multicast dst mac causes kernel to give low priority to this flow. 7848 */ 7849 static const struct rte_flow_item_eth lacp_spec = { 7850 .type = RTE_BE16(0x8809), 7851 }; 7852 static const struct rte_flow_item_eth lacp_mask = { 7853 .type = 0xffff, 7854 }; 7855 const struct rte_flow_attr attr = { 7856 .ingress = 1, 7857 }; 7858 struct rte_flow_item items[] = { 7859 { 7860 .type = RTE_FLOW_ITEM_TYPE_ETH, 7861 .spec = &lacp_spec, 7862 .mask = &lacp_mask, 7863 }, 7864 { 7865 .type = RTE_FLOW_ITEM_TYPE_END, 7866 }, 7867 }; 7868 struct rte_flow_action actions[] = { 7869 { 7870 .type = (enum rte_flow_action_type) 7871 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7872 }, 7873 { 7874 .type = RTE_FLOW_ACTION_TYPE_END, 7875 }, 7876 }; 7877 struct rte_flow_error error; 7878 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7879 &attr, items, actions, 7880 false, &error); 7881 7882 if (!flow_idx) 7883 return -rte_errno; 7884 return 0; 7885 } 7886 7887 /** 7888 * Destroy a flow. 7889 * 7890 * @see rte_flow_destroy() 7891 * @see rte_flow_ops 7892 */ 7893 int 7894 mlx5_flow_destroy(struct rte_eth_dev *dev, 7895 struct rte_flow *flow, 7896 struct rte_flow_error *error __rte_unused) 7897 { 7898 struct mlx5_priv *priv = dev->data->dev_private; 7899 7900 if (priv->sh->config.dv_flow_en == 2) 7901 return rte_flow_error_set(error, ENOTSUP, 7902 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7903 NULL, 7904 "Flow non-Q destruction not supported"); 7905 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7906 (uintptr_t)(void *)flow); 7907 return 0; 7908 } 7909 7910 /** 7911 * Destroy all flows. 7912 * 7913 * @see rte_flow_flush() 7914 * @see rte_flow_ops 7915 */ 7916 int 7917 mlx5_flow_flush(struct rte_eth_dev *dev, 7918 struct rte_flow_error *error __rte_unused) 7919 { 7920 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7921 return 0; 7922 } 7923 7924 /** 7925 * Isolated mode. 7926 * 7927 * @see rte_flow_isolate() 7928 * @see rte_flow_ops 7929 */ 7930 int 7931 mlx5_flow_isolate(struct rte_eth_dev *dev, 7932 int enable, 7933 struct rte_flow_error *error) 7934 { 7935 struct mlx5_priv *priv = dev->data->dev_private; 7936 7937 if (dev->data->dev_started) { 7938 rte_flow_error_set(error, EBUSY, 7939 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7940 NULL, 7941 "port must be stopped first"); 7942 return -rte_errno; 7943 } 7944 priv->isolated = !!enable; 7945 if (enable) 7946 dev->dev_ops = &mlx5_dev_ops_isolate; 7947 else 7948 dev->dev_ops = &mlx5_dev_ops; 7949 7950 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7951 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7952 7953 return 0; 7954 } 7955 7956 /** 7957 * Query a flow. 7958 * 7959 * @see rte_flow_query() 7960 * @see rte_flow_ops 7961 */ 7962 static int 7963 flow_drv_query(struct rte_eth_dev *dev, 7964 struct rte_flow *eflow, 7965 const struct rte_flow_action *actions, 7966 void *data, 7967 struct rte_flow_error *error) 7968 { 7969 struct mlx5_priv *priv = dev->data->dev_private; 7970 const struct mlx5_flow_driver_ops *fops; 7971 struct rte_flow *flow = NULL; 7972 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 7973 7974 if (priv->sh->config.dv_flow_en == 2) { 7975 #ifdef HAVE_MLX5_HWS_SUPPORT 7976 flow = eflow; 7977 ftype = MLX5_FLOW_TYPE_HW; 7978 #endif 7979 } else { 7980 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7981 (uintptr_t)(void *)eflow); 7982 } 7983 if (!flow) { 7984 return rte_flow_error_set(error, ENOENT, 7985 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7986 NULL, 7987 "invalid flow handle"); 7988 } 7989 if (ftype == MLX5_FLOW_TYPE_MIN) 7990 ftype = flow->drv_type; 7991 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 7992 fops = flow_get_drv_ops(ftype); 7993 7994 return fops->query(dev, flow, actions, data, error); 7995 } 7996 7997 /** 7998 * Query a flow. 7999 * 8000 * @see rte_flow_query() 8001 * @see rte_flow_ops 8002 */ 8003 int 8004 mlx5_flow_query(struct rte_eth_dev *dev, 8005 struct rte_flow *flow, 8006 const struct rte_flow_action *actions, 8007 void *data, 8008 struct rte_flow_error *error) 8009 { 8010 int ret; 8011 8012 ret = flow_drv_query(dev, flow, actions, data, 8013 error); 8014 if (ret < 0) 8015 return ret; 8016 return 0; 8017 } 8018 8019 /** 8020 * Get rte_flow callbacks. 8021 * 8022 * @param dev 8023 * Pointer to Ethernet device structure. 8024 * @param ops 8025 * Pointer to operation-specific structure. 8026 * 8027 * @return 0 8028 */ 8029 int 8030 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8031 const struct rte_flow_ops **ops) 8032 { 8033 *ops = &mlx5_flow_ops; 8034 return 0; 8035 } 8036 8037 /** 8038 * Validate meter policy actions. 8039 * Dispatcher for action type specific validation. 8040 * 8041 * @param[in] dev 8042 * Pointer to the Ethernet device structure. 8043 * @param[in] action 8044 * The meter policy action object to validate. 8045 * @param[in] attr 8046 * Attributes of flow to determine steering domain. 8047 * @param[out] is_rss 8048 * Is RSS or not. 8049 * @param[out] domain_bitmap 8050 * Domain bitmap. 8051 * @param[out] is_def_policy 8052 * Is default policy or not. 8053 * @param[out] error 8054 * Perform verbose error reporting if not NULL. Initialized in case of 8055 * error only. 8056 * 8057 * @return 8058 * 0 on success, otherwise negative errno value. 8059 */ 8060 int 8061 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8062 const struct rte_flow_action *actions[RTE_COLORS], 8063 struct rte_flow_attr *attr, 8064 bool *is_rss, 8065 uint8_t *domain_bitmap, 8066 uint8_t *policy_mode, 8067 struct rte_mtr_error *error) 8068 { 8069 const struct mlx5_flow_driver_ops *fops; 8070 8071 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8072 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8073 domain_bitmap, policy_mode, error); 8074 } 8075 8076 /** 8077 * Destroy the meter table set. 8078 * 8079 * @param[in] dev 8080 * Pointer to Ethernet device. 8081 * @param[in] mtr_policy 8082 * Meter policy struct. 8083 */ 8084 void 8085 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8086 struct mlx5_flow_meter_policy *mtr_policy) 8087 { 8088 const struct mlx5_flow_driver_ops *fops; 8089 8090 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8091 fops->destroy_mtr_acts(dev, mtr_policy); 8092 } 8093 8094 /** 8095 * Create policy action, lock free, 8096 * (mutex should be acquired by caller). 8097 * Dispatcher for action type specific call. 8098 * 8099 * @param[in] dev 8100 * Pointer to the Ethernet device structure. 8101 * @param[in] mtr_policy 8102 * Meter policy struct. 8103 * @param[in] action 8104 * Action specification used to create meter actions. 8105 * @param[in] attr 8106 * Flow rule attributes. 8107 * @param[out] error 8108 * Perform verbose error reporting if not NULL. Initialized in case of 8109 * error only. 8110 * 8111 * @return 8112 * 0 on success, otherwise negative errno value. 8113 */ 8114 int 8115 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8116 struct mlx5_flow_meter_policy *mtr_policy, 8117 const struct rte_flow_action *actions[RTE_COLORS], 8118 struct rte_flow_attr *attr, 8119 struct rte_mtr_error *error) 8120 { 8121 const struct mlx5_flow_driver_ops *fops; 8122 8123 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8124 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8125 } 8126 8127 /** 8128 * Create policy rules, lock free, 8129 * (mutex should be acquired by caller). 8130 * Dispatcher for action type specific call. 8131 * 8132 * @param[in] dev 8133 * Pointer to the Ethernet device structure. 8134 * @param[in] mtr_policy 8135 * Meter policy struct. 8136 * 8137 * @return 8138 * 0 on success, -1 otherwise. 8139 */ 8140 int 8141 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8142 struct mlx5_flow_meter_policy *mtr_policy) 8143 { 8144 const struct mlx5_flow_driver_ops *fops; 8145 8146 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8147 return fops->create_policy_rules(dev, mtr_policy); 8148 } 8149 8150 /** 8151 * Destroy policy rules, lock free, 8152 * (mutex should be acquired by caller). 8153 * Dispatcher for action type specific call. 8154 * 8155 * @param[in] dev 8156 * Pointer to the Ethernet device structure. 8157 * @param[in] mtr_policy 8158 * Meter policy struct. 8159 */ 8160 void 8161 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8162 struct mlx5_flow_meter_policy *mtr_policy) 8163 { 8164 const struct mlx5_flow_driver_ops *fops; 8165 8166 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8167 fops->destroy_policy_rules(dev, mtr_policy); 8168 } 8169 8170 /** 8171 * Destroy the default policy table set. 8172 * 8173 * @param[in] dev 8174 * Pointer to Ethernet device. 8175 */ 8176 void 8177 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8178 { 8179 const struct mlx5_flow_driver_ops *fops; 8180 8181 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8182 fops->destroy_def_policy(dev); 8183 } 8184 8185 /** 8186 * Destroy the default policy table set. 8187 * 8188 * @param[in] dev 8189 * Pointer to Ethernet device. 8190 * 8191 * @return 8192 * 0 on success, -1 otherwise. 8193 */ 8194 int 8195 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8196 { 8197 const struct mlx5_flow_driver_ops *fops; 8198 8199 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8200 return fops->create_def_policy(dev); 8201 } 8202 8203 /** 8204 * Create the needed meter and suffix tables. 8205 * 8206 * @param[in] dev 8207 * Pointer to Ethernet device. 8208 * 8209 * @return 8210 * 0 on success, -1 otherwise. 8211 */ 8212 int 8213 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8214 struct mlx5_flow_meter_info *fm, 8215 uint32_t mtr_idx, 8216 uint8_t domain_bitmap) 8217 { 8218 const struct mlx5_flow_driver_ops *fops; 8219 8220 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8221 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8222 } 8223 8224 /** 8225 * Destroy the meter table set. 8226 * 8227 * @param[in] dev 8228 * Pointer to Ethernet device. 8229 * @param[in] tbl 8230 * Pointer to the meter table set. 8231 */ 8232 void 8233 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8234 struct mlx5_flow_meter_info *fm) 8235 { 8236 const struct mlx5_flow_driver_ops *fops; 8237 8238 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8239 fops->destroy_mtr_tbls(dev, fm); 8240 } 8241 8242 /** 8243 * Destroy the global meter drop table. 8244 * 8245 * @param[in] dev 8246 * Pointer to Ethernet device. 8247 */ 8248 void 8249 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8250 { 8251 const struct mlx5_flow_driver_ops *fops; 8252 8253 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8254 fops->destroy_mtr_drop_tbls(dev); 8255 } 8256 8257 /** 8258 * Destroy the sub policy table with RX queue. 8259 * 8260 * @param[in] dev 8261 * Pointer to Ethernet device. 8262 * @param[in] mtr_policy 8263 * Pointer to meter policy table. 8264 */ 8265 void 8266 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8267 struct mlx5_flow_meter_policy *mtr_policy) 8268 { 8269 const struct mlx5_flow_driver_ops *fops; 8270 8271 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8272 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8273 } 8274 8275 /** 8276 * Allocate the needed aso flow meter id. 8277 * 8278 * @param[in] dev 8279 * Pointer to Ethernet device. 8280 * 8281 * @return 8282 * Index to aso flow meter on success, NULL otherwise. 8283 */ 8284 uint32_t 8285 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8286 { 8287 const struct mlx5_flow_driver_ops *fops; 8288 8289 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8290 return fops->create_meter(dev); 8291 } 8292 8293 /** 8294 * Free the aso flow meter id. 8295 * 8296 * @param[in] dev 8297 * Pointer to Ethernet device. 8298 * @param[in] mtr_idx 8299 * Index to aso flow meter to be free. 8300 * 8301 * @return 8302 * 0 on success. 8303 */ 8304 void 8305 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8306 { 8307 const struct mlx5_flow_driver_ops *fops; 8308 8309 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8310 fops->free_meter(dev, mtr_idx); 8311 } 8312 8313 /** 8314 * Allocate a counter. 8315 * 8316 * @param[in] dev 8317 * Pointer to Ethernet device structure. 8318 * 8319 * @return 8320 * Index to allocated counter on success, 0 otherwise. 8321 */ 8322 uint32_t 8323 mlx5_counter_alloc(struct rte_eth_dev *dev) 8324 { 8325 const struct mlx5_flow_driver_ops *fops; 8326 struct rte_flow_attr attr = { .transfer = 0 }; 8327 8328 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8329 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8330 return fops->counter_alloc(dev); 8331 } 8332 DRV_LOG(ERR, 8333 "port %u counter allocate is not supported.", 8334 dev->data->port_id); 8335 return 0; 8336 } 8337 8338 /** 8339 * Free a counter. 8340 * 8341 * @param[in] dev 8342 * Pointer to Ethernet device structure. 8343 * @param[in] cnt 8344 * Index to counter to be free. 8345 */ 8346 void 8347 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8348 { 8349 const struct mlx5_flow_driver_ops *fops; 8350 struct rte_flow_attr attr = { .transfer = 0 }; 8351 8352 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8353 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8354 fops->counter_free(dev, cnt); 8355 return; 8356 } 8357 DRV_LOG(ERR, 8358 "port %u counter free is not supported.", 8359 dev->data->port_id); 8360 } 8361 8362 /** 8363 * Query counter statistics. 8364 * 8365 * @param[in] dev 8366 * Pointer to Ethernet device structure. 8367 * @param[in] cnt 8368 * Index to counter to query. 8369 * @param[in] clear 8370 * Set to clear counter statistics. 8371 * @param[out] pkts 8372 * The counter hits packets number to save. 8373 * @param[out] bytes 8374 * The counter hits bytes number to save. 8375 * 8376 * @return 8377 * 0 on success, a negative errno value otherwise. 8378 */ 8379 int 8380 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8381 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8382 { 8383 const struct mlx5_flow_driver_ops *fops; 8384 struct rte_flow_attr attr = { .transfer = 0 }; 8385 8386 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8387 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8388 return fops->counter_query(dev, cnt, clear, pkts, 8389 bytes, action); 8390 } 8391 DRV_LOG(ERR, 8392 "port %u counter query is not supported.", 8393 dev->data->port_id); 8394 return -ENOTSUP; 8395 } 8396 8397 /** 8398 * Get information about HWS pre-configurable resources. 8399 * 8400 * @param[in] dev 8401 * Pointer to the rte_eth_dev structure. 8402 * @param[out] port_info 8403 * Pointer to port information. 8404 * @param[out] queue_info 8405 * Pointer to queue information. 8406 * @param[out] error 8407 * Pointer to error structure. 8408 * 8409 * @return 8410 * 0 on success, a negative errno value otherwise and rte_errno is set. 8411 */ 8412 static int 8413 mlx5_flow_info_get(struct rte_eth_dev *dev, 8414 struct rte_flow_port_info *port_info, 8415 struct rte_flow_queue_info *queue_info, 8416 struct rte_flow_error *error) 8417 { 8418 const struct mlx5_flow_driver_ops *fops; 8419 struct rte_flow_attr attr = {0}; 8420 8421 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8422 return rte_flow_error_set(error, ENOTSUP, 8423 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8424 NULL, 8425 "info get with incorrect steering mode"); 8426 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8427 return fops->info_get(dev, port_info, queue_info, error); 8428 } 8429 8430 /** 8431 * Configure port HWS resources. 8432 * 8433 * @param[in] dev 8434 * Pointer to the rte_eth_dev structure. 8435 * @param[in] port_attr 8436 * Port configuration attributes. 8437 * @param[in] nb_queue 8438 * Number of queue. 8439 * @param[in] queue_attr 8440 * Array that holds attributes for each flow queue. 8441 * @param[out] error 8442 * Pointer to error structure. 8443 * 8444 * @return 8445 * 0 on success, a negative errno value otherwise and rte_errno is set. 8446 */ 8447 static int 8448 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8449 const struct rte_flow_port_attr *port_attr, 8450 uint16_t nb_queue, 8451 const struct rte_flow_queue_attr *queue_attr[], 8452 struct rte_flow_error *error) 8453 { 8454 const struct mlx5_flow_driver_ops *fops; 8455 struct rte_flow_attr attr = {0}; 8456 8457 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8458 return rte_flow_error_set(error, ENOTSUP, 8459 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8460 NULL, 8461 "port configure with incorrect steering mode"); 8462 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8463 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8464 } 8465 8466 /** 8467 * Validate item template. 8468 * 8469 * @param[in] dev 8470 * Pointer to the rte_eth_dev structure. 8471 * @param[in] attr 8472 * Pointer to the item template attributes. 8473 * @param[in] items 8474 * The template item pattern. 8475 * @param[out] error 8476 * Pointer to error structure. 8477 * 8478 * @return 8479 * 0 on success, a negative errno value otherwise and rte_errno is set. 8480 */ 8481 int 8482 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8483 const struct rte_flow_pattern_template_attr *attr, 8484 const struct rte_flow_item items[], 8485 struct rte_flow_error *error) 8486 { 8487 const struct mlx5_flow_driver_ops *fops; 8488 struct rte_flow_attr fattr = {0}; 8489 8490 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8491 rte_flow_error_set(error, ENOTSUP, 8492 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8493 "pattern validate with incorrect steering mode"); 8494 return -ENOTSUP; 8495 } 8496 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8497 return fops->pattern_validate(dev, attr, items, error); 8498 } 8499 8500 /** 8501 * Create flow item template. 8502 * 8503 * @param[in] dev 8504 * Pointer to the rte_eth_dev structure. 8505 * @param[in] attr 8506 * Pointer to the item template attributes. 8507 * @param[in] items 8508 * The template item pattern. 8509 * @param[out] error 8510 * Pointer to error structure. 8511 * 8512 * @return 8513 * 0 on success, a negative errno value otherwise and rte_errno is set. 8514 */ 8515 static struct rte_flow_pattern_template * 8516 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8517 const struct rte_flow_pattern_template_attr *attr, 8518 const struct rte_flow_item items[], 8519 struct rte_flow_error *error) 8520 { 8521 const struct mlx5_flow_driver_ops *fops; 8522 struct rte_flow_attr fattr = {0}; 8523 8524 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8525 rte_flow_error_set(error, ENOTSUP, 8526 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8527 NULL, 8528 "pattern create with incorrect steering mode"); 8529 return NULL; 8530 } 8531 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8532 return fops->pattern_template_create(dev, attr, items, error); 8533 } 8534 8535 /** 8536 * Destroy flow item template. 8537 * 8538 * @param[in] dev 8539 * Pointer to the rte_eth_dev structure. 8540 * @param[in] template 8541 * Pointer to the item template to be destroyed. 8542 * @param[out] error 8543 * Pointer to error structure. 8544 * 8545 * @return 8546 * 0 on success, a negative errno value otherwise and rte_errno is set. 8547 */ 8548 static int 8549 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8550 struct rte_flow_pattern_template *template, 8551 struct rte_flow_error *error) 8552 { 8553 const struct mlx5_flow_driver_ops *fops; 8554 struct rte_flow_attr attr = {0}; 8555 8556 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8557 return rte_flow_error_set(error, ENOTSUP, 8558 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8559 NULL, 8560 "pattern destroy with incorrect steering mode"); 8561 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8562 return fops->pattern_template_destroy(dev, template, error); 8563 } 8564 8565 /** 8566 * Validate flow actions template. 8567 * 8568 * @param[in] dev 8569 * Pointer to the rte_eth_dev structure. 8570 * @param[in] attr 8571 * Pointer to the action template attributes. 8572 * @param[in] actions 8573 * Associated actions (list terminated by the END action). 8574 * @param[in] masks 8575 * List of actions that marks which of the action's member is constant. 8576 * @param[out] error 8577 * Pointer to error structure. 8578 * 8579 * @return 8580 * 0 on success, a negative errno value otherwise and rte_errno is set. 8581 */ 8582 int 8583 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8584 const struct rte_flow_actions_template_attr *attr, 8585 const struct rte_flow_action actions[], 8586 const struct rte_flow_action masks[], 8587 struct rte_flow_error *error) 8588 { 8589 const struct mlx5_flow_driver_ops *fops; 8590 struct rte_flow_attr fattr = {0}; 8591 8592 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8593 rte_flow_error_set(error, ENOTSUP, 8594 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8595 "actions validate with incorrect steering mode"); 8596 return -ENOTSUP; 8597 } 8598 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8599 return fops->actions_validate(dev, attr, actions, masks, error); 8600 } 8601 8602 /** 8603 * Create flow item template. 8604 * 8605 * @param[in] dev 8606 * Pointer to the rte_eth_dev structure. 8607 * @param[in] attr 8608 * Pointer to the action template attributes. 8609 * @param[in] actions 8610 * Associated actions (list terminated by the END action). 8611 * @param[in] masks 8612 * List of actions that marks which of the action's member is constant. 8613 * @param[out] error 8614 * Pointer to error structure. 8615 * 8616 * @return 8617 * 0 on success, a negative errno value otherwise and rte_errno is set. 8618 */ 8619 static struct rte_flow_actions_template * 8620 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8621 const struct rte_flow_actions_template_attr *attr, 8622 const struct rte_flow_action actions[], 8623 const struct rte_flow_action masks[], 8624 struct rte_flow_error *error) 8625 { 8626 const struct mlx5_flow_driver_ops *fops; 8627 struct rte_flow_attr fattr = {0}; 8628 8629 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8630 rte_flow_error_set(error, ENOTSUP, 8631 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8632 NULL, 8633 "action create with incorrect steering mode"); 8634 return NULL; 8635 } 8636 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8637 return fops->actions_template_create(dev, attr, actions, masks, error); 8638 } 8639 8640 /** 8641 * Destroy flow action template. 8642 * 8643 * @param[in] dev 8644 * Pointer to the rte_eth_dev structure. 8645 * @param[in] template 8646 * Pointer to the action template to be destroyed. 8647 * @param[out] error 8648 * Pointer to error structure. 8649 * 8650 * @return 8651 * 0 on success, a negative errno value otherwise and rte_errno is set. 8652 */ 8653 static int 8654 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8655 struct rte_flow_actions_template *template, 8656 struct rte_flow_error *error) 8657 { 8658 const struct mlx5_flow_driver_ops *fops; 8659 struct rte_flow_attr attr = {0}; 8660 8661 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8662 return rte_flow_error_set(error, ENOTSUP, 8663 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8664 NULL, 8665 "action destroy with incorrect steering mode"); 8666 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8667 return fops->actions_template_destroy(dev, template, error); 8668 } 8669 8670 /** 8671 * Create flow table. 8672 * 8673 * @param[in] dev 8674 * Pointer to the rte_eth_dev structure. 8675 * @param[in] attr 8676 * Pointer to the table attributes. 8677 * @param[in] item_templates 8678 * Item template array to be binded to the table. 8679 * @param[in] nb_item_templates 8680 * Number of item template. 8681 * @param[in] action_templates 8682 * Action template array to be binded to the table. 8683 * @param[in] nb_action_templates 8684 * Number of action template. 8685 * @param[out] error 8686 * Pointer to error structure. 8687 * 8688 * @return 8689 * Table on success, NULL otherwise and rte_errno is set. 8690 */ 8691 static struct rte_flow_template_table * 8692 mlx5_flow_table_create(struct rte_eth_dev *dev, 8693 const struct rte_flow_template_table_attr *attr, 8694 struct rte_flow_pattern_template *item_templates[], 8695 uint8_t nb_item_templates, 8696 struct rte_flow_actions_template *action_templates[], 8697 uint8_t nb_action_templates, 8698 struct rte_flow_error *error) 8699 { 8700 const struct mlx5_flow_driver_ops *fops; 8701 struct rte_flow_attr fattr = {0}; 8702 8703 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8704 rte_flow_error_set(error, ENOTSUP, 8705 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8706 NULL, 8707 "table create with incorrect steering mode"); 8708 return NULL; 8709 } 8710 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8711 return fops->template_table_create(dev, 8712 attr, 8713 item_templates, 8714 nb_item_templates, 8715 action_templates, 8716 nb_action_templates, 8717 error); 8718 } 8719 8720 /** 8721 * PMD destroy flow table. 8722 * 8723 * @param[in] dev 8724 * Pointer to the rte_eth_dev structure. 8725 * @param[in] table 8726 * Pointer to the table to be destroyed. 8727 * @param[out] error 8728 * Pointer to error structure. 8729 * 8730 * @return 8731 * 0 on success, a negative errno value otherwise and rte_errno is set. 8732 */ 8733 static int 8734 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8735 struct rte_flow_template_table *table, 8736 struct rte_flow_error *error) 8737 { 8738 const struct mlx5_flow_driver_ops *fops; 8739 struct rte_flow_attr attr = {0}; 8740 8741 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8742 return rte_flow_error_set(error, ENOTSUP, 8743 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8744 NULL, 8745 "table destroy with incorrect steering mode"); 8746 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8747 return fops->template_table_destroy(dev, table, error); 8748 } 8749 8750 /** 8751 * Enqueue flow creation. 8752 * 8753 * @param[in] dev 8754 * Pointer to the rte_eth_dev structure. 8755 * @param[in] queue_id 8756 * The queue to create the flow. 8757 * @param[in] attr 8758 * Pointer to the flow operation attributes. 8759 * @param[in] items 8760 * Items with flow spec value. 8761 * @param[in] pattern_template_index 8762 * The item pattern flow follows from the table. 8763 * @param[in] actions 8764 * Action with flow spec value. 8765 * @param[in] action_template_index 8766 * The action pattern flow follows from the table. 8767 * @param[in] user_data 8768 * Pointer to the user_data. 8769 * @param[out] error 8770 * Pointer to error structure. 8771 * 8772 * @return 8773 * Flow pointer on success, NULL otherwise and rte_errno is set. 8774 */ 8775 static struct rte_flow * 8776 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8777 uint32_t queue_id, 8778 const struct rte_flow_op_attr *attr, 8779 struct rte_flow_template_table *table, 8780 const struct rte_flow_item items[], 8781 uint8_t pattern_template_index, 8782 const struct rte_flow_action actions[], 8783 uint8_t action_template_index, 8784 void *user_data, 8785 struct rte_flow_error *error) 8786 { 8787 const struct mlx5_flow_driver_ops *fops; 8788 struct rte_flow_attr fattr = {0}; 8789 8790 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8791 rte_flow_error_set(error, ENOTSUP, 8792 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8793 NULL, 8794 "flow_q create with incorrect steering mode"); 8795 return NULL; 8796 } 8797 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8798 return fops->async_flow_create(dev, queue_id, attr, table, 8799 items, pattern_template_index, 8800 actions, action_template_index, 8801 user_data, error); 8802 } 8803 8804 /** 8805 * Enqueue flow destruction. 8806 * 8807 * @param[in] dev 8808 * Pointer to the rte_eth_dev structure. 8809 * @param[in] queue 8810 * The queue to destroy the flow. 8811 * @param[in] attr 8812 * Pointer to the flow operation attributes. 8813 * @param[in] flow 8814 * Pointer to the flow to be destroyed. 8815 * @param[in] user_data 8816 * Pointer to the user_data. 8817 * @param[out] error 8818 * Pointer to error structure. 8819 * 8820 * @return 8821 * 0 on success, negative value otherwise and rte_errno is set. 8822 */ 8823 static int 8824 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8825 uint32_t queue, 8826 const struct rte_flow_op_attr *attr, 8827 struct rte_flow *flow, 8828 void *user_data, 8829 struct rte_flow_error *error) 8830 { 8831 const struct mlx5_flow_driver_ops *fops; 8832 struct rte_flow_attr fattr = {0}; 8833 8834 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8835 return rte_flow_error_set(error, ENOTSUP, 8836 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8837 NULL, 8838 "flow_q destroy with incorrect steering mode"); 8839 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8840 return fops->async_flow_destroy(dev, queue, attr, flow, 8841 user_data, error); 8842 } 8843 8844 /** 8845 * Pull the enqueued flows. 8846 * 8847 * @param[in] dev 8848 * Pointer to the rte_eth_dev structure. 8849 * @param[in] queue 8850 * The queue to pull the result. 8851 * @param[in/out] res 8852 * Array to save the results. 8853 * @param[in] n_res 8854 * Available result with the array. 8855 * @param[out] error 8856 * Pointer to error structure. 8857 * 8858 * @return 8859 * Result number on success, negative value otherwise and rte_errno is set. 8860 */ 8861 static int 8862 mlx5_flow_pull(struct rte_eth_dev *dev, 8863 uint32_t queue, 8864 struct rte_flow_op_result res[], 8865 uint16_t n_res, 8866 struct rte_flow_error *error) 8867 { 8868 const struct mlx5_flow_driver_ops *fops; 8869 struct rte_flow_attr attr = {0}; 8870 8871 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8872 return rte_flow_error_set(error, ENOTSUP, 8873 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8874 NULL, 8875 "flow_q pull with incorrect steering mode"); 8876 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8877 return fops->pull(dev, queue, res, n_res, error); 8878 } 8879 8880 /** 8881 * Push the enqueued flows. 8882 * 8883 * @param[in] dev 8884 * Pointer to the rte_eth_dev structure. 8885 * @param[in] queue 8886 * The queue to push the flows. 8887 * @param[out] error 8888 * Pointer to error structure. 8889 * 8890 * @return 8891 * 0 on success, negative value otherwise and rte_errno is set. 8892 */ 8893 static int 8894 mlx5_flow_push(struct rte_eth_dev *dev, 8895 uint32_t queue, 8896 struct rte_flow_error *error) 8897 { 8898 const struct mlx5_flow_driver_ops *fops; 8899 struct rte_flow_attr attr = {0}; 8900 8901 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8902 return rte_flow_error_set(error, ENOTSUP, 8903 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8904 NULL, 8905 "flow_q push with incorrect steering mode"); 8906 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8907 return fops->push(dev, queue, error); 8908 } 8909 8910 /** 8911 * Create shared action. 8912 * 8913 * @param[in] dev 8914 * Pointer to the rte_eth_dev structure. 8915 * @param[in] queue 8916 * Which queue to be used.. 8917 * @param[in] attr 8918 * Operation attribute. 8919 * @param[in] conf 8920 * Indirect action configuration. 8921 * @param[in] action 8922 * rte_flow action detail. 8923 * @param[in] user_data 8924 * Pointer to the user_data. 8925 * @param[out] error 8926 * Pointer to error structure. 8927 * 8928 * @return 8929 * Action handle on success, NULL otherwise and rte_errno is set. 8930 */ 8931 static struct rte_flow_action_handle * 8932 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8933 const struct rte_flow_op_attr *attr, 8934 const struct rte_flow_indir_action_conf *conf, 8935 const struct rte_flow_action *action, 8936 void *user_data, 8937 struct rte_flow_error *error) 8938 { 8939 const struct mlx5_flow_driver_ops *fops = 8940 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8941 8942 return fops->async_action_create(dev, queue, attr, conf, action, 8943 user_data, error); 8944 } 8945 8946 /** 8947 * Update shared action. 8948 * 8949 * @param[in] dev 8950 * Pointer to the rte_eth_dev structure. 8951 * @param[in] queue 8952 * Which queue to be used.. 8953 * @param[in] attr 8954 * Operation attribute. 8955 * @param[in] handle 8956 * Action handle to be updated. 8957 * @param[in] update 8958 * Update value. 8959 * @param[in] user_data 8960 * Pointer to the user_data. 8961 * @param[out] error 8962 * Pointer to error structure. 8963 * 8964 * @return 8965 * 0 on success, negative value otherwise and rte_errno is set. 8966 */ 8967 static int 8968 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8969 const struct rte_flow_op_attr *attr, 8970 struct rte_flow_action_handle *handle, 8971 const void *update, 8972 void *user_data, 8973 struct rte_flow_error *error) 8974 { 8975 const struct mlx5_flow_driver_ops *fops = 8976 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8977 8978 return fops->async_action_update(dev, queue, attr, handle, 8979 update, user_data, error); 8980 } 8981 8982 /** 8983 * Query shared action. 8984 * 8985 * @param[in] dev 8986 * Pointer to the rte_eth_dev structure. 8987 * @param[in] queue 8988 * Which queue to be used.. 8989 * @param[in] attr 8990 * Operation attribute. 8991 * @param[in] handle 8992 * Action handle to be updated. 8993 * @param[in] data 8994 * Pointer query result data. 8995 * @param[in] user_data 8996 * Pointer to the user_data. 8997 * @param[out] error 8998 * Pointer to error structure. 8999 * 9000 * @return 9001 * 0 on success, negative value otherwise and rte_errno is set. 9002 */ 9003 static int 9004 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9005 const struct rte_flow_op_attr *attr, 9006 const struct rte_flow_action_handle *handle, 9007 void *data, 9008 void *user_data, 9009 struct rte_flow_error *error) 9010 { 9011 const struct mlx5_flow_driver_ops *fops = 9012 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9013 9014 return fops->async_action_query(dev, queue, attr, handle, 9015 data, user_data, error); 9016 } 9017 9018 /** 9019 * Destroy shared action. 9020 * 9021 * @param[in] dev 9022 * Pointer to the rte_eth_dev structure. 9023 * @param[in] queue 9024 * Which queue to be used.. 9025 * @param[in] attr 9026 * Operation attribute. 9027 * @param[in] handle 9028 * Action handle to be destroyed. 9029 * @param[in] user_data 9030 * Pointer to the user_data. 9031 * @param[out] error 9032 * Pointer to error structure. 9033 * 9034 * @return 9035 * 0 on success, negative value otherwise and rte_errno is set. 9036 */ 9037 static int 9038 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9039 const struct rte_flow_op_attr *attr, 9040 struct rte_flow_action_handle *handle, 9041 void *user_data, 9042 struct rte_flow_error *error) 9043 { 9044 const struct mlx5_flow_driver_ops *fops = 9045 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9046 9047 return fops->async_action_destroy(dev, queue, attr, handle, 9048 user_data, error); 9049 } 9050 9051 /** 9052 * Allocate a new memory for the counter values wrapped by all the needed 9053 * management. 9054 * 9055 * @param[in] sh 9056 * Pointer to mlx5_dev_ctx_shared object. 9057 * 9058 * @return 9059 * 0 on success, a negative errno value otherwise. 9060 */ 9061 static int 9062 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9063 { 9064 struct mlx5_counter_stats_mem_mng *mem_mng; 9065 volatile struct flow_counter_stats *raw_data; 9066 int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES; 9067 int size = (sizeof(struct flow_counter_stats) * 9068 MLX5_COUNTERS_PER_POOL + 9069 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9070 sizeof(struct mlx5_counter_stats_mem_mng); 9071 size_t pgsize = rte_mem_page_size(); 9072 uint8_t *mem; 9073 int ret; 9074 int i; 9075 9076 if (pgsize == (size_t)-1) { 9077 DRV_LOG(ERR, "Failed to get mem page size"); 9078 rte_errno = ENOMEM; 9079 return -ENOMEM; 9080 } 9081 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9082 if (!mem) { 9083 rte_errno = ENOMEM; 9084 return -ENOMEM; 9085 } 9086 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9087 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9088 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9089 sh->cdev->pdn, mem, size, 9090 &mem_mng->wm); 9091 if (ret) { 9092 rte_errno = errno; 9093 mlx5_free(mem); 9094 return -rte_errno; 9095 } 9096 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9097 raw_data = (volatile struct flow_counter_stats *)mem; 9098 for (i = 0; i < raws_n; ++i) { 9099 mem_mng->raws[i].mem_mng = mem_mng; 9100 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9101 } 9102 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9103 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9104 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i, 9105 next); 9106 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9107 sh->sws_cmng.mem_mng = mem_mng; 9108 return 0; 9109 } 9110 9111 /** 9112 * Set the statistic memory to the new counter pool. 9113 * 9114 * @param[in] sh 9115 * Pointer to mlx5_dev_ctx_shared object. 9116 * @param[in] pool 9117 * Pointer to the pool to set the statistic memory. 9118 * 9119 * @return 9120 * 0 on success, a negative errno value otherwise. 9121 */ 9122 static int 9123 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9124 struct mlx5_flow_counter_pool *pool) 9125 { 9126 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9127 /* Resize statistic memory once used out. */ 9128 if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) && 9129 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9130 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9131 return -1; 9132 } 9133 rte_spinlock_lock(&pool->sl); 9134 pool->raw = cmng->mem_mng->raws + pool->index % 9135 MLX5_CNT_CONTAINER_RESIZE; 9136 rte_spinlock_unlock(&pool->sl); 9137 pool->raw_hw = NULL; 9138 return 0; 9139 } 9140 9141 #define MLX5_POOL_QUERY_FREQ_US 1000000 9142 9143 /** 9144 * Set the periodic procedure for triggering asynchronous batch queries for all 9145 * the counter pools. 9146 * 9147 * @param[in] sh 9148 * Pointer to mlx5_dev_ctx_shared object. 9149 */ 9150 void 9151 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9152 { 9153 uint32_t pools_n, us; 9154 9155 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9156 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9157 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9158 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9159 sh->sws_cmng.query_thread_on = 0; 9160 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9161 } else { 9162 sh->sws_cmng.query_thread_on = 1; 9163 } 9164 } 9165 9166 /** 9167 * The periodic procedure for triggering asynchronous batch queries for all the 9168 * counter pools. This function is probably called by the host thread. 9169 * 9170 * @param[in] arg 9171 * The parameter for the alarm process. 9172 */ 9173 void 9174 mlx5_flow_query_alarm(void *arg) 9175 { 9176 struct mlx5_dev_ctx_shared *sh = arg; 9177 int ret; 9178 uint16_t pool_index = sh->sws_cmng.pool_index; 9179 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9180 struct mlx5_flow_counter_pool *pool; 9181 uint16_t n_valid; 9182 9183 if (sh->sws_cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES) 9184 goto set_alarm; 9185 rte_spinlock_lock(&cmng->pool_update_sl); 9186 pool = cmng->pools[pool_index]; 9187 n_valid = cmng->n_valid; 9188 rte_spinlock_unlock(&cmng->pool_update_sl); 9189 /* Set the statistic memory to the new created pool. */ 9190 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9191 goto set_alarm; 9192 if (pool->raw_hw) 9193 /* There is a pool query in progress. */ 9194 goto set_alarm; 9195 pool->raw_hw = 9196 LIST_FIRST(&sh->sws_cmng.free_stat_raws); 9197 if (!pool->raw_hw) 9198 /* No free counter statistics raw memory. */ 9199 goto set_alarm; 9200 /* 9201 * Identify the counters released between query trigger and query 9202 * handle more efficiently. The counter released in this gap period 9203 * should wait for a new round of query as the new arrived packets 9204 * will not be taken into account. 9205 */ 9206 pool->query_gen++; 9207 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9208 MLX5_COUNTERS_PER_POOL, 9209 NULL, NULL, 9210 pool->raw_hw->mem_mng->wm.lkey, 9211 (void *)(uintptr_t) 9212 pool->raw_hw->data, 9213 sh->devx_comp, 9214 (uint64_t)(uintptr_t)pool); 9215 if (ret) { 9216 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9217 " %d", pool->min_dcs->id); 9218 pool->raw_hw = NULL; 9219 goto set_alarm; 9220 } 9221 LIST_REMOVE(pool->raw_hw, next); 9222 sh->sws_cmng.pending_queries++; 9223 pool_index++; 9224 if (pool_index >= n_valid) 9225 pool_index = 0; 9226 set_alarm: 9227 sh->sws_cmng.pool_index = pool_index; 9228 mlx5_set_query_alarm(sh); 9229 } 9230 9231 /** 9232 * Check and callback event for new aged flow in the counter pool 9233 * 9234 * @param[in] sh 9235 * Pointer to mlx5_dev_ctx_shared object. 9236 * @param[in] pool 9237 * Pointer to Current counter pool. 9238 */ 9239 static void 9240 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9241 struct mlx5_flow_counter_pool *pool) 9242 { 9243 struct mlx5_priv *priv; 9244 struct mlx5_flow_counter *cnt; 9245 struct mlx5_age_info *age_info; 9246 struct mlx5_age_param *age_param; 9247 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9248 struct mlx5_counter_stats_raw *prev = pool->raw; 9249 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9250 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9251 uint16_t expected = AGE_CANDIDATE; 9252 uint32_t i; 9253 9254 pool->time_of_last_age_check = curr_time; 9255 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9256 cnt = MLX5_POOL_GET_CNT(pool, i); 9257 age_param = MLX5_CNT_TO_AGE(cnt); 9258 if (__atomic_load_n(&age_param->state, 9259 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9260 continue; 9261 if (cur->data[i].hits != prev->data[i].hits) { 9262 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9263 __ATOMIC_RELAXED); 9264 continue; 9265 } 9266 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9267 time_delta, 9268 __ATOMIC_RELAXED) <= age_param->timeout) 9269 continue; 9270 /** 9271 * Hold the lock first, or if between the 9272 * state AGE_TMOUT and tailq operation the 9273 * release happened, the release procedure 9274 * may delete a non-existent tailq node. 9275 */ 9276 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9277 age_info = GET_PORT_AGE_INFO(priv); 9278 rte_spinlock_lock(&age_info->aged_sl); 9279 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9280 AGE_TMOUT, false, 9281 __ATOMIC_RELAXED, 9282 __ATOMIC_RELAXED)) { 9283 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9284 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9285 } 9286 rte_spinlock_unlock(&age_info->aged_sl); 9287 } 9288 mlx5_age_event_prepare(sh); 9289 } 9290 9291 /** 9292 * Handler for the HW respond about ready values from an asynchronous batch 9293 * query. This function is probably called by the host thread. 9294 * 9295 * @param[in] sh 9296 * The pointer to the shared device context. 9297 * @param[in] async_id 9298 * The Devx async ID. 9299 * @param[in] status 9300 * The status of the completion. 9301 */ 9302 void 9303 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9304 uint64_t async_id, int status) 9305 { 9306 struct mlx5_flow_counter_pool *pool = 9307 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9308 struct mlx5_counter_stats_raw *raw_to_free; 9309 uint8_t query_gen = pool->query_gen ^ 1; 9310 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9311 enum mlx5_counter_type cnt_type = 9312 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9313 MLX5_COUNTER_TYPE_ORIGIN; 9314 9315 if (unlikely(status)) { 9316 raw_to_free = pool->raw_hw; 9317 } else { 9318 raw_to_free = pool->raw; 9319 if (pool->is_aged) 9320 mlx5_flow_aging_check(sh, pool); 9321 rte_spinlock_lock(&pool->sl); 9322 pool->raw = pool->raw_hw; 9323 rte_spinlock_unlock(&pool->sl); 9324 /* Be sure the new raw counters data is updated in memory. */ 9325 rte_io_wmb(); 9326 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9327 rte_spinlock_lock(&cmng->csl[cnt_type]); 9328 TAILQ_CONCAT(&cmng->counters[cnt_type], 9329 &pool->counters[query_gen], next); 9330 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9331 } 9332 } 9333 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9334 pool->raw_hw = NULL; 9335 sh->sws_cmng.pending_queries--; 9336 } 9337 9338 static int 9339 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9340 const struct flow_grp_info *grp_info, 9341 struct rte_flow_error *error) 9342 { 9343 if (grp_info->transfer && grp_info->external && 9344 grp_info->fdb_def_rule) { 9345 if (group == UINT32_MAX) 9346 return rte_flow_error_set 9347 (error, EINVAL, 9348 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9349 NULL, 9350 "group index not supported"); 9351 *table = group + 1; 9352 } else { 9353 *table = group; 9354 } 9355 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9356 return 0; 9357 } 9358 9359 /** 9360 * Translate the rte_flow group index to HW table value. 9361 * 9362 * If tunnel offload is disabled, all group ids converted to flow table 9363 * id using the standard method. 9364 * If tunnel offload is enabled, group id can be converted using the 9365 * standard or tunnel conversion method. Group conversion method 9366 * selection depends on flags in `grp_info` parameter: 9367 * - Internal (grp_info.external == 0) groups conversion uses the 9368 * standard method. 9369 * - Group ids in JUMP action converted with the tunnel conversion. 9370 * - Group id in rule attribute conversion depends on a rule type and 9371 * group id value: 9372 * ** non zero group attributes converted with the tunnel method 9373 * ** zero group attribute in non-tunnel rule is converted using the 9374 * standard method - there's only one root table 9375 * ** zero group attribute in steer tunnel rule is converted with the 9376 * standard method - single root table 9377 * ** zero group attribute in match tunnel rule is a special OvS 9378 * case: that value is used for portability reasons. That group 9379 * id is converted with the tunnel conversion method. 9380 * 9381 * @param[in] dev 9382 * Port device 9383 * @param[in] tunnel 9384 * PMD tunnel offload object 9385 * @param[in] group 9386 * rte_flow group index value. 9387 * @param[out] table 9388 * HW table value. 9389 * @param[in] grp_info 9390 * flags used for conversion 9391 * @param[out] error 9392 * Pointer to error structure. 9393 * 9394 * @return 9395 * 0 on success, a negative errno value otherwise and rte_errno is set. 9396 */ 9397 int 9398 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9399 const struct mlx5_flow_tunnel *tunnel, 9400 uint32_t group, uint32_t *table, 9401 const struct flow_grp_info *grp_info, 9402 struct rte_flow_error *error) 9403 { 9404 int ret; 9405 bool standard_translation; 9406 9407 if (!grp_info->skip_scale && grp_info->external && 9408 group < MLX5_MAX_TABLES_EXTERNAL) 9409 group *= MLX5_FLOW_TABLE_FACTOR; 9410 if (is_tunnel_offload_active(dev)) { 9411 standard_translation = !grp_info->external || 9412 grp_info->std_tbl_fix; 9413 } else { 9414 standard_translation = true; 9415 } 9416 DRV_LOG(DEBUG, 9417 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9418 dev->data->port_id, group, grp_info->transfer, 9419 grp_info->external, grp_info->fdb_def_rule, 9420 standard_translation ? "STANDARD" : "TUNNEL"); 9421 if (standard_translation) 9422 ret = flow_group_to_table(dev->data->port_id, group, table, 9423 grp_info, error); 9424 else 9425 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9426 table, error); 9427 9428 return ret; 9429 } 9430 9431 /** 9432 * Discover availability of metadata reg_c's. 9433 * 9434 * Iteratively use test flows to check availability. 9435 * 9436 * @param[in] dev 9437 * Pointer to the Ethernet device structure. 9438 * 9439 * @return 9440 * 0 on success, a negative errno value otherwise and rte_errno is set. 9441 */ 9442 int 9443 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9444 { 9445 struct mlx5_priv *priv = dev->data->dev_private; 9446 enum modify_reg idx; 9447 int n = 0; 9448 9449 /* reg_c[0] and reg_c[1] are reserved. */ 9450 priv->sh->flow_mreg_c[n++] = REG_C_0; 9451 priv->sh->flow_mreg_c[n++] = REG_C_1; 9452 /* Discover availability of other reg_c's. */ 9453 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9454 struct rte_flow_attr attr = { 9455 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9456 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9457 .ingress = 1, 9458 }; 9459 struct rte_flow_item items[] = { 9460 [0] = { 9461 .type = RTE_FLOW_ITEM_TYPE_END, 9462 }, 9463 }; 9464 struct rte_flow_action actions[] = { 9465 [0] = { 9466 .type = (enum rte_flow_action_type) 9467 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9468 .conf = &(struct mlx5_flow_action_copy_mreg){ 9469 .src = REG_C_1, 9470 .dst = idx, 9471 }, 9472 }, 9473 [1] = { 9474 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9475 .conf = &(struct rte_flow_action_jump){ 9476 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9477 }, 9478 }, 9479 [2] = { 9480 .type = RTE_FLOW_ACTION_TYPE_END, 9481 }, 9482 }; 9483 uint32_t flow_idx; 9484 struct rte_flow *flow; 9485 struct rte_flow_error error; 9486 9487 if (!priv->sh->config.dv_flow_en) 9488 break; 9489 /* Create internal flow, validation skips copy action. */ 9490 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9491 items, actions, false, &error); 9492 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9493 flow_idx); 9494 if (!flow) 9495 continue; 9496 priv->sh->flow_mreg_c[n++] = idx; 9497 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9498 } 9499 for (; n < MLX5_MREG_C_NUM; ++n) 9500 priv->sh->flow_mreg_c[n] = REG_NON; 9501 priv->sh->metadata_regc_check_flag = 1; 9502 return 0; 9503 } 9504 9505 int 9506 save_dump_file(const uint8_t *data, uint32_t size, 9507 uint32_t type, uint64_t id, void *arg, FILE *file) 9508 { 9509 char line[BUF_SIZE]; 9510 uint32_t out = 0; 9511 uint32_t k; 9512 uint32_t actions_num; 9513 struct rte_flow_query_count *count; 9514 9515 memset(line, 0, BUF_SIZE); 9516 switch (type) { 9517 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9518 actions_num = *(uint32_t *)(arg); 9519 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9520 type, id, actions_num); 9521 break; 9522 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9523 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9524 type, id); 9525 break; 9526 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9527 count = (struct rte_flow_query_count *)arg; 9528 fprintf(file, 9529 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9530 type, id, count->hits, count->bytes); 9531 return 0; 9532 default: 9533 return -1; 9534 } 9535 9536 for (k = 0; k < size; k++) { 9537 /* Make sure we do not overrun the line buffer length. */ 9538 if (out >= BUF_SIZE - 4) { 9539 line[out] = '\0'; 9540 break; 9541 } 9542 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9543 (data[k]) & 0xff); 9544 } 9545 fprintf(file, "%s\n", line); 9546 return 0; 9547 } 9548 9549 int 9550 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9551 struct rte_flow_query_count *count, struct rte_flow_error *error) 9552 { 9553 struct rte_flow_action action[2]; 9554 enum mlx5_flow_drv_type ftype; 9555 const struct mlx5_flow_driver_ops *fops; 9556 9557 if (!flow) { 9558 return rte_flow_error_set(error, ENOENT, 9559 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9560 NULL, 9561 "invalid flow handle"); 9562 } 9563 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9564 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9565 if (flow->counter) { 9566 memset(count, 0, sizeof(struct rte_flow_query_count)); 9567 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9568 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9569 ftype < MLX5_FLOW_TYPE_MAX); 9570 fops = flow_get_drv_ops(ftype); 9571 return fops->query(dev, flow, action, count, error); 9572 } 9573 return -1; 9574 } 9575 9576 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9577 /** 9578 * Dump flow ipool data to file 9579 * 9580 * @param[in] dev 9581 * The pointer to Ethernet device. 9582 * @param[in] file 9583 * A pointer to a file for output. 9584 * @param[out] error 9585 * Perform verbose error reporting if not NULL. PMDs initialize this 9586 * structure in case of error only. 9587 * @return 9588 * 0 on success, a negative value otherwise. 9589 */ 9590 int 9591 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9592 struct rte_flow *flow, FILE *file, 9593 struct rte_flow_error *error) 9594 { 9595 struct mlx5_priv *priv = dev->data->dev_private; 9596 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9597 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9598 uint32_t handle_idx; 9599 struct mlx5_flow_handle *dh; 9600 struct rte_flow_query_count count; 9601 uint32_t actions_num; 9602 const uint8_t *data; 9603 size_t size; 9604 uint64_t id; 9605 uint32_t type; 9606 void *action = NULL; 9607 9608 if (!flow) { 9609 return rte_flow_error_set(error, ENOENT, 9610 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9611 NULL, 9612 "invalid flow handle"); 9613 } 9614 handle_idx = flow->dev_handles; 9615 /* query counter */ 9616 if (flow->counter && 9617 (!mlx5_counter_query(dev, flow->counter, false, 9618 &count.hits, &count.bytes, &action)) && action) { 9619 id = (uint64_t)(uintptr_t)action; 9620 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9621 save_dump_file(NULL, 0, type, 9622 id, (void *)&count, file); 9623 } 9624 9625 while (handle_idx) { 9626 dh = mlx5_ipool_get(priv->sh->ipool 9627 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9628 if (!dh) 9629 continue; 9630 handle_idx = dh->next.next; 9631 9632 /* Get modify_hdr and encap_decap buf from ipools. */ 9633 encap_decap = NULL; 9634 modify_hdr = dh->dvh.modify_hdr; 9635 9636 if (dh->dvh.rix_encap_decap) { 9637 encap_decap = mlx5_ipool_get(priv->sh->ipool 9638 [MLX5_IPOOL_DECAP_ENCAP], 9639 dh->dvh.rix_encap_decap); 9640 } 9641 if (modify_hdr) { 9642 data = (const uint8_t *)modify_hdr->actions; 9643 size = (size_t)(modify_hdr->actions_num) * 8; 9644 id = (uint64_t)(uintptr_t)modify_hdr->action; 9645 actions_num = modify_hdr->actions_num; 9646 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9647 save_dump_file(data, size, type, id, 9648 (void *)(&actions_num), file); 9649 } 9650 if (encap_decap) { 9651 data = encap_decap->buf; 9652 size = encap_decap->size; 9653 id = (uint64_t)(uintptr_t)encap_decap->action; 9654 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9655 save_dump_file(data, size, type, 9656 id, NULL, file); 9657 } 9658 } 9659 return 0; 9660 } 9661 9662 /** 9663 * Dump all flow's encap_decap/modify_hdr/counter data to file 9664 * 9665 * @param[in] dev 9666 * The pointer to Ethernet device. 9667 * @param[in] file 9668 * A pointer to a file for output. 9669 * @param[out] error 9670 * Perform verbose error reporting if not NULL. PMDs initialize this 9671 * structure in case of error only. 9672 * @return 9673 * 0 on success, a negative value otherwise. 9674 */ 9675 static int 9676 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9677 FILE *file, struct rte_flow_error *error __rte_unused) 9678 { 9679 struct mlx5_priv *priv = dev->data->dev_private; 9680 struct mlx5_dev_ctx_shared *sh = priv->sh; 9681 struct mlx5_hlist *h; 9682 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9683 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9684 struct rte_flow_query_count count; 9685 uint32_t actions_num; 9686 const uint8_t *data; 9687 size_t size; 9688 uint64_t id; 9689 uint32_t type; 9690 uint32_t i; 9691 uint32_t j; 9692 struct mlx5_list_inconst *l_inconst; 9693 struct mlx5_list_entry *e; 9694 int lcore_index; 9695 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9696 uint32_t max; 9697 void *action; 9698 9699 /* encap_decap hlist is lcore_share, get global core cache. */ 9700 i = MLX5_LIST_GLOBAL; 9701 h = sh->encaps_decaps; 9702 if (h) { 9703 for (j = 0; j <= h->mask; j++) { 9704 l_inconst = &h->buckets[j].l; 9705 if (!l_inconst || !l_inconst->cache[i]) 9706 continue; 9707 9708 e = LIST_FIRST(&l_inconst->cache[i]->h); 9709 while (e) { 9710 encap_decap = 9711 (struct mlx5_flow_dv_encap_decap_resource *)e; 9712 data = encap_decap->buf; 9713 size = encap_decap->size; 9714 id = (uint64_t)(uintptr_t)encap_decap->action; 9715 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9716 save_dump_file(data, size, type, 9717 id, NULL, file); 9718 e = LIST_NEXT(e, next); 9719 } 9720 } 9721 } 9722 9723 /* get modify_hdr */ 9724 h = sh->modify_cmds; 9725 if (h) { 9726 lcore_index = rte_lcore_index(rte_lcore_id()); 9727 if (unlikely(lcore_index == -1)) { 9728 lcore_index = MLX5_LIST_NLCORE; 9729 rte_spinlock_lock(&h->l_const.lcore_lock); 9730 } 9731 i = lcore_index; 9732 9733 for (j = 0; j <= h->mask; j++) { 9734 l_inconst = &h->buckets[j].l; 9735 if (!l_inconst || !l_inconst->cache[i]) 9736 continue; 9737 9738 e = LIST_FIRST(&l_inconst->cache[i]->h); 9739 while (e) { 9740 modify_hdr = 9741 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9742 data = (const uint8_t *)modify_hdr->actions; 9743 size = (size_t)(modify_hdr->actions_num) * 8; 9744 actions_num = modify_hdr->actions_num; 9745 id = (uint64_t)(uintptr_t)modify_hdr->action; 9746 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9747 save_dump_file(data, size, type, id, 9748 (void *)(&actions_num), file); 9749 e = LIST_NEXT(e, next); 9750 } 9751 } 9752 9753 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9754 rte_spinlock_unlock(&h->l_const.lcore_lock); 9755 } 9756 9757 /* get counter */ 9758 MLX5_ASSERT(cmng->n_valid <= cmng->n); 9759 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9760 for (j = 1; j <= max; j++) { 9761 action = NULL; 9762 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9763 &count.bytes, &action)) && action) { 9764 id = (uint64_t)(uintptr_t)action; 9765 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9766 save_dump_file(NULL, 0, type, 9767 id, (void *)&count, file); 9768 } 9769 } 9770 return 0; 9771 } 9772 #endif 9773 9774 /** 9775 * Dump flow raw hw data to file 9776 * 9777 * @param[in] dev 9778 * The pointer to Ethernet device. 9779 * @param[in] file 9780 * A pointer to a file for output. 9781 * @param[out] error 9782 * Perform verbose error reporting if not NULL. PMDs initialize this 9783 * structure in case of error only. 9784 * @return 9785 * 0 on success, a negative value otherwise. 9786 */ 9787 int 9788 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9789 FILE *file, 9790 struct rte_flow_error *error __rte_unused) 9791 { 9792 struct mlx5_priv *priv = dev->data->dev_private; 9793 struct mlx5_dev_ctx_shared *sh = priv->sh; 9794 uint32_t handle_idx; 9795 int ret; 9796 struct mlx5_flow_handle *dh; 9797 struct rte_flow *flow; 9798 9799 if (!sh->config.dv_flow_en) { 9800 if (fputs("device dv flow disabled\n", file) <= 0) 9801 return -errno; 9802 return -ENOTSUP; 9803 } 9804 9805 /* dump all */ 9806 if (!flow_idx) { 9807 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9808 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9809 return -EINVAL; 9810 #endif 9811 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9812 sh->rx_domain, 9813 sh->tx_domain, file); 9814 } 9815 /* dump one */ 9816 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9817 (uintptr_t)(void *)flow_idx); 9818 if (!flow) 9819 return -EINVAL; 9820 9821 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9822 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9823 #endif 9824 handle_idx = flow->dev_handles; 9825 while (handle_idx) { 9826 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9827 handle_idx); 9828 if (!dh) 9829 return -ENOENT; 9830 if (dh->drv_flow) { 9831 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9832 file); 9833 if (ret) 9834 return -ENOENT; 9835 } 9836 handle_idx = dh->next.next; 9837 } 9838 return 0; 9839 } 9840 9841 /** 9842 * Get aged-out flows. 9843 * 9844 * @param[in] dev 9845 * Pointer to the Ethernet device structure. 9846 * @param[in] context 9847 * The address of an array of pointers to the aged-out flows contexts. 9848 * @param[in] nb_countexts 9849 * The length of context array pointers. 9850 * @param[out] error 9851 * Perform verbose error reporting if not NULL. Initialized in case of 9852 * error only. 9853 * 9854 * @return 9855 * how many contexts get in success, otherwise negative errno value. 9856 * if nb_contexts is 0, return the amount of all aged contexts. 9857 * if nb_contexts is not 0 , return the amount of aged flows reported 9858 * in the context array. 9859 */ 9860 int 9861 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9862 uint32_t nb_contexts, struct rte_flow_error *error) 9863 { 9864 const struct mlx5_flow_driver_ops *fops; 9865 struct rte_flow_attr attr = { .transfer = 0 }; 9866 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, &attr); 9867 9868 if (type == MLX5_FLOW_TYPE_DV || type == MLX5_FLOW_TYPE_HW) { 9869 fops = flow_get_drv_ops(type); 9870 return fops->get_aged_flows(dev, contexts, nb_contexts, error); 9871 } 9872 DRV_LOG(ERR, "port %u get aged flows is not supported.", 9873 dev->data->port_id); 9874 return -ENOTSUP; 9875 } 9876 9877 /** 9878 * Get aged-out flows per HWS queue. 9879 * 9880 * @param[in] dev 9881 * Pointer to the Ethernet device structure. 9882 * @param[in] queue_id 9883 * Flow queue to query. 9884 * @param[in] context 9885 * The address of an array of pointers to the aged-out flows contexts. 9886 * @param[in] nb_countexts 9887 * The length of context array pointers. 9888 * @param[out] error 9889 * Perform verbose error reporting if not NULL. Initialized in case of 9890 * error only. 9891 * 9892 * @return 9893 * how many contexts get in success, otherwise negative errno value. 9894 * if nb_contexts is 0, return the amount of all aged contexts. 9895 * if nb_contexts is not 0 , return the amount of aged flows reported 9896 * in the context array. 9897 */ 9898 int 9899 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 9900 void **contexts, uint32_t nb_contexts, 9901 struct rte_flow_error *error) 9902 { 9903 const struct mlx5_flow_driver_ops *fops; 9904 struct rte_flow_attr attr = { 0 }; 9905 9906 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 9907 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9908 return fops->get_q_aged_flows(dev, queue_id, contexts, 9909 nb_contexts, error); 9910 } 9911 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 9912 dev->data->port_id, queue_id); 9913 return rte_flow_error_set(error, ENOTSUP, 9914 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9915 "get Q aged flows with incorrect steering mode"); 9916 } 9917 9918 /* Wrapper for driver action_validate op callback */ 9919 static int 9920 flow_drv_action_validate(struct rte_eth_dev *dev, 9921 const struct rte_flow_indir_action_conf *conf, 9922 const struct rte_flow_action *action, 9923 const struct mlx5_flow_driver_ops *fops, 9924 struct rte_flow_error *error) 9925 { 9926 static const char err_msg[] = "indirect action validation unsupported"; 9927 9928 if (!fops->action_validate) { 9929 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9930 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9931 NULL, err_msg); 9932 return -rte_errno; 9933 } 9934 return fops->action_validate(dev, conf, action, error); 9935 } 9936 9937 /** 9938 * Destroys the shared action by handle. 9939 * 9940 * @param dev 9941 * Pointer to Ethernet device structure. 9942 * @param[in] handle 9943 * Handle for the indirect action object to be destroyed. 9944 * @param[out] error 9945 * Perform verbose error reporting if not NULL. PMDs initialize this 9946 * structure in case of error only. 9947 * 9948 * @return 9949 * 0 on success, a negative errno value otherwise and rte_errno is set. 9950 * 9951 * @note: wrapper for driver action_create op callback. 9952 */ 9953 static int 9954 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9955 struct rte_flow_action_handle *handle, 9956 struct rte_flow_error *error) 9957 { 9958 static const char err_msg[] = "indirect action destruction unsupported"; 9959 struct rte_flow_attr attr = { .transfer = 0 }; 9960 const struct mlx5_flow_driver_ops *fops = 9961 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9962 9963 if (!fops->action_destroy) { 9964 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9965 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9966 NULL, err_msg); 9967 return -rte_errno; 9968 } 9969 return fops->action_destroy(dev, handle, error); 9970 } 9971 9972 /* Wrapper for driver action_destroy op callback */ 9973 static int 9974 flow_drv_action_update(struct rte_eth_dev *dev, 9975 struct rte_flow_action_handle *handle, 9976 const void *update, 9977 const struct mlx5_flow_driver_ops *fops, 9978 struct rte_flow_error *error) 9979 { 9980 static const char err_msg[] = "indirect action update unsupported"; 9981 9982 if (!fops->action_update) { 9983 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9984 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9985 NULL, err_msg); 9986 return -rte_errno; 9987 } 9988 return fops->action_update(dev, handle, update, error); 9989 } 9990 9991 /* Wrapper for driver action_destroy op callback */ 9992 static int 9993 flow_drv_action_query(struct rte_eth_dev *dev, 9994 const struct rte_flow_action_handle *handle, 9995 void *data, 9996 const struct mlx5_flow_driver_ops *fops, 9997 struct rte_flow_error *error) 9998 { 9999 static const char err_msg[] = "indirect action query unsupported"; 10000 10001 if (!fops->action_query) { 10002 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10003 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10004 NULL, err_msg); 10005 return -rte_errno; 10006 } 10007 return fops->action_query(dev, handle, data, error); 10008 } 10009 10010 /** 10011 * Create indirect action for reuse in multiple flow rules. 10012 * 10013 * @param dev 10014 * Pointer to Ethernet device structure. 10015 * @param conf 10016 * Pointer to indirect action object configuration. 10017 * @param[in] action 10018 * Action configuration for indirect action object creation. 10019 * @param[out] error 10020 * Perform verbose error reporting if not NULL. PMDs initialize this 10021 * structure in case of error only. 10022 * @return 10023 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10024 */ 10025 static struct rte_flow_action_handle * 10026 mlx5_action_handle_create(struct rte_eth_dev *dev, 10027 const struct rte_flow_indir_action_conf *conf, 10028 const struct rte_flow_action *action, 10029 struct rte_flow_error *error) 10030 { 10031 static const char err_msg[] = "indirect action creation unsupported"; 10032 struct rte_flow_attr attr = { .transfer = 0 }; 10033 const struct mlx5_flow_driver_ops *fops = 10034 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10035 10036 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10037 return NULL; 10038 if (!fops->action_create) { 10039 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10040 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10041 NULL, err_msg); 10042 return NULL; 10043 } 10044 return fops->action_create(dev, conf, action, error); 10045 } 10046 10047 /** 10048 * Updates inplace the indirect action configuration pointed by *handle* 10049 * with the configuration provided as *update* argument. 10050 * The update of the indirect action configuration effects all flow rules 10051 * reusing the action via handle. 10052 * 10053 * @param dev 10054 * Pointer to Ethernet device structure. 10055 * @param[in] handle 10056 * Handle for the indirect action to be updated. 10057 * @param[in] update 10058 * Action specification used to modify the action pointed by handle. 10059 * *update* could be of same type with the action pointed by the *handle* 10060 * handle argument, or some other structures like a wrapper, depending on 10061 * the indirect action type. 10062 * @param[out] error 10063 * Perform verbose error reporting if not NULL. PMDs initialize this 10064 * structure in case of error only. 10065 * 10066 * @return 10067 * 0 on success, a negative errno value otherwise and rte_errno is set. 10068 */ 10069 static int 10070 mlx5_action_handle_update(struct rte_eth_dev *dev, 10071 struct rte_flow_action_handle *handle, 10072 const void *update, 10073 struct rte_flow_error *error) 10074 { 10075 struct rte_flow_attr attr = { .transfer = 0 }; 10076 const struct mlx5_flow_driver_ops *fops = 10077 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10078 int ret; 10079 10080 ret = flow_drv_action_validate(dev, NULL, 10081 (const struct rte_flow_action *)update, fops, error); 10082 if (ret) 10083 return ret; 10084 return flow_drv_action_update(dev, handle, update, fops, 10085 error); 10086 } 10087 10088 /** 10089 * Query the indirect action by handle. 10090 * 10091 * This function allows retrieving action-specific data such as counters. 10092 * Data is gathered by special action which may be present/referenced in 10093 * more than one flow rule definition. 10094 * 10095 * see @RTE_FLOW_ACTION_TYPE_COUNT 10096 * 10097 * @param dev 10098 * Pointer to Ethernet device structure. 10099 * @param[in] handle 10100 * Handle for the indirect action to query. 10101 * @param[in, out] data 10102 * Pointer to storage for the associated query data type. 10103 * @param[out] error 10104 * Perform verbose error reporting if not NULL. PMDs initialize this 10105 * structure in case of error only. 10106 * 10107 * @return 10108 * 0 on success, a negative errno value otherwise and rte_errno is set. 10109 */ 10110 static int 10111 mlx5_action_handle_query(struct rte_eth_dev *dev, 10112 const struct rte_flow_action_handle *handle, 10113 void *data, 10114 struct rte_flow_error *error) 10115 { 10116 struct rte_flow_attr attr = { .transfer = 0 }; 10117 const struct mlx5_flow_driver_ops *fops = 10118 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10119 10120 return flow_drv_action_query(dev, handle, data, fops, error); 10121 } 10122 10123 /** 10124 * Destroy all indirect actions (shared RSS). 10125 * 10126 * @param dev 10127 * Pointer to Ethernet device. 10128 * 10129 * @return 10130 * 0 on success, a negative errno value otherwise and rte_errno is set. 10131 */ 10132 int 10133 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10134 { 10135 struct rte_flow_error error; 10136 struct mlx5_priv *priv = dev->data->dev_private; 10137 struct mlx5_shared_action_rss *shared_rss; 10138 int ret = 0; 10139 uint32_t idx; 10140 10141 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10142 priv->rss_shared_actions, idx, shared_rss, next) { 10143 ret |= mlx5_action_handle_destroy(dev, 10144 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10145 } 10146 return ret; 10147 } 10148 10149 /** 10150 * Validate existing indirect actions against current device configuration 10151 * and attach them to device resources. 10152 * 10153 * @param dev 10154 * Pointer to Ethernet device. 10155 * 10156 * @return 10157 * 0 on success, a negative errno value otherwise and rte_errno is set. 10158 */ 10159 int 10160 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10161 { 10162 struct mlx5_priv *priv = dev->data->dev_private; 10163 int ret = 0; 10164 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10165 10166 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10167 const char *message; 10168 uint32_t queue_idx; 10169 10170 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10171 ind_tbl->queues_n, 10172 &message, &queue_idx); 10173 if (ret != 0) { 10174 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10175 dev->data->port_id, ind_tbl->queues[queue_idx], 10176 message); 10177 break; 10178 } 10179 } 10180 if (ret != 0) 10181 return ret; 10182 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10183 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10184 if (ret != 0) { 10185 DRV_LOG(ERR, "Port %u could not attach " 10186 "indirection table obj %p", 10187 dev->data->port_id, (void *)ind_tbl); 10188 goto error; 10189 } 10190 } 10191 10192 return 0; 10193 error: 10194 ind_tbl_last = ind_tbl; 10195 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10196 if (ind_tbl == ind_tbl_last) 10197 break; 10198 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10199 DRV_LOG(CRIT, "Port %u could not detach " 10200 "indirection table obj %p on rollback", 10201 dev->data->port_id, (void *)ind_tbl); 10202 } 10203 return ret; 10204 } 10205 10206 /** 10207 * Detach indirect actions of the device from its resources. 10208 * 10209 * @param dev 10210 * Pointer to Ethernet device. 10211 * 10212 * @return 10213 * 0 on success, a negative errno value otherwise and rte_errno is set. 10214 */ 10215 int 10216 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10217 { 10218 struct mlx5_priv *priv = dev->data->dev_private; 10219 int ret = 0; 10220 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10221 10222 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10223 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10224 if (ret != 0) { 10225 DRV_LOG(ERR, "Port %u could not detach " 10226 "indirection table obj %p", 10227 dev->data->port_id, (void *)ind_tbl); 10228 goto error; 10229 } 10230 } 10231 return 0; 10232 error: 10233 ind_tbl_last = ind_tbl; 10234 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10235 if (ind_tbl == ind_tbl_last) 10236 break; 10237 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10238 DRV_LOG(CRIT, "Port %u could not attach " 10239 "indirection table obj %p on rollback", 10240 dev->data->port_id, (void *)ind_tbl); 10241 } 10242 return ret; 10243 } 10244 10245 #ifndef HAVE_MLX5DV_DR 10246 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10247 #else 10248 #define MLX5_DOMAIN_SYNC_FLOW \ 10249 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10250 #endif 10251 10252 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10253 { 10254 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10255 const struct mlx5_flow_driver_ops *fops; 10256 int ret; 10257 struct rte_flow_attr attr = { .transfer = 0 }; 10258 10259 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10260 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10261 if (ret > 0) 10262 ret = -ret; 10263 return ret; 10264 } 10265 10266 const struct mlx5_flow_tunnel * 10267 mlx5_get_tof(const struct rte_flow_item *item, 10268 const struct rte_flow_action *action, 10269 enum mlx5_tof_rule_type *rule_type) 10270 { 10271 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10272 if (item->type == (typeof(item->type)) 10273 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10274 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10275 return flow_items_to_tunnel(item); 10276 } 10277 } 10278 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10279 if (action->type == (typeof(action->type)) 10280 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10281 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10282 return flow_actions_to_tunnel(action); 10283 } 10284 } 10285 return NULL; 10286 } 10287 10288 /** 10289 * tunnel offload functionality is defined for DV environment only 10290 */ 10291 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10292 __extension__ 10293 union tunnel_offload_mark { 10294 uint32_t val; 10295 struct { 10296 uint32_t app_reserve:8; 10297 uint32_t table_id:15; 10298 uint32_t transfer:1; 10299 uint32_t _unused_:8; 10300 }; 10301 }; 10302 10303 static bool 10304 mlx5_access_tunnel_offload_db 10305 (struct rte_eth_dev *dev, 10306 bool (*match)(struct rte_eth_dev *, 10307 struct mlx5_flow_tunnel *, const void *), 10308 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10309 void (*miss)(struct rte_eth_dev *, void *), 10310 void *ctx, bool lock_op); 10311 10312 static int 10313 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10314 struct rte_flow *flow, 10315 const struct rte_flow_attr *attr, 10316 const struct rte_flow_action *app_actions, 10317 uint32_t flow_idx, 10318 const struct mlx5_flow_tunnel *tunnel, 10319 struct tunnel_default_miss_ctx *ctx, 10320 struct rte_flow_error *error) 10321 { 10322 struct mlx5_priv *priv = dev->data->dev_private; 10323 struct mlx5_flow *dev_flow; 10324 struct rte_flow_attr miss_attr = *attr; 10325 const struct rte_flow_item miss_items[2] = { 10326 { 10327 .type = RTE_FLOW_ITEM_TYPE_ETH, 10328 .spec = NULL, 10329 .last = NULL, 10330 .mask = NULL 10331 }, 10332 { 10333 .type = RTE_FLOW_ITEM_TYPE_END, 10334 .spec = NULL, 10335 .last = NULL, 10336 .mask = NULL 10337 } 10338 }; 10339 union tunnel_offload_mark mark_id; 10340 struct rte_flow_action_mark miss_mark; 10341 struct rte_flow_action miss_actions[3] = { 10342 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10343 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10344 }; 10345 const struct rte_flow_action_jump *jump_data; 10346 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10347 struct flow_grp_info grp_info = { 10348 .external = 1, 10349 .transfer = attr->transfer, 10350 .fdb_def_rule = !!priv->fdb_def_rule, 10351 .std_tbl_fix = 0, 10352 }; 10353 int ret; 10354 10355 if (!attr->transfer) { 10356 uint32_t q_size; 10357 10358 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10359 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10360 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10361 0, SOCKET_ID_ANY); 10362 if (!ctx->queue) 10363 return rte_flow_error_set 10364 (error, ENOMEM, 10365 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10366 NULL, "invalid default miss RSS"); 10367 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10368 ctx->action_rss.level = 0, 10369 ctx->action_rss.types = priv->rss_conf.rss_hf, 10370 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10371 ctx->action_rss.queue_num = priv->reta_idx_n, 10372 ctx->action_rss.key = priv->rss_conf.rss_key, 10373 ctx->action_rss.queue = ctx->queue; 10374 if (!priv->reta_idx_n || !priv->rxqs_n) 10375 return rte_flow_error_set 10376 (error, EINVAL, 10377 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10378 NULL, "invalid port configuration"); 10379 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10380 ctx->action_rss.types = 0; 10381 for (i = 0; i != priv->reta_idx_n; ++i) 10382 ctx->queue[i] = (*priv->reta_idx)[i]; 10383 } else { 10384 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10385 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10386 } 10387 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10388 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10389 jump_data = app_actions->conf; 10390 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10391 miss_attr.group = jump_data->group; 10392 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10393 &flow_table, &grp_info, error); 10394 if (ret) 10395 return rte_flow_error_set(error, EINVAL, 10396 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10397 NULL, "invalid tunnel id"); 10398 mark_id.app_reserve = 0; 10399 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10400 mark_id.transfer = !!attr->transfer; 10401 mark_id._unused_ = 0; 10402 miss_mark.id = mark_id.val; 10403 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10404 miss_items, miss_actions, flow_idx, error); 10405 if (!dev_flow) 10406 return -rte_errno; 10407 dev_flow->flow = flow; 10408 dev_flow->external = true; 10409 dev_flow->tunnel = tunnel; 10410 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10411 /* Subflow object was created, we must include one in the list. */ 10412 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10413 dev_flow->handle, next); 10414 DRV_LOG(DEBUG, 10415 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10416 dev->data->port_id, tunnel->app_tunnel.type, 10417 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10418 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10419 miss_actions, error); 10420 if (!ret) 10421 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10422 error); 10423 10424 return ret; 10425 } 10426 10427 static const struct mlx5_flow_tbl_data_entry * 10428 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10429 { 10430 struct mlx5_priv *priv = dev->data->dev_private; 10431 struct mlx5_dev_ctx_shared *sh = priv->sh; 10432 struct mlx5_list_entry *he; 10433 union tunnel_offload_mark mbits = { .val = mark }; 10434 union mlx5_flow_tbl_key table_key = { 10435 { 10436 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10437 .id = 0, 10438 .reserved = 0, 10439 .dummy = 0, 10440 .is_fdb = !!mbits.transfer, 10441 .is_egress = 0, 10442 } 10443 }; 10444 struct mlx5_flow_cb_ctx ctx = { 10445 .data = &table_key.v64, 10446 }; 10447 10448 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10449 return he ? 10450 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10451 } 10452 10453 static void 10454 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10455 struct mlx5_list_entry *entry) 10456 { 10457 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10458 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10459 10460 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10461 tunnel_flow_tbl_to_id(tte->flow_table)); 10462 mlx5_free(tte); 10463 } 10464 10465 static int 10466 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10467 struct mlx5_list_entry *entry, void *cb_ctx) 10468 { 10469 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10470 union tunnel_tbl_key tbl = { 10471 .val = *(uint64_t *)(ctx->data), 10472 }; 10473 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10474 10475 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10476 } 10477 10478 static struct mlx5_list_entry * 10479 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10480 { 10481 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10482 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10483 struct tunnel_tbl_entry *tte; 10484 union tunnel_tbl_key tbl = { 10485 .val = *(uint64_t *)(ctx->data), 10486 }; 10487 10488 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10489 sizeof(*tte), 0, 10490 SOCKET_ID_ANY); 10491 if (!tte) 10492 goto err; 10493 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10494 &tte->flow_table); 10495 if (tte->flow_table >= MLX5_MAX_TABLES) { 10496 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10497 tte->flow_table); 10498 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10499 tte->flow_table); 10500 goto err; 10501 } else if (!tte->flow_table) { 10502 goto err; 10503 } 10504 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10505 tte->tunnel_id = tbl.tunnel_id; 10506 tte->group = tbl.group; 10507 return &tte->hash; 10508 err: 10509 if (tte) 10510 mlx5_free(tte); 10511 return NULL; 10512 } 10513 10514 static struct mlx5_list_entry * 10515 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10516 struct mlx5_list_entry *oentry, 10517 void *cb_ctx __rte_unused) 10518 { 10519 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10520 0, SOCKET_ID_ANY); 10521 10522 if (!tte) 10523 return NULL; 10524 memcpy(tte, oentry, sizeof(*tte)); 10525 return &tte->hash; 10526 } 10527 10528 static void 10529 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10530 struct mlx5_list_entry *entry) 10531 { 10532 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10533 10534 mlx5_free(tte); 10535 } 10536 10537 static uint32_t 10538 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10539 const struct mlx5_flow_tunnel *tunnel, 10540 uint32_t group, uint32_t *table, 10541 struct rte_flow_error *error) 10542 { 10543 struct mlx5_list_entry *he; 10544 struct tunnel_tbl_entry *tte; 10545 union tunnel_tbl_key key = { 10546 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10547 .group = group 10548 }; 10549 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10550 struct mlx5_hlist *group_hash; 10551 struct mlx5_flow_cb_ctx ctx = { 10552 .data = &key.val, 10553 }; 10554 10555 group_hash = tunnel ? tunnel->groups : thub->groups; 10556 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10557 if (!he) 10558 return rte_flow_error_set(error, EINVAL, 10559 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10560 NULL, 10561 "tunnel group index not supported"); 10562 tte = container_of(he, typeof(*tte), hash); 10563 *table = tte->flow_table; 10564 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10565 dev->data->port_id, key.tunnel_id, group, *table); 10566 return 0; 10567 } 10568 10569 static void 10570 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10571 struct mlx5_flow_tunnel *tunnel) 10572 { 10573 struct mlx5_priv *priv = dev->data->dev_private; 10574 struct mlx5_indexed_pool *ipool; 10575 10576 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10577 dev->data->port_id, tunnel->tunnel_id); 10578 LIST_REMOVE(tunnel, chain); 10579 mlx5_hlist_destroy(tunnel->groups); 10580 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10581 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10582 } 10583 10584 static bool 10585 mlx5_access_tunnel_offload_db 10586 (struct rte_eth_dev *dev, 10587 bool (*match)(struct rte_eth_dev *, 10588 struct mlx5_flow_tunnel *, const void *), 10589 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10590 void (*miss)(struct rte_eth_dev *, void *), 10591 void *ctx, bool lock_op) 10592 { 10593 bool verdict = false; 10594 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10595 struct mlx5_flow_tunnel *tunnel; 10596 10597 rte_spinlock_lock(&thub->sl); 10598 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10599 verdict = match(dev, tunnel, (const void *)ctx); 10600 if (verdict) 10601 break; 10602 } 10603 if (!lock_op) 10604 rte_spinlock_unlock(&thub->sl); 10605 if (verdict && hit) 10606 hit(dev, tunnel, ctx); 10607 if (!verdict && miss) 10608 miss(dev, ctx); 10609 if (lock_op) 10610 rte_spinlock_unlock(&thub->sl); 10611 10612 return verdict; 10613 } 10614 10615 struct tunnel_db_find_tunnel_id_ctx { 10616 uint32_t tunnel_id; 10617 struct mlx5_flow_tunnel *tunnel; 10618 }; 10619 10620 static bool 10621 find_tunnel_id_match(struct rte_eth_dev *dev, 10622 struct mlx5_flow_tunnel *tunnel, const void *x) 10623 { 10624 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10625 10626 RTE_SET_USED(dev); 10627 return tunnel->tunnel_id == ctx->tunnel_id; 10628 } 10629 10630 static void 10631 find_tunnel_id_hit(struct rte_eth_dev *dev, 10632 struct mlx5_flow_tunnel *tunnel, void *x) 10633 { 10634 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10635 RTE_SET_USED(dev); 10636 ctx->tunnel = tunnel; 10637 } 10638 10639 static struct mlx5_flow_tunnel * 10640 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10641 { 10642 struct tunnel_db_find_tunnel_id_ctx ctx = { 10643 .tunnel_id = id, 10644 }; 10645 10646 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10647 find_tunnel_id_hit, NULL, &ctx, true); 10648 10649 return ctx.tunnel; 10650 } 10651 10652 static struct mlx5_flow_tunnel * 10653 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10654 const struct rte_flow_tunnel *app_tunnel) 10655 { 10656 struct mlx5_priv *priv = dev->data->dev_private; 10657 struct mlx5_indexed_pool *ipool; 10658 struct mlx5_flow_tunnel *tunnel; 10659 uint32_t id; 10660 10661 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10662 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10663 if (!tunnel) 10664 return NULL; 10665 if (id >= MLX5_MAX_TUNNELS) { 10666 mlx5_ipool_free(ipool, id); 10667 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10668 return NULL; 10669 } 10670 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10671 priv->sh, 10672 mlx5_flow_tunnel_grp2tbl_create_cb, 10673 mlx5_flow_tunnel_grp2tbl_match_cb, 10674 mlx5_flow_tunnel_grp2tbl_remove_cb, 10675 mlx5_flow_tunnel_grp2tbl_clone_cb, 10676 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10677 if (!tunnel->groups) { 10678 mlx5_ipool_free(ipool, id); 10679 return NULL; 10680 } 10681 /* initiate new PMD tunnel */ 10682 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10683 tunnel->tunnel_id = id; 10684 tunnel->action.type = (typeof(tunnel->action.type)) 10685 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10686 tunnel->action.conf = tunnel; 10687 tunnel->item.type = (typeof(tunnel->item.type)) 10688 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10689 tunnel->item.spec = tunnel; 10690 tunnel->item.last = NULL; 10691 tunnel->item.mask = NULL; 10692 10693 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10694 dev->data->port_id, tunnel->tunnel_id); 10695 10696 return tunnel; 10697 } 10698 10699 struct tunnel_db_get_tunnel_ctx { 10700 const struct rte_flow_tunnel *app_tunnel; 10701 struct mlx5_flow_tunnel *tunnel; 10702 }; 10703 10704 static bool get_tunnel_match(struct rte_eth_dev *dev, 10705 struct mlx5_flow_tunnel *tunnel, const void *x) 10706 { 10707 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10708 10709 RTE_SET_USED(dev); 10710 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10711 sizeof(*ctx->app_tunnel)); 10712 } 10713 10714 static void get_tunnel_hit(struct rte_eth_dev *dev, 10715 struct mlx5_flow_tunnel *tunnel, void *x) 10716 { 10717 /* called under tunnel spinlock protection */ 10718 struct tunnel_db_get_tunnel_ctx *ctx = x; 10719 10720 RTE_SET_USED(dev); 10721 tunnel->refctn++; 10722 ctx->tunnel = tunnel; 10723 } 10724 10725 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10726 { 10727 /* called under tunnel spinlock protection */ 10728 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10729 struct tunnel_db_get_tunnel_ctx *ctx = x; 10730 10731 rte_spinlock_unlock(&thub->sl); 10732 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10733 rte_spinlock_lock(&thub->sl); 10734 if (ctx->tunnel) { 10735 ctx->tunnel->refctn = 1; 10736 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10737 } 10738 } 10739 10740 10741 static int 10742 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10743 const struct rte_flow_tunnel *app_tunnel, 10744 struct mlx5_flow_tunnel **tunnel) 10745 { 10746 struct tunnel_db_get_tunnel_ctx ctx = { 10747 .app_tunnel = app_tunnel, 10748 }; 10749 10750 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10751 get_tunnel_miss, &ctx, true); 10752 *tunnel = ctx.tunnel; 10753 return ctx.tunnel ? 0 : -ENOMEM; 10754 } 10755 10756 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10757 { 10758 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10759 10760 if (!thub) 10761 return; 10762 if (!LIST_EMPTY(&thub->tunnels)) 10763 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10764 mlx5_hlist_destroy(thub->groups); 10765 mlx5_free(thub); 10766 } 10767 10768 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10769 { 10770 int err; 10771 struct mlx5_flow_tunnel_hub *thub; 10772 10773 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10774 0, SOCKET_ID_ANY); 10775 if (!thub) 10776 return -ENOMEM; 10777 LIST_INIT(&thub->tunnels); 10778 rte_spinlock_init(&thub->sl); 10779 thub->groups = mlx5_hlist_create("flow groups", 64, 10780 false, true, sh, 10781 mlx5_flow_tunnel_grp2tbl_create_cb, 10782 mlx5_flow_tunnel_grp2tbl_match_cb, 10783 mlx5_flow_tunnel_grp2tbl_remove_cb, 10784 mlx5_flow_tunnel_grp2tbl_clone_cb, 10785 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10786 if (!thub->groups) { 10787 err = -rte_errno; 10788 goto err; 10789 } 10790 sh->tunnel_hub = thub; 10791 10792 return 0; 10793 10794 err: 10795 if (thub->groups) 10796 mlx5_hlist_destroy(thub->groups); 10797 if (thub) 10798 mlx5_free(thub); 10799 return err; 10800 } 10801 10802 static inline int 10803 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10804 struct rte_flow_tunnel *tunnel, 10805 struct rte_flow_error *error) 10806 { 10807 struct mlx5_priv *priv = dev->data->dev_private; 10808 10809 if (!priv->sh->config.dv_flow_en) 10810 return rte_flow_error_set(error, ENOTSUP, 10811 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10812 "flow DV interface is off"); 10813 if (!is_tunnel_offload_active(dev)) 10814 return rte_flow_error_set(error, ENOTSUP, 10815 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10816 "tunnel offload was not activated"); 10817 if (!tunnel) 10818 return rte_flow_error_set(error, EINVAL, 10819 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10820 "no application tunnel"); 10821 switch (tunnel->type) { 10822 default: 10823 return rte_flow_error_set(error, EINVAL, 10824 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10825 "unsupported tunnel type"); 10826 case RTE_FLOW_ITEM_TYPE_VXLAN: 10827 case RTE_FLOW_ITEM_TYPE_GRE: 10828 case RTE_FLOW_ITEM_TYPE_NVGRE: 10829 case RTE_FLOW_ITEM_TYPE_GENEVE: 10830 break; 10831 } 10832 return 0; 10833 } 10834 10835 static int 10836 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10837 struct rte_flow_tunnel *app_tunnel, 10838 struct rte_flow_action **actions, 10839 uint32_t *num_of_actions, 10840 struct rte_flow_error *error) 10841 { 10842 struct mlx5_flow_tunnel *tunnel; 10843 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10844 10845 if (ret) 10846 return ret; 10847 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10848 if (ret < 0) { 10849 return rte_flow_error_set(error, ret, 10850 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10851 "failed to initialize pmd tunnel"); 10852 } 10853 *actions = &tunnel->action; 10854 *num_of_actions = 1; 10855 return 0; 10856 } 10857 10858 static int 10859 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10860 struct rte_flow_tunnel *app_tunnel, 10861 struct rte_flow_item **items, 10862 uint32_t *num_of_items, 10863 struct rte_flow_error *error) 10864 { 10865 struct mlx5_flow_tunnel *tunnel; 10866 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10867 10868 if (ret) 10869 return ret; 10870 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10871 if (ret < 0) { 10872 return rte_flow_error_set(error, ret, 10873 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10874 "failed to initialize pmd tunnel"); 10875 } 10876 *items = &tunnel->item; 10877 *num_of_items = 1; 10878 return 0; 10879 } 10880 10881 struct tunnel_db_element_release_ctx { 10882 struct rte_flow_item *items; 10883 struct rte_flow_action *actions; 10884 uint32_t num_elements; 10885 struct rte_flow_error *error; 10886 int ret; 10887 }; 10888 10889 static bool 10890 tunnel_element_release_match(struct rte_eth_dev *dev, 10891 struct mlx5_flow_tunnel *tunnel, const void *x) 10892 { 10893 const struct tunnel_db_element_release_ctx *ctx = x; 10894 10895 RTE_SET_USED(dev); 10896 if (ctx->num_elements != 1) 10897 return false; 10898 else if (ctx->items) 10899 return ctx->items == &tunnel->item; 10900 else if (ctx->actions) 10901 return ctx->actions == &tunnel->action; 10902 10903 return false; 10904 } 10905 10906 static void 10907 tunnel_element_release_hit(struct rte_eth_dev *dev, 10908 struct mlx5_flow_tunnel *tunnel, void *x) 10909 { 10910 struct tunnel_db_element_release_ctx *ctx = x; 10911 ctx->ret = 0; 10912 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10913 mlx5_flow_tunnel_free(dev, tunnel); 10914 } 10915 10916 static void 10917 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10918 { 10919 struct tunnel_db_element_release_ctx *ctx = x; 10920 RTE_SET_USED(dev); 10921 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10922 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10923 "invalid argument"); 10924 } 10925 10926 static int 10927 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10928 struct rte_flow_item *pmd_items, 10929 uint32_t num_items, struct rte_flow_error *err) 10930 { 10931 struct tunnel_db_element_release_ctx ctx = { 10932 .items = pmd_items, 10933 .actions = NULL, 10934 .num_elements = num_items, 10935 .error = err, 10936 }; 10937 10938 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10939 tunnel_element_release_hit, 10940 tunnel_element_release_miss, &ctx, false); 10941 10942 return ctx.ret; 10943 } 10944 10945 static int 10946 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10947 struct rte_flow_action *pmd_actions, 10948 uint32_t num_actions, struct rte_flow_error *err) 10949 { 10950 struct tunnel_db_element_release_ctx ctx = { 10951 .items = NULL, 10952 .actions = pmd_actions, 10953 .num_elements = num_actions, 10954 .error = err, 10955 }; 10956 10957 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10958 tunnel_element_release_hit, 10959 tunnel_element_release_miss, &ctx, false); 10960 10961 return ctx.ret; 10962 } 10963 10964 static int 10965 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10966 struct rte_mbuf *m, 10967 struct rte_flow_restore_info *info, 10968 struct rte_flow_error *err) 10969 { 10970 uint64_t ol_flags = m->ol_flags; 10971 const struct mlx5_flow_tbl_data_entry *tble; 10972 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 10973 10974 if (!is_tunnel_offload_active(dev)) { 10975 info->flags = 0; 10976 return 0; 10977 } 10978 10979 if ((ol_flags & mask) != mask) 10980 goto err; 10981 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 10982 if (!tble) { 10983 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 10984 dev->data->port_id, m->hash.fdir.hi); 10985 goto err; 10986 } 10987 MLX5_ASSERT(tble->tunnel); 10988 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 10989 info->group_id = tble->group_id; 10990 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 10991 RTE_FLOW_RESTORE_INFO_GROUP_ID | 10992 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 10993 10994 return 0; 10995 10996 err: 10997 return rte_flow_error_set(err, EINVAL, 10998 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10999 "failed to get restore info"); 11000 } 11001 11002 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11003 static int 11004 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11005 __rte_unused struct rte_flow_tunnel *app_tunnel, 11006 __rte_unused struct rte_flow_action **actions, 11007 __rte_unused uint32_t *num_of_actions, 11008 __rte_unused struct rte_flow_error *error) 11009 { 11010 return -ENOTSUP; 11011 } 11012 11013 static int 11014 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11015 __rte_unused struct rte_flow_tunnel *app_tunnel, 11016 __rte_unused struct rte_flow_item **items, 11017 __rte_unused uint32_t *num_of_items, 11018 __rte_unused struct rte_flow_error *error) 11019 { 11020 return -ENOTSUP; 11021 } 11022 11023 static int 11024 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11025 __rte_unused struct rte_flow_item *pmd_items, 11026 __rte_unused uint32_t num_items, 11027 __rte_unused struct rte_flow_error *err) 11028 { 11029 return -ENOTSUP; 11030 } 11031 11032 static int 11033 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11034 __rte_unused struct rte_flow_action *pmd_action, 11035 __rte_unused uint32_t num_actions, 11036 __rte_unused struct rte_flow_error *err) 11037 { 11038 return -ENOTSUP; 11039 } 11040 11041 static int 11042 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11043 __rte_unused struct rte_mbuf *m, 11044 __rte_unused struct rte_flow_restore_info *i, 11045 __rte_unused struct rte_flow_error *err) 11046 { 11047 return -ENOTSUP; 11048 } 11049 11050 static int 11051 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11052 __rte_unused struct rte_flow *flow, 11053 __rte_unused const struct rte_flow_attr *attr, 11054 __rte_unused const struct rte_flow_action *actions, 11055 __rte_unused uint32_t flow_idx, 11056 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11057 __rte_unused struct tunnel_default_miss_ctx *ctx, 11058 __rte_unused struct rte_flow_error *error) 11059 { 11060 return -ENOTSUP; 11061 } 11062 11063 static struct mlx5_flow_tunnel * 11064 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11065 __rte_unused uint32_t id) 11066 { 11067 return NULL; 11068 } 11069 11070 static void 11071 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11072 __rte_unused struct mlx5_flow_tunnel *tunnel) 11073 { 11074 } 11075 11076 static uint32_t 11077 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11078 __rte_unused const struct mlx5_flow_tunnel *t, 11079 __rte_unused uint32_t group, 11080 __rte_unused uint32_t *table, 11081 struct rte_flow_error *error) 11082 { 11083 return rte_flow_error_set(error, ENOTSUP, 11084 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11085 "tunnel offload requires DV support"); 11086 } 11087 11088 void 11089 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11090 __rte_unused uint16_t port_id) 11091 { 11092 } 11093 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11094 11095 /* Flex flow item API */ 11096 static struct rte_flow_item_flex_handle * 11097 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11098 const struct rte_flow_item_flex_conf *conf, 11099 struct rte_flow_error *error) 11100 { 11101 static const char err_msg[] = "flex item creation unsupported"; 11102 struct mlx5_priv *priv = dev->data->dev_private; 11103 struct rte_flow_attr attr = { .transfer = 0 }; 11104 const struct mlx5_flow_driver_ops *fops = 11105 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11106 11107 if (!priv->pci_dev) { 11108 rte_flow_error_set(error, ENOTSUP, 11109 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11110 "create flex item on PF only"); 11111 return NULL; 11112 } 11113 switch (priv->pci_dev->id.device_id) { 11114 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11115 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11116 break; 11117 default: 11118 rte_flow_error_set(error, ENOTSUP, 11119 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11120 "flex item available on BlueField ports only"); 11121 return NULL; 11122 } 11123 if (!fops->item_create) { 11124 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11125 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11126 NULL, err_msg); 11127 return NULL; 11128 } 11129 return fops->item_create(dev, conf, error); 11130 } 11131 11132 static int 11133 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11134 const struct rte_flow_item_flex_handle *handle, 11135 struct rte_flow_error *error) 11136 { 11137 static const char err_msg[] = "flex item release unsupported"; 11138 struct rte_flow_attr attr = { .transfer = 0 }; 11139 const struct mlx5_flow_driver_ops *fops = 11140 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11141 11142 if (!fops->item_release) { 11143 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11144 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11145 NULL, err_msg); 11146 return -rte_errno; 11147 } 11148 return fops->item_release(dev, handle, error); 11149 } 11150 11151 static void 11152 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11153 { 11154 int ret; 11155 struct rte_flow_error error; 11156 11157 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11158 char *item_name; 11159 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11160 sizeof(item_name), 11161 (void *)(uintptr_t)item->type, &error); 11162 if (ret > 0) 11163 printf("%s ", item_name); 11164 else 11165 printf("%d\n", (int)item->type); 11166 } 11167 printf("END\n"); 11168 } 11169 11170 static int 11171 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11172 { 11173 const struct rte_flow_item_udp *spec = udp_item->spec; 11174 const struct rte_flow_item_udp *mask = udp_item->mask; 11175 uint16_t udp_dport = 0; 11176 11177 if (spec != NULL) { 11178 if (!mask) 11179 mask = &rte_flow_item_udp_mask; 11180 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11181 mask->hdr.dst_port); 11182 } 11183 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11184 } 11185 11186 static const struct mlx5_flow_expand_node * 11187 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11188 unsigned int item_idx, 11189 const struct mlx5_flow_expand_node graph[], 11190 const struct mlx5_flow_expand_node *node) 11191 { 11192 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11193 11194 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11195 node != NULL && 11196 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11197 /* 11198 * The expansion node is VXLAN and it is also the last 11199 * expandable item in the pattern, so need to continue 11200 * expansion of the inner tunnel. 11201 */ 11202 MLX5_ASSERT(item_idx > 0); 11203 prev_item = pattern + item_idx - 1; 11204 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11205 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11206 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11207 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11208 } 11209 return node; 11210 } 11211 11212 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11213 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11214 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11215 }; 11216 11217 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11218 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11219 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11220 { 9, 10, 11 }, { 12, 13, 14 }, 11221 }; 11222 11223 /** 11224 * Discover the number of available flow priorities. 11225 * 11226 * @param dev 11227 * Ethernet device. 11228 * 11229 * @return 11230 * On success, number of available flow priorities. 11231 * On failure, a negative errno-style code and rte_errno is set. 11232 */ 11233 int 11234 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11235 { 11236 static const uint16_t vprio[] = {8, 16}; 11237 const struct mlx5_priv *priv = dev->data->dev_private; 11238 const struct mlx5_flow_driver_ops *fops; 11239 enum mlx5_flow_drv_type type; 11240 int ret; 11241 11242 type = mlx5_flow_os_get_type(); 11243 if (type == MLX5_FLOW_TYPE_MAX) { 11244 type = MLX5_FLOW_TYPE_VERBS; 11245 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11246 type = MLX5_FLOW_TYPE_DV; 11247 } 11248 fops = flow_get_drv_ops(type); 11249 if (fops->discover_priorities == NULL) { 11250 DRV_LOG(ERR, "Priority discovery not supported"); 11251 rte_errno = ENOTSUP; 11252 return -rte_errno; 11253 } 11254 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11255 if (ret < 0) 11256 return ret; 11257 switch (ret) { 11258 case 8: 11259 ret = RTE_DIM(priority_map_3); 11260 break; 11261 case 16: 11262 ret = RTE_DIM(priority_map_5); 11263 break; 11264 default: 11265 rte_errno = ENOTSUP; 11266 DRV_LOG(ERR, 11267 "port %u maximum priority: %d expected 8/16", 11268 dev->data->port_id, ret); 11269 return -rte_errno; 11270 } 11271 DRV_LOG(INFO, "port %u supported flow priorities:" 11272 " 0-%d for ingress or egress root table," 11273 " 0-%d for non-root table or transfer root table.", 11274 dev->data->port_id, ret - 2, 11275 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11276 return ret; 11277 } 11278 11279 /** 11280 * Adjust flow priority based on the highest layer and the request priority. 11281 * 11282 * @param[in] dev 11283 * Pointer to the Ethernet device structure. 11284 * @param[in] priority 11285 * The rule base priority. 11286 * @param[in] subpriority 11287 * The priority based on the items. 11288 * 11289 * @return 11290 * The new priority. 11291 */ 11292 uint32_t 11293 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11294 uint32_t subpriority) 11295 { 11296 uint32_t res = 0; 11297 struct mlx5_priv *priv = dev->data->dev_private; 11298 11299 switch (priv->sh->flow_max_priority) { 11300 case RTE_DIM(priority_map_3): 11301 res = priority_map_3[priority][subpriority]; 11302 break; 11303 case RTE_DIM(priority_map_5): 11304 res = priority_map_5[priority][subpriority]; 11305 break; 11306 } 11307 return res; 11308 } 11309 11310 /** 11311 * Get the priority for sending traffic to kernel table. 11312 * 11313 * @param[in] dev 11314 * Pointer to the Ethernet device structure. 11315 * 11316 * @return 11317 * On success: the value of priority for sending traffic to kernel table 11318 * On failure: -1 11319 */ 11320 uint32_t 11321 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11322 { 11323 struct mlx5_priv *priv = dev->data->dev_private; 11324 uint32_t res; 11325 11326 switch (priv->sh->flow_max_priority) { 11327 case RTE_DIM(priority_map_5): 11328 res = 15; 11329 break; 11330 case RTE_DIM(priority_map_3): 11331 res = 7; 11332 break; 11333 default: 11334 DRV_LOG(ERR, 11335 "port %u maximum priority: %d expected 8/16", 11336 dev->data->port_id, priv->sh->flow_max_priority); 11337 res = (uint32_t)-1; 11338 } 11339 return res; 11340 } 11341 11342 /** 11343 * Get the E-Switch Manager vport id. 11344 * 11345 * @param[in] dev 11346 * Pointer to the Ethernet device structure. 11347 * 11348 * @return 11349 * The vport id. 11350 */ 11351 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11352 { 11353 struct mlx5_priv *priv = dev->data->dev_private; 11354 struct mlx5_common_device *cdev = priv->sh->cdev; 11355 11356 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11357 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11358 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11359 11360 if (priv->pci_dev == NULL) 11361 return 0; 11362 switch (priv->pci_dev->id.device_id) { 11363 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 11364 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11365 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11366 /* 11367 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11368 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11369 * ECPF vport ID is selected and not the host port (0) in any BF case. 11370 */ 11371 return (int16_t)MLX5_ECPF_VPORT_ID; 11372 default: 11373 return MLX5_PF_VPORT_ID; 11374 } 11375 } 11376 11377 /** 11378 * Parse item to get the vport id. 11379 * 11380 * @param[in] dev 11381 * Pointer to the Ethernet device structure. 11382 * @param[in] item 11383 * The src port id match item. 11384 * @param[out] vport_id 11385 * Pointer to put the vport id. 11386 * @param[out] all_ports 11387 * Indicate if the item matches all ports. 11388 * @param[out] error 11389 * Pointer to error structure. 11390 * 11391 * @return 11392 * 0 on success, a negative errno value otherwise and rte_errno is set. 11393 */ 11394 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11395 const struct rte_flow_item *item, 11396 uint16_t *vport_id, 11397 bool *all_ports, 11398 struct rte_flow_error *error) 11399 { 11400 struct mlx5_priv *port_priv; 11401 const struct rte_flow_item_port_id *pid_v; 11402 uint32_t esw_mgr_port; 11403 11404 if (item->type != RTE_FLOW_ITEM_TYPE_PORT_ID && 11405 item->type != RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) 11406 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11407 NULL, "Incorrect item type."); 11408 pid_v = item->spec; 11409 if (!pid_v) { 11410 if (all_ports) 11411 *all_ports = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT); 11412 return 0; 11413 } 11414 if (all_ports) 11415 *all_ports = false; 11416 esw_mgr_port = (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) ? 11417 MLX5_REPRESENTED_PORT_ESW_MGR : MLX5_PORT_ESW_MGR; 11418 if (pid_v->id == esw_mgr_port) { 11419 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11420 } else { 11421 port_priv = mlx5_port_to_eswitch_info(pid_v->id, false); 11422 if (!port_priv) 11423 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11424 NULL, "Failed to get port info."); 11425 *vport_id = port_priv->representor_id; 11426 } 11427 11428 return 0; 11429 } 11430 11431 int 11432 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11433 uint16_t *proxy_port_id, 11434 struct rte_flow_error *error) 11435 { 11436 const struct mlx5_priv *priv = dev->data->dev_private; 11437 uint16_t port_id; 11438 11439 if (!priv->sh->config.dv_esw_en) 11440 return rte_flow_error_set(error, EINVAL, 11441 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11442 NULL, 11443 "unable to provide a proxy port" 11444 " without E-Switch configured"); 11445 if (!priv->master && !priv->representor) 11446 return rte_flow_error_set(error, EINVAL, 11447 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11448 NULL, 11449 "unable to provide a proxy port" 11450 " for port which is not a master" 11451 " or a representor port"); 11452 if (priv->master) { 11453 *proxy_port_id = dev->data->port_id; 11454 return 0; 11455 } 11456 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11457 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11458 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11459 11460 if (port_priv->master && 11461 port_priv->domain_id == priv->domain_id) { 11462 *proxy_port_id = port_id; 11463 return 0; 11464 } 11465 } 11466 return rte_flow_error_set(error, ENODEV, 11467 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11468 NULL, "unable to find a proxy port"); 11469 } 11470