1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static bool 168 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 169 { 170 switch (item->type) { 171 case RTE_FLOW_ITEM_TYPE_ETH: 172 case RTE_FLOW_ITEM_TYPE_VLAN: 173 case RTE_FLOW_ITEM_TYPE_IPV4: 174 case RTE_FLOW_ITEM_TYPE_IPV6: 175 case RTE_FLOW_ITEM_TYPE_UDP: 176 case RTE_FLOW_ITEM_TYPE_TCP: 177 case RTE_FLOW_ITEM_TYPE_ESP: 178 case RTE_FLOW_ITEM_TYPE_ICMP: 179 case RTE_FLOW_ITEM_TYPE_ICMP6: 180 case RTE_FLOW_ITEM_TYPE_VXLAN: 181 case RTE_FLOW_ITEM_TYPE_NVGRE: 182 case RTE_FLOW_ITEM_TYPE_GRE: 183 case RTE_FLOW_ITEM_TYPE_GENEVE: 184 case RTE_FLOW_ITEM_TYPE_MPLS: 185 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 186 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 187 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 188 case RTE_FLOW_ITEM_TYPE_GTP: 189 return true; 190 default: 191 break; 192 } 193 return false; 194 } 195 196 /** 197 * Network Service Header (NSH) and its next protocol values 198 * are described in RFC-8393. 199 */ 200 static enum rte_flow_item_type 201 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 202 { 203 enum rte_flow_item_type type; 204 205 switch (proto_mask & proto_spec) { 206 case 0: 207 type = RTE_FLOW_ITEM_TYPE_VOID; 208 break; 209 case RTE_VXLAN_GPE_TYPE_IPV4: 210 type = RTE_FLOW_ITEM_TYPE_IPV4; 211 break; 212 case RTE_VXLAN_GPE_TYPE_IPV6: 213 type = RTE_VXLAN_GPE_TYPE_IPV6; 214 break; 215 case RTE_VXLAN_GPE_TYPE_ETH: 216 type = RTE_FLOW_ITEM_TYPE_ETH; 217 break; 218 default: 219 type = RTE_FLOW_ITEM_TYPE_END; 220 } 221 return type; 222 } 223 224 static enum rte_flow_item_type 225 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 226 { 227 enum rte_flow_item_type type; 228 229 switch (proto_mask & proto_spec) { 230 case 0: 231 type = RTE_FLOW_ITEM_TYPE_VOID; 232 break; 233 case IPPROTO_UDP: 234 type = RTE_FLOW_ITEM_TYPE_UDP; 235 break; 236 case IPPROTO_TCP: 237 type = RTE_FLOW_ITEM_TYPE_TCP; 238 break; 239 case IPPROTO_IPIP: 240 type = RTE_FLOW_ITEM_TYPE_IPV4; 241 break; 242 case IPPROTO_IPV6: 243 type = RTE_FLOW_ITEM_TYPE_IPV6; 244 break; 245 case IPPROTO_ESP: 246 type = RTE_FLOW_ITEM_TYPE_ESP; 247 break; 248 default: 249 type = RTE_FLOW_ITEM_TYPE_END; 250 } 251 return type; 252 } 253 254 static enum rte_flow_item_type 255 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 256 rte_be16_t type_mask, bool is_tunnel) 257 { 258 enum rte_flow_item_type type; 259 260 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 261 case 0: 262 type = RTE_FLOW_ITEM_TYPE_VOID; 263 break; 264 case RTE_ETHER_TYPE_TEB: 265 type = is_tunnel ? 266 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 267 break; 268 case RTE_ETHER_TYPE_VLAN: 269 type = !is_tunnel ? 270 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 271 break; 272 case RTE_ETHER_TYPE_IPV4: 273 type = RTE_FLOW_ITEM_TYPE_IPV4; 274 break; 275 case RTE_ETHER_TYPE_IPV6: 276 type = RTE_FLOW_ITEM_TYPE_IPV6; 277 break; 278 default: 279 type = RTE_FLOW_ITEM_TYPE_END; 280 } 281 return type; 282 } 283 284 static enum rte_flow_item_type 285 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 286 { 287 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 288 do { \ 289 const void *m = item->mask; \ 290 const void *s = item->spec; \ 291 mask = m ? \ 292 ((const struct rte_flow_item_##type *)m)->fld : \ 293 rte_flow_item_##type##_mask.fld; \ 294 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 295 } while (0) 296 297 enum rte_flow_item_type ret; 298 uint16_t spec, mask; 299 300 if (item == NULL || item->spec == NULL) 301 return RTE_FLOW_ITEM_TYPE_VOID; 302 switch (item->type) { 303 case RTE_FLOW_ITEM_TYPE_ETH: 304 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 305 if (!mask) 306 return RTE_FLOW_ITEM_TYPE_VOID; 307 ret = mlx5_ethertype_to_item_type(spec, mask, false); 308 break; 309 case RTE_FLOW_ITEM_TYPE_VLAN: 310 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 311 if (!mask) 312 return RTE_FLOW_ITEM_TYPE_VOID; 313 ret = mlx5_ethertype_to_item_type(spec, mask, false); 314 break; 315 case RTE_FLOW_ITEM_TYPE_IPV4: 316 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 317 if (!mask) 318 return RTE_FLOW_ITEM_TYPE_VOID; 319 ret = mlx5_inet_proto_to_item_type(spec, mask); 320 break; 321 case RTE_FLOW_ITEM_TYPE_IPV6: 322 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 323 if (!mask) 324 return RTE_FLOW_ITEM_TYPE_VOID; 325 ret = mlx5_inet_proto_to_item_type(spec, mask); 326 break; 327 case RTE_FLOW_ITEM_TYPE_GENEVE: 328 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 329 ret = mlx5_ethertype_to_item_type(spec, mask, true); 330 break; 331 case RTE_FLOW_ITEM_TYPE_GRE: 332 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 333 ret = mlx5_ethertype_to_item_type(spec, mask, true); 334 break; 335 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 336 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 337 ret = mlx5_nsh_proto_to_item_type(spec, mask); 338 break; 339 default: 340 ret = RTE_FLOW_ITEM_TYPE_VOID; 341 break; 342 } 343 return ret; 344 #undef MLX5_XSET_ITEM_MASK_SPEC 345 } 346 347 static const int * 348 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 349 const int *next_node) 350 { 351 const struct mlx5_flow_expand_node *node = NULL; 352 const int *next = next_node; 353 354 while (next && *next) { 355 /* 356 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 357 * flag set, because they were not found in the flow pattern. 358 */ 359 node = &graph[*next]; 360 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 361 break; 362 next = node->next; 363 } 364 return next; 365 } 366 367 #define MLX5_RSS_EXP_ELT_N 16 368 369 /** 370 * Expand RSS flows into several possible flows according to the RSS hash 371 * fields requested and the driver capabilities. 372 * 373 * @param[out] buf 374 * Buffer to store the result expansion. 375 * @param[in] size 376 * Buffer size in bytes. If 0, @p buf can be NULL. 377 * @param[in] pattern 378 * User flow pattern. 379 * @param[in] types 380 * RSS types to expand (see RTE_ETH_RSS_* definitions). 381 * @param[in] graph 382 * Input graph to expand @p pattern according to @p types. 383 * @param[in] graph_root_index 384 * Index of root node in @p graph, typically 0. 385 * 386 * @return 387 * A positive value representing the size of @p buf in bytes regardless of 388 * @p size on success, a negative errno value otherwise and rte_errno is 389 * set, the following errors are defined: 390 * 391 * -E2BIG: graph-depth @p graph is too deep. 392 * -EINVAL: @p size has not enough space for expanded pattern. 393 */ 394 static int 395 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 396 const struct rte_flow_item *pattern, uint64_t types, 397 const struct mlx5_flow_expand_node graph[], 398 int graph_root_index) 399 { 400 const struct rte_flow_item *item; 401 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 402 const int *next_node; 403 const int *stack[MLX5_RSS_EXP_ELT_N]; 404 int stack_pos = 0; 405 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 406 unsigned int i, item_idx, last_expand_item_idx = 0; 407 size_t lsize; 408 size_t user_pattern_size = 0; 409 void *addr = NULL; 410 const struct mlx5_flow_expand_node *next = NULL; 411 struct rte_flow_item missed_item; 412 int missed = 0; 413 int elt = 0; 414 const struct rte_flow_item *last_expand_item = NULL; 415 416 memset(&missed_item, 0, sizeof(missed_item)); 417 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 418 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 419 if (lsize > size) 420 return -EINVAL; 421 buf->entry[0].priority = 0; 422 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 423 buf->entries = 0; 424 addr = buf->entry[0].pattern; 425 for (item = pattern, item_idx = 0; 426 item->type != RTE_FLOW_ITEM_TYPE_END; 427 item++, item_idx++) { 428 if (!mlx5_flow_is_rss_expandable_item(item)) { 429 user_pattern_size += sizeof(*item); 430 continue; 431 } 432 last_expand_item = item; 433 last_expand_item_idx = item_idx; 434 i = 0; 435 while (node->next && node->next[i]) { 436 next = &graph[node->next[i]]; 437 if (next->type == item->type) 438 break; 439 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 440 node = next; 441 i = 0; 442 } else { 443 ++i; 444 } 445 } 446 if (next) 447 node = next; 448 user_pattern_size += sizeof(*item); 449 } 450 user_pattern_size += sizeof(*item); /* Handle END item. */ 451 lsize += user_pattern_size; 452 if (lsize > size) 453 return -EINVAL; 454 /* Copy the user pattern in the first entry of the buffer. */ 455 rte_memcpy(addr, pattern, user_pattern_size); 456 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 457 buf->entries = 1; 458 /* Start expanding. */ 459 memset(flow_items, 0, sizeof(flow_items)); 460 user_pattern_size -= sizeof(*item); 461 /* 462 * Check if the last valid item has spec set, need complete pattern, 463 * and the pattern can be used for expansion. 464 */ 465 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 466 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 467 /* Item type END indicates expansion is not required. */ 468 return lsize; 469 } 470 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 471 next = NULL; 472 missed = 1; 473 i = 0; 474 while (node->next && node->next[i]) { 475 next = &graph[node->next[i]]; 476 if (next->type == missed_item.type) { 477 flow_items[0].type = missed_item.type; 478 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 479 break; 480 } 481 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 482 node = next; 483 i = 0; 484 } else { 485 ++i; 486 } 487 next = NULL; 488 } 489 } 490 if (next && missed) { 491 elt = 2; /* missed item + item end. */ 492 node = next; 493 lsize += elt * sizeof(*item) + user_pattern_size; 494 if (lsize > size) 495 return -EINVAL; 496 if (node->rss_types & types) { 497 buf->entry[buf->entries].priority = 1; 498 buf->entry[buf->entries].pattern = addr; 499 buf->entries++; 500 rte_memcpy(addr, buf->entry[0].pattern, 501 user_pattern_size); 502 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 503 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 504 addr = (void *)(((uintptr_t)addr) + 505 elt * sizeof(*item)); 506 } 507 } else if (last_expand_item != NULL) { 508 node = mlx5_flow_expand_rss_adjust_node(pattern, 509 last_expand_item_idx, graph, node); 510 } 511 memset(flow_items, 0, sizeof(flow_items)); 512 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 513 node->next); 514 stack[stack_pos] = next_node; 515 node = next_node ? &graph[*next_node] : NULL; 516 while (node) { 517 flow_items[stack_pos].type = node->type; 518 if (node->rss_types & types) { 519 size_t n; 520 /* 521 * compute the number of items to copy from the 522 * expansion and copy it. 523 * When the stack_pos is 0, there are 1 element in it, 524 * plus the addition END item. 525 */ 526 elt = stack_pos + 2; 527 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 528 lsize += elt * sizeof(*item) + user_pattern_size; 529 if (lsize > size) 530 return -EINVAL; 531 n = elt * sizeof(*item); 532 buf->entry[buf->entries].priority = 533 stack_pos + 1 + missed; 534 buf->entry[buf->entries].pattern = addr; 535 buf->entries++; 536 rte_memcpy(addr, buf->entry[0].pattern, 537 user_pattern_size); 538 addr = (void *)(((uintptr_t)addr) + 539 user_pattern_size); 540 rte_memcpy(addr, &missed_item, 541 missed * sizeof(*item)); 542 addr = (void *)(((uintptr_t)addr) + 543 missed * sizeof(*item)); 544 rte_memcpy(addr, flow_items, n); 545 addr = (void *)(((uintptr_t)addr) + n); 546 } 547 /* Go deeper. */ 548 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 549 node->next) { 550 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 551 node->next); 552 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 553 rte_errno = E2BIG; 554 return -rte_errno; 555 } 556 stack[stack_pos] = next_node; 557 } else if (*(next_node + 1)) { 558 /* Follow up with the next possibility. */ 559 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 560 ++next_node); 561 } else if (!stack_pos) { 562 /* 563 * Completing the traverse over the different paths. 564 * The next_node is advanced to the terminator. 565 */ 566 ++next_node; 567 } else { 568 /* Move to the next path. */ 569 while (stack_pos) { 570 next_node = stack[--stack_pos]; 571 next_node++; 572 if (*next_node) 573 break; 574 } 575 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 576 next_node); 577 stack[stack_pos] = next_node; 578 } 579 node = next_node && *next_node ? &graph[*next_node] : NULL; 580 }; 581 return lsize; 582 } 583 584 /** 585 * Expand SQN flows into several possible flows according to the Tx queue 586 * number 587 * 588 * @param[in] buf 589 * Buffer to store the result expansion. 590 * @param[in] size 591 * Buffer size in bytes. If 0, @p buf can be NULL. 592 * @param[in] pattern 593 * User flow pattern. 594 * @param[in] sq_specs 595 * Buffer to store sq spec. 596 * 597 * @return 598 * 0 for success and negative value for failure 599 * 600 */ 601 static int 602 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 603 const struct rte_flow_item *pattern, 604 struct mlx5_rte_flow_item_sq *sq_specs) 605 { 606 const struct rte_flow_item *item; 607 bool port_representor = false; 608 size_t user_pattern_size = 0; 609 struct rte_eth_dev *dev; 610 struct mlx5_priv *priv; 611 void *addr = NULL; 612 uint16_t port_id; 613 size_t lsize; 614 int elt = 2; 615 uint16_t i; 616 617 buf->entries = 0; 618 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 619 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 620 const struct rte_flow_item_ethdev *pid_v = item->spec; 621 622 if (!pid_v) 623 return 0; 624 port_id = pid_v->port_id; 625 port_representor = true; 626 } 627 user_pattern_size += sizeof(*item); 628 } 629 if (!port_representor) 630 return 0; 631 dev = &rte_eth_devices[port_id]; 632 priv = dev->data->dev_private; 633 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 634 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 635 sizeof(buf->entry[0]) * priv->txqs_n; 636 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 637 return -EINVAL; 638 addr = buf->entry[0].pattern; 639 for (i = 0; i != priv->txqs_n; ++i) { 640 struct rte_flow_item pattern_add[] = { 641 { 642 .type = (enum rte_flow_item_type) 643 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 644 .spec = &sq_specs[i], 645 }, 646 { 647 .type = RTE_FLOW_ITEM_TYPE_END, 648 }, 649 }; 650 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 651 652 if (txq == NULL) 653 return -EINVAL; 654 buf->entry[i].pattern = addr; 655 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 656 mlx5_txq_release(dev, i); 657 rte_memcpy(addr, pattern, user_pattern_size); 658 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 659 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 660 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 661 buf->entries++; 662 } 663 return 0; 664 } 665 666 enum mlx5_expansion { 667 MLX5_EXPANSION_ROOT, 668 MLX5_EXPANSION_ROOT_OUTER, 669 MLX5_EXPANSION_OUTER_ETH, 670 MLX5_EXPANSION_OUTER_VLAN, 671 MLX5_EXPANSION_OUTER_IPV4, 672 MLX5_EXPANSION_OUTER_IPV4_UDP, 673 MLX5_EXPANSION_OUTER_IPV4_TCP, 674 MLX5_EXPANSION_OUTER_IPV4_ESP, 675 MLX5_EXPANSION_OUTER_IPV4_ICMP, 676 MLX5_EXPANSION_OUTER_IPV6, 677 MLX5_EXPANSION_OUTER_IPV6_UDP, 678 MLX5_EXPANSION_OUTER_IPV6_TCP, 679 MLX5_EXPANSION_OUTER_IPV6_ESP, 680 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 681 MLX5_EXPANSION_VXLAN, 682 MLX5_EXPANSION_STD_VXLAN, 683 MLX5_EXPANSION_L3_VXLAN, 684 MLX5_EXPANSION_VXLAN_GPE, 685 MLX5_EXPANSION_GRE, 686 MLX5_EXPANSION_NVGRE, 687 MLX5_EXPANSION_GRE_KEY, 688 MLX5_EXPANSION_MPLS, 689 MLX5_EXPANSION_ETH, 690 MLX5_EXPANSION_VLAN, 691 MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV4_UDP, 693 MLX5_EXPANSION_IPV4_TCP, 694 MLX5_EXPANSION_IPV4_ESP, 695 MLX5_EXPANSION_IPV4_ICMP, 696 MLX5_EXPANSION_IPV6, 697 MLX5_EXPANSION_IPV6_UDP, 698 MLX5_EXPANSION_IPV6_TCP, 699 MLX5_EXPANSION_IPV6_ESP, 700 MLX5_EXPANSION_IPV6_ICMP6, 701 MLX5_EXPANSION_IPV6_FRAG_EXT, 702 MLX5_EXPANSION_GTP, 703 MLX5_EXPANSION_GENEVE, 704 }; 705 706 /** Supported expansion of items. */ 707 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 708 [MLX5_EXPANSION_ROOT] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 710 MLX5_EXPANSION_IPV4, 711 MLX5_EXPANSION_IPV6), 712 .type = RTE_FLOW_ITEM_TYPE_END, 713 }, 714 [MLX5_EXPANSION_ROOT_OUTER] = { 715 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 716 MLX5_EXPANSION_OUTER_IPV4, 717 MLX5_EXPANSION_OUTER_IPV6), 718 .type = RTE_FLOW_ITEM_TYPE_END, 719 }, 720 [MLX5_EXPANSION_OUTER_ETH] = { 721 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 722 .type = RTE_FLOW_ITEM_TYPE_ETH, 723 .rss_types = 0, 724 }, 725 [MLX5_EXPANSION_OUTER_VLAN] = { 726 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 727 MLX5_EXPANSION_OUTER_IPV6), 728 .type = RTE_FLOW_ITEM_TYPE_VLAN, 729 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 730 }, 731 [MLX5_EXPANSION_OUTER_IPV4] = { 732 .next = MLX5_FLOW_EXPAND_RSS_NEXT 733 (MLX5_EXPANSION_OUTER_IPV4_UDP, 734 MLX5_EXPANSION_OUTER_IPV4_TCP, 735 MLX5_EXPANSION_OUTER_IPV4_ESP, 736 MLX5_EXPANSION_OUTER_IPV4_ICMP, 737 MLX5_EXPANSION_GRE, 738 MLX5_EXPANSION_NVGRE, 739 MLX5_EXPANSION_IPV4, 740 MLX5_EXPANSION_IPV6), 741 .type = RTE_FLOW_ITEM_TYPE_IPV4, 742 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 743 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 744 }, 745 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 746 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 747 MLX5_EXPANSION_VXLAN_GPE, 748 MLX5_EXPANSION_MPLS, 749 MLX5_EXPANSION_GENEVE, 750 MLX5_EXPANSION_GTP), 751 .type = RTE_FLOW_ITEM_TYPE_UDP, 752 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 753 }, 754 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 755 .type = RTE_FLOW_ITEM_TYPE_TCP, 756 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 757 }, 758 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 759 .type = RTE_FLOW_ITEM_TYPE_ESP, 760 .rss_types = RTE_ETH_RSS_ESP, 761 }, 762 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 763 .type = RTE_FLOW_ITEM_TYPE_ICMP, 764 }, 765 [MLX5_EXPANSION_OUTER_IPV6] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT 767 (MLX5_EXPANSION_OUTER_IPV6_UDP, 768 MLX5_EXPANSION_OUTER_IPV6_TCP, 769 MLX5_EXPANSION_OUTER_IPV6_ESP, 770 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 771 MLX5_EXPANSION_IPV4, 772 MLX5_EXPANSION_IPV6, 773 MLX5_EXPANSION_GRE, 774 MLX5_EXPANSION_NVGRE), 775 .type = RTE_FLOW_ITEM_TYPE_IPV6, 776 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 777 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 778 }, 779 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 780 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 781 MLX5_EXPANSION_VXLAN_GPE, 782 MLX5_EXPANSION_MPLS, 783 MLX5_EXPANSION_GENEVE, 784 MLX5_EXPANSION_GTP), 785 .type = RTE_FLOW_ITEM_TYPE_UDP, 786 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 787 }, 788 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 789 .type = RTE_FLOW_ITEM_TYPE_TCP, 790 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 791 }, 792 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 793 .type = RTE_FLOW_ITEM_TYPE_ESP, 794 .rss_types = RTE_ETH_RSS_ESP, 795 }, 796 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 797 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 798 }, 799 [MLX5_EXPANSION_VXLAN] = { 800 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 801 MLX5_EXPANSION_IPV4, 802 MLX5_EXPANSION_IPV6), 803 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 804 }, 805 [MLX5_EXPANSION_STD_VXLAN] = { 806 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 807 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 808 }, 809 [MLX5_EXPANSION_L3_VXLAN] = { 810 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 811 MLX5_EXPANSION_IPV6), 812 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 813 }, 814 [MLX5_EXPANSION_VXLAN_GPE] = { 815 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 816 MLX5_EXPANSION_IPV4, 817 MLX5_EXPANSION_IPV6), 818 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 819 }, 820 [MLX5_EXPANSION_GRE] = { 821 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 822 MLX5_EXPANSION_IPV4, 823 MLX5_EXPANSION_IPV6, 824 MLX5_EXPANSION_GRE_KEY, 825 MLX5_EXPANSION_MPLS), 826 .type = RTE_FLOW_ITEM_TYPE_GRE, 827 }, 828 [MLX5_EXPANSION_GRE_KEY] = { 829 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 830 MLX5_EXPANSION_IPV6, 831 MLX5_EXPANSION_MPLS), 832 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 833 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 834 }, 835 [MLX5_EXPANSION_NVGRE] = { 836 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 837 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 838 }, 839 [MLX5_EXPANSION_MPLS] = { 840 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 841 MLX5_EXPANSION_IPV6, 842 MLX5_EXPANSION_ETH), 843 .type = RTE_FLOW_ITEM_TYPE_MPLS, 844 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 845 }, 846 [MLX5_EXPANSION_ETH] = { 847 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 848 .type = RTE_FLOW_ITEM_TYPE_ETH, 849 }, 850 [MLX5_EXPANSION_VLAN] = { 851 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 852 MLX5_EXPANSION_IPV6), 853 .type = RTE_FLOW_ITEM_TYPE_VLAN, 854 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 855 }, 856 [MLX5_EXPANSION_IPV4] = { 857 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 858 MLX5_EXPANSION_IPV4_TCP, 859 MLX5_EXPANSION_IPV4_ESP, 860 MLX5_EXPANSION_IPV4_ICMP), 861 .type = RTE_FLOW_ITEM_TYPE_IPV4, 862 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 863 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 864 }, 865 [MLX5_EXPANSION_IPV4_UDP] = { 866 .type = RTE_FLOW_ITEM_TYPE_UDP, 867 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 868 }, 869 [MLX5_EXPANSION_IPV4_TCP] = { 870 .type = RTE_FLOW_ITEM_TYPE_TCP, 871 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 872 }, 873 [MLX5_EXPANSION_IPV4_ESP] = { 874 .type = RTE_FLOW_ITEM_TYPE_ESP, 875 .rss_types = RTE_ETH_RSS_ESP, 876 }, 877 [MLX5_EXPANSION_IPV4_ICMP] = { 878 .type = RTE_FLOW_ITEM_TYPE_ICMP, 879 }, 880 [MLX5_EXPANSION_IPV6] = { 881 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 882 MLX5_EXPANSION_IPV6_TCP, 883 MLX5_EXPANSION_IPV6_ESP, 884 MLX5_EXPANSION_IPV6_ICMP6, 885 MLX5_EXPANSION_IPV6_FRAG_EXT), 886 .type = RTE_FLOW_ITEM_TYPE_IPV6, 887 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 888 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 889 }, 890 [MLX5_EXPANSION_IPV6_UDP] = { 891 .type = RTE_FLOW_ITEM_TYPE_UDP, 892 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 893 }, 894 [MLX5_EXPANSION_IPV6_TCP] = { 895 .type = RTE_FLOW_ITEM_TYPE_TCP, 896 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 897 }, 898 [MLX5_EXPANSION_IPV6_ESP] = { 899 .type = RTE_FLOW_ITEM_TYPE_ESP, 900 .rss_types = RTE_ETH_RSS_ESP, 901 }, 902 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 903 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 904 }, 905 [MLX5_EXPANSION_IPV6_ICMP6] = { 906 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 907 }, 908 [MLX5_EXPANSION_GTP] = { 909 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 910 MLX5_EXPANSION_IPV6), 911 .type = RTE_FLOW_ITEM_TYPE_GTP, 912 }, 913 [MLX5_EXPANSION_GENEVE] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 915 MLX5_EXPANSION_IPV4, 916 MLX5_EXPANSION_IPV6), 917 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 918 }, 919 }; 920 921 static struct rte_flow_action_handle * 922 mlx5_action_handle_create(struct rte_eth_dev *dev, 923 const struct rte_flow_indir_action_conf *conf, 924 const struct rte_flow_action *action, 925 struct rte_flow_error *error); 926 static int mlx5_action_handle_destroy 927 (struct rte_eth_dev *dev, 928 struct rte_flow_action_handle *handle, 929 struct rte_flow_error *error); 930 static int mlx5_action_handle_update 931 (struct rte_eth_dev *dev, 932 struct rte_flow_action_handle *handle, 933 const void *update, 934 struct rte_flow_error *error); 935 static int mlx5_action_handle_query 936 (struct rte_eth_dev *dev, 937 const struct rte_flow_action_handle *handle, 938 void *data, 939 struct rte_flow_error *error); 940 static int 941 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 942 struct rte_flow_tunnel *app_tunnel, 943 struct rte_flow_action **actions, 944 uint32_t *num_of_actions, 945 struct rte_flow_error *error); 946 static int 947 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 948 struct rte_flow_tunnel *app_tunnel, 949 struct rte_flow_item **items, 950 uint32_t *num_of_items, 951 struct rte_flow_error *error); 952 static int 953 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 954 struct rte_flow_item *pmd_items, 955 uint32_t num_items, struct rte_flow_error *err); 956 static int 957 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 958 struct rte_flow_action *pmd_actions, 959 uint32_t num_actions, 960 struct rte_flow_error *err); 961 static int 962 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 963 struct rte_mbuf *m, 964 struct rte_flow_restore_info *info, 965 struct rte_flow_error *err); 966 static struct rte_flow_item_flex_handle * 967 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 968 const struct rte_flow_item_flex_conf *conf, 969 struct rte_flow_error *error); 970 static int 971 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 972 const struct rte_flow_item_flex_handle *handle, 973 struct rte_flow_error *error); 974 static int 975 mlx5_flow_info_get(struct rte_eth_dev *dev, 976 struct rte_flow_port_info *port_info, 977 struct rte_flow_queue_info *queue_info, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_port_configure(struct rte_eth_dev *dev, 981 const struct rte_flow_port_attr *port_attr, 982 uint16_t nb_queue, 983 const struct rte_flow_queue_attr *queue_attr[], 984 struct rte_flow_error *err); 985 986 static struct rte_flow_pattern_template * 987 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 988 const struct rte_flow_pattern_template_attr *attr, 989 const struct rte_flow_item items[], 990 struct rte_flow_error *error); 991 992 static int 993 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 994 struct rte_flow_pattern_template *template, 995 struct rte_flow_error *error); 996 static struct rte_flow_actions_template * 997 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 998 const struct rte_flow_actions_template_attr *attr, 999 const struct rte_flow_action actions[], 1000 const struct rte_flow_action masks[], 1001 struct rte_flow_error *error); 1002 static int 1003 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1004 struct rte_flow_actions_template *template, 1005 struct rte_flow_error *error); 1006 1007 static struct rte_flow_template_table * 1008 mlx5_flow_table_create(struct rte_eth_dev *dev, 1009 const struct rte_flow_template_table_attr *attr, 1010 struct rte_flow_pattern_template *item_templates[], 1011 uint8_t nb_item_templates, 1012 struct rte_flow_actions_template *action_templates[], 1013 uint8_t nb_action_templates, 1014 struct rte_flow_error *error); 1015 static int 1016 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1017 struct rte_flow_template_table *table, 1018 struct rte_flow_error *error); 1019 static struct rte_flow * 1020 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1021 uint32_t queue, 1022 const struct rte_flow_op_attr *attr, 1023 struct rte_flow_template_table *table, 1024 const struct rte_flow_item items[], 1025 uint8_t pattern_template_index, 1026 const struct rte_flow_action actions[], 1027 uint8_t action_template_index, 1028 void *user_data, 1029 struct rte_flow_error *error); 1030 static int 1031 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1032 uint32_t queue, 1033 const struct rte_flow_op_attr *attr, 1034 struct rte_flow *flow, 1035 void *user_data, 1036 struct rte_flow_error *error); 1037 static int 1038 mlx5_flow_pull(struct rte_eth_dev *dev, 1039 uint32_t queue, 1040 struct rte_flow_op_result res[], 1041 uint16_t n_res, 1042 struct rte_flow_error *error); 1043 static int 1044 mlx5_flow_push(struct rte_eth_dev *dev, 1045 uint32_t queue, 1046 struct rte_flow_error *error); 1047 1048 static struct rte_flow_action_handle * 1049 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1050 const struct rte_flow_op_attr *attr, 1051 const struct rte_flow_indir_action_conf *conf, 1052 const struct rte_flow_action *action, 1053 void *user_data, 1054 struct rte_flow_error *error); 1055 1056 static int 1057 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1058 const struct rte_flow_op_attr *attr, 1059 struct rte_flow_action_handle *handle, 1060 const void *update, 1061 void *user_data, 1062 struct rte_flow_error *error); 1063 1064 static int 1065 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1066 const struct rte_flow_op_attr *attr, 1067 struct rte_flow_action_handle *handle, 1068 void *user_data, 1069 struct rte_flow_error *error); 1070 1071 static int 1072 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1073 const struct rte_flow_op_attr *attr, 1074 const struct rte_flow_action_handle *handle, 1075 void *data, 1076 void *user_data, 1077 struct rte_flow_error *error); 1078 1079 static const struct rte_flow_ops mlx5_flow_ops = { 1080 .validate = mlx5_flow_validate, 1081 .create = mlx5_flow_create, 1082 .destroy = mlx5_flow_destroy, 1083 .flush = mlx5_flow_flush, 1084 .isolate = mlx5_flow_isolate, 1085 .query = mlx5_flow_query, 1086 .dev_dump = mlx5_flow_dev_dump, 1087 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1088 .get_aged_flows = mlx5_flow_get_aged_flows, 1089 .action_handle_create = mlx5_action_handle_create, 1090 .action_handle_destroy = mlx5_action_handle_destroy, 1091 .action_handle_update = mlx5_action_handle_update, 1092 .action_handle_query = mlx5_action_handle_query, 1093 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1094 .tunnel_match = mlx5_flow_tunnel_match, 1095 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1096 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1097 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1098 .flex_item_create = mlx5_flow_flex_item_create, 1099 .flex_item_release = mlx5_flow_flex_item_release, 1100 .info_get = mlx5_flow_info_get, 1101 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1102 .configure = mlx5_flow_port_configure, 1103 .pattern_template_create = mlx5_flow_pattern_template_create, 1104 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1105 .actions_template_create = mlx5_flow_actions_template_create, 1106 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1107 .template_table_create = mlx5_flow_table_create, 1108 .template_table_destroy = mlx5_flow_table_destroy, 1109 .async_create = mlx5_flow_async_flow_create, 1110 .async_destroy = mlx5_flow_async_flow_destroy, 1111 .pull = mlx5_flow_pull, 1112 .push = mlx5_flow_push, 1113 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1114 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1115 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1116 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1117 }; 1118 1119 /* Tunnel information. */ 1120 struct mlx5_flow_tunnel_info { 1121 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1122 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1123 }; 1124 1125 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1126 { 1127 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1128 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1129 }, 1130 { 1131 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1132 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1133 }, 1134 { 1135 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1136 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1137 }, 1138 { 1139 .tunnel = MLX5_FLOW_LAYER_GRE, 1140 .ptype = RTE_PTYPE_TUNNEL_GRE, 1141 }, 1142 { 1143 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1144 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1145 }, 1146 { 1147 .tunnel = MLX5_FLOW_LAYER_MPLS, 1148 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1149 }, 1150 { 1151 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1152 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1153 }, 1154 { 1155 .tunnel = MLX5_FLOW_LAYER_IPIP, 1156 .ptype = RTE_PTYPE_TUNNEL_IP, 1157 }, 1158 { 1159 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1160 .ptype = RTE_PTYPE_TUNNEL_IP, 1161 }, 1162 { 1163 .tunnel = MLX5_FLOW_LAYER_GTP, 1164 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1165 }, 1166 }; 1167 1168 1169 1170 /** 1171 * Translate tag ID to register. 1172 * 1173 * @param[in] dev 1174 * Pointer to the Ethernet device structure. 1175 * @param[in] feature 1176 * The feature that request the register. 1177 * @param[in] id 1178 * The request register ID. 1179 * @param[out] error 1180 * Error description in case of any. 1181 * 1182 * @return 1183 * The request register on success, a negative errno 1184 * value otherwise and rte_errno is set. 1185 */ 1186 int 1187 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1188 enum mlx5_feature_name feature, 1189 uint32_t id, 1190 struct rte_flow_error *error) 1191 { 1192 struct mlx5_priv *priv = dev->data->dev_private; 1193 struct mlx5_sh_config *config = &priv->sh->config; 1194 enum modify_reg start_reg; 1195 bool skip_mtr_reg = false; 1196 1197 switch (feature) { 1198 case MLX5_HAIRPIN_RX: 1199 return REG_B; 1200 case MLX5_HAIRPIN_TX: 1201 return REG_A; 1202 case MLX5_METADATA_RX: 1203 switch (config->dv_xmeta_en) { 1204 case MLX5_XMETA_MODE_LEGACY: 1205 return REG_B; 1206 case MLX5_XMETA_MODE_META16: 1207 return REG_C_0; 1208 case MLX5_XMETA_MODE_META32: 1209 return REG_C_1; 1210 case MLX5_XMETA_MODE_META32_HWS: 1211 return REG_C_1; 1212 } 1213 break; 1214 case MLX5_METADATA_TX: 1215 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1216 return REG_C_1; 1217 } else { 1218 return REG_A; 1219 } 1220 case MLX5_METADATA_FDB: 1221 switch (config->dv_xmeta_en) { 1222 case MLX5_XMETA_MODE_LEGACY: 1223 return REG_NON; 1224 case MLX5_XMETA_MODE_META16: 1225 return REG_C_0; 1226 case MLX5_XMETA_MODE_META32: 1227 return REG_C_1; 1228 case MLX5_XMETA_MODE_META32_HWS: 1229 return REG_C_1; 1230 } 1231 break; 1232 case MLX5_FLOW_MARK: 1233 switch (config->dv_xmeta_en) { 1234 case MLX5_XMETA_MODE_LEGACY: 1235 case MLX5_XMETA_MODE_META32_HWS: 1236 return REG_NON; 1237 case MLX5_XMETA_MODE_META16: 1238 return REG_C_1; 1239 case MLX5_XMETA_MODE_META32: 1240 return REG_C_0; 1241 } 1242 break; 1243 case MLX5_MTR_ID: 1244 /* 1245 * If meter color and meter id share one register, flow match 1246 * should use the meter color register for match. 1247 */ 1248 if (priv->mtr_reg_share) 1249 return priv->mtr_color_reg; 1250 else 1251 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1252 REG_C_3; 1253 case MLX5_MTR_COLOR: 1254 case MLX5_ASO_FLOW_HIT: 1255 case MLX5_ASO_CONNTRACK: 1256 case MLX5_SAMPLE_ID: 1257 /* All features use the same REG_C. */ 1258 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1259 return priv->mtr_color_reg; 1260 case MLX5_COPY_MARK: 1261 /* 1262 * Metadata COPY_MARK register using is in meter suffix sub 1263 * flow while with meter. It's safe to share the same register. 1264 */ 1265 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1266 case MLX5_APP_TAG: 1267 /* 1268 * If meter is enable, it will engage the register for color 1269 * match and flow match. If meter color match is not using the 1270 * REG_C_2, need to skip the REG_C_x be used by meter color 1271 * match. 1272 * If meter is disable, free to use all available registers. 1273 */ 1274 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1275 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1276 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1277 if (id > (uint32_t)(REG_C_7 - start_reg)) 1278 return rte_flow_error_set(error, EINVAL, 1279 RTE_FLOW_ERROR_TYPE_ITEM, 1280 NULL, "invalid tag id"); 1281 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1282 return rte_flow_error_set(error, ENOTSUP, 1283 RTE_FLOW_ERROR_TYPE_ITEM, 1284 NULL, "unsupported tag id"); 1285 /* 1286 * This case means meter is using the REG_C_x great than 2. 1287 * Take care not to conflict with meter color REG_C_x. 1288 * If the available index REG_C_y >= REG_C_x, skip the 1289 * color register. 1290 */ 1291 if (skip_mtr_reg && priv->sh->flow_mreg_c 1292 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1293 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1294 return rte_flow_error_set(error, EINVAL, 1295 RTE_FLOW_ERROR_TYPE_ITEM, 1296 NULL, "invalid tag id"); 1297 if (priv->sh->flow_mreg_c 1298 [id + 1 + start_reg - REG_C_0] != REG_NON) 1299 return priv->sh->flow_mreg_c 1300 [id + 1 + start_reg - REG_C_0]; 1301 return rte_flow_error_set(error, ENOTSUP, 1302 RTE_FLOW_ERROR_TYPE_ITEM, 1303 NULL, "unsupported tag id"); 1304 } 1305 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1306 } 1307 MLX5_ASSERT(false); 1308 return rte_flow_error_set(error, EINVAL, 1309 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1310 NULL, "invalid feature name"); 1311 } 1312 1313 /** 1314 * Check extensive flow metadata register support. 1315 * 1316 * @param dev 1317 * Pointer to rte_eth_dev structure. 1318 * 1319 * @return 1320 * True if device supports extensive flow metadata register, otherwise false. 1321 */ 1322 bool 1323 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1324 { 1325 struct mlx5_priv *priv = dev->data->dev_private; 1326 1327 /* 1328 * Having available reg_c can be regarded inclusively as supporting 1329 * extensive flow metadata register, which could mean, 1330 * - metadata register copy action by modify header. 1331 * - 16 modify header actions is supported. 1332 * - reg_c's are preserved across different domain (FDB and NIC) on 1333 * packet loopback by flow lookup miss. 1334 */ 1335 return priv->sh->flow_mreg_c[2] != REG_NON; 1336 } 1337 1338 /** 1339 * Get the lowest priority. 1340 * 1341 * @param[in] dev 1342 * Pointer to the Ethernet device structure. 1343 * @param[in] attributes 1344 * Pointer to device flow rule attributes. 1345 * 1346 * @return 1347 * The value of lowest priority of flow. 1348 */ 1349 uint32_t 1350 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1351 const struct rte_flow_attr *attr) 1352 { 1353 struct mlx5_priv *priv = dev->data->dev_private; 1354 1355 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1356 return priv->sh->flow_max_priority - 2; 1357 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1358 } 1359 1360 /** 1361 * Calculate matcher priority of the flow. 1362 * 1363 * @param[in] dev 1364 * Pointer to the Ethernet device structure. 1365 * @param[in] attr 1366 * Pointer to device flow rule attributes. 1367 * @param[in] subpriority 1368 * The priority based on the items. 1369 * @param[in] external 1370 * Flow is user flow. 1371 * @return 1372 * The matcher priority of the flow. 1373 */ 1374 uint16_t 1375 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1376 const struct rte_flow_attr *attr, 1377 uint32_t subpriority, bool external) 1378 { 1379 uint16_t priority = (uint16_t)attr->priority; 1380 struct mlx5_priv *priv = dev->data->dev_private; 1381 1382 /* NIC root rules */ 1383 if (!attr->group && !attr->transfer) { 1384 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1385 priority = priv->sh->flow_max_priority - 1; 1386 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1387 /* FDB root rules */ 1388 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1389 attr->group == 0 && 1390 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1391 return (priv->sh->flow_max_priority - 1) * 3; 1392 } 1393 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1394 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1395 return priority * 3 + subpriority; 1396 } 1397 1398 /** 1399 * Verify the @p item specifications (spec, last, mask) are compatible with the 1400 * NIC capabilities. 1401 * 1402 * @param[in] item 1403 * Item specification. 1404 * @param[in] mask 1405 * @p item->mask or flow default bit-masks. 1406 * @param[in] nic_mask 1407 * Bit-masks covering supported fields by the NIC to compare with user mask. 1408 * @param[in] size 1409 * Bit-masks size in bytes. 1410 * @param[in] range_accepted 1411 * True if range of values is accepted for specific fields, false otherwise. 1412 * @param[out] error 1413 * Pointer to error structure. 1414 * 1415 * @return 1416 * 0 on success, a negative errno value otherwise and rte_errno is set. 1417 */ 1418 int 1419 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1420 const uint8_t *mask, 1421 const uint8_t *nic_mask, 1422 unsigned int size, 1423 bool range_accepted, 1424 struct rte_flow_error *error) 1425 { 1426 unsigned int i; 1427 1428 MLX5_ASSERT(nic_mask); 1429 for (i = 0; i < size; ++i) 1430 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1431 return rte_flow_error_set(error, ENOTSUP, 1432 RTE_FLOW_ERROR_TYPE_ITEM, 1433 item, 1434 "mask enables non supported" 1435 " bits"); 1436 if (!item->spec && (item->mask || item->last)) 1437 return rte_flow_error_set(error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ITEM, item, 1439 "mask/last without a spec is not" 1440 " supported"); 1441 if (item->spec && item->last && !range_accepted) { 1442 uint8_t spec[size]; 1443 uint8_t last[size]; 1444 unsigned int i; 1445 int ret; 1446 1447 for (i = 0; i < size; ++i) { 1448 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1449 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1450 } 1451 ret = memcmp(spec, last, size); 1452 if (ret != 0) 1453 return rte_flow_error_set(error, EINVAL, 1454 RTE_FLOW_ERROR_TYPE_ITEM, 1455 item, 1456 "range is not valid"); 1457 } 1458 return 0; 1459 } 1460 1461 /** 1462 * Adjust the hash fields according to the @p flow information. 1463 * 1464 * @param[in] dev_flow. 1465 * Pointer to the mlx5_flow. 1466 * @param[in] tunnel 1467 * 1 when the hash field is for a tunnel item. 1468 * @param[in] layer_types 1469 * RTE_ETH_RSS_* types. 1470 * @param[in] hash_fields 1471 * Item hash fields. 1472 * 1473 * @return 1474 * The hash fields that should be used. 1475 */ 1476 uint64_t 1477 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1478 int tunnel __rte_unused, uint64_t layer_types, 1479 uint64_t hash_fields) 1480 { 1481 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1482 int rss_request_inner = rss_desc->level >= 2; 1483 1484 /* Check RSS hash level for tunnel. */ 1485 if (tunnel && rss_request_inner) 1486 hash_fields |= IBV_RX_HASH_INNER; 1487 else if (tunnel || rss_request_inner) 1488 return 0; 1489 #endif 1490 /* Check if requested layer matches RSS hash fields. */ 1491 if (!(rss_desc->types & layer_types)) 1492 return 0; 1493 return hash_fields; 1494 } 1495 1496 /** 1497 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1498 * if several tunnel rules are used on this queue, the tunnel ptype will be 1499 * cleared. 1500 * 1501 * @param rxq_ctrl 1502 * Rx queue to update. 1503 */ 1504 static void 1505 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1506 { 1507 unsigned int i; 1508 uint32_t tunnel_ptype = 0; 1509 1510 /* Look up for the ptype to use. */ 1511 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1512 if (!rxq_ctrl->flow_tunnels_n[i]) 1513 continue; 1514 if (!tunnel_ptype) { 1515 tunnel_ptype = tunnels_info[i].ptype; 1516 } else { 1517 tunnel_ptype = 0; 1518 break; 1519 } 1520 } 1521 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1522 } 1523 1524 /** 1525 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1526 * flow. 1527 * 1528 * @param[in] dev 1529 * Pointer to the Ethernet device structure. 1530 * @param[in] dev_handle 1531 * Pointer to device flow handle structure. 1532 */ 1533 void 1534 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1535 struct mlx5_flow_handle *dev_handle) 1536 { 1537 struct mlx5_priv *priv = dev->data->dev_private; 1538 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1539 struct mlx5_ind_table_obj *ind_tbl = NULL; 1540 unsigned int i; 1541 1542 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1543 struct mlx5_hrxq *hrxq; 1544 1545 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1546 dev_handle->rix_hrxq); 1547 if (hrxq) 1548 ind_tbl = hrxq->ind_table; 1549 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1550 struct mlx5_shared_action_rss *shared_rss; 1551 1552 shared_rss = mlx5_ipool_get 1553 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1554 dev_handle->rix_srss); 1555 if (shared_rss) 1556 ind_tbl = shared_rss->ind_tbl; 1557 } 1558 if (!ind_tbl) 1559 return; 1560 for (i = 0; i != ind_tbl->queues_n; ++i) { 1561 int idx = ind_tbl->queues[i]; 1562 struct mlx5_rxq_ctrl *rxq_ctrl; 1563 1564 if (mlx5_is_external_rxq(dev, idx)) 1565 continue; 1566 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1567 MLX5_ASSERT(rxq_ctrl != NULL); 1568 if (rxq_ctrl == NULL) 1569 continue; 1570 /* 1571 * To support metadata register copy on Tx loopback, 1572 * this must be always enabled (metadata may arive 1573 * from other port - not from local flows only. 1574 */ 1575 if (tunnel) { 1576 unsigned int j; 1577 1578 /* Increase the counter matching the flow. */ 1579 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1580 if ((tunnels_info[j].tunnel & 1581 dev_handle->layers) == 1582 tunnels_info[j].tunnel) { 1583 rxq_ctrl->flow_tunnels_n[j]++; 1584 break; 1585 } 1586 } 1587 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1588 } 1589 } 1590 } 1591 1592 static void 1593 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1594 { 1595 struct mlx5_priv *priv = dev->data->dev_private; 1596 struct mlx5_rxq_ctrl *rxq_ctrl; 1597 uint16_t port_id; 1598 1599 if (priv->sh->shared_mark_enabled) 1600 return; 1601 if (priv->master || priv->representor) { 1602 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1603 struct mlx5_priv *opriv = 1604 rte_eth_devices[port_id].data->dev_private; 1605 1606 if (!opriv || 1607 opriv->sh != priv->sh || 1608 opriv->domain_id != priv->domain_id || 1609 opriv->mark_enabled) 1610 continue; 1611 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1612 rxq_ctrl->rxq.mark = 1; 1613 } 1614 opriv->mark_enabled = 1; 1615 } 1616 } else { 1617 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1618 rxq_ctrl->rxq.mark = 1; 1619 } 1620 priv->mark_enabled = 1; 1621 } 1622 priv->sh->shared_mark_enabled = 1; 1623 } 1624 1625 /** 1626 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1627 * 1628 * @param[in] dev 1629 * Pointer to the Ethernet device structure. 1630 * @param[in] flow 1631 * Pointer to flow structure. 1632 */ 1633 static void 1634 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1635 { 1636 struct mlx5_priv *priv = dev->data->dev_private; 1637 uint32_t handle_idx; 1638 struct mlx5_flow_handle *dev_handle; 1639 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1640 1641 MLX5_ASSERT(wks); 1642 if (wks->mark) 1643 flow_rxq_mark_flag_set(dev); 1644 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1645 handle_idx, dev_handle, next) 1646 flow_drv_rxq_flags_set(dev, dev_handle); 1647 } 1648 1649 /** 1650 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1651 * device flow if no other flow uses it with the same kind of request. 1652 * 1653 * @param dev 1654 * Pointer to Ethernet device. 1655 * @param[in] dev_handle 1656 * Pointer to the device flow handle structure. 1657 */ 1658 static void 1659 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1660 struct mlx5_flow_handle *dev_handle) 1661 { 1662 struct mlx5_priv *priv = dev->data->dev_private; 1663 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1664 struct mlx5_ind_table_obj *ind_tbl = NULL; 1665 unsigned int i; 1666 1667 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1668 struct mlx5_hrxq *hrxq; 1669 1670 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1671 dev_handle->rix_hrxq); 1672 if (hrxq) 1673 ind_tbl = hrxq->ind_table; 1674 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1675 struct mlx5_shared_action_rss *shared_rss; 1676 1677 shared_rss = mlx5_ipool_get 1678 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1679 dev_handle->rix_srss); 1680 if (shared_rss) 1681 ind_tbl = shared_rss->ind_tbl; 1682 } 1683 if (!ind_tbl) 1684 return; 1685 MLX5_ASSERT(dev->data->dev_started); 1686 for (i = 0; i != ind_tbl->queues_n; ++i) { 1687 int idx = ind_tbl->queues[i]; 1688 struct mlx5_rxq_ctrl *rxq_ctrl; 1689 1690 if (mlx5_is_external_rxq(dev, idx)) 1691 continue; 1692 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1693 MLX5_ASSERT(rxq_ctrl != NULL); 1694 if (rxq_ctrl == NULL) 1695 continue; 1696 if (tunnel) { 1697 unsigned int j; 1698 1699 /* Decrease the counter matching the flow. */ 1700 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1701 if ((tunnels_info[j].tunnel & 1702 dev_handle->layers) == 1703 tunnels_info[j].tunnel) { 1704 rxq_ctrl->flow_tunnels_n[j]--; 1705 break; 1706 } 1707 } 1708 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1709 } 1710 } 1711 } 1712 1713 /** 1714 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1715 * @p flow if no other flow uses it with the same kind of request. 1716 * 1717 * @param dev 1718 * Pointer to Ethernet device. 1719 * @param[in] flow 1720 * Pointer to the flow. 1721 */ 1722 static void 1723 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1724 { 1725 struct mlx5_priv *priv = dev->data->dev_private; 1726 uint32_t handle_idx; 1727 struct mlx5_flow_handle *dev_handle; 1728 1729 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1730 handle_idx, dev_handle, next) 1731 flow_drv_rxq_flags_trim(dev, dev_handle); 1732 } 1733 1734 /** 1735 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1736 * 1737 * @param dev 1738 * Pointer to Ethernet device. 1739 */ 1740 static void 1741 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1742 { 1743 struct mlx5_priv *priv = dev->data->dev_private; 1744 unsigned int i; 1745 1746 for (i = 0; i != priv->rxqs_n; ++i) { 1747 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1748 unsigned int j; 1749 1750 if (rxq == NULL || rxq->ctrl == NULL) 1751 continue; 1752 rxq->ctrl->rxq.mark = 0; 1753 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1754 rxq->ctrl->flow_tunnels_n[j] = 0; 1755 rxq->ctrl->rxq.tunnel = 0; 1756 } 1757 priv->mark_enabled = 0; 1758 priv->sh->shared_mark_enabled = 0; 1759 } 1760 1761 /** 1762 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1763 * 1764 * @param[in] dev 1765 * Pointer to the Ethernet device structure. 1766 */ 1767 void 1768 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1769 { 1770 struct mlx5_priv *priv = dev->data->dev_private; 1771 unsigned int i; 1772 1773 for (i = 0; i != priv->rxqs_n; ++i) { 1774 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1775 struct mlx5_rxq_data *data; 1776 1777 if (rxq == NULL || rxq->ctrl == NULL) 1778 continue; 1779 data = &rxq->ctrl->rxq; 1780 if (!rte_flow_dynf_metadata_avail()) { 1781 data->dynf_meta = 0; 1782 data->flow_meta_mask = 0; 1783 data->flow_meta_offset = -1; 1784 data->flow_meta_port_mask = 0; 1785 } else { 1786 data->dynf_meta = 1; 1787 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1788 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1789 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1790 } 1791 } 1792 } 1793 1794 /* 1795 * return a pointer to the desired action in the list of actions. 1796 * 1797 * @param[in] actions 1798 * The list of actions to search the action in. 1799 * @param[in] action 1800 * The action to find. 1801 * 1802 * @return 1803 * Pointer to the action in the list, if found. NULL otherwise. 1804 */ 1805 const struct rte_flow_action * 1806 mlx5_flow_find_action(const struct rte_flow_action *actions, 1807 enum rte_flow_action_type action) 1808 { 1809 if (actions == NULL) 1810 return NULL; 1811 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1812 if (actions->type == action) 1813 return actions; 1814 return NULL; 1815 } 1816 1817 /* 1818 * Validate the flag action. 1819 * 1820 * @param[in] action_flags 1821 * Bit-fields that holds the actions detected until now. 1822 * @param[in] attr 1823 * Attributes of flow that includes this action. 1824 * @param[out] error 1825 * Pointer to error structure. 1826 * 1827 * @return 1828 * 0 on success, a negative errno value otherwise and rte_errno is set. 1829 */ 1830 int 1831 mlx5_flow_validate_action_flag(uint64_t action_flags, 1832 const struct rte_flow_attr *attr, 1833 struct rte_flow_error *error) 1834 { 1835 if (action_flags & MLX5_FLOW_ACTION_MARK) 1836 return rte_flow_error_set(error, EINVAL, 1837 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1838 "can't mark and flag in same flow"); 1839 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1840 return rte_flow_error_set(error, EINVAL, 1841 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1842 "can't have 2 flag" 1843 " actions in same flow"); 1844 if (attr->egress) 1845 return rte_flow_error_set(error, ENOTSUP, 1846 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1847 "flag action not supported for " 1848 "egress"); 1849 return 0; 1850 } 1851 1852 /* 1853 * Validate the mark action. 1854 * 1855 * @param[in] action 1856 * Pointer to the queue action. 1857 * @param[in] action_flags 1858 * Bit-fields that holds the actions detected until now. 1859 * @param[in] attr 1860 * Attributes of flow that includes this action. 1861 * @param[out] error 1862 * Pointer to error structure. 1863 * 1864 * @return 1865 * 0 on success, a negative errno value otherwise and rte_errno is set. 1866 */ 1867 int 1868 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1869 uint64_t action_flags, 1870 const struct rte_flow_attr *attr, 1871 struct rte_flow_error *error) 1872 { 1873 const struct rte_flow_action_mark *mark = action->conf; 1874 1875 if (!mark) 1876 return rte_flow_error_set(error, EINVAL, 1877 RTE_FLOW_ERROR_TYPE_ACTION, 1878 action, 1879 "configuration cannot be null"); 1880 if (mark->id >= MLX5_FLOW_MARK_MAX) 1881 return rte_flow_error_set(error, EINVAL, 1882 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1883 &mark->id, 1884 "mark id must in 0 <= id < " 1885 RTE_STR(MLX5_FLOW_MARK_MAX)); 1886 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1887 return rte_flow_error_set(error, EINVAL, 1888 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1889 "can't flag and mark in same flow"); 1890 if (action_flags & MLX5_FLOW_ACTION_MARK) 1891 return rte_flow_error_set(error, EINVAL, 1892 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1893 "can't have 2 mark actions in same" 1894 " flow"); 1895 if (attr->egress) 1896 return rte_flow_error_set(error, ENOTSUP, 1897 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1898 "mark action not supported for " 1899 "egress"); 1900 return 0; 1901 } 1902 1903 /* 1904 * Validate the drop action. 1905 * 1906 * @param[in] action_flags 1907 * Bit-fields that holds the actions detected until now. 1908 * @param[in] attr 1909 * Attributes of flow that includes this action. 1910 * @param[out] error 1911 * Pointer to error structure. 1912 * 1913 * @return 1914 * 0 on success, a negative errno value otherwise and rte_errno is set. 1915 */ 1916 int 1917 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1918 const struct rte_flow_attr *attr, 1919 struct rte_flow_error *error) 1920 { 1921 if (attr->egress) 1922 return rte_flow_error_set(error, ENOTSUP, 1923 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1924 "drop action not supported for " 1925 "egress"); 1926 return 0; 1927 } 1928 1929 /* 1930 * Validate the queue action. 1931 * 1932 * @param[in] action 1933 * Pointer to the queue action. 1934 * @param[in] action_flags 1935 * Bit-fields that holds the actions detected until now. 1936 * @param[in] dev 1937 * Pointer to the Ethernet device structure. 1938 * @param[in] attr 1939 * Attributes of flow that includes this action. 1940 * @param[out] error 1941 * Pointer to error structure. 1942 * 1943 * @return 1944 * 0 on success, a negative errno value otherwise and rte_errno is set. 1945 */ 1946 int 1947 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1948 uint64_t action_flags, 1949 struct rte_eth_dev *dev, 1950 const struct rte_flow_attr *attr, 1951 struct rte_flow_error *error) 1952 { 1953 struct mlx5_priv *priv = dev->data->dev_private; 1954 const struct rte_flow_action_queue *queue = action->conf; 1955 1956 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1957 return rte_flow_error_set(error, EINVAL, 1958 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1959 "can't have 2 fate actions in" 1960 " same flow"); 1961 if (attr->egress) 1962 return rte_flow_error_set(error, ENOTSUP, 1963 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1964 "queue action not supported for egress."); 1965 if (mlx5_is_external_rxq(dev, queue->index)) 1966 return 0; 1967 if (!priv->rxqs_n) 1968 return rte_flow_error_set(error, EINVAL, 1969 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1970 NULL, "No Rx queues configured"); 1971 if (queue->index >= priv->rxqs_n) 1972 return rte_flow_error_set(error, EINVAL, 1973 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1974 &queue->index, 1975 "queue index out of range"); 1976 if (mlx5_rxq_get(dev, queue->index) == NULL) 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1979 &queue->index, 1980 "queue is not configured"); 1981 return 0; 1982 } 1983 1984 /** 1985 * Validate queue numbers for device RSS. 1986 * 1987 * @param[in] dev 1988 * Configured device. 1989 * @param[in] queues 1990 * Array of queue numbers. 1991 * @param[in] queues_n 1992 * Size of the @p queues array. 1993 * @param[out] error 1994 * On error, filled with a textual error description. 1995 * @param[out] queue_idx 1996 * On error, filled with an offending queue index in @p queues array. 1997 * 1998 * @return 1999 * 0 on success, a negative errno code on error. 2000 */ 2001 static int 2002 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2003 const uint16_t *queues, uint32_t queues_n, 2004 const char **error, uint32_t *queue_idx) 2005 { 2006 const struct mlx5_priv *priv = dev->data->dev_private; 2007 bool is_hairpin = false; 2008 bool is_ext_rss = false; 2009 uint32_t i; 2010 2011 for (i = 0; i != queues_n; ++i) { 2012 struct mlx5_rxq_ctrl *rxq_ctrl; 2013 2014 if (mlx5_is_external_rxq(dev, queues[0])) { 2015 is_ext_rss = true; 2016 continue; 2017 } 2018 if (is_ext_rss) { 2019 *error = "Combining external and regular RSS queues is not supported"; 2020 *queue_idx = i; 2021 return -ENOTSUP; 2022 } 2023 if (queues[i] >= priv->rxqs_n) { 2024 *error = "queue index out of range"; 2025 *queue_idx = i; 2026 return -EINVAL; 2027 } 2028 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2029 if (rxq_ctrl == NULL) { 2030 *error = "queue is not configured"; 2031 *queue_idx = i; 2032 return -EINVAL; 2033 } 2034 if (i == 0 && rxq_ctrl->is_hairpin) 2035 is_hairpin = true; 2036 if (is_hairpin != rxq_ctrl->is_hairpin) { 2037 *error = "combining hairpin and regular RSS queues is not supported"; 2038 *queue_idx = i; 2039 return -ENOTSUP; 2040 } 2041 } 2042 return 0; 2043 } 2044 2045 /* 2046 * Validate the rss action. 2047 * 2048 * @param[in] dev 2049 * Pointer to the Ethernet device structure. 2050 * @param[in] action 2051 * Pointer to the queue action. 2052 * @param[out] error 2053 * Pointer to error structure. 2054 * 2055 * @return 2056 * 0 on success, a negative errno value otherwise and rte_errno is set. 2057 */ 2058 int 2059 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2060 const struct rte_flow_action *action, 2061 struct rte_flow_error *error) 2062 { 2063 struct mlx5_priv *priv = dev->data->dev_private; 2064 const struct rte_flow_action_rss *rss = action->conf; 2065 int ret; 2066 const char *message; 2067 uint32_t queue_idx; 2068 2069 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2070 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2071 return rte_flow_error_set(error, ENOTSUP, 2072 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2073 &rss->func, 2074 "RSS hash function not supported"); 2075 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2076 if (rss->level > 2) 2077 #else 2078 if (rss->level > 1) 2079 #endif 2080 return rte_flow_error_set(error, ENOTSUP, 2081 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2082 &rss->level, 2083 "tunnel RSS is not supported"); 2084 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2085 if (rss->key_len == 0 && rss->key != NULL) 2086 return rte_flow_error_set(error, ENOTSUP, 2087 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2088 &rss->key_len, 2089 "RSS hash key length 0"); 2090 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2091 return rte_flow_error_set(error, ENOTSUP, 2092 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2093 &rss->key_len, 2094 "RSS hash key too small"); 2095 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2096 return rte_flow_error_set(error, ENOTSUP, 2097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2098 &rss->key_len, 2099 "RSS hash key too large"); 2100 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2101 return rte_flow_error_set(error, ENOTSUP, 2102 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2103 &rss->queue_num, 2104 "number of queues too large"); 2105 if (rss->types & MLX5_RSS_HF_MASK) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2108 &rss->types, 2109 "some RSS protocols are not" 2110 " supported"); 2111 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2112 !(rss->types & RTE_ETH_RSS_IP)) 2113 return rte_flow_error_set(error, EINVAL, 2114 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2115 "L3 partial RSS requested but L3 RSS" 2116 " type not specified"); 2117 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2118 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2119 return rte_flow_error_set(error, EINVAL, 2120 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2121 "L4 partial RSS requested but L4 RSS" 2122 " type not specified"); 2123 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2124 return rte_flow_error_set(error, EINVAL, 2125 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2126 NULL, "No Rx queues configured"); 2127 if (!rss->queue_num) 2128 return rte_flow_error_set(error, EINVAL, 2129 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2130 NULL, "No queues configured"); 2131 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2132 &message, &queue_idx); 2133 if (ret != 0) { 2134 return rte_flow_error_set(error, -ret, 2135 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2136 &rss->queue[queue_idx], message); 2137 } 2138 return 0; 2139 } 2140 2141 /* 2142 * Validate the rss action. 2143 * 2144 * @param[in] action 2145 * Pointer to the queue action. 2146 * @param[in] action_flags 2147 * Bit-fields that holds the actions detected until now. 2148 * @param[in] dev 2149 * Pointer to the Ethernet device structure. 2150 * @param[in] attr 2151 * Attributes of flow that includes this action. 2152 * @param[in] item_flags 2153 * Items that were detected. 2154 * @param[out] error 2155 * Pointer to error structure. 2156 * 2157 * @return 2158 * 0 on success, a negative errno value otherwise and rte_errno is set. 2159 */ 2160 int 2161 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2162 uint64_t action_flags, 2163 struct rte_eth_dev *dev, 2164 const struct rte_flow_attr *attr, 2165 uint64_t item_flags, 2166 struct rte_flow_error *error) 2167 { 2168 const struct rte_flow_action_rss *rss = action->conf; 2169 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2170 int ret; 2171 2172 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2175 "can't have 2 fate actions" 2176 " in same flow"); 2177 ret = mlx5_validate_action_rss(dev, action, error); 2178 if (ret) 2179 return ret; 2180 if (attr->egress) 2181 return rte_flow_error_set(error, ENOTSUP, 2182 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2183 "rss action not supported for " 2184 "egress"); 2185 if (rss->level > 1 && !tunnel) 2186 return rte_flow_error_set(error, EINVAL, 2187 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2188 "inner RSS is not supported for " 2189 "non-tunnel flows"); 2190 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2191 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2192 return rte_flow_error_set(error, EINVAL, 2193 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2194 "RSS on eCPRI is not supported now"); 2195 } 2196 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2197 !(item_flags & 2198 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2199 rss->level > 1) 2200 return rte_flow_error_set(error, EINVAL, 2201 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2202 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2203 return 0; 2204 } 2205 2206 /* 2207 * Validate the default miss action. 2208 * 2209 * @param[in] action_flags 2210 * Bit-fields that holds the actions detected until now. 2211 * @param[out] error 2212 * Pointer to error structure. 2213 * 2214 * @return 2215 * 0 on success, a negative errno value otherwise and rte_errno is set. 2216 */ 2217 int 2218 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2219 const struct rte_flow_attr *attr, 2220 struct rte_flow_error *error) 2221 { 2222 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2225 "can't have 2 fate actions in" 2226 " same flow"); 2227 if (attr->egress) 2228 return rte_flow_error_set(error, ENOTSUP, 2229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2230 "default miss action not supported " 2231 "for egress"); 2232 if (attr->group) 2233 return rte_flow_error_set(error, ENOTSUP, 2234 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2235 "only group 0 is supported"); 2236 if (attr->transfer) 2237 return rte_flow_error_set(error, ENOTSUP, 2238 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2239 NULL, "transfer is not supported"); 2240 return 0; 2241 } 2242 2243 /* 2244 * Validate the count action. 2245 * 2246 * @param[in] dev 2247 * Pointer to the Ethernet device structure. 2248 * @param[in] attr 2249 * Attributes of flow that includes this action. 2250 * @param[out] error 2251 * Pointer to error structure. 2252 * 2253 * @return 2254 * 0 on success, a negative errno value otherwise and rte_errno is set. 2255 */ 2256 int 2257 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2258 const struct rte_flow_attr *attr, 2259 struct rte_flow_error *error) 2260 { 2261 if (attr->egress) 2262 return rte_flow_error_set(error, ENOTSUP, 2263 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2264 "count action not supported for " 2265 "egress"); 2266 return 0; 2267 } 2268 2269 /* 2270 * Validate the ASO CT action. 2271 * 2272 * @param[in] dev 2273 * Pointer to the Ethernet device structure. 2274 * @param[in] conntrack 2275 * Pointer to the CT action profile. 2276 * @param[out] error 2277 * Pointer to error structure. 2278 * 2279 * @return 2280 * 0 on success, a negative errno value otherwise and rte_errno is set. 2281 */ 2282 int 2283 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2284 const struct rte_flow_action_conntrack *conntrack, 2285 struct rte_flow_error *error) 2286 { 2287 RTE_SET_USED(dev); 2288 2289 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2290 return rte_flow_error_set(error, EINVAL, 2291 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2292 "Invalid CT state"); 2293 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2296 "Invalid last TCP packet flag"); 2297 return 0; 2298 } 2299 2300 /** 2301 * Validate ICMP6 item. 2302 * 2303 * @param[in] item 2304 * Item specification. 2305 * @param[in] item_flags 2306 * Bit-fields that holds the items detected until now. 2307 * @param[in] ext_vlan_sup 2308 * Whether extended VLAN features are supported or not. 2309 * @param[out] error 2310 * Pointer to error structure. 2311 * 2312 * @return 2313 * 0 on success, a negative errno value otherwise and rte_errno is set. 2314 */ 2315 int 2316 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2317 uint64_t item_flags, 2318 uint8_t target_protocol, 2319 struct rte_flow_error *error) 2320 { 2321 const struct rte_flow_item_icmp6 *mask = item->mask; 2322 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2323 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2324 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2325 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2326 MLX5_FLOW_LAYER_OUTER_L4; 2327 int ret; 2328 2329 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2330 return rte_flow_error_set(error, EINVAL, 2331 RTE_FLOW_ERROR_TYPE_ITEM, item, 2332 "protocol filtering not compatible" 2333 " with ICMP6 layer"); 2334 if (!(item_flags & l3m)) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "IPv6 is mandatory to filter on" 2338 " ICMP6"); 2339 if (item_flags & l4m) 2340 return rte_flow_error_set(error, EINVAL, 2341 RTE_FLOW_ERROR_TYPE_ITEM, item, 2342 "multiple L4 layers not supported"); 2343 if (!mask) 2344 mask = &rte_flow_item_icmp6_mask; 2345 ret = mlx5_flow_item_acceptable 2346 (item, (const uint8_t *)mask, 2347 (const uint8_t *)&rte_flow_item_icmp6_mask, 2348 sizeof(struct rte_flow_item_icmp6), 2349 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2350 if (ret < 0) 2351 return ret; 2352 return 0; 2353 } 2354 2355 /** 2356 * Validate ICMP item. 2357 * 2358 * @param[in] item 2359 * Item specification. 2360 * @param[in] item_flags 2361 * Bit-fields that holds the items detected until now. 2362 * @param[out] error 2363 * Pointer to error structure. 2364 * 2365 * @return 2366 * 0 on success, a negative errno value otherwise and rte_errno is set. 2367 */ 2368 int 2369 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2370 uint64_t item_flags, 2371 uint8_t target_protocol, 2372 struct rte_flow_error *error) 2373 { 2374 const struct rte_flow_item_icmp *mask = item->mask; 2375 const struct rte_flow_item_icmp nic_mask = { 2376 .hdr.icmp_type = 0xff, 2377 .hdr.icmp_code = 0xff, 2378 .hdr.icmp_ident = RTE_BE16(0xffff), 2379 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2380 }; 2381 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2382 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2383 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2384 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2385 MLX5_FLOW_LAYER_OUTER_L4; 2386 int ret; 2387 2388 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2389 return rte_flow_error_set(error, EINVAL, 2390 RTE_FLOW_ERROR_TYPE_ITEM, item, 2391 "protocol filtering not compatible" 2392 " with ICMP layer"); 2393 if (!(item_flags & l3m)) 2394 return rte_flow_error_set(error, EINVAL, 2395 RTE_FLOW_ERROR_TYPE_ITEM, item, 2396 "IPv4 is mandatory to filter" 2397 " on ICMP"); 2398 if (item_flags & l4m) 2399 return rte_flow_error_set(error, EINVAL, 2400 RTE_FLOW_ERROR_TYPE_ITEM, item, 2401 "multiple L4 layers not supported"); 2402 if (!mask) 2403 mask = &nic_mask; 2404 ret = mlx5_flow_item_acceptable 2405 (item, (const uint8_t *)mask, 2406 (const uint8_t *)&nic_mask, 2407 sizeof(struct rte_flow_item_icmp), 2408 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2409 if (ret < 0) 2410 return ret; 2411 return 0; 2412 } 2413 2414 /** 2415 * Validate Ethernet item. 2416 * 2417 * @param[in] item 2418 * Item specification. 2419 * @param[in] item_flags 2420 * Bit-fields that holds the items detected until now. 2421 * @param[out] error 2422 * Pointer to error structure. 2423 * 2424 * @return 2425 * 0 on success, a negative errno value otherwise and rte_errno is set. 2426 */ 2427 int 2428 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2429 uint64_t item_flags, bool ext_vlan_sup, 2430 struct rte_flow_error *error) 2431 { 2432 const struct rte_flow_item_eth *mask = item->mask; 2433 const struct rte_flow_item_eth nic_mask = { 2434 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2435 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2436 .hdr.ether_type = RTE_BE16(0xffff), 2437 .has_vlan = ext_vlan_sup ? 1 : 0, 2438 }; 2439 int ret; 2440 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2441 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2442 MLX5_FLOW_LAYER_OUTER_L2; 2443 2444 if (item_flags & ethm) 2445 return rte_flow_error_set(error, ENOTSUP, 2446 RTE_FLOW_ERROR_TYPE_ITEM, item, 2447 "multiple L2 layers not supported"); 2448 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2449 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2450 return rte_flow_error_set(error, EINVAL, 2451 RTE_FLOW_ERROR_TYPE_ITEM, item, 2452 "L2 layer should not follow " 2453 "L3 layers"); 2454 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2455 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2456 return rte_flow_error_set(error, EINVAL, 2457 RTE_FLOW_ERROR_TYPE_ITEM, item, 2458 "L2 layer should not follow VLAN"); 2459 if (item_flags & MLX5_FLOW_LAYER_GTP) 2460 return rte_flow_error_set(error, EINVAL, 2461 RTE_FLOW_ERROR_TYPE_ITEM, item, 2462 "L2 layer should not follow GTP"); 2463 if (!mask) 2464 mask = &rte_flow_item_eth_mask; 2465 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2466 (const uint8_t *)&nic_mask, 2467 sizeof(struct rte_flow_item_eth), 2468 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2469 return ret; 2470 } 2471 2472 /** 2473 * Validate VLAN item. 2474 * 2475 * @param[in] item 2476 * Item specification. 2477 * @param[in] item_flags 2478 * Bit-fields that holds the items detected until now. 2479 * @param[in] dev 2480 * Ethernet device flow is being created on. 2481 * @param[out] error 2482 * Pointer to error structure. 2483 * 2484 * @return 2485 * 0 on success, a negative errno value otherwise and rte_errno is set. 2486 */ 2487 int 2488 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2489 uint64_t item_flags, 2490 struct rte_eth_dev *dev, 2491 struct rte_flow_error *error) 2492 { 2493 const struct rte_flow_item_vlan *spec = item->spec; 2494 const struct rte_flow_item_vlan *mask = item->mask; 2495 const struct rte_flow_item_vlan nic_mask = { 2496 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2497 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2498 }; 2499 uint16_t vlan_tag = 0; 2500 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2501 int ret; 2502 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2503 MLX5_FLOW_LAYER_INNER_L4) : 2504 (MLX5_FLOW_LAYER_OUTER_L3 | 2505 MLX5_FLOW_LAYER_OUTER_L4); 2506 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2507 MLX5_FLOW_LAYER_OUTER_VLAN; 2508 2509 if (item_flags & vlanm) 2510 return rte_flow_error_set(error, EINVAL, 2511 RTE_FLOW_ERROR_TYPE_ITEM, item, 2512 "multiple VLAN layers not supported"); 2513 else if ((item_flags & l34m) != 0) 2514 return rte_flow_error_set(error, EINVAL, 2515 RTE_FLOW_ERROR_TYPE_ITEM, item, 2516 "VLAN cannot follow L3/L4 layer"); 2517 if (!mask) 2518 mask = &rte_flow_item_vlan_mask; 2519 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2520 (const uint8_t *)&nic_mask, 2521 sizeof(struct rte_flow_item_vlan), 2522 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2523 if (ret) 2524 return ret; 2525 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2526 struct mlx5_priv *priv = dev->data->dev_private; 2527 2528 if (priv->vmwa_context) { 2529 /* 2530 * Non-NULL context means we have a virtual machine 2531 * and SR-IOV enabled, we have to create VLAN interface 2532 * to make hypervisor to setup E-Switch vport 2533 * context correctly. We avoid creating the multiple 2534 * VLAN interfaces, so we cannot support VLAN tag mask. 2535 */ 2536 return rte_flow_error_set(error, EINVAL, 2537 RTE_FLOW_ERROR_TYPE_ITEM, 2538 item, 2539 "VLAN tag mask is not" 2540 " supported in virtual" 2541 " environment"); 2542 } 2543 } 2544 if (spec) { 2545 vlan_tag = spec->hdr.vlan_tci; 2546 vlan_tag &= mask->hdr.vlan_tci; 2547 } 2548 /* 2549 * From verbs perspective an empty VLAN is equivalent 2550 * to a packet without VLAN layer. 2551 */ 2552 if (!vlan_tag) 2553 return rte_flow_error_set(error, EINVAL, 2554 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2555 item->spec, 2556 "VLAN cannot be empty"); 2557 return 0; 2558 } 2559 2560 /** 2561 * Validate IPV4 item. 2562 * 2563 * @param[in] item 2564 * Item specification. 2565 * @param[in] item_flags 2566 * Bit-fields that holds the items detected until now. 2567 * @param[in] last_item 2568 * Previous validated item in the pattern items. 2569 * @param[in] ether_type 2570 * Type in the ethernet layer header (including dot1q). 2571 * @param[in] acc_mask 2572 * Acceptable mask, if NULL default internal default mask 2573 * will be used to check whether item fields are supported. 2574 * @param[in] range_accepted 2575 * True if range of values is accepted for specific fields, false otherwise. 2576 * @param[out] error 2577 * Pointer to error structure. 2578 * 2579 * @return 2580 * 0 on success, a negative errno value otherwise and rte_errno is set. 2581 */ 2582 int 2583 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2584 uint64_t item_flags, 2585 uint64_t last_item, 2586 uint16_t ether_type, 2587 const struct rte_flow_item_ipv4 *acc_mask, 2588 bool range_accepted, 2589 struct rte_flow_error *error) 2590 { 2591 const struct rte_flow_item_ipv4 *mask = item->mask; 2592 const struct rte_flow_item_ipv4 *spec = item->spec; 2593 const struct rte_flow_item_ipv4 nic_mask = { 2594 .hdr = { 2595 .src_addr = RTE_BE32(0xffffffff), 2596 .dst_addr = RTE_BE32(0xffffffff), 2597 .type_of_service = 0xff, 2598 .next_proto_id = 0xff, 2599 }, 2600 }; 2601 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2602 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2603 MLX5_FLOW_LAYER_OUTER_L3; 2604 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2605 MLX5_FLOW_LAYER_OUTER_L4; 2606 int ret; 2607 uint8_t next_proto = 0xFF; 2608 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2609 MLX5_FLOW_LAYER_OUTER_VLAN | 2610 MLX5_FLOW_LAYER_INNER_VLAN); 2611 2612 if ((last_item & l2_vlan) && ether_type && 2613 ether_type != RTE_ETHER_TYPE_IPV4) 2614 return rte_flow_error_set(error, EINVAL, 2615 RTE_FLOW_ERROR_TYPE_ITEM, item, 2616 "IPv4 cannot follow L2/VLAN layer " 2617 "which ether type is not IPv4"); 2618 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2619 if (mask && spec) 2620 next_proto = mask->hdr.next_proto_id & 2621 spec->hdr.next_proto_id; 2622 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2623 return rte_flow_error_set(error, EINVAL, 2624 RTE_FLOW_ERROR_TYPE_ITEM, 2625 item, 2626 "multiple tunnel " 2627 "not supported"); 2628 } 2629 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2630 return rte_flow_error_set(error, EINVAL, 2631 RTE_FLOW_ERROR_TYPE_ITEM, item, 2632 "wrong tunnel type - IPv6 specified " 2633 "but IPv4 item provided"); 2634 if (item_flags & l3m) 2635 return rte_flow_error_set(error, ENOTSUP, 2636 RTE_FLOW_ERROR_TYPE_ITEM, item, 2637 "multiple L3 layers not supported"); 2638 else if (item_flags & l4m) 2639 return rte_flow_error_set(error, EINVAL, 2640 RTE_FLOW_ERROR_TYPE_ITEM, item, 2641 "L3 cannot follow an L4 layer."); 2642 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2643 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2644 return rte_flow_error_set(error, EINVAL, 2645 RTE_FLOW_ERROR_TYPE_ITEM, item, 2646 "L3 cannot follow an NVGRE layer."); 2647 if (!mask) 2648 mask = &rte_flow_item_ipv4_mask; 2649 else if (mask->hdr.next_proto_id != 0 && 2650 mask->hdr.next_proto_id != 0xff) 2651 return rte_flow_error_set(error, EINVAL, 2652 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2653 "partial mask is not supported" 2654 " for protocol"); 2655 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2656 acc_mask ? (const uint8_t *)acc_mask 2657 : (const uint8_t *)&nic_mask, 2658 sizeof(struct rte_flow_item_ipv4), 2659 range_accepted, error); 2660 if (ret < 0) 2661 return ret; 2662 return 0; 2663 } 2664 2665 /** 2666 * Validate IPV6 item. 2667 * 2668 * @param[in] item 2669 * Item specification. 2670 * @param[in] item_flags 2671 * Bit-fields that holds the items detected until now. 2672 * @param[in] last_item 2673 * Previous validated item in the pattern items. 2674 * @param[in] ether_type 2675 * Type in the ethernet layer header (including dot1q). 2676 * @param[in] acc_mask 2677 * Acceptable mask, if NULL default internal default mask 2678 * will be used to check whether item fields are supported. 2679 * @param[out] error 2680 * Pointer to error structure. 2681 * 2682 * @return 2683 * 0 on success, a negative errno value otherwise and rte_errno is set. 2684 */ 2685 int 2686 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2687 uint64_t item_flags, 2688 uint64_t last_item, 2689 uint16_t ether_type, 2690 const struct rte_flow_item_ipv6 *acc_mask, 2691 struct rte_flow_error *error) 2692 { 2693 const struct rte_flow_item_ipv6 *mask = item->mask; 2694 const struct rte_flow_item_ipv6 *spec = item->spec; 2695 const struct rte_flow_item_ipv6 nic_mask = { 2696 .hdr = { 2697 .src_addr = 2698 "\xff\xff\xff\xff\xff\xff\xff\xff" 2699 "\xff\xff\xff\xff\xff\xff\xff\xff", 2700 .dst_addr = 2701 "\xff\xff\xff\xff\xff\xff\xff\xff" 2702 "\xff\xff\xff\xff\xff\xff\xff\xff", 2703 .vtc_flow = RTE_BE32(0xffffffff), 2704 .proto = 0xff, 2705 }, 2706 }; 2707 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2708 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2709 MLX5_FLOW_LAYER_OUTER_L3; 2710 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2711 MLX5_FLOW_LAYER_OUTER_L4; 2712 int ret; 2713 uint8_t next_proto = 0xFF; 2714 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2715 MLX5_FLOW_LAYER_OUTER_VLAN | 2716 MLX5_FLOW_LAYER_INNER_VLAN); 2717 2718 if ((last_item & l2_vlan) && ether_type && 2719 ether_type != RTE_ETHER_TYPE_IPV6) 2720 return rte_flow_error_set(error, EINVAL, 2721 RTE_FLOW_ERROR_TYPE_ITEM, item, 2722 "IPv6 cannot follow L2/VLAN layer " 2723 "which ether type is not IPv6"); 2724 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2725 next_proto = spec->hdr.proto; 2726 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2727 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2728 return rte_flow_error_set(error, EINVAL, 2729 RTE_FLOW_ERROR_TYPE_ITEM, 2730 item, 2731 "multiple tunnel " 2732 "not supported"); 2733 } 2734 if (next_proto == IPPROTO_HOPOPTS || 2735 next_proto == IPPROTO_ROUTING || 2736 next_proto == IPPROTO_FRAGMENT || 2737 next_proto == IPPROTO_ESP || 2738 next_proto == IPPROTO_AH || 2739 next_proto == IPPROTO_DSTOPTS) 2740 return rte_flow_error_set(error, EINVAL, 2741 RTE_FLOW_ERROR_TYPE_ITEM, item, 2742 "IPv6 proto (next header) should " 2743 "not be set as extension header"); 2744 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2745 return rte_flow_error_set(error, EINVAL, 2746 RTE_FLOW_ERROR_TYPE_ITEM, item, 2747 "wrong tunnel type - IPv4 specified " 2748 "but IPv6 item provided"); 2749 if (item_flags & l3m) 2750 return rte_flow_error_set(error, ENOTSUP, 2751 RTE_FLOW_ERROR_TYPE_ITEM, item, 2752 "multiple L3 layers not supported"); 2753 else if (item_flags & l4m) 2754 return rte_flow_error_set(error, EINVAL, 2755 RTE_FLOW_ERROR_TYPE_ITEM, item, 2756 "L3 cannot follow an L4 layer."); 2757 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2758 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2759 return rte_flow_error_set(error, EINVAL, 2760 RTE_FLOW_ERROR_TYPE_ITEM, item, 2761 "L3 cannot follow an NVGRE layer."); 2762 if (!mask) 2763 mask = &rte_flow_item_ipv6_mask; 2764 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2765 acc_mask ? (const uint8_t *)acc_mask 2766 : (const uint8_t *)&nic_mask, 2767 sizeof(struct rte_flow_item_ipv6), 2768 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2769 if (ret < 0) 2770 return ret; 2771 return 0; 2772 } 2773 2774 /** 2775 * Validate UDP item. 2776 * 2777 * @param[in] item 2778 * Item specification. 2779 * @param[in] item_flags 2780 * Bit-fields that holds the items detected until now. 2781 * @param[in] target_protocol 2782 * The next protocol in the previous item. 2783 * @param[in] flow_mask 2784 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2785 * @param[out] error 2786 * Pointer to error structure. 2787 * 2788 * @return 2789 * 0 on success, a negative errno value otherwise and rte_errno is set. 2790 */ 2791 int 2792 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2793 uint64_t item_flags, 2794 uint8_t target_protocol, 2795 struct rte_flow_error *error) 2796 { 2797 const struct rte_flow_item_udp *mask = item->mask; 2798 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2799 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2800 MLX5_FLOW_LAYER_OUTER_L3; 2801 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2802 MLX5_FLOW_LAYER_OUTER_L4; 2803 int ret; 2804 2805 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2806 return rte_flow_error_set(error, EINVAL, 2807 RTE_FLOW_ERROR_TYPE_ITEM, item, 2808 "protocol filtering not compatible" 2809 " with UDP layer"); 2810 if (!(item_flags & l3m)) 2811 return rte_flow_error_set(error, EINVAL, 2812 RTE_FLOW_ERROR_TYPE_ITEM, item, 2813 "L3 is mandatory to filter on L4"); 2814 if (item_flags & l4m) 2815 return rte_flow_error_set(error, EINVAL, 2816 RTE_FLOW_ERROR_TYPE_ITEM, item, 2817 "multiple L4 layers not supported"); 2818 if (!mask) 2819 mask = &rte_flow_item_udp_mask; 2820 ret = mlx5_flow_item_acceptable 2821 (item, (const uint8_t *)mask, 2822 (const uint8_t *)&rte_flow_item_udp_mask, 2823 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2824 error); 2825 if (ret < 0) 2826 return ret; 2827 return 0; 2828 } 2829 2830 /** 2831 * Validate TCP item. 2832 * 2833 * @param[in] item 2834 * Item specification. 2835 * @param[in] item_flags 2836 * Bit-fields that holds the items detected until now. 2837 * @param[in] target_protocol 2838 * The next protocol in the previous item. 2839 * @param[out] error 2840 * Pointer to error structure. 2841 * 2842 * @return 2843 * 0 on success, a negative errno value otherwise and rte_errno is set. 2844 */ 2845 int 2846 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2847 uint64_t item_flags, 2848 uint8_t target_protocol, 2849 const struct rte_flow_item_tcp *flow_mask, 2850 struct rte_flow_error *error) 2851 { 2852 const struct rte_flow_item_tcp *mask = item->mask; 2853 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2854 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2855 MLX5_FLOW_LAYER_OUTER_L3; 2856 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2857 MLX5_FLOW_LAYER_OUTER_L4; 2858 int ret; 2859 2860 MLX5_ASSERT(flow_mask); 2861 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2862 return rte_flow_error_set(error, EINVAL, 2863 RTE_FLOW_ERROR_TYPE_ITEM, item, 2864 "protocol filtering not compatible" 2865 " with TCP layer"); 2866 if (!(item_flags & l3m)) 2867 return rte_flow_error_set(error, EINVAL, 2868 RTE_FLOW_ERROR_TYPE_ITEM, item, 2869 "L3 is mandatory to filter on L4"); 2870 if (item_flags & l4m) 2871 return rte_flow_error_set(error, EINVAL, 2872 RTE_FLOW_ERROR_TYPE_ITEM, item, 2873 "multiple L4 layers not supported"); 2874 if (!mask) 2875 mask = &rte_flow_item_tcp_mask; 2876 ret = mlx5_flow_item_acceptable 2877 (item, (const uint8_t *)mask, 2878 (const uint8_t *)flow_mask, 2879 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2880 error); 2881 if (ret < 0) 2882 return ret; 2883 return 0; 2884 } 2885 2886 /** 2887 * Validate VXLAN item. 2888 * 2889 * @param[in] dev 2890 * Pointer to the Ethernet device structure. 2891 * @param[in] udp_dport 2892 * UDP destination port 2893 * @param[in] item 2894 * Item specification. 2895 * @param[in] item_flags 2896 * Bit-fields that holds the items detected until now. 2897 * @param root 2898 * Whether action is on root table. 2899 * @param[out] error 2900 * Pointer to error structure. 2901 * 2902 * @return 2903 * 0 on success, a negative errno value otherwise and rte_errno is set. 2904 */ 2905 int 2906 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2907 uint16_t udp_dport, 2908 const struct rte_flow_item *item, 2909 uint64_t item_flags, 2910 bool root, 2911 struct rte_flow_error *error) 2912 { 2913 const struct rte_flow_item_vxlan *spec = item->spec; 2914 const struct rte_flow_item_vxlan *mask = item->mask; 2915 int ret; 2916 struct mlx5_priv *priv = dev->data->dev_private; 2917 union vni { 2918 uint32_t vlan_id; 2919 uint8_t vni[4]; 2920 } id = { .vlan_id = 0, }; 2921 const struct rte_flow_item_vxlan nic_mask = { 2922 .hdr.vni = "\xff\xff\xff", 2923 .hdr.rsvd1 = 0xff, 2924 }; 2925 const struct rte_flow_item_vxlan *valid_mask; 2926 2927 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2928 return rte_flow_error_set(error, ENOTSUP, 2929 RTE_FLOW_ERROR_TYPE_ITEM, item, 2930 "multiple tunnel layers not" 2931 " supported"); 2932 valid_mask = &rte_flow_item_vxlan_mask; 2933 /* 2934 * Verify only UDPv4 is present as defined in 2935 * https://tools.ietf.org/html/rfc7348 2936 */ 2937 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2938 return rte_flow_error_set(error, EINVAL, 2939 RTE_FLOW_ERROR_TYPE_ITEM, item, 2940 "no outer UDP layer found"); 2941 if (!mask) 2942 mask = &rte_flow_item_vxlan_mask; 2943 2944 if (priv->sh->steering_format_version != 2945 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 2946 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 2947 /* non-root table */ 2948 if (!root && priv->sh->misc5_cap) 2949 valid_mask = &nic_mask; 2950 /* Group zero in NIC domain */ 2951 if (!root && priv->sh->tunnel_header_0_1) 2952 valid_mask = &nic_mask; 2953 } 2954 ret = mlx5_flow_item_acceptable 2955 (item, (const uint8_t *)mask, 2956 (const uint8_t *)valid_mask, 2957 sizeof(struct rte_flow_item_vxlan), 2958 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2959 if (ret < 0) 2960 return ret; 2961 if (spec) { 2962 memcpy(&id.vni[1], spec->hdr.vni, 3); 2963 memcpy(&id.vni[1], mask->hdr.vni, 3); 2964 } 2965 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 2966 return rte_flow_error_set(error, ENOTSUP, 2967 RTE_FLOW_ERROR_TYPE_ITEM, item, 2968 "VXLAN tunnel must be fully defined"); 2969 return 0; 2970 } 2971 2972 /** 2973 * Validate VXLAN_GPE item. 2974 * 2975 * @param[in] item 2976 * Item specification. 2977 * @param[in] item_flags 2978 * Bit-fields that holds the items detected until now. 2979 * @param[in] priv 2980 * Pointer to the private data structure. 2981 * @param[in] target_protocol 2982 * The next protocol in the previous item. 2983 * @param[out] error 2984 * Pointer to error structure. 2985 * 2986 * @return 2987 * 0 on success, a negative errno value otherwise and rte_errno is set. 2988 */ 2989 int 2990 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 2991 uint64_t item_flags, 2992 struct rte_eth_dev *dev, 2993 struct rte_flow_error *error) 2994 { 2995 struct mlx5_priv *priv = dev->data->dev_private; 2996 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 2997 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 2998 int ret; 2999 union vni { 3000 uint32_t vlan_id; 3001 uint8_t vni[4]; 3002 } id = { .vlan_id = 0, }; 3003 3004 if (!priv->sh->config.l3_vxlan_en) 3005 return rte_flow_error_set(error, ENOTSUP, 3006 RTE_FLOW_ERROR_TYPE_ITEM, item, 3007 "L3 VXLAN is not enabled by device" 3008 " parameter and/or not configured in" 3009 " firmware"); 3010 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3011 return rte_flow_error_set(error, ENOTSUP, 3012 RTE_FLOW_ERROR_TYPE_ITEM, item, 3013 "multiple tunnel layers not" 3014 " supported"); 3015 /* 3016 * Verify only UDPv4 is present as defined in 3017 * https://tools.ietf.org/html/rfc7348 3018 */ 3019 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3020 return rte_flow_error_set(error, EINVAL, 3021 RTE_FLOW_ERROR_TYPE_ITEM, item, 3022 "no outer UDP layer found"); 3023 if (!mask) 3024 mask = &rte_flow_item_vxlan_gpe_mask; 3025 ret = mlx5_flow_item_acceptable 3026 (item, (const uint8_t *)mask, 3027 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 3028 sizeof(struct rte_flow_item_vxlan_gpe), 3029 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3030 if (ret < 0) 3031 return ret; 3032 if (spec) { 3033 if (spec->hdr.proto) 3034 return rte_flow_error_set(error, ENOTSUP, 3035 RTE_FLOW_ERROR_TYPE_ITEM, 3036 item, 3037 "VxLAN-GPE protocol" 3038 " not supported"); 3039 memcpy(&id.vni[1], spec->hdr.vni, 3); 3040 memcpy(&id.vni[1], mask->hdr.vni, 3); 3041 } 3042 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3043 return rte_flow_error_set(error, ENOTSUP, 3044 RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "VXLAN-GPE tunnel must be fully" 3046 " defined"); 3047 return 0; 3048 } 3049 /** 3050 * Validate GRE Key item. 3051 * 3052 * @param[in] item 3053 * Item specification. 3054 * @param[in] item_flags 3055 * Bit flags to mark detected items. 3056 * @param[in] gre_item 3057 * Pointer to gre_item 3058 * @param[out] error 3059 * Pointer to error structure. 3060 * 3061 * @return 3062 * 0 on success, a negative errno value otherwise and rte_errno is set. 3063 */ 3064 int 3065 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3066 uint64_t item_flags, 3067 const struct rte_flow_item *gre_item, 3068 struct rte_flow_error *error) 3069 { 3070 const rte_be32_t *mask = item->mask; 3071 int ret = 0; 3072 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3073 const struct rte_flow_item_gre *gre_spec; 3074 const struct rte_flow_item_gre *gre_mask; 3075 3076 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3077 return rte_flow_error_set(error, ENOTSUP, 3078 RTE_FLOW_ERROR_TYPE_ITEM, item, 3079 "Multiple GRE key not support"); 3080 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3081 return rte_flow_error_set(error, ENOTSUP, 3082 RTE_FLOW_ERROR_TYPE_ITEM, item, 3083 "No preceding GRE header"); 3084 if (item_flags & MLX5_FLOW_LAYER_INNER) 3085 return rte_flow_error_set(error, ENOTSUP, 3086 RTE_FLOW_ERROR_TYPE_ITEM, item, 3087 "GRE key following a wrong item"); 3088 gre_mask = gre_item->mask; 3089 if (!gre_mask) 3090 gre_mask = &rte_flow_item_gre_mask; 3091 gre_spec = gre_item->spec; 3092 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3093 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3094 return rte_flow_error_set(error, EINVAL, 3095 RTE_FLOW_ERROR_TYPE_ITEM, item, 3096 "Key bit must be on"); 3097 3098 if (!mask) 3099 mask = &gre_key_default_mask; 3100 ret = mlx5_flow_item_acceptable 3101 (item, (const uint8_t *)mask, 3102 (const uint8_t *)&gre_key_default_mask, 3103 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3104 return ret; 3105 } 3106 3107 /** 3108 * Validate GRE optional item. 3109 * 3110 * @param[in] dev 3111 * Pointer to the Ethernet device structure. 3112 * @param[in] item 3113 * Item specification. 3114 * @param[in] item_flags 3115 * Bit flags to mark detected items. 3116 * @param[in] attr 3117 * Flow rule attributes. 3118 * @param[in] gre_item 3119 * Pointer to gre_item 3120 * @param[out] error 3121 * Pointer to error structure. 3122 * 3123 * @return 3124 * 0 on success, a negative errno value otherwise and rte_errno is set. 3125 */ 3126 int 3127 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3128 const struct rte_flow_item *item, 3129 uint64_t item_flags, 3130 const struct rte_flow_attr *attr, 3131 const struct rte_flow_item *gre_item, 3132 struct rte_flow_error *error) 3133 { 3134 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3135 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3136 const struct rte_flow_item_gre_opt *spec = item->spec; 3137 const struct rte_flow_item_gre_opt *mask = item->mask; 3138 struct mlx5_priv *priv = dev->data->dev_private; 3139 int ret = 0; 3140 struct rte_flow_item_gre_opt nic_mask = { 3141 .checksum_rsvd = { 3142 .checksum = RTE_BE16(UINT16_MAX), 3143 .reserved1 = 0x0, 3144 }, 3145 .key = { 3146 .key = RTE_BE32(UINT32_MAX), 3147 }, 3148 .sequence = { 3149 .sequence = RTE_BE32(UINT32_MAX), 3150 }, 3151 }; 3152 3153 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3154 return rte_flow_error_set(error, ENOTSUP, 3155 RTE_FLOW_ERROR_TYPE_ITEM, item, 3156 "No preceding GRE header"); 3157 if (item_flags & MLX5_FLOW_LAYER_INNER) 3158 return rte_flow_error_set(error, ENOTSUP, 3159 RTE_FLOW_ERROR_TYPE_ITEM, item, 3160 "GRE option following a wrong item"); 3161 if (!spec || !mask) 3162 return rte_flow_error_set(error, EINVAL, 3163 RTE_FLOW_ERROR_TYPE_ITEM, item, 3164 "At least one field gre_option(checksum/key/sequence) must be specified"); 3165 if (!gre_mask) 3166 gre_mask = &rte_flow_item_gre_mask; 3167 if (mask->checksum_rsvd.checksum) 3168 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3169 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3170 return rte_flow_error_set(error, EINVAL, 3171 RTE_FLOW_ERROR_TYPE_ITEM, 3172 item, 3173 "Checksum bit must be on"); 3174 if (mask->key.key) 3175 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3176 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3177 return rte_flow_error_set(error, EINVAL, 3178 RTE_FLOW_ERROR_TYPE_ITEM, 3179 item, "Key bit must be on"); 3180 if (mask->sequence.sequence) 3181 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3182 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3183 return rte_flow_error_set(error, EINVAL, 3184 RTE_FLOW_ERROR_TYPE_ITEM, 3185 item, 3186 "Sequence bit must be on"); 3187 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3188 if (priv->sh->steering_format_version == 3189 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3190 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3191 !priv->sh->misc5_cap) || 3192 (!(priv->sh->tunnel_header_0_1 && 3193 priv->sh->tunnel_header_2_3) && 3194 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3195 return rte_flow_error_set(error, EINVAL, 3196 RTE_FLOW_ERROR_TYPE_ITEM, 3197 item, 3198 "Checksum/Sequence not supported"); 3199 } 3200 ret = mlx5_flow_item_acceptable 3201 (item, (const uint8_t *)mask, 3202 (const uint8_t *)&nic_mask, 3203 sizeof(struct rte_flow_item_gre_opt), 3204 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3205 return ret; 3206 } 3207 3208 /** 3209 * Validate GRE item. 3210 * 3211 * @param[in] item 3212 * Item specification. 3213 * @param[in] item_flags 3214 * Bit flags to mark detected items. 3215 * @param[in] target_protocol 3216 * The next protocol in the previous item. 3217 * @param[out] error 3218 * Pointer to error structure. 3219 * 3220 * @return 3221 * 0 on success, a negative errno value otherwise and rte_errno is set. 3222 */ 3223 int 3224 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3225 uint64_t item_flags, 3226 uint8_t target_protocol, 3227 struct rte_flow_error *error) 3228 { 3229 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3230 const struct rte_flow_item_gre *mask = item->mask; 3231 int ret; 3232 const struct rte_flow_item_gre nic_mask = { 3233 .c_rsvd0_ver = RTE_BE16(0xB000), 3234 .protocol = RTE_BE16(UINT16_MAX), 3235 }; 3236 3237 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3238 return rte_flow_error_set(error, EINVAL, 3239 RTE_FLOW_ERROR_TYPE_ITEM, item, 3240 "protocol filtering not compatible" 3241 " with this GRE layer"); 3242 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3243 return rte_flow_error_set(error, ENOTSUP, 3244 RTE_FLOW_ERROR_TYPE_ITEM, item, 3245 "multiple tunnel layers not" 3246 " supported"); 3247 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3248 return rte_flow_error_set(error, ENOTSUP, 3249 RTE_FLOW_ERROR_TYPE_ITEM, item, 3250 "L3 Layer is missing"); 3251 if (!mask) 3252 mask = &rte_flow_item_gre_mask; 3253 ret = mlx5_flow_item_acceptable 3254 (item, (const uint8_t *)mask, 3255 (const uint8_t *)&nic_mask, 3256 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3257 error); 3258 if (ret < 0) 3259 return ret; 3260 #ifndef HAVE_MLX5DV_DR 3261 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3262 if (spec && (spec->protocol & mask->protocol)) 3263 return rte_flow_error_set(error, ENOTSUP, 3264 RTE_FLOW_ERROR_TYPE_ITEM, item, 3265 "without MPLS support the" 3266 " specification cannot be used for" 3267 " filtering"); 3268 #endif 3269 #endif 3270 return 0; 3271 } 3272 3273 /** 3274 * Validate Geneve item. 3275 * 3276 * @param[in] item 3277 * Item specification. 3278 * @param[in] itemFlags 3279 * Bit-fields that holds the items detected until now. 3280 * @param[in] enPriv 3281 * Pointer to the private data structure. 3282 * @param[out] error 3283 * Pointer to error structure. 3284 * 3285 * @return 3286 * 0 on success, a negative errno value otherwise and rte_errno is set. 3287 */ 3288 3289 int 3290 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3291 uint64_t item_flags, 3292 struct rte_eth_dev *dev, 3293 struct rte_flow_error *error) 3294 { 3295 struct mlx5_priv *priv = dev->data->dev_private; 3296 const struct rte_flow_item_geneve *spec = item->spec; 3297 const struct rte_flow_item_geneve *mask = item->mask; 3298 int ret; 3299 uint16_t gbhdr; 3300 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3301 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3302 const struct rte_flow_item_geneve nic_mask = { 3303 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3304 .vni = "\xff\xff\xff", 3305 .protocol = RTE_BE16(UINT16_MAX), 3306 }; 3307 3308 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3309 return rte_flow_error_set(error, ENOTSUP, 3310 RTE_FLOW_ERROR_TYPE_ITEM, item, 3311 "L3 Geneve is not enabled by device" 3312 " parameter and/or not configured in" 3313 " firmware"); 3314 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3315 return rte_flow_error_set(error, ENOTSUP, 3316 RTE_FLOW_ERROR_TYPE_ITEM, item, 3317 "multiple tunnel layers not" 3318 " supported"); 3319 /* 3320 * Verify only UDPv4 is present as defined in 3321 * https://tools.ietf.org/html/rfc7348 3322 */ 3323 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3324 return rte_flow_error_set(error, EINVAL, 3325 RTE_FLOW_ERROR_TYPE_ITEM, item, 3326 "no outer UDP layer found"); 3327 if (!mask) 3328 mask = &rte_flow_item_geneve_mask; 3329 ret = mlx5_flow_item_acceptable 3330 (item, (const uint8_t *)mask, 3331 (const uint8_t *)&nic_mask, 3332 sizeof(struct rte_flow_item_geneve), 3333 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3334 if (ret) 3335 return ret; 3336 if (spec) { 3337 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3338 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3339 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3340 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3341 return rte_flow_error_set(error, ENOTSUP, 3342 RTE_FLOW_ERROR_TYPE_ITEM, 3343 item, 3344 "Geneve protocol unsupported" 3345 " fields are being used"); 3346 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3347 return rte_flow_error_set 3348 (error, ENOTSUP, 3349 RTE_FLOW_ERROR_TYPE_ITEM, 3350 item, 3351 "Unsupported Geneve options length"); 3352 } 3353 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3354 return rte_flow_error_set 3355 (error, ENOTSUP, 3356 RTE_FLOW_ERROR_TYPE_ITEM, item, 3357 "Geneve tunnel must be fully defined"); 3358 return 0; 3359 } 3360 3361 /** 3362 * Validate Geneve TLV option item. 3363 * 3364 * @param[in] item 3365 * Item specification. 3366 * @param[in] last_item 3367 * Previous validated item in the pattern items. 3368 * @param[in] geneve_item 3369 * Previous GENEVE item specification. 3370 * @param[in] dev 3371 * Pointer to the rte_eth_dev structure. 3372 * @param[out] error 3373 * Pointer to error structure. 3374 * 3375 * @return 3376 * 0 on success, a negative errno value otherwise and rte_errno is set. 3377 */ 3378 int 3379 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3380 uint64_t last_item, 3381 const struct rte_flow_item *geneve_item, 3382 struct rte_eth_dev *dev, 3383 struct rte_flow_error *error) 3384 { 3385 struct mlx5_priv *priv = dev->data->dev_private; 3386 struct mlx5_dev_ctx_shared *sh = priv->sh; 3387 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3388 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3389 uint8_t data_max_supported = 3390 hca_attr->max_geneve_tlv_option_data_len * 4; 3391 const struct rte_flow_item_geneve *geneve_spec; 3392 const struct rte_flow_item_geneve *geneve_mask; 3393 const struct rte_flow_item_geneve_opt *spec = item->spec; 3394 const struct rte_flow_item_geneve_opt *mask = item->mask; 3395 unsigned int i; 3396 unsigned int data_len; 3397 uint8_t tlv_option_len; 3398 uint16_t optlen_m, optlen_v; 3399 const struct rte_flow_item_geneve_opt full_mask = { 3400 .option_class = RTE_BE16(0xffff), 3401 .option_type = 0xff, 3402 .option_len = 0x1f, 3403 }; 3404 3405 if (!mask) 3406 mask = &rte_flow_item_geneve_opt_mask; 3407 if (!spec) 3408 return rte_flow_error_set 3409 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3410 "Geneve TLV opt class/type/length must be specified"); 3411 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3412 return rte_flow_error_set 3413 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3414 "Geneve TLV opt length exceeds the limit (31)"); 3415 /* Check if class type and length masks are full. */ 3416 if (full_mask.option_class != mask->option_class || 3417 full_mask.option_type != mask->option_type || 3418 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3419 return rte_flow_error_set 3420 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3421 "Geneve TLV opt class/type/length masks must be full"); 3422 /* Check if length is supported */ 3423 if ((uint32_t)spec->option_len > 3424 hca_attr->max_geneve_tlv_option_data_len) 3425 return rte_flow_error_set 3426 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3427 "Geneve TLV opt length not supported"); 3428 if (hca_attr->max_geneve_tlv_options > 1) 3429 DRV_LOG(DEBUG, 3430 "max_geneve_tlv_options supports more than 1 option"); 3431 /* Check GENEVE item preceding. */ 3432 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3433 return rte_flow_error_set 3434 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3435 "Geneve opt item must be preceded with Geneve item"); 3436 geneve_spec = geneve_item->spec; 3437 geneve_mask = geneve_item->mask ? geneve_item->mask : 3438 &rte_flow_item_geneve_mask; 3439 /* Check if GENEVE TLV option size doesn't exceed option length */ 3440 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3441 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3442 tlv_option_len = spec->option_len & mask->option_len; 3443 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3444 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3445 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3446 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3447 if ((optlen_v & optlen_m) <= tlv_option_len) 3448 return rte_flow_error_set 3449 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3450 "GENEVE TLV option length exceeds optlen"); 3451 } 3452 /* Check if length is 0 or data is 0. */ 3453 if (spec->data == NULL || spec->option_len == 0) 3454 return rte_flow_error_set 3455 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3456 "Geneve TLV opt with zero data/length not supported"); 3457 /* Check not all data & mask are 0. */ 3458 data_len = spec->option_len * 4; 3459 if (mask->data == NULL) { 3460 for (i = 0; i < data_len; i++) 3461 if (spec->data[i]) 3462 break; 3463 if (i == data_len) 3464 return rte_flow_error_set(error, ENOTSUP, 3465 RTE_FLOW_ERROR_TYPE_ITEM, item, 3466 "Can't match on Geneve option data 0"); 3467 } else { 3468 for (i = 0; i < data_len; i++) 3469 if (spec->data[i] & mask->data[i]) 3470 break; 3471 if (i == data_len) 3472 return rte_flow_error_set(error, ENOTSUP, 3473 RTE_FLOW_ERROR_TYPE_ITEM, item, 3474 "Can't match on Geneve option data and mask 0"); 3475 /* Check data mask supported. */ 3476 for (i = data_max_supported; i < data_len ; i++) 3477 if (mask->data[i]) 3478 return rte_flow_error_set(error, ENOTSUP, 3479 RTE_FLOW_ERROR_TYPE_ITEM, item, 3480 "Data mask is of unsupported size"); 3481 } 3482 /* Check GENEVE option is supported in NIC. */ 3483 if (!hca_attr->geneve_tlv_opt) 3484 return rte_flow_error_set 3485 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3486 "Geneve TLV opt not supported"); 3487 /* Check if we already have geneve option with different type/class. */ 3488 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3489 geneve_opt_resource = sh->geneve_tlv_option_resource; 3490 if (geneve_opt_resource != NULL) 3491 if (geneve_opt_resource->option_class != spec->option_class || 3492 geneve_opt_resource->option_type != spec->option_type || 3493 geneve_opt_resource->length != spec->option_len) { 3494 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3495 return rte_flow_error_set(error, ENOTSUP, 3496 RTE_FLOW_ERROR_TYPE_ITEM, item, 3497 "Only one Geneve TLV option supported"); 3498 } 3499 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3500 return 0; 3501 } 3502 3503 /** 3504 * Validate MPLS item. 3505 * 3506 * @param[in] dev 3507 * Pointer to the rte_eth_dev structure. 3508 * @param[in] item 3509 * Item specification. 3510 * @param[in] item_flags 3511 * Bit-fields that holds the items detected until now. 3512 * @param[in] prev_layer 3513 * The protocol layer indicated in previous item. 3514 * @param[out] error 3515 * Pointer to error structure. 3516 * 3517 * @return 3518 * 0 on success, a negative errno value otherwise and rte_errno is set. 3519 */ 3520 int 3521 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3522 const struct rte_flow_item *item __rte_unused, 3523 uint64_t item_flags __rte_unused, 3524 uint64_t prev_layer __rte_unused, 3525 struct rte_flow_error *error) 3526 { 3527 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3528 const struct rte_flow_item_mpls *mask = item->mask; 3529 struct mlx5_priv *priv = dev->data->dev_private; 3530 int ret; 3531 3532 if (!priv->sh->dev_cap.mpls_en) 3533 return rte_flow_error_set(error, ENOTSUP, 3534 RTE_FLOW_ERROR_TYPE_ITEM, item, 3535 "MPLS not supported or" 3536 " disabled in firmware" 3537 " configuration."); 3538 /* MPLS over UDP, GRE is allowed */ 3539 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3540 MLX5_FLOW_LAYER_GRE | 3541 MLX5_FLOW_LAYER_GRE_KEY))) 3542 return rte_flow_error_set(error, EINVAL, 3543 RTE_FLOW_ERROR_TYPE_ITEM, item, 3544 "protocol filtering not compatible" 3545 " with MPLS layer"); 3546 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3547 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3548 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3549 return rte_flow_error_set(error, ENOTSUP, 3550 RTE_FLOW_ERROR_TYPE_ITEM, item, 3551 "multiple tunnel layers not" 3552 " supported"); 3553 if (!mask) 3554 mask = &rte_flow_item_mpls_mask; 3555 ret = mlx5_flow_item_acceptable 3556 (item, (const uint8_t *)mask, 3557 (const uint8_t *)&rte_flow_item_mpls_mask, 3558 sizeof(struct rte_flow_item_mpls), 3559 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3560 if (ret < 0) 3561 return ret; 3562 return 0; 3563 #else 3564 return rte_flow_error_set(error, ENOTSUP, 3565 RTE_FLOW_ERROR_TYPE_ITEM, item, 3566 "MPLS is not supported by Verbs, please" 3567 " update."); 3568 #endif 3569 } 3570 3571 /** 3572 * Validate NVGRE item. 3573 * 3574 * @param[in] item 3575 * Item specification. 3576 * @param[in] item_flags 3577 * Bit flags to mark detected items. 3578 * @param[in] target_protocol 3579 * The next protocol in the previous item. 3580 * @param[out] error 3581 * Pointer to error structure. 3582 * 3583 * @return 3584 * 0 on success, a negative errno value otherwise and rte_errno is set. 3585 */ 3586 int 3587 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3588 uint64_t item_flags, 3589 uint8_t target_protocol, 3590 struct rte_flow_error *error) 3591 { 3592 const struct rte_flow_item_nvgre *mask = item->mask; 3593 int ret; 3594 3595 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3596 return rte_flow_error_set(error, EINVAL, 3597 RTE_FLOW_ERROR_TYPE_ITEM, item, 3598 "protocol filtering not compatible" 3599 " with this GRE layer"); 3600 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3601 return rte_flow_error_set(error, ENOTSUP, 3602 RTE_FLOW_ERROR_TYPE_ITEM, item, 3603 "multiple tunnel layers not" 3604 " supported"); 3605 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3606 return rte_flow_error_set(error, ENOTSUP, 3607 RTE_FLOW_ERROR_TYPE_ITEM, item, 3608 "L3 Layer is missing"); 3609 if (!mask) 3610 mask = &rte_flow_item_nvgre_mask; 3611 ret = mlx5_flow_item_acceptable 3612 (item, (const uint8_t *)mask, 3613 (const uint8_t *)&rte_flow_item_nvgre_mask, 3614 sizeof(struct rte_flow_item_nvgre), 3615 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3616 if (ret < 0) 3617 return ret; 3618 return 0; 3619 } 3620 3621 /** 3622 * Validate eCPRI item. 3623 * 3624 * @param[in] item 3625 * Item specification. 3626 * @param[in] item_flags 3627 * Bit-fields that holds the items detected until now. 3628 * @param[in] last_item 3629 * Previous validated item in the pattern items. 3630 * @param[in] ether_type 3631 * Type in the ethernet layer header (including dot1q). 3632 * @param[in] acc_mask 3633 * Acceptable mask, if NULL default internal default mask 3634 * will be used to check whether item fields are supported. 3635 * @param[out] error 3636 * Pointer to error structure. 3637 * 3638 * @return 3639 * 0 on success, a negative errno value otherwise and rte_errno is set. 3640 */ 3641 int 3642 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3643 uint64_t item_flags, 3644 uint64_t last_item, 3645 uint16_t ether_type, 3646 const struct rte_flow_item_ecpri *acc_mask, 3647 struct rte_flow_error *error) 3648 { 3649 const struct rte_flow_item_ecpri *mask = item->mask; 3650 const struct rte_flow_item_ecpri nic_mask = { 3651 .hdr = { 3652 .common = { 3653 .u32 = 3654 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3655 .type = 0xFF, 3656 }).u32), 3657 }, 3658 .dummy[0] = 0xFFFFFFFF, 3659 }, 3660 }; 3661 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3662 MLX5_FLOW_LAYER_OUTER_VLAN); 3663 struct rte_flow_item_ecpri mask_lo; 3664 3665 if (!(last_item & outer_l2_vlan) && 3666 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3667 return rte_flow_error_set(error, EINVAL, 3668 RTE_FLOW_ERROR_TYPE_ITEM, item, 3669 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3670 if ((last_item & outer_l2_vlan) && ether_type && 3671 ether_type != RTE_ETHER_TYPE_ECPRI) 3672 return rte_flow_error_set(error, EINVAL, 3673 RTE_FLOW_ERROR_TYPE_ITEM, item, 3674 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3675 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3676 return rte_flow_error_set(error, EINVAL, 3677 RTE_FLOW_ERROR_TYPE_ITEM, item, 3678 "eCPRI with tunnel is not supported right now"); 3679 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3680 return rte_flow_error_set(error, ENOTSUP, 3681 RTE_FLOW_ERROR_TYPE_ITEM, item, 3682 "multiple L3 layers not supported"); 3683 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3684 return rte_flow_error_set(error, EINVAL, 3685 RTE_FLOW_ERROR_TYPE_ITEM, item, 3686 "eCPRI cannot coexist with a TCP layer"); 3687 /* In specification, eCPRI could be over UDP layer. */ 3688 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3689 return rte_flow_error_set(error, EINVAL, 3690 RTE_FLOW_ERROR_TYPE_ITEM, item, 3691 "eCPRI over UDP layer is not yet supported right now"); 3692 /* Mask for type field in common header could be zero. */ 3693 if (!mask) 3694 mask = &rte_flow_item_ecpri_mask; 3695 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3696 /* Input mask is in big-endian format. */ 3697 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3698 return rte_flow_error_set(error, EINVAL, 3699 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3700 "partial mask is not supported for protocol"); 3701 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3702 return rte_flow_error_set(error, EINVAL, 3703 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3704 "message header mask must be after a type mask"); 3705 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3706 acc_mask ? (const uint8_t *)acc_mask 3707 : (const uint8_t *)&nic_mask, 3708 sizeof(struct rte_flow_item_ecpri), 3709 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3710 } 3711 3712 static int 3713 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3714 const struct rte_flow_attr *attr __rte_unused, 3715 const struct rte_flow_item items[] __rte_unused, 3716 const struct rte_flow_action actions[] __rte_unused, 3717 bool external __rte_unused, 3718 int hairpin __rte_unused, 3719 struct rte_flow_error *error) 3720 { 3721 return rte_flow_error_set(error, ENOTSUP, 3722 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3723 } 3724 3725 static struct mlx5_flow * 3726 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3727 const struct rte_flow_attr *attr __rte_unused, 3728 const struct rte_flow_item items[] __rte_unused, 3729 const struct rte_flow_action actions[] __rte_unused, 3730 struct rte_flow_error *error) 3731 { 3732 rte_flow_error_set(error, ENOTSUP, 3733 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3734 return NULL; 3735 } 3736 3737 static int 3738 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3739 struct mlx5_flow *dev_flow __rte_unused, 3740 const struct rte_flow_attr *attr __rte_unused, 3741 const struct rte_flow_item items[] __rte_unused, 3742 const struct rte_flow_action actions[] __rte_unused, 3743 struct rte_flow_error *error) 3744 { 3745 return rte_flow_error_set(error, ENOTSUP, 3746 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3747 } 3748 3749 static int 3750 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3751 struct rte_flow *flow __rte_unused, 3752 struct rte_flow_error *error) 3753 { 3754 return rte_flow_error_set(error, ENOTSUP, 3755 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3756 } 3757 3758 static void 3759 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3760 struct rte_flow *flow __rte_unused) 3761 { 3762 } 3763 3764 static void 3765 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3766 struct rte_flow *flow __rte_unused) 3767 { 3768 } 3769 3770 static int 3771 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3772 struct rte_flow *flow __rte_unused, 3773 const struct rte_flow_action *actions __rte_unused, 3774 void *data __rte_unused, 3775 struct rte_flow_error *error) 3776 { 3777 return rte_flow_error_set(error, ENOTSUP, 3778 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3779 } 3780 3781 static int 3782 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3783 uint32_t domains __rte_unused, 3784 uint32_t flags __rte_unused) 3785 { 3786 return 0; 3787 } 3788 3789 /* Void driver to protect from null pointer reference. */ 3790 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3791 .validate = flow_null_validate, 3792 .prepare = flow_null_prepare, 3793 .translate = flow_null_translate, 3794 .apply = flow_null_apply, 3795 .remove = flow_null_remove, 3796 .destroy = flow_null_destroy, 3797 .query = flow_null_query, 3798 .sync_domain = flow_null_sync_domain, 3799 }; 3800 3801 /** 3802 * Select flow driver type according to flow attributes and device 3803 * configuration. 3804 * 3805 * @param[in] dev 3806 * Pointer to the dev structure. 3807 * @param[in] attr 3808 * Pointer to the flow attributes. 3809 * 3810 * @return 3811 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3812 */ 3813 static enum mlx5_flow_drv_type 3814 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3815 { 3816 struct mlx5_priv *priv = dev->data->dev_private; 3817 /* The OS can determine first a specific flow type (DV, VERBS) */ 3818 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3819 3820 if (type != MLX5_FLOW_TYPE_MAX) 3821 return type; 3822 /* 3823 * Currently when dv_flow_en == 2, only HW steering engine is 3824 * supported. New engines can also be chosen here if ready. 3825 */ 3826 if (priv->sh->config.dv_flow_en == 2) 3827 return MLX5_FLOW_TYPE_HW; 3828 if (!attr) 3829 return MLX5_FLOW_TYPE_MIN; 3830 /* If no OS specific type - continue with DV/VERBS selection */ 3831 if (attr->transfer && priv->sh->config.dv_esw_en) 3832 type = MLX5_FLOW_TYPE_DV; 3833 if (!attr->transfer) 3834 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3835 MLX5_FLOW_TYPE_VERBS; 3836 return type; 3837 } 3838 3839 #define flow_get_drv_ops(type) flow_drv_ops[type] 3840 3841 /** 3842 * Flow driver validation API. This abstracts calling driver specific functions. 3843 * The type of flow driver is determined according to flow attributes. 3844 * 3845 * @param[in] dev 3846 * Pointer to the dev structure. 3847 * @param[in] attr 3848 * Pointer to the flow attributes. 3849 * @param[in] items 3850 * Pointer to the list of items. 3851 * @param[in] actions 3852 * Pointer to the list of actions. 3853 * @param[in] external 3854 * This flow rule is created by request external to PMD. 3855 * @param[in] hairpin 3856 * Number of hairpin TX actions, 0 means classic flow. 3857 * @param[out] error 3858 * Pointer to the error structure. 3859 * 3860 * @return 3861 * 0 on success, a negative errno value otherwise and rte_errno is set. 3862 */ 3863 static inline int 3864 flow_drv_validate(struct rte_eth_dev *dev, 3865 const struct rte_flow_attr *attr, 3866 const struct rte_flow_item items[], 3867 const struct rte_flow_action actions[], 3868 bool external, int hairpin, struct rte_flow_error *error) 3869 { 3870 const struct mlx5_flow_driver_ops *fops; 3871 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3872 3873 fops = flow_get_drv_ops(type); 3874 return fops->validate(dev, attr, items, actions, external, 3875 hairpin, error); 3876 } 3877 3878 /** 3879 * Flow driver preparation API. This abstracts calling driver specific 3880 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3881 * calculates the size of memory required for device flow, allocates the memory, 3882 * initializes the device flow and returns the pointer. 3883 * 3884 * @note 3885 * This function initializes device flow structure such as dv or verbs in 3886 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3887 * rest. For example, adding returning device flow to flow->dev_flow list and 3888 * setting backward reference to the flow should be done out of this function. 3889 * layers field is not filled either. 3890 * 3891 * @param[in] dev 3892 * Pointer to the dev structure. 3893 * @param[in] attr 3894 * Pointer to the flow attributes. 3895 * @param[in] items 3896 * Pointer to the list of items. 3897 * @param[in] actions 3898 * Pointer to the list of actions. 3899 * @param[in] flow_idx 3900 * This memory pool index to the flow. 3901 * @param[out] error 3902 * Pointer to the error structure. 3903 * 3904 * @return 3905 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 3906 */ 3907 static inline struct mlx5_flow * 3908 flow_drv_prepare(struct rte_eth_dev *dev, 3909 const struct rte_flow *flow, 3910 const struct rte_flow_attr *attr, 3911 const struct rte_flow_item items[], 3912 const struct rte_flow_action actions[], 3913 uint32_t flow_idx, 3914 struct rte_flow_error *error) 3915 { 3916 const struct mlx5_flow_driver_ops *fops; 3917 enum mlx5_flow_drv_type type = flow->drv_type; 3918 struct mlx5_flow *mlx5_flow = NULL; 3919 3920 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3921 fops = flow_get_drv_ops(type); 3922 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 3923 if (mlx5_flow) 3924 mlx5_flow->flow_idx = flow_idx; 3925 return mlx5_flow; 3926 } 3927 3928 /** 3929 * Flow driver translation API. This abstracts calling driver specific 3930 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3931 * translates a generic flow into a driver flow. flow_drv_prepare() must 3932 * precede. 3933 * 3934 * @note 3935 * dev_flow->layers could be filled as a result of parsing during translation 3936 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 3937 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 3938 * flow->actions could be overwritten even though all the expanded dev_flows 3939 * have the same actions. 3940 * 3941 * @param[in] dev 3942 * Pointer to the rte dev structure. 3943 * @param[in, out] dev_flow 3944 * Pointer to the mlx5 flow. 3945 * @param[in] attr 3946 * Pointer to the flow attributes. 3947 * @param[in] items 3948 * Pointer to the list of items. 3949 * @param[in] actions 3950 * Pointer to the list of actions. 3951 * @param[out] error 3952 * Pointer to the error structure. 3953 * 3954 * @return 3955 * 0 on success, a negative errno value otherwise and rte_errno is set. 3956 */ 3957 static inline int 3958 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 3959 const struct rte_flow_attr *attr, 3960 const struct rte_flow_item items[], 3961 const struct rte_flow_action actions[], 3962 struct rte_flow_error *error) 3963 { 3964 const struct mlx5_flow_driver_ops *fops; 3965 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 3966 3967 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3968 fops = flow_get_drv_ops(type); 3969 return fops->translate(dev, dev_flow, attr, items, actions, error); 3970 } 3971 3972 /** 3973 * Flow driver apply API. This abstracts calling driver specific functions. 3974 * Parent flow (rte_flow) should have driver type (drv_type). It applies 3975 * translated driver flows on to device. flow_drv_translate() must precede. 3976 * 3977 * @param[in] dev 3978 * Pointer to Ethernet device structure. 3979 * @param[in, out] flow 3980 * Pointer to flow structure. 3981 * @param[out] error 3982 * Pointer to error structure. 3983 * 3984 * @return 3985 * 0 on success, a negative errno value otherwise and rte_errno is set. 3986 */ 3987 static inline int 3988 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 3989 struct rte_flow_error *error) 3990 { 3991 const struct mlx5_flow_driver_ops *fops; 3992 enum mlx5_flow_drv_type type = flow->drv_type; 3993 3994 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 3995 fops = flow_get_drv_ops(type); 3996 return fops->apply(dev, flow, error); 3997 } 3998 3999 /** 4000 * Flow driver destroy API. This abstracts calling driver specific functions. 4001 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4002 * on device and releases resources of the flow. 4003 * 4004 * @param[in] dev 4005 * Pointer to Ethernet device. 4006 * @param[in, out] flow 4007 * Pointer to flow structure. 4008 */ 4009 static inline void 4010 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4011 { 4012 const struct mlx5_flow_driver_ops *fops; 4013 enum mlx5_flow_drv_type type = flow->drv_type; 4014 4015 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4016 fops = flow_get_drv_ops(type); 4017 fops->destroy(dev, flow); 4018 } 4019 4020 /** 4021 * Flow driver find RSS policy tbl API. This abstracts calling driver 4022 * specific functions. Parent flow (rte_flow) should have driver 4023 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4024 * 4025 * @param[in] dev 4026 * Pointer to Ethernet device. 4027 * @param[in, out] flow 4028 * Pointer to flow structure. 4029 * @param[in] policy 4030 * Pointer to meter policy table. 4031 * @param[in] rss_desc 4032 * Pointer to rss_desc 4033 */ 4034 static struct mlx5_flow_meter_sub_policy * 4035 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4036 struct rte_flow *flow, 4037 struct mlx5_flow_meter_policy *policy, 4038 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4039 { 4040 const struct mlx5_flow_driver_ops *fops; 4041 enum mlx5_flow_drv_type type = flow->drv_type; 4042 4043 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4044 fops = flow_get_drv_ops(type); 4045 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4046 } 4047 4048 /** 4049 * Flow driver color tag rule API. This abstracts calling driver 4050 * specific functions. Parent flow (rte_flow) should have driver 4051 * type (drv_type). It will create the color tag rules in hierarchy meter. 4052 * 4053 * @param[in] dev 4054 * Pointer to Ethernet device. 4055 * @param[in, out] flow 4056 * Pointer to flow structure. 4057 * @param[in] fm 4058 * Pointer to flow meter structure. 4059 * @param[in] src_port 4060 * The src port this extra rule should use. 4061 * @param[in] item 4062 * The src port id match item. 4063 * @param[out] error 4064 * Pointer to error structure. 4065 */ 4066 static int 4067 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4068 struct rte_flow *flow, 4069 struct mlx5_flow_meter_info *fm, 4070 int32_t src_port, 4071 const struct rte_flow_item *item, 4072 struct rte_flow_error *error) 4073 { 4074 const struct mlx5_flow_driver_ops *fops; 4075 enum mlx5_flow_drv_type type = flow->drv_type; 4076 4077 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4078 fops = flow_get_drv_ops(type); 4079 return fops->meter_hierarchy_rule_create(dev, fm, 4080 src_port, item, error); 4081 } 4082 4083 /** 4084 * Get RSS action from the action list. 4085 * 4086 * @param[in] dev 4087 * Pointer to Ethernet device. 4088 * @param[in] actions 4089 * Pointer to the list of actions. 4090 * @param[in] flow 4091 * Parent flow structure pointer. 4092 * 4093 * @return 4094 * Pointer to the RSS action if exist, else return NULL. 4095 */ 4096 static const struct rte_flow_action_rss* 4097 flow_get_rss_action(struct rte_eth_dev *dev, 4098 const struct rte_flow_action actions[]) 4099 { 4100 struct mlx5_priv *priv = dev->data->dev_private; 4101 const struct rte_flow_action_rss *rss = NULL; 4102 struct mlx5_meter_policy_action_container *acg; 4103 struct mlx5_meter_policy_action_container *acy; 4104 4105 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4106 switch (actions->type) { 4107 case RTE_FLOW_ACTION_TYPE_RSS: 4108 rss = actions->conf; 4109 break; 4110 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4111 { 4112 const struct rte_flow_action_sample *sample = 4113 actions->conf; 4114 const struct rte_flow_action *act = sample->actions; 4115 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4116 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4117 rss = act->conf; 4118 break; 4119 } 4120 case RTE_FLOW_ACTION_TYPE_METER: 4121 { 4122 uint32_t mtr_idx; 4123 struct mlx5_flow_meter_info *fm; 4124 struct mlx5_flow_meter_policy *policy; 4125 const struct rte_flow_action_meter *mtr = actions->conf; 4126 4127 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4128 if (fm && !fm->def_policy) { 4129 policy = mlx5_flow_meter_policy_find(dev, 4130 fm->policy_id, NULL); 4131 MLX5_ASSERT(policy); 4132 if (policy->is_hierarchy) { 4133 policy = 4134 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4135 policy); 4136 if (!policy) 4137 return NULL; 4138 } 4139 if (policy->is_rss) { 4140 acg = 4141 &policy->act_cnt[RTE_COLOR_GREEN]; 4142 acy = 4143 &policy->act_cnt[RTE_COLOR_YELLOW]; 4144 if (acg->fate_action == 4145 MLX5_FLOW_FATE_SHARED_RSS) 4146 rss = acg->rss->conf; 4147 else if (acy->fate_action == 4148 MLX5_FLOW_FATE_SHARED_RSS) 4149 rss = acy->rss->conf; 4150 } 4151 } 4152 break; 4153 } 4154 default: 4155 break; 4156 } 4157 } 4158 return rss; 4159 } 4160 4161 /** 4162 * Get ASO age action by index. 4163 * 4164 * @param[in] dev 4165 * Pointer to the Ethernet device structure. 4166 * @param[in] age_idx 4167 * Index to the ASO age action. 4168 * 4169 * @return 4170 * The specified ASO age action. 4171 */ 4172 struct mlx5_aso_age_action* 4173 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4174 { 4175 uint16_t pool_idx = age_idx & UINT16_MAX; 4176 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4177 struct mlx5_priv *priv = dev->data->dev_private; 4178 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4179 struct mlx5_aso_age_pool *pool; 4180 4181 rte_rwlock_read_lock(&mng->resize_rwl); 4182 pool = mng->pools[pool_idx]; 4183 rte_rwlock_read_unlock(&mng->resize_rwl); 4184 return &pool->actions[offset - 1]; 4185 } 4186 4187 /* maps indirect action to translated direct in some actions array */ 4188 struct mlx5_translated_action_handle { 4189 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4190 int index; /**< Index in related array of rte_flow_action. */ 4191 }; 4192 4193 /** 4194 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4195 * direct action if translation possible. 4196 * This functionality used to run same execution path for both direct and 4197 * indirect actions on flow create. All necessary preparations for indirect 4198 * action handling should be performed on *handle* actions list returned 4199 * from this call. 4200 * 4201 * @param[in] dev 4202 * Pointer to Ethernet device. 4203 * @param[in] actions 4204 * List of actions to translate. 4205 * @param[out] handle 4206 * List to store translated indirect action object handles. 4207 * @param[in, out] indir_n 4208 * Size of *handle* array. On return should be updated with number of 4209 * indirect actions retrieved from the *actions* list. 4210 * @param[out] translated_actions 4211 * List of actions where all indirect actions were translated to direct 4212 * if possible. NULL if no translation took place. 4213 * @param[out] error 4214 * Pointer to the error structure. 4215 * 4216 * @return 4217 * 0 on success, a negative errno value otherwise and rte_errno is set. 4218 */ 4219 static int 4220 flow_action_handles_translate(struct rte_eth_dev *dev, 4221 const struct rte_flow_action actions[], 4222 struct mlx5_translated_action_handle *handle, 4223 int *indir_n, 4224 struct rte_flow_action **translated_actions, 4225 struct rte_flow_error *error) 4226 { 4227 struct mlx5_priv *priv = dev->data->dev_private; 4228 struct rte_flow_action *translated = NULL; 4229 size_t actions_size; 4230 int n; 4231 int copied_n = 0; 4232 struct mlx5_translated_action_handle *handle_end = NULL; 4233 4234 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4235 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4236 continue; 4237 if (copied_n == *indir_n) { 4238 return rte_flow_error_set 4239 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4240 NULL, "too many shared actions"); 4241 } 4242 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4243 sizeof(actions[n].conf)); 4244 handle[copied_n].index = n; 4245 copied_n++; 4246 } 4247 n++; 4248 *indir_n = copied_n; 4249 if (!copied_n) 4250 return 0; 4251 actions_size = sizeof(struct rte_flow_action) * n; 4252 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4253 if (!translated) { 4254 rte_errno = ENOMEM; 4255 return -ENOMEM; 4256 } 4257 memcpy(translated, actions, actions_size); 4258 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4259 struct mlx5_shared_action_rss *shared_rss; 4260 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4261 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4262 uint32_t idx = act_idx & 4263 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4264 4265 switch (type) { 4266 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4267 shared_rss = mlx5_ipool_get 4268 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4269 translated[handle->index].type = 4270 RTE_FLOW_ACTION_TYPE_RSS; 4271 translated[handle->index].conf = 4272 &shared_rss->origin; 4273 break; 4274 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4275 translated[handle->index].type = 4276 (enum rte_flow_action_type) 4277 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4278 translated[handle->index].conf = (void *)(uintptr_t)idx; 4279 break; 4280 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4281 translated[handle->index].type = 4282 (enum rte_flow_action_type) 4283 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4284 translated[handle->index].conf = (void *)(uintptr_t)idx; 4285 break; 4286 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4287 if (priv->sh->flow_hit_aso_en) { 4288 translated[handle->index].type = 4289 (enum rte_flow_action_type) 4290 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4291 translated[handle->index].conf = 4292 (void *)(uintptr_t)idx; 4293 break; 4294 } 4295 /* Fall-through */ 4296 case MLX5_INDIRECT_ACTION_TYPE_CT: 4297 if (priv->sh->ct_aso_en) { 4298 translated[handle->index].type = 4299 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4300 translated[handle->index].conf = 4301 (void *)(uintptr_t)idx; 4302 break; 4303 } 4304 /* Fall-through */ 4305 default: 4306 mlx5_free(translated); 4307 return rte_flow_error_set 4308 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4309 NULL, "invalid indirect action type"); 4310 } 4311 } 4312 *translated_actions = translated; 4313 return 0; 4314 } 4315 4316 /** 4317 * Get Shared RSS action from the action list. 4318 * 4319 * @param[in] dev 4320 * Pointer to Ethernet device. 4321 * @param[in] shared 4322 * Pointer to the list of actions. 4323 * @param[in] shared_n 4324 * Actions list length. 4325 * 4326 * @return 4327 * The MLX5 RSS action ID if exists, otherwise return 0. 4328 */ 4329 static uint32_t 4330 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4331 struct mlx5_translated_action_handle *handle, 4332 int shared_n) 4333 { 4334 struct mlx5_translated_action_handle *handle_end; 4335 struct mlx5_priv *priv = dev->data->dev_private; 4336 struct mlx5_shared_action_rss *shared_rss; 4337 4338 4339 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4340 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4341 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4342 uint32_t idx = act_idx & 4343 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4344 switch (type) { 4345 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4346 shared_rss = mlx5_ipool_get 4347 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4348 idx); 4349 __atomic_add_fetch(&shared_rss->refcnt, 1, 4350 __ATOMIC_RELAXED); 4351 return idx; 4352 default: 4353 break; 4354 } 4355 } 4356 return 0; 4357 } 4358 4359 static unsigned int 4360 find_graph_root(uint32_t rss_level) 4361 { 4362 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4363 MLX5_EXPANSION_ROOT_OUTER; 4364 } 4365 4366 /** 4367 * Get layer flags from the prefix flow. 4368 * 4369 * Some flows may be split to several subflows, the prefix subflow gets the 4370 * match items and the suffix sub flow gets the actions. 4371 * Some actions need the user defined match item flags to get the detail for 4372 * the action. 4373 * This function helps the suffix flow to get the item layer flags from prefix 4374 * subflow. 4375 * 4376 * @param[in] dev_flow 4377 * Pointer the created prefix subflow. 4378 * 4379 * @return 4380 * The layers get from prefix subflow. 4381 */ 4382 static inline uint64_t 4383 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4384 { 4385 uint64_t layers = 0; 4386 4387 /* 4388 * Layers bits could be localization, but usually the compiler will 4389 * help to do the optimization work for source code. 4390 * If no decap actions, use the layers directly. 4391 */ 4392 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4393 return dev_flow->handle->layers; 4394 /* Convert L3 layers with decap action. */ 4395 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4396 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4397 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4398 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4399 /* Convert L4 layers with decap action. */ 4400 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4401 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4402 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4403 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4404 return layers; 4405 } 4406 4407 /** 4408 * Get metadata split action information. 4409 * 4410 * @param[in] actions 4411 * Pointer to the list of actions. 4412 * @param[out] qrss 4413 * Pointer to the return pointer. 4414 * @param[out] qrss_type 4415 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4416 * if no QUEUE/RSS is found. 4417 * @param[out] encap_idx 4418 * Pointer to the index of the encap action if exists, otherwise the last 4419 * action index. 4420 * 4421 * @return 4422 * Total number of actions. 4423 */ 4424 static int 4425 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4426 const struct rte_flow_action **qrss, 4427 int *encap_idx) 4428 { 4429 const struct rte_flow_action_raw_encap *raw_encap; 4430 int actions_n = 0; 4431 int raw_decap_idx = -1; 4432 4433 *encap_idx = -1; 4434 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4435 switch (actions->type) { 4436 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4437 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4438 *encap_idx = actions_n; 4439 break; 4440 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4441 raw_decap_idx = actions_n; 4442 break; 4443 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4444 raw_encap = actions->conf; 4445 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4446 *encap_idx = raw_decap_idx != -1 ? 4447 raw_decap_idx : actions_n; 4448 break; 4449 case RTE_FLOW_ACTION_TYPE_QUEUE: 4450 case RTE_FLOW_ACTION_TYPE_RSS: 4451 *qrss = actions; 4452 break; 4453 default: 4454 break; 4455 } 4456 actions_n++; 4457 } 4458 if (*encap_idx == -1) 4459 *encap_idx = actions_n; 4460 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4461 return actions_n + 1; 4462 } 4463 4464 /** 4465 * Check if the action will change packet. 4466 * 4467 * @param dev 4468 * Pointer to Ethernet device. 4469 * @param[in] type 4470 * action type. 4471 * 4472 * @return 4473 * true if action will change packet, false otherwise. 4474 */ 4475 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4476 enum rte_flow_action_type type) 4477 { 4478 struct mlx5_priv *priv = dev->data->dev_private; 4479 4480 switch (type) { 4481 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4482 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4483 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4484 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4485 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4486 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4487 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4488 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4489 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4490 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4491 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4492 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4493 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4494 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4495 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4496 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4497 case RTE_FLOW_ACTION_TYPE_SET_META: 4498 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4499 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4500 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4501 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4502 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4503 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4504 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4505 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4506 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4507 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4508 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4509 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4510 return true; 4511 case RTE_FLOW_ACTION_TYPE_FLAG: 4512 case RTE_FLOW_ACTION_TYPE_MARK: 4513 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4514 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4515 return true; 4516 else 4517 return false; 4518 default: 4519 return false; 4520 } 4521 } 4522 4523 /** 4524 * Check meter action from the action list. 4525 * 4526 * @param dev 4527 * Pointer to Ethernet device. 4528 * @param[in] actions 4529 * Pointer to the list of actions. 4530 * @param[out] has_mtr 4531 * Pointer to the meter exist flag. 4532 * @param[out] has_modify 4533 * Pointer to the flag showing there's packet change action. 4534 * @param[out] meter_id 4535 * Pointer to the meter id. 4536 * 4537 * @return 4538 * Total number of actions. 4539 */ 4540 static int 4541 flow_check_meter_action(struct rte_eth_dev *dev, 4542 const struct rte_flow_action actions[], 4543 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4544 { 4545 const struct rte_flow_action_meter *mtr = NULL; 4546 int actions_n = 0; 4547 4548 MLX5_ASSERT(has_mtr); 4549 *has_mtr = false; 4550 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4551 switch (actions->type) { 4552 case RTE_FLOW_ACTION_TYPE_METER: 4553 mtr = actions->conf; 4554 *meter_id = mtr->mtr_id; 4555 *has_mtr = true; 4556 break; 4557 default: 4558 break; 4559 } 4560 if (!*has_mtr) 4561 *has_modify |= flow_check_modify_action_type(dev, 4562 actions->type); 4563 actions_n++; 4564 } 4565 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4566 return actions_n + 1; 4567 } 4568 4569 /** 4570 * Check if the flow should be split due to hairpin. 4571 * The reason for the split is that in current HW we can't 4572 * support encap and push-vlan on Rx, so if a flow contains 4573 * these actions we move it to Tx. 4574 * 4575 * @param dev 4576 * Pointer to Ethernet device. 4577 * @param[in] attr 4578 * Flow rule attributes. 4579 * @param[in] actions 4580 * Associated actions (list terminated by the END action). 4581 * 4582 * @return 4583 * > 0 the number of actions and the flow should be split, 4584 * 0 when no split required. 4585 */ 4586 static int 4587 flow_check_hairpin_split(struct rte_eth_dev *dev, 4588 const struct rte_flow_attr *attr, 4589 const struct rte_flow_action actions[]) 4590 { 4591 int queue_action = 0; 4592 int action_n = 0; 4593 int split = 0; 4594 int push_vlan = 0; 4595 const struct rte_flow_action_queue *queue; 4596 const struct rte_flow_action_rss *rss; 4597 const struct rte_flow_action_raw_encap *raw_encap; 4598 const struct rte_eth_hairpin_conf *conf; 4599 4600 if (!attr->ingress) 4601 return 0; 4602 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4603 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4604 push_vlan = 1; 4605 switch (actions->type) { 4606 case RTE_FLOW_ACTION_TYPE_QUEUE: 4607 queue = actions->conf; 4608 if (queue == NULL) 4609 return 0; 4610 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4611 if (conf == NULL || conf->tx_explicit != 0) 4612 return 0; 4613 queue_action = 1; 4614 action_n++; 4615 break; 4616 case RTE_FLOW_ACTION_TYPE_RSS: 4617 rss = actions->conf; 4618 if (rss == NULL || rss->queue_num == 0) 4619 return 0; 4620 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4621 if (conf == NULL || conf->tx_explicit != 0) 4622 return 0; 4623 queue_action = 1; 4624 action_n++; 4625 break; 4626 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4627 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4628 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4629 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4630 split++; 4631 action_n++; 4632 break; 4633 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4634 if (push_vlan) 4635 split++; 4636 action_n++; 4637 break; 4638 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4639 raw_encap = actions->conf; 4640 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4641 split++; 4642 action_n++; 4643 break; 4644 default: 4645 action_n++; 4646 break; 4647 } 4648 } 4649 if (split && queue_action) 4650 return action_n; 4651 return 0; 4652 } 4653 4654 /* Declare flow create/destroy prototype in advance. */ 4655 static uint32_t 4656 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4657 const struct rte_flow_attr *attr, 4658 const struct rte_flow_item items[], 4659 const struct rte_flow_action actions[], 4660 bool external, struct rte_flow_error *error); 4661 4662 static void 4663 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4664 uint32_t flow_idx); 4665 4666 int 4667 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4668 struct mlx5_list_entry *entry, void *cb_ctx) 4669 { 4670 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4671 struct mlx5_flow_mreg_copy_resource *mcp_res = 4672 container_of(entry, typeof(*mcp_res), hlist_ent); 4673 4674 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4675 } 4676 4677 struct mlx5_list_entry * 4678 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4679 { 4680 struct rte_eth_dev *dev = tool_ctx; 4681 struct mlx5_priv *priv = dev->data->dev_private; 4682 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4683 struct mlx5_flow_mreg_copy_resource *mcp_res; 4684 struct rte_flow_error *error = ctx->error; 4685 uint32_t idx = 0; 4686 int ret; 4687 uint32_t mark_id = *(uint32_t *)(ctx->data); 4688 struct rte_flow_attr attr = { 4689 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4690 .ingress = 1, 4691 }; 4692 struct mlx5_rte_flow_item_tag tag_spec = { 4693 .data = mark_id, 4694 }; 4695 struct rte_flow_item items[] = { 4696 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4697 }; 4698 struct rte_flow_action_mark ftag = { 4699 .id = mark_id, 4700 }; 4701 struct mlx5_flow_action_copy_mreg cp_mreg = { 4702 .dst = REG_B, 4703 .src = REG_NON, 4704 }; 4705 struct rte_flow_action_jump jump = { 4706 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4707 }; 4708 struct rte_flow_action actions[] = { 4709 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4710 }; 4711 4712 /* Fill the register fields in the flow. */ 4713 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4714 if (ret < 0) 4715 return NULL; 4716 tag_spec.id = ret; 4717 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4718 if (ret < 0) 4719 return NULL; 4720 cp_mreg.src = ret; 4721 /* Provide the full width of FLAG specific value. */ 4722 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4723 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4724 /* Build a new flow. */ 4725 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4726 items[0] = (struct rte_flow_item){ 4727 .type = (enum rte_flow_item_type) 4728 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4729 .spec = &tag_spec, 4730 }; 4731 items[1] = (struct rte_flow_item){ 4732 .type = RTE_FLOW_ITEM_TYPE_END, 4733 }; 4734 actions[0] = (struct rte_flow_action){ 4735 .type = (enum rte_flow_action_type) 4736 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4737 .conf = &ftag, 4738 }; 4739 actions[1] = (struct rte_flow_action){ 4740 .type = (enum rte_flow_action_type) 4741 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4742 .conf = &cp_mreg, 4743 }; 4744 actions[2] = (struct rte_flow_action){ 4745 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4746 .conf = &jump, 4747 }; 4748 actions[3] = (struct rte_flow_action){ 4749 .type = RTE_FLOW_ACTION_TYPE_END, 4750 }; 4751 } else { 4752 /* Default rule, wildcard match. */ 4753 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4754 items[0] = (struct rte_flow_item){ 4755 .type = RTE_FLOW_ITEM_TYPE_END, 4756 }; 4757 actions[0] = (struct rte_flow_action){ 4758 .type = (enum rte_flow_action_type) 4759 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4760 .conf = &cp_mreg, 4761 }; 4762 actions[1] = (struct rte_flow_action){ 4763 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4764 .conf = &jump, 4765 }; 4766 actions[2] = (struct rte_flow_action){ 4767 .type = RTE_FLOW_ACTION_TYPE_END, 4768 }; 4769 } 4770 /* Build a new entry. */ 4771 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4772 if (!mcp_res) { 4773 rte_errno = ENOMEM; 4774 return NULL; 4775 } 4776 mcp_res->idx = idx; 4777 mcp_res->mark_id = mark_id; 4778 /* 4779 * The copy Flows are not included in any list. There 4780 * ones are referenced from other Flows and can not 4781 * be applied, removed, deleted in arbitrary order 4782 * by list traversing. 4783 */ 4784 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4785 &attr, items, actions, false, error); 4786 if (!mcp_res->rix_flow) { 4787 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4788 return NULL; 4789 } 4790 return &mcp_res->hlist_ent; 4791 } 4792 4793 struct mlx5_list_entry * 4794 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4795 void *cb_ctx __rte_unused) 4796 { 4797 struct rte_eth_dev *dev = tool_ctx; 4798 struct mlx5_priv *priv = dev->data->dev_private; 4799 struct mlx5_flow_mreg_copy_resource *mcp_res; 4800 uint32_t idx = 0; 4801 4802 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4803 if (!mcp_res) { 4804 rte_errno = ENOMEM; 4805 return NULL; 4806 } 4807 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4808 mcp_res->idx = idx; 4809 return &mcp_res->hlist_ent; 4810 } 4811 4812 void 4813 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4814 { 4815 struct mlx5_flow_mreg_copy_resource *mcp_res = 4816 container_of(entry, typeof(*mcp_res), hlist_ent); 4817 struct rte_eth_dev *dev = tool_ctx; 4818 struct mlx5_priv *priv = dev->data->dev_private; 4819 4820 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4821 } 4822 4823 /** 4824 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4825 * 4826 * As mark_id is unique, if there's already a registered flow for the mark_id, 4827 * return by increasing the reference counter of the resource. Otherwise, create 4828 * the resource (mcp_res) and flow. 4829 * 4830 * Flow looks like, 4831 * - If ingress port is ANY and reg_c[1] is mark_id, 4832 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4833 * 4834 * For default flow (zero mark_id), flow is like, 4835 * - If ingress port is ANY, 4836 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4837 * 4838 * @param dev 4839 * Pointer to Ethernet device. 4840 * @param mark_id 4841 * ID of MARK action, zero means default flow for META. 4842 * @param[out] error 4843 * Perform verbose error reporting if not NULL. 4844 * 4845 * @return 4846 * Associated resource on success, NULL otherwise and rte_errno is set. 4847 */ 4848 static struct mlx5_flow_mreg_copy_resource * 4849 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4850 struct rte_flow_error *error) 4851 { 4852 struct mlx5_priv *priv = dev->data->dev_private; 4853 struct mlx5_list_entry *entry; 4854 struct mlx5_flow_cb_ctx ctx = { 4855 .dev = dev, 4856 .error = error, 4857 .data = &mark_id, 4858 }; 4859 4860 /* Check if already registered. */ 4861 MLX5_ASSERT(priv->mreg_cp_tbl); 4862 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4863 if (!entry) 4864 return NULL; 4865 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4866 hlist_ent); 4867 } 4868 4869 void 4870 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4871 { 4872 struct mlx5_flow_mreg_copy_resource *mcp_res = 4873 container_of(entry, typeof(*mcp_res), hlist_ent); 4874 struct rte_eth_dev *dev = tool_ctx; 4875 struct mlx5_priv *priv = dev->data->dev_private; 4876 4877 MLX5_ASSERT(mcp_res->rix_flow); 4878 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4879 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4880 } 4881 4882 /** 4883 * Release flow in RX_CP_TBL. 4884 * 4885 * @param dev 4886 * Pointer to Ethernet device. 4887 * @flow 4888 * Parent flow for wich copying is provided. 4889 */ 4890 static void 4891 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4892 struct rte_flow *flow) 4893 { 4894 struct mlx5_flow_mreg_copy_resource *mcp_res; 4895 struct mlx5_priv *priv = dev->data->dev_private; 4896 4897 if (!flow->rix_mreg_copy) 4898 return; 4899 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 4900 flow->rix_mreg_copy); 4901 if (!mcp_res || !priv->mreg_cp_tbl) 4902 return; 4903 MLX5_ASSERT(mcp_res->rix_flow); 4904 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 4905 flow->rix_mreg_copy = 0; 4906 } 4907 4908 /** 4909 * Remove the default copy action from RX_CP_TBL. 4910 * 4911 * This functions is called in the mlx5_dev_start(). No thread safe 4912 * is guaranteed. 4913 * 4914 * @param dev 4915 * Pointer to Ethernet device. 4916 */ 4917 static void 4918 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 4919 { 4920 struct mlx5_list_entry *entry; 4921 struct mlx5_priv *priv = dev->data->dev_private; 4922 struct mlx5_flow_cb_ctx ctx; 4923 uint32_t mark_id; 4924 4925 /* Check if default flow is registered. */ 4926 if (!priv->mreg_cp_tbl) 4927 return; 4928 mark_id = MLX5_DEFAULT_COPY_ID; 4929 ctx.data = &mark_id; 4930 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 4931 if (!entry) 4932 return; 4933 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 4934 } 4935 4936 /** 4937 * Add the default copy action in in RX_CP_TBL. 4938 * 4939 * This functions is called in the mlx5_dev_start(). No thread safe 4940 * is guaranteed. 4941 * 4942 * @param dev 4943 * Pointer to Ethernet device. 4944 * @param[out] error 4945 * Perform verbose error reporting if not NULL. 4946 * 4947 * @return 4948 * 0 for success, negative value otherwise and rte_errno is set. 4949 */ 4950 static int 4951 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 4952 struct rte_flow_error *error) 4953 { 4954 struct mlx5_priv *priv = dev->data->dev_private; 4955 struct mlx5_flow_mreg_copy_resource *mcp_res; 4956 struct mlx5_flow_cb_ctx ctx; 4957 uint32_t mark_id; 4958 4959 /* Check whether extensive metadata feature is engaged. */ 4960 if (!priv->sh->config.dv_flow_en || 4961 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 4962 !mlx5_flow_ext_mreg_supported(dev) || 4963 !priv->sh->dv_regc0_mask) 4964 return 0; 4965 /* 4966 * Add default mreg copy flow may be called multiple time, but 4967 * only be called once in stop. Avoid register it twice. 4968 */ 4969 mark_id = MLX5_DEFAULT_COPY_ID; 4970 ctx.data = &mark_id; 4971 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 4972 return 0; 4973 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 4974 if (!mcp_res) 4975 return -rte_errno; 4976 return 0; 4977 } 4978 4979 /** 4980 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4981 * 4982 * All the flow having Q/RSS action should be split by 4983 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 4984 * performs the following, 4985 * - CQE->flow_tag := reg_c[1] (MARK) 4986 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 4987 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 4988 * but there should be a flow per each MARK ID set by MARK action. 4989 * 4990 * For the aforementioned reason, if there's a MARK action in flow's action 4991 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 4992 * the MARK ID to CQE's flow_tag like, 4993 * - If reg_c[1] is mark_id, 4994 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4995 * 4996 * For SET_META action which stores value in reg_c[0], as the destination is 4997 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 4998 * MARK ID means the default flow. The default flow looks like, 4999 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5000 * 5001 * @param dev 5002 * Pointer to Ethernet device. 5003 * @param flow 5004 * Pointer to flow structure. 5005 * @param[in] actions 5006 * Pointer to the list of actions. 5007 * @param[out] error 5008 * Perform verbose error reporting if not NULL. 5009 * 5010 * @return 5011 * 0 on success, negative value otherwise and rte_errno is set. 5012 */ 5013 static int 5014 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5015 struct rte_flow *flow, 5016 const struct rte_flow_action *actions, 5017 struct rte_flow_error *error) 5018 { 5019 struct mlx5_priv *priv = dev->data->dev_private; 5020 struct mlx5_sh_config *config = &priv->sh->config; 5021 struct mlx5_flow_mreg_copy_resource *mcp_res; 5022 const struct rte_flow_action_mark *mark; 5023 5024 /* Check whether extensive metadata feature is engaged. */ 5025 if (!config->dv_flow_en || 5026 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5027 !mlx5_flow_ext_mreg_supported(dev) || 5028 !priv->sh->dv_regc0_mask) 5029 return 0; 5030 /* Find MARK action. */ 5031 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5032 switch (actions->type) { 5033 case RTE_FLOW_ACTION_TYPE_FLAG: 5034 mcp_res = flow_mreg_add_copy_action 5035 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5036 if (!mcp_res) 5037 return -rte_errno; 5038 flow->rix_mreg_copy = mcp_res->idx; 5039 return 0; 5040 case RTE_FLOW_ACTION_TYPE_MARK: 5041 mark = (const struct rte_flow_action_mark *) 5042 actions->conf; 5043 mcp_res = 5044 flow_mreg_add_copy_action(dev, mark->id, error); 5045 if (!mcp_res) 5046 return -rte_errno; 5047 flow->rix_mreg_copy = mcp_res->idx; 5048 return 0; 5049 default: 5050 break; 5051 } 5052 } 5053 return 0; 5054 } 5055 5056 #define MLX5_MAX_SPLIT_ACTIONS 24 5057 #define MLX5_MAX_SPLIT_ITEMS 24 5058 5059 /** 5060 * Split the hairpin flow. 5061 * Since HW can't support encap and push-vlan on Rx, we move these 5062 * actions to Tx. 5063 * If the count action is after the encap then we also 5064 * move the count action. in this case the count will also measure 5065 * the outer bytes. 5066 * 5067 * @param dev 5068 * Pointer to Ethernet device. 5069 * @param[in] actions 5070 * Associated actions (list terminated by the END action). 5071 * @param[out] actions_rx 5072 * Rx flow actions. 5073 * @param[out] actions_tx 5074 * Tx flow actions.. 5075 * @param[out] pattern_tx 5076 * The pattern items for the Tx flow. 5077 * @param[out] flow_id 5078 * The flow ID connected to this flow. 5079 * 5080 * @return 5081 * 0 on success. 5082 */ 5083 static int 5084 flow_hairpin_split(struct rte_eth_dev *dev, 5085 const struct rte_flow_action actions[], 5086 struct rte_flow_action actions_rx[], 5087 struct rte_flow_action actions_tx[], 5088 struct rte_flow_item pattern_tx[], 5089 uint32_t flow_id) 5090 { 5091 const struct rte_flow_action_raw_encap *raw_encap; 5092 const struct rte_flow_action_raw_decap *raw_decap; 5093 struct mlx5_rte_flow_action_set_tag *set_tag; 5094 struct rte_flow_action *tag_action; 5095 struct mlx5_rte_flow_item_tag *tag_item; 5096 struct rte_flow_item *item; 5097 char *addr; 5098 int push_vlan = 0; 5099 int encap = 0; 5100 5101 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5102 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5103 push_vlan = 1; 5104 switch (actions->type) { 5105 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5106 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5107 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5108 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5109 rte_memcpy(actions_tx, actions, 5110 sizeof(struct rte_flow_action)); 5111 actions_tx++; 5112 break; 5113 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5114 if (push_vlan) { 5115 rte_memcpy(actions_tx, actions, 5116 sizeof(struct rte_flow_action)); 5117 actions_tx++; 5118 } else { 5119 rte_memcpy(actions_rx, actions, 5120 sizeof(struct rte_flow_action)); 5121 actions_rx++; 5122 } 5123 break; 5124 case RTE_FLOW_ACTION_TYPE_COUNT: 5125 if (encap) { 5126 rte_memcpy(actions_tx, actions, 5127 sizeof(struct rte_flow_action)); 5128 actions_tx++; 5129 } else { 5130 rte_memcpy(actions_rx, actions, 5131 sizeof(struct rte_flow_action)); 5132 actions_rx++; 5133 } 5134 break; 5135 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5136 raw_encap = actions->conf; 5137 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5138 memcpy(actions_tx, actions, 5139 sizeof(struct rte_flow_action)); 5140 actions_tx++; 5141 encap = 1; 5142 } else { 5143 rte_memcpy(actions_rx, actions, 5144 sizeof(struct rte_flow_action)); 5145 actions_rx++; 5146 } 5147 break; 5148 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5149 raw_decap = actions->conf; 5150 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5151 memcpy(actions_tx, actions, 5152 sizeof(struct rte_flow_action)); 5153 actions_tx++; 5154 } else { 5155 rte_memcpy(actions_rx, actions, 5156 sizeof(struct rte_flow_action)); 5157 actions_rx++; 5158 } 5159 break; 5160 default: 5161 rte_memcpy(actions_rx, actions, 5162 sizeof(struct rte_flow_action)); 5163 actions_rx++; 5164 break; 5165 } 5166 } 5167 /* Add set meta action and end action for the Rx flow. */ 5168 tag_action = actions_rx; 5169 tag_action->type = (enum rte_flow_action_type) 5170 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5171 actions_rx++; 5172 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5173 actions_rx++; 5174 set_tag = (void *)actions_rx; 5175 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5176 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5177 .data = flow_id, 5178 }; 5179 MLX5_ASSERT(set_tag->id > REG_NON); 5180 tag_action->conf = set_tag; 5181 /* Create Tx item list. */ 5182 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5183 addr = (void *)&pattern_tx[2]; 5184 item = pattern_tx; 5185 item->type = (enum rte_flow_item_type) 5186 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5187 tag_item = (void *)addr; 5188 tag_item->data = flow_id; 5189 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5190 MLX5_ASSERT(set_tag->id > REG_NON); 5191 item->spec = tag_item; 5192 addr += sizeof(struct mlx5_rte_flow_item_tag); 5193 tag_item = (void *)addr; 5194 tag_item->data = UINT32_MAX; 5195 tag_item->id = UINT16_MAX; 5196 item->mask = tag_item; 5197 item->last = NULL; 5198 item++; 5199 item->type = RTE_FLOW_ITEM_TYPE_END; 5200 return 0; 5201 } 5202 5203 /** 5204 * The last stage of splitting chain, just creates the subflow 5205 * without any modification. 5206 * 5207 * @param[in] dev 5208 * Pointer to Ethernet device. 5209 * @param[in] flow 5210 * Parent flow structure pointer. 5211 * @param[in, out] sub_flow 5212 * Pointer to return the created subflow, may be NULL. 5213 * @param[in] attr 5214 * Flow rule attributes. 5215 * @param[in] items 5216 * Pattern specification (list terminated by the END pattern item). 5217 * @param[in] actions 5218 * Associated actions (list terminated by the END action). 5219 * @param[in] flow_split_info 5220 * Pointer to flow split info structure. 5221 * @param[out] error 5222 * Perform verbose error reporting if not NULL. 5223 * @return 5224 * 0 on success, negative value otherwise 5225 */ 5226 static int 5227 flow_create_split_inner(struct rte_eth_dev *dev, 5228 struct rte_flow *flow, 5229 struct mlx5_flow **sub_flow, 5230 const struct rte_flow_attr *attr, 5231 const struct rte_flow_item items[], 5232 const struct rte_flow_action actions[], 5233 struct mlx5_flow_split_info *flow_split_info, 5234 struct rte_flow_error *error) 5235 { 5236 struct mlx5_flow *dev_flow; 5237 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5238 5239 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5240 flow_split_info->flow_idx, error); 5241 if (!dev_flow) 5242 return -rte_errno; 5243 dev_flow->flow = flow; 5244 dev_flow->external = flow_split_info->external; 5245 dev_flow->skip_scale = flow_split_info->skip_scale; 5246 /* Subflow object was created, we must include one in the list. */ 5247 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5248 dev_flow->handle, next); 5249 /* 5250 * If dev_flow is as one of the suffix flow, some actions in suffix 5251 * flow may need some user defined item layer flags, and pass the 5252 * Metadata rxq mark flag to suffix flow as well. 5253 */ 5254 if (flow_split_info->prefix_layers) 5255 dev_flow->handle->layers = flow_split_info->prefix_layers; 5256 if (flow_split_info->prefix_mark) { 5257 MLX5_ASSERT(wks); 5258 wks->mark = 1; 5259 } 5260 if (sub_flow) 5261 *sub_flow = dev_flow; 5262 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5263 dev_flow->dv.table_id = flow_split_info->table_id; 5264 #endif 5265 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5266 } 5267 5268 /** 5269 * Get the sub policy of a meter. 5270 * 5271 * @param[in] dev 5272 * Pointer to Ethernet device. 5273 * @param[in] flow 5274 * Parent flow structure pointer. 5275 * @param wks 5276 * Pointer to thread flow work space. 5277 * @param[in] attr 5278 * Flow rule attributes. 5279 * @param[in] items 5280 * Pattern specification (list terminated by the END pattern item). 5281 * @param[out] error 5282 * Perform verbose error reporting if not NULL. 5283 * 5284 * @return 5285 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5286 */ 5287 static struct mlx5_flow_meter_sub_policy * 5288 get_meter_sub_policy(struct rte_eth_dev *dev, 5289 struct rte_flow *flow, 5290 struct mlx5_flow_workspace *wks, 5291 const struct rte_flow_attr *attr, 5292 const struct rte_flow_item items[], 5293 struct rte_flow_error *error) 5294 { 5295 struct mlx5_flow_meter_policy *policy; 5296 struct mlx5_flow_meter_policy *final_policy; 5297 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5298 5299 policy = wks->policy; 5300 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5301 if (final_policy->is_rss || final_policy->is_queue) { 5302 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5303 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5304 uint32_t i; 5305 5306 /* 5307 * This is a tmp dev_flow, 5308 * no need to register any matcher for it in translate. 5309 */ 5310 wks->skip_matcher_reg = 1; 5311 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5312 struct mlx5_flow dev_flow = {0}; 5313 struct mlx5_flow_handle dev_handle = { {0} }; 5314 uint8_t fate = final_policy->act_cnt[i].fate_action; 5315 5316 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5317 const struct rte_flow_action_rss *rss_act = 5318 final_policy->act_cnt[i].rss->conf; 5319 struct rte_flow_action rss_actions[2] = { 5320 [0] = { 5321 .type = RTE_FLOW_ACTION_TYPE_RSS, 5322 .conf = rss_act, 5323 }, 5324 [1] = { 5325 .type = RTE_FLOW_ACTION_TYPE_END, 5326 .conf = NULL, 5327 } 5328 }; 5329 5330 dev_flow.handle = &dev_handle; 5331 dev_flow.ingress = attr->ingress; 5332 dev_flow.flow = flow; 5333 dev_flow.external = 0; 5334 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5335 dev_flow.dv.transfer = attr->transfer; 5336 #endif 5337 /** 5338 * Translate RSS action to get rss hash fields. 5339 */ 5340 if (flow_drv_translate(dev, &dev_flow, attr, 5341 items, rss_actions, error)) 5342 goto exit; 5343 rss_desc_v[i] = wks->rss_desc; 5344 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5345 rss_desc_v[i].hash_fields = 5346 dev_flow.hash_fields; 5347 rss_desc_v[i].queue_num = 5348 rss_desc_v[i].hash_fields ? 5349 rss_desc_v[i].queue_num : 1; 5350 rss_desc_v[i].tunnel = 5351 !!(dev_flow.handle->layers & 5352 MLX5_FLOW_LAYER_TUNNEL); 5353 /* Use the RSS queues in the containers. */ 5354 rss_desc_v[i].queue = 5355 (uint16_t *)(uintptr_t)rss_act->queue; 5356 rss_desc[i] = &rss_desc_v[i]; 5357 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5358 /* This is queue action. */ 5359 rss_desc_v[i] = wks->rss_desc; 5360 rss_desc_v[i].key_len = 0; 5361 rss_desc_v[i].hash_fields = 0; 5362 rss_desc_v[i].queue = 5363 &final_policy->act_cnt[i].queue; 5364 rss_desc_v[i].queue_num = 1; 5365 rss_desc[i] = &rss_desc_v[i]; 5366 } else { 5367 rss_desc[i] = NULL; 5368 } 5369 } 5370 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5371 flow, policy, rss_desc); 5372 } else { 5373 enum mlx5_meter_domain mtr_domain = 5374 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5375 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5376 MLX5_MTR_DOMAIN_INGRESS); 5377 sub_policy = policy->sub_policys[mtr_domain][0]; 5378 } 5379 if (!sub_policy) 5380 rte_flow_error_set(error, EINVAL, 5381 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5382 "Failed to get meter sub-policy."); 5383 exit: 5384 return sub_policy; 5385 } 5386 5387 /** 5388 * Split the meter flow. 5389 * 5390 * As meter flow will split to three sub flow, other than meter 5391 * action, the other actions make sense to only meter accepts 5392 * the packet. If it need to be dropped, no other additional 5393 * actions should be take. 5394 * 5395 * One kind of special action which decapsulates the L3 tunnel 5396 * header will be in the prefix sub flow, as not to take the 5397 * L3 tunnel header into account. 5398 * 5399 * @param[in] dev 5400 * Pointer to Ethernet device. 5401 * @param[in] flow 5402 * Parent flow structure pointer. 5403 * @param wks 5404 * Pointer to thread flow work space. 5405 * @param[in] attr 5406 * Flow rule attributes. 5407 * @param[in] items 5408 * Pattern specification (list terminated by the END pattern item). 5409 * @param[out] sfx_items 5410 * Suffix flow match items (list terminated by the END pattern item). 5411 * @param[in] actions 5412 * Associated actions (list terminated by the END action). 5413 * @param[out] actions_sfx 5414 * Suffix flow actions. 5415 * @param[out] actions_pre 5416 * Prefix flow actions. 5417 * @param[out] mtr_flow_id 5418 * Pointer to meter flow id. 5419 * @param[out] error 5420 * Perform verbose error reporting if not NULL. 5421 * 5422 * @return 5423 * 0 on success, a negative errno value otherwise and rte_errno is set. 5424 */ 5425 static int 5426 flow_meter_split_prep(struct rte_eth_dev *dev, 5427 struct rte_flow *flow, 5428 struct mlx5_flow_workspace *wks, 5429 const struct rte_flow_attr *attr, 5430 const struct rte_flow_item items[], 5431 struct rte_flow_item sfx_items[], 5432 const struct rte_flow_action actions[], 5433 struct rte_flow_action actions_sfx[], 5434 struct rte_flow_action actions_pre[], 5435 uint32_t *mtr_flow_id, 5436 struct rte_flow_error *error) 5437 { 5438 struct mlx5_priv *priv = dev->data->dev_private; 5439 struct mlx5_flow_meter_info *fm = wks->fm; 5440 struct rte_flow_action *tag_action = NULL; 5441 struct rte_flow_item *tag_item; 5442 struct mlx5_rte_flow_action_set_tag *set_tag; 5443 const struct rte_flow_action_raw_encap *raw_encap; 5444 const struct rte_flow_action_raw_decap *raw_decap; 5445 struct mlx5_rte_flow_item_tag *tag_item_spec; 5446 struct mlx5_rte_flow_item_tag *tag_item_mask; 5447 uint32_t tag_id = 0; 5448 struct rte_flow_item *vlan_item_dst = NULL; 5449 const struct rte_flow_item *vlan_item_src = NULL; 5450 const struct rte_flow_item *orig_items = items; 5451 struct rte_flow_action *hw_mtr_action; 5452 struct rte_flow_action *action_pre_head = NULL; 5453 uint16_t flow_src_port = priv->representor_id; 5454 bool mtr_first; 5455 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5456 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5457 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5458 uint32_t flow_id = 0; 5459 uint32_t flow_id_reversed = 0; 5460 uint8_t flow_id_bits = 0; 5461 bool after_meter = false; 5462 int shift; 5463 5464 /* Prepare the suffix subflow items. */ 5465 tag_item = sfx_items++; 5466 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5467 int item_type = items->type; 5468 5469 switch (item_type) { 5470 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5471 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5472 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5473 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5474 return -rte_errno; 5475 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5476 flow_src_port != priv->representor_id) { 5477 if (flow_drv_mtr_hierarchy_rule_create(dev, 5478 flow, fm, 5479 flow_src_port, 5480 items, 5481 error)) 5482 return -rte_errno; 5483 } 5484 memcpy(sfx_items, items, sizeof(*sfx_items)); 5485 sfx_items++; 5486 break; 5487 case RTE_FLOW_ITEM_TYPE_VLAN: 5488 /* Determine if copy vlan item below. */ 5489 vlan_item_src = items; 5490 vlan_item_dst = sfx_items++; 5491 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5492 break; 5493 default: 5494 break; 5495 } 5496 } 5497 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5498 sfx_items++; 5499 mtr_first = priv->sh->meter_aso_en && 5500 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5501 /* For ASO meter, meter must be before tag in TX direction. */ 5502 if (mtr_first) { 5503 action_pre_head = actions_pre++; 5504 /* Leave space for tag action. */ 5505 tag_action = actions_pre++; 5506 } 5507 /* Prepare the actions for prefix and suffix flow. */ 5508 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5509 struct rte_flow_action *action_cur = NULL; 5510 5511 switch (actions->type) { 5512 case RTE_FLOW_ACTION_TYPE_METER: 5513 if (mtr_first) { 5514 action_cur = action_pre_head; 5515 } else { 5516 /* Leave space for tag action. */ 5517 tag_action = actions_pre++; 5518 action_cur = actions_pre++; 5519 } 5520 after_meter = true; 5521 break; 5522 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5523 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5524 action_cur = actions_pre++; 5525 break; 5526 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5527 raw_encap = actions->conf; 5528 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5529 action_cur = actions_pre++; 5530 break; 5531 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5532 raw_decap = actions->conf; 5533 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5534 action_cur = actions_pre++; 5535 break; 5536 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5537 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5538 if (vlan_item_dst && vlan_item_src) { 5539 memcpy(vlan_item_dst, vlan_item_src, 5540 sizeof(*vlan_item_dst)); 5541 /* 5542 * Convert to internal match item, it is used 5543 * for vlan push and set vid. 5544 */ 5545 vlan_item_dst->type = (enum rte_flow_item_type) 5546 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5547 } 5548 break; 5549 case RTE_FLOW_ACTION_TYPE_COUNT: 5550 if (fm->def_policy) 5551 action_cur = after_meter ? 5552 actions_sfx++ : actions_pre++; 5553 break; 5554 default: 5555 break; 5556 } 5557 if (!action_cur) 5558 action_cur = (fm->def_policy) ? 5559 actions_sfx++ : actions_pre++; 5560 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5561 } 5562 /* Add end action to the actions. */ 5563 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5564 if (priv->sh->meter_aso_en) { 5565 /** 5566 * For ASO meter, need to add an extra jump action explicitly, 5567 * to jump from meter to policer table. 5568 */ 5569 struct mlx5_flow_meter_sub_policy *sub_policy; 5570 struct mlx5_flow_tbl_data_entry *tbl_data; 5571 5572 if (!fm->def_policy) { 5573 sub_policy = get_meter_sub_policy(dev, flow, wks, 5574 attr, orig_items, 5575 error); 5576 if (!sub_policy) 5577 return -rte_errno; 5578 } else { 5579 enum mlx5_meter_domain mtr_domain = 5580 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5581 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5582 MLX5_MTR_DOMAIN_INGRESS); 5583 5584 sub_policy = 5585 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5586 } 5587 tbl_data = container_of(sub_policy->tbl_rsc, 5588 struct mlx5_flow_tbl_data_entry, tbl); 5589 hw_mtr_action = actions_pre++; 5590 hw_mtr_action->type = (enum rte_flow_action_type) 5591 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5592 hw_mtr_action->conf = tbl_data->jump.action; 5593 } 5594 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5595 actions_pre++; 5596 if (!tag_action) 5597 return rte_flow_error_set(error, ENOMEM, 5598 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5599 NULL, "No tag action space."); 5600 if (!mtr_flow_id) { 5601 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5602 goto exit; 5603 } 5604 /* Only default-policy Meter creates mtr flow id. */ 5605 if (fm->def_policy) { 5606 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5607 if (!tag_id) 5608 return rte_flow_error_set(error, ENOMEM, 5609 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5610 "Failed to allocate meter flow id."); 5611 flow_id = tag_id - 1; 5612 flow_id_bits = (!flow_id) ? 1 : 5613 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5614 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5615 mtr_reg_bits) { 5616 mlx5_ipool_free(fm->flow_ipool, tag_id); 5617 return rte_flow_error_set(error, EINVAL, 5618 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5619 "Meter flow id exceeds max limit."); 5620 } 5621 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5622 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5623 } 5624 /* Build tag actions and items for meter_id/meter flow_id. */ 5625 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5626 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5627 tag_item_mask = tag_item_spec + 1; 5628 /* Both flow_id and meter_id share the same register. */ 5629 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5630 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5631 0, error), 5632 .offset = mtr_id_offset, 5633 .length = mtr_reg_bits, 5634 .data = flow->meter, 5635 }; 5636 /* 5637 * The color Reg bits used by flow_id are growing from 5638 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5639 */ 5640 for (shift = 0; shift < flow_id_bits; shift++) 5641 flow_id_reversed = (flow_id_reversed << 1) | 5642 ((flow_id >> shift) & 0x1); 5643 set_tag->data |= 5644 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5645 tag_item_spec->id = set_tag->id; 5646 tag_item_spec->data = set_tag->data << mtr_id_offset; 5647 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5648 tag_action->type = (enum rte_flow_action_type) 5649 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5650 tag_action->conf = set_tag; 5651 tag_item->type = (enum rte_flow_item_type) 5652 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5653 tag_item->spec = tag_item_spec; 5654 tag_item->last = NULL; 5655 tag_item->mask = tag_item_mask; 5656 exit: 5657 if (mtr_flow_id) 5658 *mtr_flow_id = tag_id; 5659 return 0; 5660 } 5661 5662 /** 5663 * Split action list having QUEUE/RSS for metadata register copy. 5664 * 5665 * Once Q/RSS action is detected in user's action list, the flow action 5666 * should be split in order to copy metadata registers, which will happen in 5667 * RX_CP_TBL like, 5668 * - CQE->flow_tag := reg_c[1] (MARK) 5669 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5670 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5671 * This is because the last action of each flow must be a terminal action 5672 * (QUEUE, RSS or DROP). 5673 * 5674 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5675 * stored and kept in the mlx5_flow structure per each sub_flow. 5676 * 5677 * The Q/RSS action is replaced with, 5678 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5679 * And the following JUMP action is added at the end, 5680 * - JUMP, to RX_CP_TBL. 5681 * 5682 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5683 * flow_create_split_metadata() routine. The flow will look like, 5684 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5685 * 5686 * @param dev 5687 * Pointer to Ethernet device. 5688 * @param[out] split_actions 5689 * Pointer to store split actions to jump to CP_TBL. 5690 * @param[in] actions 5691 * Pointer to the list of original flow actions. 5692 * @param[in] qrss 5693 * Pointer to the Q/RSS action. 5694 * @param[in] actions_n 5695 * Number of original actions. 5696 * @param[in] mtr_sfx 5697 * Check if it is in meter suffix table. 5698 * @param[out] error 5699 * Perform verbose error reporting if not NULL. 5700 * 5701 * @return 5702 * non-zero unique flow_id on success, otherwise 0 and 5703 * error/rte_error are set. 5704 */ 5705 static uint32_t 5706 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5707 struct rte_flow_action *split_actions, 5708 const struct rte_flow_action *actions, 5709 const struct rte_flow_action *qrss, 5710 int actions_n, int mtr_sfx, 5711 struct rte_flow_error *error) 5712 { 5713 struct mlx5_priv *priv = dev->data->dev_private; 5714 struct mlx5_rte_flow_action_set_tag *set_tag; 5715 struct rte_flow_action_jump *jump; 5716 const int qrss_idx = qrss - actions; 5717 uint32_t flow_id = 0; 5718 int ret = 0; 5719 5720 /* 5721 * Given actions will be split 5722 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5723 * - Add jump to mreg CP_TBL. 5724 * As a result, there will be one more action. 5725 */ 5726 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5727 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5728 ++actions_n; 5729 set_tag = (void *)(split_actions + actions_n); 5730 /* 5731 * If we are not the meter suffix flow, add the tag action. 5732 * Since meter suffix flow already has the tag added. 5733 */ 5734 if (!mtr_sfx) { 5735 /* 5736 * Allocate the new subflow ID. This one is unique within 5737 * device and not shared with representors. Otherwise, 5738 * we would have to resolve multi-thread access synch 5739 * issue. Each flow on the shared device is appended 5740 * with source vport identifier, so the resulting 5741 * flows will be unique in the shared (by master and 5742 * representors) domain even if they have coinciding 5743 * IDs. 5744 */ 5745 mlx5_ipool_malloc(priv->sh->ipool 5746 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5747 if (!flow_id) 5748 return rte_flow_error_set(error, ENOMEM, 5749 RTE_FLOW_ERROR_TYPE_ACTION, 5750 NULL, "can't allocate id " 5751 "for split Q/RSS subflow"); 5752 /* Internal SET_TAG action to set flow ID. */ 5753 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5754 .data = flow_id, 5755 }; 5756 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5757 if (ret < 0) 5758 return ret; 5759 set_tag->id = ret; 5760 /* Construct new actions array. */ 5761 /* Replace QUEUE/RSS action. */ 5762 split_actions[qrss_idx] = (struct rte_flow_action){ 5763 .type = (enum rte_flow_action_type) 5764 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5765 .conf = set_tag, 5766 }; 5767 } else { 5768 /* 5769 * If we are the suffix flow of meter, tag already exist. 5770 * Set the QUEUE/RSS action to void. 5771 */ 5772 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5773 } 5774 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5775 jump = (void *)(set_tag + 1); 5776 *jump = (struct rte_flow_action_jump){ 5777 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5778 }; 5779 split_actions[actions_n - 2] = (struct rte_flow_action){ 5780 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5781 .conf = jump, 5782 }; 5783 split_actions[actions_n - 1] = (struct rte_flow_action){ 5784 .type = RTE_FLOW_ACTION_TYPE_END, 5785 }; 5786 return flow_id; 5787 } 5788 5789 /** 5790 * Extend the given action list for Tx metadata copy. 5791 * 5792 * Copy the given action list to the ext_actions and add flow metadata register 5793 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5794 * 5795 * @param[out] ext_actions 5796 * Pointer to the extended action list. 5797 * @param[in] actions 5798 * Pointer to the list of actions. 5799 * @param[in] actions_n 5800 * Number of actions in the list. 5801 * @param[out] error 5802 * Perform verbose error reporting if not NULL. 5803 * @param[in] encap_idx 5804 * The encap action index. 5805 * 5806 * @return 5807 * 0 on success, negative value otherwise 5808 */ 5809 static int 5810 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5811 struct rte_flow_action *ext_actions, 5812 const struct rte_flow_action *actions, 5813 int actions_n, struct rte_flow_error *error, 5814 int encap_idx) 5815 { 5816 struct mlx5_flow_action_copy_mreg *cp_mreg = 5817 (struct mlx5_flow_action_copy_mreg *) 5818 (ext_actions + actions_n + 1); 5819 int ret; 5820 5821 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5822 if (ret < 0) 5823 return ret; 5824 cp_mreg->dst = ret; 5825 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5826 if (ret < 0) 5827 return ret; 5828 cp_mreg->src = ret; 5829 if (encap_idx != 0) 5830 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5831 if (encap_idx == actions_n - 1) { 5832 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5833 .type = (enum rte_flow_action_type) 5834 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5835 .conf = cp_mreg, 5836 }; 5837 ext_actions[actions_n] = (struct rte_flow_action){ 5838 .type = RTE_FLOW_ACTION_TYPE_END, 5839 }; 5840 } else { 5841 ext_actions[encap_idx] = (struct rte_flow_action){ 5842 .type = (enum rte_flow_action_type) 5843 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5844 .conf = cp_mreg, 5845 }; 5846 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5847 sizeof(*ext_actions) * (actions_n - encap_idx)); 5848 } 5849 return 0; 5850 } 5851 5852 /** 5853 * Check the match action from the action list. 5854 * 5855 * @param[in] actions 5856 * Pointer to the list of actions. 5857 * @param[in] attr 5858 * Flow rule attributes. 5859 * @param[in] action 5860 * The action to be check if exist. 5861 * @param[out] match_action_pos 5862 * Pointer to the position of the matched action if exists, otherwise is -1. 5863 * @param[out] qrss_action_pos 5864 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5865 * @param[out] modify_after_mirror 5866 * Pointer to the flag of modify action after FDB mirroring. 5867 * 5868 * @return 5869 * > 0 the total number of actions. 5870 * 0 if not found match action in action list. 5871 */ 5872 static int 5873 flow_check_match_action(const struct rte_flow_action actions[], 5874 const struct rte_flow_attr *attr, 5875 enum rte_flow_action_type action, 5876 int *match_action_pos, int *qrss_action_pos, 5877 int *modify_after_mirror) 5878 { 5879 const struct rte_flow_action_sample *sample; 5880 const struct rte_flow_action_raw_decap *decap; 5881 int actions_n = 0; 5882 uint32_t ratio = 0; 5883 int sub_type = 0; 5884 int flag = 0; 5885 int fdb_mirror = 0; 5886 5887 *match_action_pos = -1; 5888 *qrss_action_pos = -1; 5889 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5890 if (actions->type == action) { 5891 flag = 1; 5892 *match_action_pos = actions_n; 5893 } 5894 switch (actions->type) { 5895 case RTE_FLOW_ACTION_TYPE_QUEUE: 5896 case RTE_FLOW_ACTION_TYPE_RSS: 5897 *qrss_action_pos = actions_n; 5898 break; 5899 case RTE_FLOW_ACTION_TYPE_SAMPLE: 5900 sample = actions->conf; 5901 ratio = sample->ratio; 5902 sub_type = ((const struct rte_flow_action *) 5903 (sample->actions))->type; 5904 if (ratio == 1 && attr->transfer && 5905 sub_type != RTE_FLOW_ACTION_TYPE_END) 5906 fdb_mirror = 1; 5907 break; 5908 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 5909 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 5910 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 5911 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 5912 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 5913 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 5914 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 5915 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 5916 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 5917 case RTE_FLOW_ACTION_TYPE_SET_TTL: 5918 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 5919 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 5920 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 5921 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 5922 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 5923 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 5924 case RTE_FLOW_ACTION_TYPE_FLAG: 5925 case RTE_FLOW_ACTION_TYPE_MARK: 5926 case RTE_FLOW_ACTION_TYPE_SET_META: 5927 case RTE_FLOW_ACTION_TYPE_SET_TAG: 5928 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 5929 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5930 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5931 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5932 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5933 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5934 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 5935 case RTE_FLOW_ACTION_TYPE_METER: 5936 if (fdb_mirror) 5937 *modify_after_mirror = 1; 5938 break; 5939 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5940 decap = actions->conf; 5941 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 5942 ; 5943 actions_n++; 5944 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 5945 const struct rte_flow_action_raw_encap *encap = 5946 actions->conf; 5947 if (decap->size <= 5948 MLX5_ENCAPSULATION_DECISION_SIZE && 5949 encap->size > 5950 MLX5_ENCAPSULATION_DECISION_SIZE) 5951 /* L3 encap. */ 5952 break; 5953 } 5954 if (fdb_mirror) 5955 *modify_after_mirror = 1; 5956 break; 5957 default: 5958 break; 5959 } 5960 actions_n++; 5961 } 5962 if (flag && fdb_mirror && !*modify_after_mirror) { 5963 /* FDB mirroring uses the destination array to implement 5964 * instead of FLOW_SAMPLER object. 5965 */ 5966 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 5967 flag = 0; 5968 } 5969 /* Count RTE_FLOW_ACTION_TYPE_END. */ 5970 return flag ? actions_n + 1 : 0; 5971 } 5972 5973 #define SAMPLE_SUFFIX_ITEM 3 5974 5975 /** 5976 * Split the sample flow. 5977 * 5978 * As sample flow will split to two sub flow, sample flow with 5979 * sample action, the other actions will move to new suffix flow. 5980 * 5981 * Also add unique tag id with tag action in the sample flow, 5982 * the same tag id will be as match in the suffix flow. 5983 * 5984 * @param dev 5985 * Pointer to Ethernet device. 5986 * @param[in] add_tag 5987 * Add extra tag action flag. 5988 * @param[out] sfx_items 5989 * Suffix flow match items (list terminated by the END pattern item). 5990 * @param[in] actions 5991 * Associated actions (list terminated by the END action). 5992 * @param[out] actions_sfx 5993 * Suffix flow actions. 5994 * @param[out] actions_pre 5995 * Prefix flow actions. 5996 * @param[in] actions_n 5997 * The total number of actions. 5998 * @param[in] sample_action_pos 5999 * The sample action position. 6000 * @param[in] qrss_action_pos 6001 * The Queue/RSS action position. 6002 * @param[in] jump_table 6003 * Add extra jump action flag. 6004 * @param[out] error 6005 * Perform verbose error reporting if not NULL. 6006 * 6007 * @return 6008 * 0 on success, or unique flow_id, a negative errno value 6009 * otherwise and rte_errno is set. 6010 */ 6011 static int 6012 flow_sample_split_prep(struct rte_eth_dev *dev, 6013 int add_tag, 6014 const struct rte_flow_item items[], 6015 struct rte_flow_item sfx_items[], 6016 const struct rte_flow_action actions[], 6017 struct rte_flow_action actions_sfx[], 6018 struct rte_flow_action actions_pre[], 6019 int actions_n, 6020 int sample_action_pos, 6021 int qrss_action_pos, 6022 int jump_table, 6023 struct rte_flow_error *error) 6024 { 6025 struct mlx5_priv *priv = dev->data->dev_private; 6026 struct mlx5_rte_flow_action_set_tag *set_tag; 6027 struct mlx5_rte_flow_item_tag *tag_spec; 6028 struct mlx5_rte_flow_item_tag *tag_mask; 6029 struct rte_flow_action_jump *jump_action; 6030 uint32_t tag_id = 0; 6031 int append_index = 0; 6032 int set_tag_idx = -1; 6033 int index; 6034 int ret; 6035 6036 if (sample_action_pos < 0) 6037 return rte_flow_error_set(error, EINVAL, 6038 RTE_FLOW_ERROR_TYPE_ACTION, 6039 NULL, "invalid position of sample " 6040 "action in list"); 6041 /* Prepare the actions for prefix and suffix flow. */ 6042 if (add_tag) { 6043 /* Update the new added tag action index preceding 6044 * the PUSH_VLAN or ENCAP action. 6045 */ 6046 const struct rte_flow_action_raw_encap *raw_encap; 6047 const struct rte_flow_action *action = actions; 6048 int encap_idx; 6049 int action_idx = 0; 6050 int raw_decap_idx = -1; 6051 int push_vlan_idx = -1; 6052 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6053 switch (action->type) { 6054 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6055 raw_decap_idx = action_idx; 6056 break; 6057 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6058 raw_encap = action->conf; 6059 if (raw_encap->size > 6060 MLX5_ENCAPSULATION_DECISION_SIZE) { 6061 encap_idx = raw_decap_idx != -1 ? 6062 raw_decap_idx : action_idx; 6063 if (encap_idx < sample_action_pos && 6064 push_vlan_idx == -1) 6065 set_tag_idx = encap_idx; 6066 } 6067 break; 6068 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6069 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6070 encap_idx = action_idx; 6071 if (encap_idx < sample_action_pos && 6072 push_vlan_idx == -1) 6073 set_tag_idx = encap_idx; 6074 break; 6075 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6076 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6077 if (action_idx < sample_action_pos && 6078 push_vlan_idx == -1) { 6079 set_tag_idx = action_idx; 6080 push_vlan_idx = action_idx; 6081 } 6082 break; 6083 default: 6084 break; 6085 } 6086 action_idx++; 6087 } 6088 } 6089 /* Prepare the actions for prefix and suffix flow. */ 6090 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6091 index = qrss_action_pos; 6092 /* Put the preceding the Queue/RSS action into prefix flow. */ 6093 if (index != 0) 6094 memcpy(actions_pre, actions, 6095 sizeof(struct rte_flow_action) * index); 6096 /* Put others preceding the sample action into prefix flow. */ 6097 if (sample_action_pos > index + 1) 6098 memcpy(actions_pre + index, actions + index + 1, 6099 sizeof(struct rte_flow_action) * 6100 (sample_action_pos - index - 1)); 6101 index = sample_action_pos - 1; 6102 /* Put Queue/RSS action into Suffix flow. */ 6103 memcpy(actions_sfx, actions + qrss_action_pos, 6104 sizeof(struct rte_flow_action)); 6105 actions_sfx++; 6106 } else if (add_tag && set_tag_idx >= 0) { 6107 if (set_tag_idx > 0) 6108 memcpy(actions_pre, actions, 6109 sizeof(struct rte_flow_action) * set_tag_idx); 6110 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6111 sizeof(struct rte_flow_action) * 6112 (sample_action_pos - set_tag_idx)); 6113 index = sample_action_pos; 6114 } else { 6115 index = sample_action_pos; 6116 if (index != 0) 6117 memcpy(actions_pre, actions, 6118 sizeof(struct rte_flow_action) * index); 6119 } 6120 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6121 * For CX6DX and above, metadata registers Cx preserve their value, 6122 * add an extra tag action for NIC-RX and E-Switch Domain. 6123 */ 6124 if (add_tag) { 6125 /* Prepare the prefix tag action. */ 6126 append_index++; 6127 set_tag = (void *)(actions_pre + actions_n + append_index); 6128 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6129 /* Trust VF/SF on CX5 not supported meter so that the reserved 6130 * metadata regC is REG_NON, back to use application tag 6131 * index 0. 6132 */ 6133 if (unlikely(ret == REG_NON)) 6134 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6135 if (ret < 0) 6136 return ret; 6137 mlx5_ipool_malloc(priv->sh->ipool 6138 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6139 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6140 .id = ret, 6141 .data = tag_id, 6142 }; 6143 /* Prepare the suffix subflow items. */ 6144 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6145 tag_spec->data = tag_id; 6146 tag_spec->id = set_tag->id; 6147 tag_mask = tag_spec + 1; 6148 tag_mask->data = UINT32_MAX; 6149 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6150 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6151 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6152 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6153 memcpy(sfx_items, items, sizeof(*sfx_items)); 6154 sfx_items++; 6155 break; 6156 } 6157 } 6158 sfx_items[0] = (struct rte_flow_item){ 6159 .type = (enum rte_flow_item_type) 6160 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6161 .spec = tag_spec, 6162 .last = NULL, 6163 .mask = tag_mask, 6164 }; 6165 sfx_items[1] = (struct rte_flow_item){ 6166 .type = (enum rte_flow_item_type) 6167 RTE_FLOW_ITEM_TYPE_END, 6168 }; 6169 /* Prepare the tag action in prefix subflow. */ 6170 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6171 actions_pre[set_tag_idx] = 6172 (struct rte_flow_action){ 6173 .type = (enum rte_flow_action_type) 6174 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6175 .conf = set_tag, 6176 }; 6177 /* Update next sample position due to add one tag action */ 6178 index += 1; 6179 } 6180 /* Copy the sample action into prefix flow. */ 6181 memcpy(actions_pre + index, actions + sample_action_pos, 6182 sizeof(struct rte_flow_action)); 6183 index += 1; 6184 /* For the modify action after the sample action in E-Switch mirroring, 6185 * Add the extra jump action in prefix subflow and jump into the next 6186 * table, then do the modify action in the new table. 6187 */ 6188 if (jump_table) { 6189 /* Prepare the prefix jump action. */ 6190 append_index++; 6191 jump_action = (void *)(actions_pre + actions_n + append_index); 6192 jump_action->group = jump_table; 6193 actions_pre[index++] = 6194 (struct rte_flow_action){ 6195 .type = (enum rte_flow_action_type) 6196 RTE_FLOW_ACTION_TYPE_JUMP, 6197 .conf = jump_action, 6198 }; 6199 } 6200 actions_pre[index] = (struct rte_flow_action){ 6201 .type = (enum rte_flow_action_type) 6202 RTE_FLOW_ACTION_TYPE_END, 6203 }; 6204 /* Put the actions after sample into Suffix flow. */ 6205 memcpy(actions_sfx, actions + sample_action_pos + 1, 6206 sizeof(struct rte_flow_action) * 6207 (actions_n - sample_action_pos - 1)); 6208 return tag_id; 6209 } 6210 6211 /** 6212 * The splitting for metadata feature. 6213 * 6214 * - Q/RSS action on NIC Rx should be split in order to pass by 6215 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6216 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6217 * 6218 * - All the actions on NIC Tx should have a mreg copy action to 6219 * copy reg_a from WQE to reg_c[0]. 6220 * 6221 * @param dev 6222 * Pointer to Ethernet device. 6223 * @param[in] flow 6224 * Parent flow structure pointer. 6225 * @param[in] attr 6226 * Flow rule attributes. 6227 * @param[in] items 6228 * Pattern specification (list terminated by the END pattern item). 6229 * @param[in] actions 6230 * Associated actions (list terminated by the END action). 6231 * @param[in] flow_split_info 6232 * Pointer to flow split info structure. 6233 * @param[out] error 6234 * Perform verbose error reporting if not NULL. 6235 * @return 6236 * 0 on success, negative value otherwise 6237 */ 6238 static int 6239 flow_create_split_metadata(struct rte_eth_dev *dev, 6240 struct rte_flow *flow, 6241 const struct rte_flow_attr *attr, 6242 const struct rte_flow_item items[], 6243 const struct rte_flow_action actions[], 6244 struct mlx5_flow_split_info *flow_split_info, 6245 struct rte_flow_error *error) 6246 { 6247 struct mlx5_priv *priv = dev->data->dev_private; 6248 struct mlx5_sh_config *config = &priv->sh->config; 6249 const struct rte_flow_action *qrss = NULL; 6250 struct rte_flow_action *ext_actions = NULL; 6251 struct mlx5_flow *dev_flow = NULL; 6252 uint32_t qrss_id = 0; 6253 int mtr_sfx = 0; 6254 size_t act_size; 6255 int actions_n; 6256 int encap_idx; 6257 int ret; 6258 6259 /* Check whether extensive metadata feature is engaged. */ 6260 if (!config->dv_flow_en || 6261 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6262 !mlx5_flow_ext_mreg_supported(dev)) 6263 return flow_create_split_inner(dev, flow, NULL, attr, items, 6264 actions, flow_split_info, error); 6265 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6266 &encap_idx); 6267 if (qrss) { 6268 /* Exclude hairpin flows from splitting. */ 6269 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6270 const struct rte_flow_action_queue *queue; 6271 6272 queue = qrss->conf; 6273 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6274 qrss = NULL; 6275 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6276 const struct rte_flow_action_rss *rss; 6277 6278 rss = qrss->conf; 6279 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6280 qrss = NULL; 6281 } 6282 } 6283 if (qrss) { 6284 /* Check if it is in meter suffix table. */ 6285 mtr_sfx = attr->group == 6286 ((attr->transfer && priv->fdb_def_rule) ? 6287 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6288 MLX5_FLOW_TABLE_LEVEL_METER); 6289 /* 6290 * Q/RSS action on NIC Rx should be split in order to pass by 6291 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6292 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6293 */ 6294 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6295 sizeof(struct rte_flow_action_set_tag) + 6296 sizeof(struct rte_flow_action_jump); 6297 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6298 SOCKET_ID_ANY); 6299 if (!ext_actions) 6300 return rte_flow_error_set(error, ENOMEM, 6301 RTE_FLOW_ERROR_TYPE_ACTION, 6302 NULL, "no memory to split " 6303 "metadata flow"); 6304 /* 6305 * Create the new actions list with removed Q/RSS action 6306 * and appended set tag and jump to register copy table 6307 * (RX_CP_TBL). We should preallocate unique tag ID here 6308 * in advance, because it is needed for set tag action. 6309 */ 6310 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6311 qrss, actions_n, 6312 mtr_sfx, error); 6313 if (!mtr_sfx && !qrss_id) { 6314 ret = -rte_errno; 6315 goto exit; 6316 } 6317 } else if (attr->egress) { 6318 /* 6319 * All the actions on NIC Tx should have a metadata register 6320 * copy action to copy reg_a from WQE to reg_c[meta] 6321 */ 6322 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6323 sizeof(struct mlx5_flow_action_copy_mreg); 6324 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6325 SOCKET_ID_ANY); 6326 if (!ext_actions) 6327 return rte_flow_error_set(error, ENOMEM, 6328 RTE_FLOW_ERROR_TYPE_ACTION, 6329 NULL, "no memory to split " 6330 "metadata flow"); 6331 /* Create the action list appended with copy register. */ 6332 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6333 actions_n, error, encap_idx); 6334 if (ret < 0) 6335 goto exit; 6336 } 6337 /* Add the unmodified original or prefix subflow. */ 6338 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6339 items, ext_actions ? ext_actions : 6340 actions, flow_split_info, error); 6341 if (ret < 0) 6342 goto exit; 6343 MLX5_ASSERT(dev_flow); 6344 if (qrss) { 6345 const struct rte_flow_attr q_attr = { 6346 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6347 .ingress = 1, 6348 }; 6349 /* Internal PMD action to set register. */ 6350 struct mlx5_rte_flow_item_tag q_tag_spec = { 6351 .data = qrss_id, 6352 .id = REG_NON, 6353 }; 6354 struct rte_flow_item q_items[] = { 6355 { 6356 .type = (enum rte_flow_item_type) 6357 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6358 .spec = &q_tag_spec, 6359 .last = NULL, 6360 .mask = NULL, 6361 }, 6362 { 6363 .type = RTE_FLOW_ITEM_TYPE_END, 6364 }, 6365 }; 6366 struct rte_flow_action q_actions[] = { 6367 { 6368 .type = qrss->type, 6369 .conf = qrss->conf, 6370 }, 6371 { 6372 .type = RTE_FLOW_ACTION_TYPE_END, 6373 }, 6374 }; 6375 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6376 6377 /* 6378 * Configure the tag item only if there is no meter subflow. 6379 * Since tag is already marked in the meter suffix subflow 6380 * we can just use the meter suffix items as is. 6381 */ 6382 if (qrss_id) { 6383 /* Not meter subflow. */ 6384 MLX5_ASSERT(!mtr_sfx); 6385 /* 6386 * Put unique id in prefix flow due to it is destroyed 6387 * after suffix flow and id will be freed after there 6388 * is no actual flows with this id and identifier 6389 * reallocation becomes possible (for example, for 6390 * other flows in other threads). 6391 */ 6392 dev_flow->handle->split_flow_id = qrss_id; 6393 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6394 error); 6395 if (ret < 0) 6396 goto exit; 6397 q_tag_spec.id = ret; 6398 } 6399 dev_flow = NULL; 6400 /* Add suffix subflow to execute Q/RSS. */ 6401 flow_split_info->prefix_layers = layers; 6402 flow_split_info->prefix_mark = 0; 6403 flow_split_info->table_id = 0; 6404 ret = flow_create_split_inner(dev, flow, &dev_flow, 6405 &q_attr, mtr_sfx ? items : 6406 q_items, q_actions, 6407 flow_split_info, error); 6408 if (ret < 0) 6409 goto exit; 6410 /* qrss ID should be freed if failed. */ 6411 qrss_id = 0; 6412 MLX5_ASSERT(dev_flow); 6413 } 6414 6415 exit: 6416 /* 6417 * We do not destroy the partially created sub_flows in case of error. 6418 * These ones are included into parent flow list and will be destroyed 6419 * by flow_drv_destroy. 6420 */ 6421 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6422 qrss_id); 6423 mlx5_free(ext_actions); 6424 return ret; 6425 } 6426 6427 /** 6428 * Create meter internal drop flow with the original pattern. 6429 * 6430 * @param dev 6431 * Pointer to Ethernet device. 6432 * @param[in] flow 6433 * Parent flow structure pointer. 6434 * @param[in] attr 6435 * Flow rule attributes. 6436 * @param[in] items 6437 * Pattern specification (list terminated by the END pattern item). 6438 * @param[in] flow_split_info 6439 * Pointer to flow split info structure. 6440 * @param[in] fm 6441 * Pointer to flow meter structure. 6442 * @param[out] error 6443 * Perform verbose error reporting if not NULL. 6444 * @return 6445 * 0 on success, negative value otherwise 6446 */ 6447 static uint32_t 6448 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6449 struct rte_flow *flow, 6450 const struct rte_flow_attr *attr, 6451 const struct rte_flow_item items[], 6452 struct mlx5_flow_split_info *flow_split_info, 6453 struct mlx5_flow_meter_info *fm, 6454 struct rte_flow_error *error) 6455 { 6456 struct mlx5_flow *dev_flow = NULL; 6457 struct rte_flow_attr drop_attr = *attr; 6458 struct rte_flow_action drop_actions[3]; 6459 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6460 6461 MLX5_ASSERT(fm->drop_cnt); 6462 drop_actions[0].type = 6463 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6464 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6465 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6466 drop_actions[1].conf = NULL; 6467 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6468 drop_actions[2].conf = NULL; 6469 drop_split_info.external = false; 6470 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6471 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6472 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6473 return flow_create_split_inner(dev, flow, &dev_flow, 6474 &drop_attr, items, drop_actions, 6475 &drop_split_info, error); 6476 } 6477 6478 /** 6479 * The splitting for meter feature. 6480 * 6481 * - The meter flow will be split to two flows as prefix and 6482 * suffix flow. The packets make sense only it pass the prefix 6483 * meter action. 6484 * 6485 * - Reg_C_5 is used for the packet to match betweend prefix and 6486 * suffix flow. 6487 * 6488 * @param dev 6489 * Pointer to Ethernet device. 6490 * @param[in] flow 6491 * Parent flow structure pointer. 6492 * @param[in] attr 6493 * Flow rule attributes. 6494 * @param[in] items 6495 * Pattern specification (list terminated by the END pattern item). 6496 * @param[in] actions 6497 * Associated actions (list terminated by the END action). 6498 * @param[in] flow_split_info 6499 * Pointer to flow split info structure. 6500 * @param[out] error 6501 * Perform verbose error reporting if not NULL. 6502 * @return 6503 * 0 on success, negative value otherwise 6504 */ 6505 static int 6506 flow_create_split_meter(struct rte_eth_dev *dev, 6507 struct rte_flow *flow, 6508 const struct rte_flow_attr *attr, 6509 const struct rte_flow_item items[], 6510 const struct rte_flow_action actions[], 6511 struct mlx5_flow_split_info *flow_split_info, 6512 struct rte_flow_error *error) 6513 { 6514 struct mlx5_priv *priv = dev->data->dev_private; 6515 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6516 struct rte_flow_action *sfx_actions = NULL; 6517 struct rte_flow_action *pre_actions = NULL; 6518 struct rte_flow_item *sfx_items = NULL; 6519 struct mlx5_flow *dev_flow = NULL; 6520 struct rte_flow_attr sfx_attr = *attr; 6521 struct mlx5_flow_meter_info *fm = NULL; 6522 uint8_t skip_scale_restore; 6523 bool has_mtr = false; 6524 bool has_modify = false; 6525 bool set_mtr_reg = true; 6526 bool is_mtr_hierarchy = false; 6527 uint32_t meter_id = 0; 6528 uint32_t mtr_idx = 0; 6529 uint32_t mtr_flow_id = 0; 6530 size_t act_size; 6531 size_t item_size; 6532 int actions_n = 0; 6533 int ret = 0; 6534 6535 if (priv->mtr_en) 6536 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6537 &has_modify, &meter_id); 6538 if (has_mtr) { 6539 if (flow->meter) { 6540 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6541 if (!fm) 6542 return rte_flow_error_set(error, EINVAL, 6543 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6544 NULL, "Meter not found."); 6545 } else { 6546 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6547 if (!fm) 6548 return rte_flow_error_set(error, EINVAL, 6549 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6550 NULL, "Meter not found."); 6551 ret = mlx5_flow_meter_attach(priv, fm, 6552 &sfx_attr, error); 6553 if (ret) 6554 return -rte_errno; 6555 flow->meter = mtr_idx; 6556 } 6557 MLX5_ASSERT(wks); 6558 wks->fm = fm; 6559 if (!fm->def_policy) { 6560 wks->policy = mlx5_flow_meter_policy_find(dev, 6561 fm->policy_id, 6562 NULL); 6563 MLX5_ASSERT(wks->policy); 6564 if (wks->policy->mark) 6565 wks->mark = 1; 6566 if (wks->policy->is_hierarchy) { 6567 wks->final_policy = 6568 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6569 wks->policy); 6570 if (!wks->final_policy) 6571 return rte_flow_error_set(error, 6572 EINVAL, 6573 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6574 "Failed to find terminal policy of hierarchy."); 6575 is_mtr_hierarchy = true; 6576 } 6577 } 6578 /* 6579 * If it isn't default-policy Meter, and 6580 * 1. Not meter hierarchy and there's no action in flow to change 6581 * packet (modify/encap/decap etc.), OR 6582 * 2. No drop count needed for this meter. 6583 * Then no need to use regC to save meter id anymore. 6584 */ 6585 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6586 set_mtr_reg = false; 6587 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6588 #define METER_PREFIX_ACTION 7 6589 act_size = (sizeof(struct rte_flow_action) * 6590 (actions_n + METER_PREFIX_ACTION)) + 6591 sizeof(struct mlx5_rte_flow_action_set_tag); 6592 /* Suffix items: tag, vlan, port id, end. */ 6593 #define METER_SUFFIX_ITEM 4 6594 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6595 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6596 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6597 0, SOCKET_ID_ANY); 6598 if (!sfx_actions) 6599 return rte_flow_error_set(error, ENOMEM, 6600 RTE_FLOW_ERROR_TYPE_ACTION, 6601 NULL, "no memory to split " 6602 "meter flow"); 6603 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6604 act_size); 6605 /* There's no suffix flow for meter of non-default policy. */ 6606 if (!fm->def_policy) 6607 pre_actions = sfx_actions + 1; 6608 else 6609 pre_actions = sfx_actions + actions_n; 6610 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6611 items, sfx_items, actions, 6612 sfx_actions, pre_actions, 6613 (set_mtr_reg ? &mtr_flow_id : NULL), 6614 error); 6615 if (ret) { 6616 ret = -rte_errno; 6617 goto exit; 6618 } 6619 /* Add the prefix subflow. */ 6620 skip_scale_restore = flow_split_info->skip_scale; 6621 flow_split_info->skip_scale |= 6622 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6623 ret = flow_create_split_inner(dev, flow, &dev_flow, 6624 attr, items, pre_actions, 6625 flow_split_info, error); 6626 flow_split_info->skip_scale = skip_scale_restore; 6627 if (ret) { 6628 if (mtr_flow_id) 6629 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6630 ret = -rte_errno; 6631 goto exit; 6632 } 6633 if (mtr_flow_id) { 6634 dev_flow->handle->split_flow_id = mtr_flow_id; 6635 dev_flow->handle->is_meter_flow_id = 1; 6636 } 6637 if (!fm->def_policy) { 6638 if (!set_mtr_reg && fm->drop_cnt) 6639 ret = 6640 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6641 &sfx_attr, items, 6642 flow_split_info, 6643 fm, error); 6644 goto exit; 6645 } 6646 /* Setting the sfx group atrr. */ 6647 sfx_attr.group = sfx_attr.transfer ? 6648 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6649 MLX5_FLOW_TABLE_LEVEL_METER; 6650 flow_split_info->prefix_layers = 6651 flow_get_prefix_layer_flags(dev_flow); 6652 flow_split_info->prefix_mark |= wks->mark; 6653 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6654 } 6655 /* Add the prefix subflow. */ 6656 ret = flow_create_split_metadata(dev, flow, 6657 &sfx_attr, sfx_items ? 6658 sfx_items : items, 6659 sfx_actions ? sfx_actions : actions, 6660 flow_split_info, error); 6661 exit: 6662 if (sfx_actions) 6663 mlx5_free(sfx_actions); 6664 return ret; 6665 } 6666 6667 /** 6668 * The splitting for sample feature. 6669 * 6670 * Once Sample action is detected in the action list, the flow actions should 6671 * be split into prefix sub flow and suffix sub flow. 6672 * 6673 * The original items remain in the prefix sub flow, all actions preceding the 6674 * sample action and the sample action itself will be copied to the prefix 6675 * sub flow, the actions following the sample action will be copied to the 6676 * suffix sub flow, Queue action always be located in the suffix sub flow. 6677 * 6678 * In order to make the packet from prefix sub flow matches with suffix sub 6679 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6680 * flow uses tag item with the unique flow id. 6681 * 6682 * @param dev 6683 * Pointer to Ethernet device. 6684 * @param[in] flow 6685 * Parent flow structure pointer. 6686 * @param[in] attr 6687 * Flow rule attributes. 6688 * @param[in] items 6689 * Pattern specification (list terminated by the END pattern item). 6690 * @param[in] actions 6691 * Associated actions (list terminated by the END action). 6692 * @param[in] flow_split_info 6693 * Pointer to flow split info structure. 6694 * @param[out] error 6695 * Perform verbose error reporting if not NULL. 6696 * @return 6697 * 0 on success, negative value otherwise 6698 */ 6699 static int 6700 flow_create_split_sample(struct rte_eth_dev *dev, 6701 struct rte_flow *flow, 6702 const struct rte_flow_attr *attr, 6703 const struct rte_flow_item items[], 6704 const struct rte_flow_action actions[], 6705 struct mlx5_flow_split_info *flow_split_info, 6706 struct rte_flow_error *error) 6707 { 6708 struct mlx5_priv *priv = dev->data->dev_private; 6709 struct rte_flow_action *sfx_actions = NULL; 6710 struct rte_flow_action *pre_actions = NULL; 6711 struct rte_flow_item *sfx_items = NULL; 6712 struct mlx5_flow *dev_flow = NULL; 6713 struct rte_flow_attr sfx_attr = *attr; 6714 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6715 struct mlx5_flow_dv_sample_resource *sample_res; 6716 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6717 struct mlx5_flow_tbl_resource *sfx_tbl; 6718 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6719 #endif 6720 size_t act_size; 6721 size_t item_size; 6722 uint32_t fdb_tx = 0; 6723 int32_t tag_id = 0; 6724 int actions_n = 0; 6725 int sample_action_pos; 6726 int qrss_action_pos; 6727 int add_tag = 0; 6728 int modify_after_mirror = 0; 6729 uint16_t jump_table = 0; 6730 const uint32_t next_ft_step = 1; 6731 int ret = 0; 6732 struct mlx5_priv *item_port_priv = NULL; 6733 const struct rte_flow_item *item; 6734 6735 if (priv->sampler_en) 6736 actions_n = flow_check_match_action(actions, attr, 6737 RTE_FLOW_ACTION_TYPE_SAMPLE, 6738 &sample_action_pos, &qrss_action_pos, 6739 &modify_after_mirror); 6740 if (actions_n) { 6741 /* The prefix actions must includes sample, tag, end. */ 6742 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6743 + sizeof(struct mlx5_rte_flow_action_set_tag); 6744 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6745 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6746 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6747 item_size), 0, SOCKET_ID_ANY); 6748 if (!sfx_actions) 6749 return rte_flow_error_set(error, ENOMEM, 6750 RTE_FLOW_ERROR_TYPE_ACTION, 6751 NULL, "no memory to split " 6752 "sample flow"); 6753 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 6754 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6755 const struct rte_flow_item_port_id *spec; 6756 6757 spec = (const struct rte_flow_item_port_id *)item->spec; 6758 if (spec) 6759 item_port_priv = 6760 mlx5_port_to_eswitch_info(spec->id, true); 6761 break; 6762 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6763 const struct rte_flow_item_ethdev *spec; 6764 6765 spec = (const struct rte_flow_item_ethdev *)item->spec; 6766 if (spec) 6767 item_port_priv = 6768 mlx5_port_to_eswitch_info(spec->port_id, true); 6769 break; 6770 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 6771 const struct rte_flow_item_ethdev *spec; 6772 6773 spec = (const struct rte_flow_item_ethdev *)item->spec; 6774 if (spec) 6775 item_port_priv = 6776 mlx5_port_to_eswitch_info(spec->port_id, true); 6777 break; 6778 } 6779 } 6780 /* The representor_id is UINT16_MAX for uplink. */ 6781 fdb_tx = (attr->transfer && 6782 flow_source_vport_representor(priv, item_port_priv)); 6783 /* 6784 * When reg_c_preserve is set, metadata registers Cx preserve 6785 * their value even through packet duplication. 6786 */ 6787 add_tag = (!fdb_tx || 6788 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6789 if (add_tag) 6790 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6791 + act_size); 6792 if (modify_after_mirror) 6793 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6794 next_ft_step; 6795 pre_actions = sfx_actions + actions_n; 6796 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6797 actions, sfx_actions, 6798 pre_actions, actions_n, 6799 sample_action_pos, 6800 qrss_action_pos, jump_table, 6801 error); 6802 if (tag_id < 0 || (add_tag && !tag_id)) { 6803 ret = -rte_errno; 6804 goto exit; 6805 } 6806 if (modify_after_mirror) 6807 flow_split_info->skip_scale = 6808 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6809 /* Add the prefix subflow. */ 6810 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6811 items, pre_actions, 6812 flow_split_info, error); 6813 if (ret) { 6814 ret = -rte_errno; 6815 goto exit; 6816 } 6817 dev_flow->handle->split_flow_id = tag_id; 6818 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6819 if (!modify_after_mirror) { 6820 /* Set the sfx group attr. */ 6821 sample_res = (struct mlx5_flow_dv_sample_resource *) 6822 dev_flow->dv.sample_res; 6823 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6824 sample_res->normal_path_tbl; 6825 sfx_tbl_data = container_of(sfx_tbl, 6826 struct mlx5_flow_tbl_data_entry, 6827 tbl); 6828 sfx_attr.group = sfx_attr.transfer ? 6829 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6830 } else { 6831 MLX5_ASSERT(attr->transfer); 6832 sfx_attr.group = jump_table; 6833 } 6834 flow_split_info->prefix_layers = 6835 flow_get_prefix_layer_flags(dev_flow); 6836 MLX5_ASSERT(wks); 6837 flow_split_info->prefix_mark |= wks->mark; 6838 /* Suffix group level already be scaled with factor, set 6839 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6840 * again in translation. 6841 */ 6842 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6843 #endif 6844 } 6845 /* Add the suffix subflow. */ 6846 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6847 sfx_items ? sfx_items : items, 6848 sfx_actions ? sfx_actions : actions, 6849 flow_split_info, error); 6850 exit: 6851 if (sfx_actions) 6852 mlx5_free(sfx_actions); 6853 return ret; 6854 } 6855 6856 /** 6857 * Split the flow to subflow set. The splitters might be linked 6858 * in the chain, like this: 6859 * flow_create_split_outer() calls: 6860 * flow_create_split_meter() calls: 6861 * flow_create_split_metadata(meter_subflow_0) calls: 6862 * flow_create_split_inner(metadata_subflow_0) 6863 * flow_create_split_inner(metadata_subflow_1) 6864 * flow_create_split_inner(metadata_subflow_2) 6865 * flow_create_split_metadata(meter_subflow_1) calls: 6866 * flow_create_split_inner(metadata_subflow_0) 6867 * flow_create_split_inner(metadata_subflow_1) 6868 * flow_create_split_inner(metadata_subflow_2) 6869 * 6870 * This provide flexible way to add new levels of flow splitting. 6871 * The all of successfully created subflows are included to the 6872 * parent flow dev_flow list. 6873 * 6874 * @param dev 6875 * Pointer to Ethernet device. 6876 * @param[in] flow 6877 * Parent flow structure pointer. 6878 * @param[in] attr 6879 * Flow rule attributes. 6880 * @param[in] items 6881 * Pattern specification (list terminated by the END pattern item). 6882 * @param[in] actions 6883 * Associated actions (list terminated by the END action). 6884 * @param[in] flow_split_info 6885 * Pointer to flow split info structure. 6886 * @param[out] error 6887 * Perform verbose error reporting if not NULL. 6888 * @return 6889 * 0 on success, negative value otherwise 6890 */ 6891 static int 6892 flow_create_split_outer(struct rte_eth_dev *dev, 6893 struct rte_flow *flow, 6894 const struct rte_flow_attr *attr, 6895 const struct rte_flow_item items[], 6896 const struct rte_flow_action actions[], 6897 struct mlx5_flow_split_info *flow_split_info, 6898 struct rte_flow_error *error) 6899 { 6900 int ret; 6901 6902 ret = flow_create_split_sample(dev, flow, attr, items, 6903 actions, flow_split_info, error); 6904 MLX5_ASSERT(ret <= 0); 6905 return ret; 6906 } 6907 6908 static inline struct mlx5_flow_tunnel * 6909 flow_tunnel_from_rule(const struct mlx5_flow *flow) 6910 { 6911 struct mlx5_flow_tunnel *tunnel; 6912 6913 #pragma GCC diagnostic push 6914 #pragma GCC diagnostic ignored "-Wcast-qual" 6915 tunnel = (typeof(tunnel))flow->tunnel; 6916 #pragma GCC diagnostic pop 6917 6918 return tunnel; 6919 } 6920 6921 /** 6922 * Adjust flow RSS workspace if needed. 6923 * 6924 * @param wks 6925 * Pointer to thread flow work space. 6926 * @param rss_desc 6927 * Pointer to RSS descriptor. 6928 * @param[in] nrssq_num 6929 * New RSS queue number. 6930 * 6931 * @return 6932 * 0 on success, -1 otherwise and rte_errno is set. 6933 */ 6934 static int 6935 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 6936 struct mlx5_flow_rss_desc *rss_desc, 6937 uint32_t nrssq_num) 6938 { 6939 if (likely(nrssq_num <= wks->rssq_num)) 6940 return 0; 6941 rss_desc->queue = realloc(rss_desc->queue, 6942 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 6943 if (!rss_desc->queue) { 6944 rte_errno = ENOMEM; 6945 return -1; 6946 } 6947 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 6948 return 0; 6949 } 6950 6951 /** 6952 * Create a flow and add it to @p list. 6953 * 6954 * @param dev 6955 * Pointer to Ethernet device. 6956 * @param list 6957 * Pointer to a TAILQ flow list. If this parameter NULL, 6958 * no list insertion occurred, flow is just created, 6959 * this is caller's responsibility to track the 6960 * created flow. 6961 * @param[in] attr 6962 * Flow rule attributes. 6963 * @param[in] items 6964 * Pattern specification (list terminated by the END pattern item). 6965 * @param[in] actions 6966 * Associated actions (list terminated by the END action). 6967 * @param[in] external 6968 * This flow rule is created by request external to PMD. 6969 * @param[out] error 6970 * Perform verbose error reporting if not NULL. 6971 * 6972 * @return 6973 * A flow index on success, 0 otherwise and rte_errno is set. 6974 */ 6975 static uint32_t 6976 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 6977 const struct rte_flow_attr *attr, 6978 const struct rte_flow_item items[], 6979 const struct rte_flow_action original_actions[], 6980 bool external, struct rte_flow_error *error) 6981 { 6982 struct mlx5_priv *priv = dev->data->dev_private; 6983 struct rte_flow *flow = NULL; 6984 struct mlx5_flow *dev_flow; 6985 const struct rte_flow_action_rss *rss = NULL; 6986 struct mlx5_translated_action_handle 6987 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 6988 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 6989 union { 6990 struct mlx5_flow_expand_rss buf; 6991 uint8_t buffer[8192]; 6992 } expand_buffer; 6993 union { 6994 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6995 uint8_t buffer[2048]; 6996 } actions_rx; 6997 union { 6998 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 6999 uint8_t buffer[2048]; 7000 } actions_hairpin_tx; 7001 union { 7002 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7003 uint8_t buffer[2048]; 7004 } items_tx; 7005 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7006 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7007 struct mlx5_flow_rss_desc *rss_desc; 7008 const struct rte_flow_action *p_actions_rx; 7009 uint32_t i; 7010 uint32_t idx = 0; 7011 int hairpin_flow; 7012 struct rte_flow_attr attr_tx = { .priority = 0 }; 7013 const struct rte_flow_action *actions; 7014 struct rte_flow_action *translated_actions = NULL; 7015 struct mlx5_flow_tunnel *tunnel; 7016 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7017 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7018 struct mlx5_flow_split_info flow_split_info = { 7019 .external = !!external, 7020 .skip_scale = 0, 7021 .flow_idx = 0, 7022 .prefix_mark = 0, 7023 .prefix_layers = 0, 7024 .table_id = 0 7025 }; 7026 int ret; 7027 7028 MLX5_ASSERT(wks); 7029 rss_desc = &wks->rss_desc; 7030 ret = flow_action_handles_translate(dev, original_actions, 7031 indir_actions, 7032 &indir_actions_n, 7033 &translated_actions, error); 7034 if (ret < 0) { 7035 MLX5_ASSERT(translated_actions == NULL); 7036 return 0; 7037 } 7038 actions = translated_actions ? translated_actions : original_actions; 7039 p_actions_rx = actions; 7040 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7041 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7042 external, hairpin_flow, error); 7043 if (ret < 0) 7044 goto error_before_hairpin_split; 7045 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7046 if (!flow) { 7047 rte_errno = ENOMEM; 7048 goto error_before_hairpin_split; 7049 } 7050 if (hairpin_flow > 0) { 7051 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7052 rte_errno = EINVAL; 7053 goto error_before_hairpin_split; 7054 } 7055 flow_hairpin_split(dev, actions, actions_rx.actions, 7056 actions_hairpin_tx.actions, items_tx.items, 7057 idx); 7058 p_actions_rx = actions_rx.actions; 7059 } 7060 flow_split_info.flow_idx = idx; 7061 flow->drv_type = flow_get_drv_type(dev, attr); 7062 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7063 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7064 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7065 /* RSS Action only works on NIC RX domain */ 7066 if (attr->ingress) 7067 rss = flow_get_rss_action(dev, p_actions_rx); 7068 if (rss) { 7069 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 7070 return 0; 7071 /* 7072 * The following information is required by 7073 * mlx5_flow_hashfields_adjust() in advance. 7074 */ 7075 rss_desc->level = rss->level; 7076 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7077 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7078 } 7079 flow->dev_handles = 0; 7080 if (rss && rss->types) { 7081 unsigned int graph_root; 7082 7083 graph_root = find_graph_root(rss->level); 7084 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7085 items, rss->types, 7086 mlx5_support_expansion, graph_root); 7087 MLX5_ASSERT(ret > 0 && 7088 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7089 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7090 for (i = 0; i < buf->entries; ++i) 7091 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7092 } 7093 } else { 7094 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7095 sizeof(expand_buffer.buffer), 7096 items, sq_specs); 7097 if (ret) { 7098 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7099 NULL, "not enough memory for rte_flow"); 7100 goto error; 7101 } 7102 if (buf->entries == 0) { 7103 buf->entries = 1; 7104 buf->entry[0].pattern = (void *)(uintptr_t)items; 7105 } 7106 } 7107 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7108 indir_actions_n); 7109 for (i = 0; i < buf->entries; ++i) { 7110 /* Initialize flow split data. */ 7111 flow_split_info.prefix_layers = 0; 7112 flow_split_info.prefix_mark = 0; 7113 flow_split_info.skip_scale = 0; 7114 /* 7115 * The splitter may create multiple dev_flows, 7116 * depending on configuration. In the simplest 7117 * case it just creates unmodified original flow. 7118 */ 7119 ret = flow_create_split_outer(dev, flow, attr, 7120 buf->entry[i].pattern, 7121 p_actions_rx, &flow_split_info, 7122 error); 7123 if (ret < 0) 7124 goto error; 7125 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7126 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7127 p_actions_rx, 7128 idx, 7129 wks->flows[0].tunnel, 7130 &default_miss_ctx, 7131 error); 7132 if (ret < 0) { 7133 mlx5_free(default_miss_ctx.queue); 7134 goto error; 7135 } 7136 } 7137 } 7138 /* Create the tx flow. */ 7139 if (hairpin_flow) { 7140 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7141 attr_tx.ingress = 0; 7142 attr_tx.egress = 1; 7143 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7144 actions_hairpin_tx.actions, 7145 idx, error); 7146 if (!dev_flow) 7147 goto error; 7148 dev_flow->flow = flow; 7149 dev_flow->external = 0; 7150 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7151 dev_flow->handle, next); 7152 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7153 items_tx.items, 7154 actions_hairpin_tx.actions, error); 7155 if (ret < 0) 7156 goto error; 7157 } 7158 /* 7159 * Update the metadata register copy table. If extensive 7160 * metadata feature is enabled and registers are supported 7161 * we might create the extra rte_flow for each unique 7162 * MARK/FLAG action ID. 7163 * 7164 * The table is updated for ingress and transfer flows only, because 7165 * the egress Flows belong to the different device and 7166 * copy table should be updated in peer NIC Rx domain. 7167 */ 7168 if ((attr->ingress || attr->transfer) && 7169 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7170 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7171 if (ret) 7172 goto error; 7173 } 7174 /* 7175 * If the flow is external (from application) OR device is started, 7176 * OR mreg discover, then apply immediately. 7177 */ 7178 if (external || dev->data->dev_started || 7179 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7180 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7181 ret = flow_drv_apply(dev, flow, error); 7182 if (ret < 0) 7183 goto error; 7184 } 7185 flow->type = type; 7186 flow_rxq_flags_set(dev, flow); 7187 rte_free(translated_actions); 7188 tunnel = flow_tunnel_from_rule(wks->flows); 7189 if (tunnel) { 7190 flow->tunnel = 1; 7191 flow->tunnel_id = tunnel->tunnel_id; 7192 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7193 mlx5_free(default_miss_ctx.queue); 7194 } 7195 mlx5_flow_pop_thread_workspace(); 7196 return idx; 7197 error: 7198 MLX5_ASSERT(flow); 7199 ret = rte_errno; /* Save rte_errno before cleanup. */ 7200 flow_mreg_del_copy_action(dev, flow); 7201 flow_drv_destroy(dev, flow); 7202 if (rss_desc->shared_rss) 7203 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7204 mlx5_ipool_get 7205 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7206 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7207 mlx5_ipool_free(priv->flows[type], idx); 7208 rte_errno = ret; /* Restore rte_errno. */ 7209 ret = rte_errno; 7210 rte_errno = ret; 7211 error_before_hairpin_split: 7212 mlx5_flow_pop_thread_workspace(); 7213 rte_free(translated_actions); 7214 return 0; 7215 } 7216 7217 /** 7218 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7219 * incoming packets to table 1. 7220 * 7221 * Other flow rules, requested for group n, will be created in 7222 * e-switch table n+1. 7223 * Jump action to e-switch group n will be created to group n+1. 7224 * 7225 * Used when working in switchdev mode, to utilise advantages of table 1 7226 * and above. 7227 * 7228 * @param dev 7229 * Pointer to Ethernet device. 7230 * 7231 * @return 7232 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7233 */ 7234 struct rte_flow * 7235 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7236 { 7237 const struct rte_flow_attr attr = { 7238 .group = 0, 7239 .priority = 0, 7240 .ingress = 0, 7241 .egress = 0, 7242 .transfer = 1, 7243 }; 7244 const struct rte_flow_item pattern = { 7245 .type = RTE_FLOW_ITEM_TYPE_END, 7246 }; 7247 struct rte_flow_action_jump jump = { 7248 .group = 1, 7249 }; 7250 const struct rte_flow_action actions[] = { 7251 { 7252 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7253 .conf = &jump, 7254 }, 7255 { 7256 .type = RTE_FLOW_ACTION_TYPE_END, 7257 }, 7258 }; 7259 struct rte_flow_error error; 7260 7261 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7262 &attr, &pattern, 7263 actions, false, &error); 7264 } 7265 7266 /** 7267 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7268 * and sq number, directs all packets to peer vport. 7269 * 7270 * @param dev 7271 * Pointer to Ethernet device. 7272 * @param sq_num 7273 * SQ number. 7274 * 7275 * @return 7276 * Flow ID on success, 0 otherwise and rte_errno is set. 7277 */ 7278 uint32_t 7279 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7280 { 7281 struct rte_flow_attr attr = { 7282 .group = 0, 7283 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7284 .ingress = 0, 7285 .egress = 0, 7286 .transfer = 1, 7287 }; 7288 struct rte_flow_item_port_id port_spec = { 7289 .id = MLX5_PORT_ESW_MGR, 7290 }; 7291 struct mlx5_rte_flow_item_sq sq_spec = { 7292 .queue = sq_num, 7293 }; 7294 struct rte_flow_item pattern[] = { 7295 { 7296 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7297 .spec = &port_spec, 7298 }, 7299 { 7300 .type = (enum rte_flow_item_type) 7301 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7302 .spec = &sq_spec, 7303 }, 7304 { 7305 .type = RTE_FLOW_ITEM_TYPE_END, 7306 }, 7307 }; 7308 struct rte_flow_action_jump jump = { 7309 .group = 1, 7310 }; 7311 struct rte_flow_action_port_id port = { 7312 .id = dev->data->port_id, 7313 }; 7314 struct rte_flow_action actions[] = { 7315 { 7316 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7317 .conf = &jump, 7318 }, 7319 { 7320 .type = RTE_FLOW_ACTION_TYPE_END, 7321 }, 7322 }; 7323 struct rte_flow_error error; 7324 7325 /* 7326 * Creates group 0, highest priority jump flow. 7327 * Matches txq to bypass kernel packets. 7328 */ 7329 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7330 false, &error) == 0) 7331 return 0; 7332 /* Create group 1, lowest priority redirect flow for txq. */ 7333 attr.group = 1; 7334 actions[0].conf = &port; 7335 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7336 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7337 actions, false, &error); 7338 } 7339 7340 /** 7341 * Validate a flow supported by the NIC. 7342 * 7343 * @see rte_flow_validate() 7344 * @see rte_flow_ops 7345 */ 7346 int 7347 mlx5_flow_validate(struct rte_eth_dev *dev, 7348 const struct rte_flow_attr *attr, 7349 const struct rte_flow_item items[], 7350 const struct rte_flow_action original_actions[], 7351 struct rte_flow_error *error) 7352 { 7353 int hairpin_flow; 7354 struct mlx5_translated_action_handle 7355 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7356 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7357 const struct rte_flow_action *actions; 7358 struct rte_flow_action *translated_actions = NULL; 7359 int ret = flow_action_handles_translate(dev, original_actions, 7360 indir_actions, 7361 &indir_actions_n, 7362 &translated_actions, error); 7363 7364 if (ret) 7365 return ret; 7366 actions = translated_actions ? translated_actions : original_actions; 7367 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7368 ret = flow_drv_validate(dev, attr, items, actions, 7369 true, hairpin_flow, error); 7370 rte_free(translated_actions); 7371 return ret; 7372 } 7373 7374 /** 7375 * Create a flow. 7376 * 7377 * @see rte_flow_create() 7378 * @see rte_flow_ops 7379 */ 7380 struct rte_flow * 7381 mlx5_flow_create(struct rte_eth_dev *dev, 7382 const struct rte_flow_attr *attr, 7383 const struct rte_flow_item items[], 7384 const struct rte_flow_action actions[], 7385 struct rte_flow_error *error) 7386 { 7387 struct mlx5_priv *priv = dev->data->dev_private; 7388 7389 if (priv->sh->config.dv_flow_en == 2) { 7390 rte_flow_error_set(error, ENOTSUP, 7391 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7392 NULL, 7393 "Flow non-Q creation not supported"); 7394 return NULL; 7395 } 7396 /* 7397 * If the device is not started yet, it is not allowed to created a 7398 * flow from application. PMD default flows and traffic control flows 7399 * are not affected. 7400 */ 7401 if (unlikely(!dev->data->dev_started)) { 7402 DRV_LOG(DEBUG, "port %u is not started when " 7403 "inserting a flow", dev->data->port_id); 7404 rte_flow_error_set(error, ENODEV, 7405 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7406 NULL, 7407 "port not started"); 7408 return NULL; 7409 } 7410 7411 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7412 attr, items, actions, 7413 true, error); 7414 } 7415 7416 /** 7417 * Destroy a flow in a list. 7418 * 7419 * @param dev 7420 * Pointer to Ethernet device. 7421 * @param[in] flow_idx 7422 * Index of flow to destroy. 7423 */ 7424 static void 7425 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7426 uint32_t flow_idx) 7427 { 7428 struct mlx5_priv *priv = dev->data->dev_private; 7429 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7430 7431 if (!flow) 7432 return; 7433 MLX5_ASSERT(flow->type == type); 7434 /* 7435 * Update RX queue flags only if port is started, otherwise it is 7436 * already clean. 7437 */ 7438 if (dev->data->dev_started) 7439 flow_rxq_flags_trim(dev, flow); 7440 flow_drv_destroy(dev, flow); 7441 if (flow->tunnel) { 7442 struct mlx5_flow_tunnel *tunnel; 7443 7444 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7445 RTE_VERIFY(tunnel); 7446 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7447 mlx5_flow_tunnel_free(dev, tunnel); 7448 } 7449 flow_mreg_del_copy_action(dev, flow); 7450 mlx5_ipool_free(priv->flows[type], flow_idx); 7451 } 7452 7453 /** 7454 * Destroy all flows. 7455 * 7456 * @param dev 7457 * Pointer to Ethernet device. 7458 * @param type 7459 * Flow type to be flushed. 7460 * @param active 7461 * If flushing is called actively. 7462 */ 7463 void 7464 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7465 bool active) 7466 { 7467 struct mlx5_priv *priv = dev->data->dev_private; 7468 uint32_t num_flushed = 0, fidx = 1; 7469 struct rte_flow *flow; 7470 7471 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7472 if (priv->sh->config.dv_flow_en == 2 && 7473 type == MLX5_FLOW_TYPE_GEN) { 7474 flow_hw_q_flow_flush(dev, NULL); 7475 return; 7476 } 7477 #endif 7478 7479 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7480 flow_list_destroy(dev, type, fidx); 7481 num_flushed++; 7482 } 7483 if (active) { 7484 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7485 dev->data->port_id, num_flushed); 7486 } 7487 } 7488 7489 /** 7490 * Stop all default actions for flows. 7491 * 7492 * @param dev 7493 * Pointer to Ethernet device. 7494 */ 7495 void 7496 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7497 { 7498 flow_mreg_del_default_copy_action(dev); 7499 flow_rxq_flags_clear(dev); 7500 } 7501 7502 /** 7503 * Set rxq flag. 7504 * 7505 * @param[in] dev 7506 * Pointer to the rte_eth_dev structure. 7507 * @param[in] enable 7508 * Flag to enable or not. 7509 */ 7510 void 7511 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 7512 { 7513 struct mlx5_priv *priv = dev->data->dev_private; 7514 unsigned int i; 7515 7516 if ((!priv->mark_enabled && !enable) || 7517 (priv->mark_enabled && enable)) 7518 return; 7519 for (i = 0; i < priv->rxqs_n; ++i) { 7520 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 7521 7522 /* With RXQ start/stop feature, RXQ might be stopped. */ 7523 if (!rxq_ctrl) 7524 continue; 7525 rxq_ctrl->rxq.mark = enable; 7526 } 7527 priv->mark_enabled = enable; 7528 } 7529 7530 /** 7531 * Start all default actions for flows. 7532 * 7533 * @param dev 7534 * Pointer to Ethernet device. 7535 * @return 7536 * 0 on success, a negative errno value otherwise and rte_errno is set. 7537 */ 7538 int 7539 mlx5_flow_start_default(struct rte_eth_dev *dev) 7540 { 7541 struct rte_flow_error error; 7542 7543 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7544 return flow_mreg_add_default_copy_action(dev, &error); 7545 } 7546 7547 /** 7548 * Release key of thread specific flow workspace data. 7549 */ 7550 void 7551 flow_release_workspace(void *data) 7552 { 7553 struct mlx5_flow_workspace *wks = data; 7554 struct mlx5_flow_workspace *next; 7555 7556 while (wks) { 7557 next = wks->next; 7558 free(wks->rss_desc.queue); 7559 free(wks); 7560 wks = next; 7561 } 7562 } 7563 7564 /** 7565 * Get thread specific current flow workspace. 7566 * 7567 * @return pointer to thread specific flow workspace data, NULL on error. 7568 */ 7569 struct mlx5_flow_workspace* 7570 mlx5_flow_get_thread_workspace(void) 7571 { 7572 struct mlx5_flow_workspace *data; 7573 7574 data = mlx5_flow_os_get_specific_workspace(); 7575 MLX5_ASSERT(data && data->inuse); 7576 if (!data || !data->inuse) 7577 DRV_LOG(ERR, "flow workspace not initialized."); 7578 return data; 7579 } 7580 7581 /** 7582 * Allocate and init new flow workspace. 7583 * 7584 * @return pointer to flow workspace data, NULL on error. 7585 */ 7586 static struct mlx5_flow_workspace* 7587 flow_alloc_thread_workspace(void) 7588 { 7589 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7590 7591 if (!data) { 7592 DRV_LOG(ERR, "Failed to allocate flow workspace " 7593 "memory."); 7594 return NULL; 7595 } 7596 data->rss_desc.queue = calloc(1, 7597 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7598 if (!data->rss_desc.queue) 7599 goto err; 7600 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7601 return data; 7602 err: 7603 free(data->rss_desc.queue); 7604 free(data); 7605 return NULL; 7606 } 7607 7608 /** 7609 * Get new thread specific flow workspace. 7610 * 7611 * If current workspace inuse, create new one and set as current. 7612 * 7613 * @return pointer to thread specific flow workspace data, NULL on error. 7614 */ 7615 struct mlx5_flow_workspace* 7616 mlx5_flow_push_thread_workspace(void) 7617 { 7618 struct mlx5_flow_workspace *curr; 7619 struct mlx5_flow_workspace *data; 7620 7621 curr = mlx5_flow_os_get_specific_workspace(); 7622 if (!curr) { 7623 data = flow_alloc_thread_workspace(); 7624 if (!data) 7625 return NULL; 7626 } else if (!curr->inuse) { 7627 data = curr; 7628 } else if (curr->next) { 7629 data = curr->next; 7630 } else { 7631 data = flow_alloc_thread_workspace(); 7632 if (!data) 7633 return NULL; 7634 curr->next = data; 7635 data->prev = curr; 7636 } 7637 data->inuse = 1; 7638 data->flow_idx = 0; 7639 /* Set as current workspace */ 7640 if (mlx5_flow_os_set_specific_workspace(data)) 7641 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7642 return data; 7643 } 7644 7645 /** 7646 * Close current thread specific flow workspace. 7647 * 7648 * If previous workspace available, set it as current. 7649 * 7650 * @return pointer to thread specific flow workspace data, NULL on error. 7651 */ 7652 void 7653 mlx5_flow_pop_thread_workspace(void) 7654 { 7655 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7656 7657 if (!data) 7658 return; 7659 if (!data->inuse) { 7660 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7661 return; 7662 } 7663 data->inuse = 0; 7664 if (!data->prev) 7665 return; 7666 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7667 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7668 } 7669 7670 /** 7671 * Verify the flow list is empty 7672 * 7673 * @param dev 7674 * Pointer to Ethernet device. 7675 * 7676 * @return the number of flows not released. 7677 */ 7678 int 7679 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7680 { 7681 struct mlx5_priv *priv = dev->data->dev_private; 7682 struct rte_flow *flow; 7683 uint32_t idx = 0; 7684 int ret = 0, i; 7685 7686 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7687 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7688 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7689 dev->data->port_id, (void *)flow); 7690 ret++; 7691 } 7692 } 7693 return ret; 7694 } 7695 7696 /** 7697 * Enable default hairpin egress flow. 7698 * 7699 * @param dev 7700 * Pointer to Ethernet device. 7701 * @param sq_num 7702 * The SQ hw number. 7703 * 7704 * @return 7705 * 0 on success, a negative errno value otherwise and rte_errno is set. 7706 */ 7707 int 7708 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7709 uint32_t sq_num) 7710 { 7711 const struct rte_flow_attr attr = { 7712 .egress = 1, 7713 .priority = 0, 7714 }; 7715 struct mlx5_rte_flow_item_sq queue_spec = { 7716 .queue = sq_num, 7717 }; 7718 struct mlx5_rte_flow_item_sq queue_mask = { 7719 .queue = UINT32_MAX, 7720 }; 7721 struct rte_flow_item items[] = { 7722 { 7723 .type = (enum rte_flow_item_type) 7724 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7725 .spec = &queue_spec, 7726 .last = NULL, 7727 .mask = &queue_mask, 7728 }, 7729 { 7730 .type = RTE_FLOW_ITEM_TYPE_END, 7731 }, 7732 }; 7733 struct rte_flow_action_jump jump = { 7734 .group = MLX5_HAIRPIN_TX_TABLE, 7735 }; 7736 struct rte_flow_action actions[2]; 7737 uint32_t flow_idx; 7738 struct rte_flow_error error; 7739 7740 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7741 actions[0].conf = &jump; 7742 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7743 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7744 &attr, items, actions, false, &error); 7745 if (!flow_idx) { 7746 DRV_LOG(DEBUG, 7747 "Failed to create ctrl flow: rte_errno(%d)," 7748 " type(%d), message(%s)", 7749 rte_errno, error.type, 7750 error.message ? error.message : " (no stated reason)"); 7751 return -rte_errno; 7752 } 7753 return 0; 7754 } 7755 7756 /** 7757 * Enable a control flow configured from the control plane. 7758 * 7759 * @param dev 7760 * Pointer to Ethernet device. 7761 * @param eth_spec 7762 * An Ethernet flow spec to apply. 7763 * @param eth_mask 7764 * An Ethernet flow mask to apply. 7765 * @param vlan_spec 7766 * A VLAN flow spec to apply. 7767 * @param vlan_mask 7768 * A VLAN flow mask to apply. 7769 * 7770 * @return 7771 * 0 on success, a negative errno value otherwise and rte_errno is set. 7772 */ 7773 int 7774 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7775 struct rte_flow_item_eth *eth_spec, 7776 struct rte_flow_item_eth *eth_mask, 7777 struct rte_flow_item_vlan *vlan_spec, 7778 struct rte_flow_item_vlan *vlan_mask) 7779 { 7780 struct mlx5_priv *priv = dev->data->dev_private; 7781 const struct rte_flow_attr attr = { 7782 .ingress = 1, 7783 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7784 }; 7785 struct rte_flow_item items[] = { 7786 { 7787 .type = RTE_FLOW_ITEM_TYPE_ETH, 7788 .spec = eth_spec, 7789 .last = NULL, 7790 .mask = eth_mask, 7791 }, 7792 { 7793 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7794 RTE_FLOW_ITEM_TYPE_END, 7795 .spec = vlan_spec, 7796 .last = NULL, 7797 .mask = vlan_mask, 7798 }, 7799 { 7800 .type = RTE_FLOW_ITEM_TYPE_END, 7801 }, 7802 }; 7803 uint16_t queue[priv->reta_idx_n]; 7804 struct rte_flow_action_rss action_rss = { 7805 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7806 .level = 0, 7807 .types = priv->rss_conf.rss_hf, 7808 .key_len = priv->rss_conf.rss_key_len, 7809 .queue_num = priv->reta_idx_n, 7810 .key = priv->rss_conf.rss_key, 7811 .queue = queue, 7812 }; 7813 struct rte_flow_action actions[] = { 7814 { 7815 .type = RTE_FLOW_ACTION_TYPE_RSS, 7816 .conf = &action_rss, 7817 }, 7818 { 7819 .type = RTE_FLOW_ACTION_TYPE_END, 7820 }, 7821 }; 7822 uint32_t flow_idx; 7823 struct rte_flow_error error; 7824 unsigned int i; 7825 7826 if (!priv->reta_idx_n || !priv->rxqs_n) { 7827 return 0; 7828 } 7829 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7830 action_rss.types = 0; 7831 for (i = 0; i != priv->reta_idx_n; ++i) 7832 queue[i] = (*priv->reta_idx)[i]; 7833 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7834 &attr, items, actions, false, &error); 7835 if (!flow_idx) 7836 return -rte_errno; 7837 return 0; 7838 } 7839 7840 /** 7841 * Enable a flow control configured from the control plane. 7842 * 7843 * @param dev 7844 * Pointer to Ethernet device. 7845 * @param eth_spec 7846 * An Ethernet flow spec to apply. 7847 * @param eth_mask 7848 * An Ethernet flow mask to apply. 7849 * 7850 * @return 7851 * 0 on success, a negative errno value otherwise and rte_errno is set. 7852 */ 7853 int 7854 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7855 struct rte_flow_item_eth *eth_spec, 7856 struct rte_flow_item_eth *eth_mask) 7857 { 7858 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7859 } 7860 7861 /** 7862 * Create default miss flow rule matching lacp traffic 7863 * 7864 * @param dev 7865 * Pointer to Ethernet device. 7866 * @param eth_spec 7867 * An Ethernet flow spec to apply. 7868 * 7869 * @return 7870 * 0 on success, a negative errno value otherwise and rte_errno is set. 7871 */ 7872 int 7873 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7874 { 7875 /* 7876 * The LACP matching is done by only using ether type since using 7877 * a multicast dst mac causes kernel to give low priority to this flow. 7878 */ 7879 static const struct rte_flow_item_eth lacp_spec = { 7880 .hdr.ether_type = RTE_BE16(0x8809), 7881 }; 7882 static const struct rte_flow_item_eth lacp_mask = { 7883 .hdr.ether_type = 0xffff, 7884 }; 7885 const struct rte_flow_attr attr = { 7886 .ingress = 1, 7887 }; 7888 struct rte_flow_item items[] = { 7889 { 7890 .type = RTE_FLOW_ITEM_TYPE_ETH, 7891 .spec = &lacp_spec, 7892 .mask = &lacp_mask, 7893 }, 7894 { 7895 .type = RTE_FLOW_ITEM_TYPE_END, 7896 }, 7897 }; 7898 struct rte_flow_action actions[] = { 7899 { 7900 .type = (enum rte_flow_action_type) 7901 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 7902 }, 7903 { 7904 .type = RTE_FLOW_ACTION_TYPE_END, 7905 }, 7906 }; 7907 struct rte_flow_error error; 7908 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7909 &attr, items, actions, 7910 false, &error); 7911 7912 if (!flow_idx) 7913 return -rte_errno; 7914 return 0; 7915 } 7916 7917 /** 7918 * Destroy a flow. 7919 * 7920 * @see rte_flow_destroy() 7921 * @see rte_flow_ops 7922 */ 7923 int 7924 mlx5_flow_destroy(struct rte_eth_dev *dev, 7925 struct rte_flow *flow, 7926 struct rte_flow_error *error __rte_unused) 7927 { 7928 struct mlx5_priv *priv = dev->data->dev_private; 7929 7930 if (priv->sh->config.dv_flow_en == 2) 7931 return rte_flow_error_set(error, ENOTSUP, 7932 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7933 NULL, 7934 "Flow non-Q destruction not supported"); 7935 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7936 (uintptr_t)(void *)flow); 7937 return 0; 7938 } 7939 7940 /** 7941 * Destroy all flows. 7942 * 7943 * @see rte_flow_flush() 7944 * @see rte_flow_ops 7945 */ 7946 int 7947 mlx5_flow_flush(struct rte_eth_dev *dev, 7948 struct rte_flow_error *error __rte_unused) 7949 { 7950 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 7951 return 0; 7952 } 7953 7954 /** 7955 * Isolated mode. 7956 * 7957 * @see rte_flow_isolate() 7958 * @see rte_flow_ops 7959 */ 7960 int 7961 mlx5_flow_isolate(struct rte_eth_dev *dev, 7962 int enable, 7963 struct rte_flow_error *error) 7964 { 7965 struct mlx5_priv *priv = dev->data->dev_private; 7966 7967 if (dev->data->dev_started) { 7968 rte_flow_error_set(error, EBUSY, 7969 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7970 NULL, 7971 "port must be stopped first"); 7972 return -rte_errno; 7973 } 7974 priv->isolated = !!enable; 7975 if (enable) 7976 dev->dev_ops = &mlx5_dev_ops_isolate; 7977 else 7978 dev->dev_ops = &mlx5_dev_ops; 7979 7980 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 7981 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 7982 7983 return 0; 7984 } 7985 7986 /** 7987 * Query a flow. 7988 * 7989 * @see rte_flow_query() 7990 * @see rte_flow_ops 7991 */ 7992 static int 7993 flow_drv_query(struct rte_eth_dev *dev, 7994 struct rte_flow *eflow, 7995 const struct rte_flow_action *actions, 7996 void *data, 7997 struct rte_flow_error *error) 7998 { 7999 struct mlx5_priv *priv = dev->data->dev_private; 8000 const struct mlx5_flow_driver_ops *fops; 8001 struct rte_flow *flow = NULL; 8002 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8003 8004 if (priv->sh->config.dv_flow_en == 2) { 8005 #ifdef HAVE_MLX5_HWS_SUPPORT 8006 flow = eflow; 8007 ftype = MLX5_FLOW_TYPE_HW; 8008 #endif 8009 } else { 8010 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8011 (uintptr_t)(void *)eflow); 8012 } 8013 if (!flow) { 8014 return rte_flow_error_set(error, ENOENT, 8015 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8016 NULL, 8017 "invalid flow handle"); 8018 } 8019 if (ftype == MLX5_FLOW_TYPE_MIN) 8020 ftype = flow->drv_type; 8021 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8022 fops = flow_get_drv_ops(ftype); 8023 8024 return fops->query(dev, flow, actions, data, error); 8025 } 8026 8027 /** 8028 * Query a flow. 8029 * 8030 * @see rte_flow_query() 8031 * @see rte_flow_ops 8032 */ 8033 int 8034 mlx5_flow_query(struct rte_eth_dev *dev, 8035 struct rte_flow *flow, 8036 const struct rte_flow_action *actions, 8037 void *data, 8038 struct rte_flow_error *error) 8039 { 8040 int ret; 8041 8042 ret = flow_drv_query(dev, flow, actions, data, 8043 error); 8044 if (ret < 0) 8045 return ret; 8046 return 0; 8047 } 8048 8049 /** 8050 * Get rte_flow callbacks. 8051 * 8052 * @param dev 8053 * Pointer to Ethernet device structure. 8054 * @param ops 8055 * Pointer to operation-specific structure. 8056 * 8057 * @return 0 8058 */ 8059 int 8060 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8061 const struct rte_flow_ops **ops) 8062 { 8063 *ops = &mlx5_flow_ops; 8064 return 0; 8065 } 8066 8067 /** 8068 * Validate meter policy actions. 8069 * Dispatcher for action type specific validation. 8070 * 8071 * @param[in] dev 8072 * Pointer to the Ethernet device structure. 8073 * @param[in] action 8074 * The meter policy action object to validate. 8075 * @param[in] attr 8076 * Attributes of flow to determine steering domain. 8077 * @param[out] is_rss 8078 * Is RSS or not. 8079 * @param[out] domain_bitmap 8080 * Domain bitmap. 8081 * @param[out] is_def_policy 8082 * Is default policy or not. 8083 * @param[out] error 8084 * Perform verbose error reporting if not NULL. Initialized in case of 8085 * error only. 8086 * 8087 * @return 8088 * 0 on success, otherwise negative errno value. 8089 */ 8090 int 8091 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8092 const struct rte_flow_action *actions[RTE_COLORS], 8093 struct rte_flow_attr *attr, 8094 bool *is_rss, 8095 uint8_t *domain_bitmap, 8096 uint8_t *policy_mode, 8097 struct rte_mtr_error *error) 8098 { 8099 const struct mlx5_flow_driver_ops *fops; 8100 8101 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8102 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8103 domain_bitmap, policy_mode, error); 8104 } 8105 8106 /** 8107 * Destroy the meter table set. 8108 * 8109 * @param[in] dev 8110 * Pointer to Ethernet device. 8111 * @param[in] mtr_policy 8112 * Meter policy struct. 8113 */ 8114 void 8115 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8116 struct mlx5_flow_meter_policy *mtr_policy) 8117 { 8118 const struct mlx5_flow_driver_ops *fops; 8119 8120 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8121 fops->destroy_mtr_acts(dev, mtr_policy); 8122 } 8123 8124 /** 8125 * Create policy action, lock free, 8126 * (mutex should be acquired by caller). 8127 * Dispatcher for action type specific call. 8128 * 8129 * @param[in] dev 8130 * Pointer to the Ethernet device structure. 8131 * @param[in] mtr_policy 8132 * Meter policy struct. 8133 * @param[in] action 8134 * Action specification used to create meter actions. 8135 * @param[in] attr 8136 * Flow rule attributes. 8137 * @param[out] error 8138 * Perform verbose error reporting if not NULL. Initialized in case of 8139 * error only. 8140 * 8141 * @return 8142 * 0 on success, otherwise negative errno value. 8143 */ 8144 int 8145 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8146 struct mlx5_flow_meter_policy *mtr_policy, 8147 const struct rte_flow_action *actions[RTE_COLORS], 8148 struct rte_flow_attr *attr, 8149 struct rte_mtr_error *error) 8150 { 8151 const struct mlx5_flow_driver_ops *fops; 8152 8153 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8154 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8155 } 8156 8157 /** 8158 * Create policy rules, lock free, 8159 * (mutex should be acquired by caller). 8160 * Dispatcher for action type specific call. 8161 * 8162 * @param[in] dev 8163 * Pointer to the Ethernet device structure. 8164 * @param[in] mtr_policy 8165 * Meter policy struct. 8166 * 8167 * @return 8168 * 0 on success, -1 otherwise. 8169 */ 8170 int 8171 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8172 struct mlx5_flow_meter_policy *mtr_policy) 8173 { 8174 const struct mlx5_flow_driver_ops *fops; 8175 8176 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8177 return fops->create_policy_rules(dev, mtr_policy); 8178 } 8179 8180 /** 8181 * Destroy policy rules, lock free, 8182 * (mutex should be acquired by caller). 8183 * Dispatcher for action type specific call. 8184 * 8185 * @param[in] dev 8186 * Pointer to the Ethernet device structure. 8187 * @param[in] mtr_policy 8188 * Meter policy struct. 8189 */ 8190 void 8191 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8192 struct mlx5_flow_meter_policy *mtr_policy) 8193 { 8194 const struct mlx5_flow_driver_ops *fops; 8195 8196 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8197 fops->destroy_policy_rules(dev, mtr_policy); 8198 } 8199 8200 /** 8201 * Destroy the default policy table set. 8202 * 8203 * @param[in] dev 8204 * Pointer to Ethernet device. 8205 */ 8206 void 8207 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8208 { 8209 const struct mlx5_flow_driver_ops *fops; 8210 8211 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8212 fops->destroy_def_policy(dev); 8213 } 8214 8215 /** 8216 * Destroy the default policy table set. 8217 * 8218 * @param[in] dev 8219 * Pointer to Ethernet device. 8220 * 8221 * @return 8222 * 0 on success, -1 otherwise. 8223 */ 8224 int 8225 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8226 { 8227 const struct mlx5_flow_driver_ops *fops; 8228 8229 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8230 return fops->create_def_policy(dev); 8231 } 8232 8233 /** 8234 * Create the needed meter and suffix tables. 8235 * 8236 * @param[in] dev 8237 * Pointer to Ethernet device. 8238 * 8239 * @return 8240 * 0 on success, -1 otherwise. 8241 */ 8242 int 8243 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8244 struct mlx5_flow_meter_info *fm, 8245 uint32_t mtr_idx, 8246 uint8_t domain_bitmap) 8247 { 8248 const struct mlx5_flow_driver_ops *fops; 8249 8250 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8251 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8252 } 8253 8254 /** 8255 * Destroy the meter table set. 8256 * 8257 * @param[in] dev 8258 * Pointer to Ethernet device. 8259 * @param[in] tbl 8260 * Pointer to the meter table set. 8261 */ 8262 void 8263 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8264 struct mlx5_flow_meter_info *fm) 8265 { 8266 const struct mlx5_flow_driver_ops *fops; 8267 8268 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8269 fops->destroy_mtr_tbls(dev, fm); 8270 } 8271 8272 /** 8273 * Destroy the global meter drop table. 8274 * 8275 * @param[in] dev 8276 * Pointer to Ethernet device. 8277 */ 8278 void 8279 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8280 { 8281 const struct mlx5_flow_driver_ops *fops; 8282 8283 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8284 fops->destroy_mtr_drop_tbls(dev); 8285 } 8286 8287 /** 8288 * Destroy the sub policy table with RX queue. 8289 * 8290 * @param[in] dev 8291 * Pointer to Ethernet device. 8292 * @param[in] mtr_policy 8293 * Pointer to meter policy table. 8294 */ 8295 void 8296 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8297 struct mlx5_flow_meter_policy *mtr_policy) 8298 { 8299 const struct mlx5_flow_driver_ops *fops; 8300 8301 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8302 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8303 } 8304 8305 /** 8306 * Allocate the needed aso flow meter id. 8307 * 8308 * @param[in] dev 8309 * Pointer to Ethernet device. 8310 * 8311 * @return 8312 * Index to aso flow meter on success, NULL otherwise. 8313 */ 8314 uint32_t 8315 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8316 { 8317 const struct mlx5_flow_driver_ops *fops; 8318 8319 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8320 return fops->create_meter(dev); 8321 } 8322 8323 /** 8324 * Free the aso flow meter id. 8325 * 8326 * @param[in] dev 8327 * Pointer to Ethernet device. 8328 * @param[in] mtr_idx 8329 * Index to aso flow meter to be free. 8330 * 8331 * @return 8332 * 0 on success. 8333 */ 8334 void 8335 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8336 { 8337 const struct mlx5_flow_driver_ops *fops; 8338 8339 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8340 fops->free_meter(dev, mtr_idx); 8341 } 8342 8343 /** 8344 * Allocate a counter. 8345 * 8346 * @param[in] dev 8347 * Pointer to Ethernet device structure. 8348 * 8349 * @return 8350 * Index to allocated counter on success, 0 otherwise. 8351 */ 8352 uint32_t 8353 mlx5_counter_alloc(struct rte_eth_dev *dev) 8354 { 8355 const struct mlx5_flow_driver_ops *fops; 8356 struct rte_flow_attr attr = { .transfer = 0 }; 8357 8358 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8359 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8360 return fops->counter_alloc(dev); 8361 } 8362 DRV_LOG(ERR, 8363 "port %u counter allocate is not supported.", 8364 dev->data->port_id); 8365 return 0; 8366 } 8367 8368 /** 8369 * Free a counter. 8370 * 8371 * @param[in] dev 8372 * Pointer to Ethernet device structure. 8373 * @param[in] cnt 8374 * Index to counter to be free. 8375 */ 8376 void 8377 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8378 { 8379 const struct mlx5_flow_driver_ops *fops; 8380 struct rte_flow_attr attr = { .transfer = 0 }; 8381 8382 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8383 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8384 fops->counter_free(dev, cnt); 8385 return; 8386 } 8387 DRV_LOG(ERR, 8388 "port %u counter free is not supported.", 8389 dev->data->port_id); 8390 } 8391 8392 /** 8393 * Query counter statistics. 8394 * 8395 * @param[in] dev 8396 * Pointer to Ethernet device structure. 8397 * @param[in] cnt 8398 * Index to counter to query. 8399 * @param[in] clear 8400 * Set to clear counter statistics. 8401 * @param[out] pkts 8402 * The counter hits packets number to save. 8403 * @param[out] bytes 8404 * The counter hits bytes number to save. 8405 * 8406 * @return 8407 * 0 on success, a negative errno value otherwise. 8408 */ 8409 int 8410 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8411 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8412 { 8413 const struct mlx5_flow_driver_ops *fops; 8414 struct rte_flow_attr attr = { .transfer = 0 }; 8415 8416 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_DV) { 8417 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8418 return fops->counter_query(dev, cnt, clear, pkts, 8419 bytes, action); 8420 } 8421 DRV_LOG(ERR, 8422 "port %u counter query is not supported.", 8423 dev->data->port_id); 8424 return -ENOTSUP; 8425 } 8426 8427 /** 8428 * Get information about HWS pre-configurable resources. 8429 * 8430 * @param[in] dev 8431 * Pointer to the rte_eth_dev structure. 8432 * @param[out] port_info 8433 * Pointer to port information. 8434 * @param[out] queue_info 8435 * Pointer to queue information. 8436 * @param[out] error 8437 * Pointer to error structure. 8438 * 8439 * @return 8440 * 0 on success, a negative errno value otherwise and rte_errno is set. 8441 */ 8442 static int 8443 mlx5_flow_info_get(struct rte_eth_dev *dev, 8444 struct rte_flow_port_info *port_info, 8445 struct rte_flow_queue_info *queue_info, 8446 struct rte_flow_error *error) 8447 { 8448 const struct mlx5_flow_driver_ops *fops; 8449 struct rte_flow_attr attr = {0}; 8450 8451 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8452 return rte_flow_error_set(error, ENOTSUP, 8453 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8454 NULL, 8455 "info get with incorrect steering mode"); 8456 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8457 return fops->info_get(dev, port_info, queue_info, error); 8458 } 8459 8460 /** 8461 * Configure port HWS resources. 8462 * 8463 * @param[in] dev 8464 * Pointer to the rte_eth_dev structure. 8465 * @param[in] port_attr 8466 * Port configuration attributes. 8467 * @param[in] nb_queue 8468 * Number of queue. 8469 * @param[in] queue_attr 8470 * Array that holds attributes for each flow queue. 8471 * @param[out] error 8472 * Pointer to error structure. 8473 * 8474 * @return 8475 * 0 on success, a negative errno value otherwise and rte_errno is set. 8476 */ 8477 static int 8478 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8479 const struct rte_flow_port_attr *port_attr, 8480 uint16_t nb_queue, 8481 const struct rte_flow_queue_attr *queue_attr[], 8482 struct rte_flow_error *error) 8483 { 8484 const struct mlx5_flow_driver_ops *fops; 8485 struct rte_flow_attr attr = {0}; 8486 8487 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8488 return rte_flow_error_set(error, ENOTSUP, 8489 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8490 NULL, 8491 "port configure with incorrect steering mode"); 8492 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8493 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8494 } 8495 8496 /** 8497 * Validate item template. 8498 * 8499 * @param[in] dev 8500 * Pointer to the rte_eth_dev structure. 8501 * @param[in] attr 8502 * Pointer to the item template attributes. 8503 * @param[in] items 8504 * The template item pattern. 8505 * @param[out] error 8506 * Pointer to error structure. 8507 * 8508 * @return 8509 * 0 on success, a negative errno value otherwise and rte_errno is set. 8510 */ 8511 int 8512 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8513 const struct rte_flow_pattern_template_attr *attr, 8514 const struct rte_flow_item items[], 8515 struct rte_flow_error *error) 8516 { 8517 const struct mlx5_flow_driver_ops *fops; 8518 struct rte_flow_attr fattr = {0}; 8519 8520 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8521 rte_flow_error_set(error, ENOTSUP, 8522 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8523 "pattern validate with incorrect steering mode"); 8524 return -ENOTSUP; 8525 } 8526 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8527 return fops->pattern_validate(dev, attr, items, error); 8528 } 8529 8530 /** 8531 * Create flow item template. 8532 * 8533 * @param[in] dev 8534 * Pointer to the rte_eth_dev structure. 8535 * @param[in] attr 8536 * Pointer to the item template attributes. 8537 * @param[in] items 8538 * The template item pattern. 8539 * @param[out] error 8540 * Pointer to error structure. 8541 * 8542 * @return 8543 * 0 on success, a negative errno value otherwise and rte_errno is set. 8544 */ 8545 static struct rte_flow_pattern_template * 8546 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8547 const struct rte_flow_pattern_template_attr *attr, 8548 const struct rte_flow_item items[], 8549 struct rte_flow_error *error) 8550 { 8551 const struct mlx5_flow_driver_ops *fops; 8552 struct rte_flow_attr fattr = {0}; 8553 8554 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8555 rte_flow_error_set(error, ENOTSUP, 8556 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8557 NULL, 8558 "pattern create with incorrect steering mode"); 8559 return NULL; 8560 } 8561 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8562 return fops->pattern_template_create(dev, attr, items, error); 8563 } 8564 8565 /** 8566 * Destroy flow item template. 8567 * 8568 * @param[in] dev 8569 * Pointer to the rte_eth_dev structure. 8570 * @param[in] template 8571 * Pointer to the item template to be destroyed. 8572 * @param[out] error 8573 * Pointer to error structure. 8574 * 8575 * @return 8576 * 0 on success, a negative errno value otherwise and rte_errno is set. 8577 */ 8578 static int 8579 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8580 struct rte_flow_pattern_template *template, 8581 struct rte_flow_error *error) 8582 { 8583 const struct mlx5_flow_driver_ops *fops; 8584 struct rte_flow_attr attr = {0}; 8585 8586 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8587 return rte_flow_error_set(error, ENOTSUP, 8588 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8589 NULL, 8590 "pattern destroy with incorrect steering mode"); 8591 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8592 return fops->pattern_template_destroy(dev, template, error); 8593 } 8594 8595 /** 8596 * Validate flow actions template. 8597 * 8598 * @param[in] dev 8599 * Pointer to the rte_eth_dev structure. 8600 * @param[in] attr 8601 * Pointer to the action template attributes. 8602 * @param[in] actions 8603 * Associated actions (list terminated by the END action). 8604 * @param[in] masks 8605 * List of actions that marks which of the action's member is constant. 8606 * @param[out] error 8607 * Pointer to error structure. 8608 * 8609 * @return 8610 * 0 on success, a negative errno value otherwise and rte_errno is set. 8611 */ 8612 int 8613 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8614 const struct rte_flow_actions_template_attr *attr, 8615 const struct rte_flow_action actions[], 8616 const struct rte_flow_action masks[], 8617 struct rte_flow_error *error) 8618 { 8619 const struct mlx5_flow_driver_ops *fops; 8620 struct rte_flow_attr fattr = {0}; 8621 8622 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8623 rte_flow_error_set(error, ENOTSUP, 8624 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8625 "actions validate with incorrect steering mode"); 8626 return -ENOTSUP; 8627 } 8628 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8629 return fops->actions_validate(dev, attr, actions, masks, error); 8630 } 8631 8632 /** 8633 * Create flow item template. 8634 * 8635 * @param[in] dev 8636 * Pointer to the rte_eth_dev structure. 8637 * @param[in] attr 8638 * Pointer to the action template attributes. 8639 * @param[in] actions 8640 * Associated actions (list terminated by the END action). 8641 * @param[in] masks 8642 * List of actions that marks which of the action's member is constant. 8643 * @param[out] error 8644 * Pointer to error structure. 8645 * 8646 * @return 8647 * 0 on success, a negative errno value otherwise and rte_errno is set. 8648 */ 8649 static struct rte_flow_actions_template * 8650 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8651 const struct rte_flow_actions_template_attr *attr, 8652 const struct rte_flow_action actions[], 8653 const struct rte_flow_action masks[], 8654 struct rte_flow_error *error) 8655 { 8656 const struct mlx5_flow_driver_ops *fops; 8657 struct rte_flow_attr fattr = {0}; 8658 8659 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8660 rte_flow_error_set(error, ENOTSUP, 8661 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8662 NULL, 8663 "action create with incorrect steering mode"); 8664 return NULL; 8665 } 8666 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8667 return fops->actions_template_create(dev, attr, actions, masks, error); 8668 } 8669 8670 /** 8671 * Destroy flow action template. 8672 * 8673 * @param[in] dev 8674 * Pointer to the rte_eth_dev structure. 8675 * @param[in] template 8676 * Pointer to the action template to be destroyed. 8677 * @param[out] error 8678 * Pointer to error structure. 8679 * 8680 * @return 8681 * 0 on success, a negative errno value otherwise and rte_errno is set. 8682 */ 8683 static int 8684 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8685 struct rte_flow_actions_template *template, 8686 struct rte_flow_error *error) 8687 { 8688 const struct mlx5_flow_driver_ops *fops; 8689 struct rte_flow_attr attr = {0}; 8690 8691 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8692 return rte_flow_error_set(error, ENOTSUP, 8693 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8694 NULL, 8695 "action destroy with incorrect steering mode"); 8696 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8697 return fops->actions_template_destroy(dev, template, error); 8698 } 8699 8700 /** 8701 * Create flow table. 8702 * 8703 * @param[in] dev 8704 * Pointer to the rte_eth_dev structure. 8705 * @param[in] attr 8706 * Pointer to the table attributes. 8707 * @param[in] item_templates 8708 * Item template array to be binded to the table. 8709 * @param[in] nb_item_templates 8710 * Number of item template. 8711 * @param[in] action_templates 8712 * Action template array to be binded to the table. 8713 * @param[in] nb_action_templates 8714 * Number of action template. 8715 * @param[out] error 8716 * Pointer to error structure. 8717 * 8718 * @return 8719 * Table on success, NULL otherwise and rte_errno is set. 8720 */ 8721 static struct rte_flow_template_table * 8722 mlx5_flow_table_create(struct rte_eth_dev *dev, 8723 const struct rte_flow_template_table_attr *attr, 8724 struct rte_flow_pattern_template *item_templates[], 8725 uint8_t nb_item_templates, 8726 struct rte_flow_actions_template *action_templates[], 8727 uint8_t nb_action_templates, 8728 struct rte_flow_error *error) 8729 { 8730 const struct mlx5_flow_driver_ops *fops; 8731 struct rte_flow_attr fattr = {0}; 8732 8733 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8734 rte_flow_error_set(error, ENOTSUP, 8735 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8736 NULL, 8737 "table create with incorrect steering mode"); 8738 return NULL; 8739 } 8740 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8741 return fops->template_table_create(dev, 8742 attr, 8743 item_templates, 8744 nb_item_templates, 8745 action_templates, 8746 nb_action_templates, 8747 error); 8748 } 8749 8750 /** 8751 * PMD destroy flow table. 8752 * 8753 * @param[in] dev 8754 * Pointer to the rte_eth_dev structure. 8755 * @param[in] table 8756 * Pointer to the table to be destroyed. 8757 * @param[out] error 8758 * Pointer to error structure. 8759 * 8760 * @return 8761 * 0 on success, a negative errno value otherwise and rte_errno is set. 8762 */ 8763 static int 8764 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8765 struct rte_flow_template_table *table, 8766 struct rte_flow_error *error) 8767 { 8768 const struct mlx5_flow_driver_ops *fops; 8769 struct rte_flow_attr attr = {0}; 8770 8771 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8772 return rte_flow_error_set(error, ENOTSUP, 8773 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8774 NULL, 8775 "table destroy with incorrect steering mode"); 8776 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8777 return fops->template_table_destroy(dev, table, error); 8778 } 8779 8780 /** 8781 * Enqueue flow creation. 8782 * 8783 * @param[in] dev 8784 * Pointer to the rte_eth_dev structure. 8785 * @param[in] queue_id 8786 * The queue to create the flow. 8787 * @param[in] attr 8788 * Pointer to the flow operation attributes. 8789 * @param[in] items 8790 * Items with flow spec value. 8791 * @param[in] pattern_template_index 8792 * The item pattern flow follows from the table. 8793 * @param[in] actions 8794 * Action with flow spec value. 8795 * @param[in] action_template_index 8796 * The action pattern flow follows from the table. 8797 * @param[in] user_data 8798 * Pointer to the user_data. 8799 * @param[out] error 8800 * Pointer to error structure. 8801 * 8802 * @return 8803 * Flow pointer on success, NULL otherwise and rte_errno is set. 8804 */ 8805 static struct rte_flow * 8806 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8807 uint32_t queue_id, 8808 const struct rte_flow_op_attr *attr, 8809 struct rte_flow_template_table *table, 8810 const struct rte_flow_item items[], 8811 uint8_t pattern_template_index, 8812 const struct rte_flow_action actions[], 8813 uint8_t action_template_index, 8814 void *user_data, 8815 struct rte_flow_error *error) 8816 { 8817 const struct mlx5_flow_driver_ops *fops; 8818 struct rte_flow_attr fattr = {0}; 8819 8820 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8821 rte_flow_error_set(error, ENOTSUP, 8822 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8823 NULL, 8824 "flow_q create with incorrect steering mode"); 8825 return NULL; 8826 } 8827 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8828 return fops->async_flow_create(dev, queue_id, attr, table, 8829 items, pattern_template_index, 8830 actions, action_template_index, 8831 user_data, error); 8832 } 8833 8834 /** 8835 * Enqueue flow destruction. 8836 * 8837 * @param[in] dev 8838 * Pointer to the rte_eth_dev structure. 8839 * @param[in] queue 8840 * The queue to destroy the flow. 8841 * @param[in] attr 8842 * Pointer to the flow operation attributes. 8843 * @param[in] flow 8844 * Pointer to the flow to be destroyed. 8845 * @param[in] user_data 8846 * Pointer to the user_data. 8847 * @param[out] error 8848 * Pointer to error structure. 8849 * 8850 * @return 8851 * 0 on success, negative value otherwise and rte_errno is set. 8852 */ 8853 static int 8854 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8855 uint32_t queue, 8856 const struct rte_flow_op_attr *attr, 8857 struct rte_flow *flow, 8858 void *user_data, 8859 struct rte_flow_error *error) 8860 { 8861 const struct mlx5_flow_driver_ops *fops; 8862 struct rte_flow_attr fattr = {0}; 8863 8864 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8865 return rte_flow_error_set(error, ENOTSUP, 8866 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8867 NULL, 8868 "flow_q destroy with incorrect steering mode"); 8869 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8870 return fops->async_flow_destroy(dev, queue, attr, flow, 8871 user_data, error); 8872 } 8873 8874 /** 8875 * Pull the enqueued flows. 8876 * 8877 * @param[in] dev 8878 * Pointer to the rte_eth_dev structure. 8879 * @param[in] queue 8880 * The queue to pull the result. 8881 * @param[in/out] res 8882 * Array to save the results. 8883 * @param[in] n_res 8884 * Available result with the array. 8885 * @param[out] error 8886 * Pointer to error structure. 8887 * 8888 * @return 8889 * Result number on success, negative value otherwise and rte_errno is set. 8890 */ 8891 static int 8892 mlx5_flow_pull(struct rte_eth_dev *dev, 8893 uint32_t queue, 8894 struct rte_flow_op_result res[], 8895 uint16_t n_res, 8896 struct rte_flow_error *error) 8897 { 8898 const struct mlx5_flow_driver_ops *fops; 8899 struct rte_flow_attr attr = {0}; 8900 8901 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8902 return rte_flow_error_set(error, ENOTSUP, 8903 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8904 NULL, 8905 "flow_q pull with incorrect steering mode"); 8906 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8907 return fops->pull(dev, queue, res, n_res, error); 8908 } 8909 8910 /** 8911 * Push the enqueued flows. 8912 * 8913 * @param[in] dev 8914 * Pointer to the rte_eth_dev structure. 8915 * @param[in] queue 8916 * The queue to push the flows. 8917 * @param[out] error 8918 * Pointer to error structure. 8919 * 8920 * @return 8921 * 0 on success, negative value otherwise and rte_errno is set. 8922 */ 8923 static int 8924 mlx5_flow_push(struct rte_eth_dev *dev, 8925 uint32_t queue, 8926 struct rte_flow_error *error) 8927 { 8928 const struct mlx5_flow_driver_ops *fops; 8929 struct rte_flow_attr attr = {0}; 8930 8931 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8932 return rte_flow_error_set(error, ENOTSUP, 8933 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8934 NULL, 8935 "flow_q push with incorrect steering mode"); 8936 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8937 return fops->push(dev, queue, error); 8938 } 8939 8940 /** 8941 * Create shared action. 8942 * 8943 * @param[in] dev 8944 * Pointer to the rte_eth_dev structure. 8945 * @param[in] queue 8946 * Which queue to be used.. 8947 * @param[in] attr 8948 * Operation attribute. 8949 * @param[in] conf 8950 * Indirect action configuration. 8951 * @param[in] action 8952 * rte_flow action detail. 8953 * @param[in] user_data 8954 * Pointer to the user_data. 8955 * @param[out] error 8956 * Pointer to error structure. 8957 * 8958 * @return 8959 * Action handle on success, NULL otherwise and rte_errno is set. 8960 */ 8961 static struct rte_flow_action_handle * 8962 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 8963 const struct rte_flow_op_attr *attr, 8964 const struct rte_flow_indir_action_conf *conf, 8965 const struct rte_flow_action *action, 8966 void *user_data, 8967 struct rte_flow_error *error) 8968 { 8969 const struct mlx5_flow_driver_ops *fops = 8970 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8971 8972 return fops->async_action_create(dev, queue, attr, conf, action, 8973 user_data, error); 8974 } 8975 8976 /** 8977 * Update shared action. 8978 * 8979 * @param[in] dev 8980 * Pointer to the rte_eth_dev structure. 8981 * @param[in] queue 8982 * Which queue to be used.. 8983 * @param[in] attr 8984 * Operation attribute. 8985 * @param[in] handle 8986 * Action handle to be updated. 8987 * @param[in] update 8988 * Update value. 8989 * @param[in] user_data 8990 * Pointer to the user_data. 8991 * @param[out] error 8992 * Pointer to error structure. 8993 * 8994 * @return 8995 * 0 on success, negative value otherwise and rte_errno is set. 8996 */ 8997 static int 8998 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 8999 const struct rte_flow_op_attr *attr, 9000 struct rte_flow_action_handle *handle, 9001 const void *update, 9002 void *user_data, 9003 struct rte_flow_error *error) 9004 { 9005 const struct mlx5_flow_driver_ops *fops = 9006 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9007 9008 return fops->async_action_update(dev, queue, attr, handle, 9009 update, user_data, error); 9010 } 9011 9012 /** 9013 * Query shared action. 9014 * 9015 * @param[in] dev 9016 * Pointer to the rte_eth_dev structure. 9017 * @param[in] queue 9018 * Which queue to be used.. 9019 * @param[in] attr 9020 * Operation attribute. 9021 * @param[in] handle 9022 * Action handle to be updated. 9023 * @param[in] data 9024 * Pointer query result data. 9025 * @param[in] user_data 9026 * Pointer to the user_data. 9027 * @param[out] error 9028 * Pointer to error structure. 9029 * 9030 * @return 9031 * 0 on success, negative value otherwise and rte_errno is set. 9032 */ 9033 static int 9034 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9035 const struct rte_flow_op_attr *attr, 9036 const struct rte_flow_action_handle *handle, 9037 void *data, 9038 void *user_data, 9039 struct rte_flow_error *error) 9040 { 9041 const struct mlx5_flow_driver_ops *fops = 9042 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9043 9044 return fops->async_action_query(dev, queue, attr, handle, 9045 data, user_data, error); 9046 } 9047 9048 /** 9049 * Destroy shared action. 9050 * 9051 * @param[in] dev 9052 * Pointer to the rte_eth_dev structure. 9053 * @param[in] queue 9054 * Which queue to be used.. 9055 * @param[in] attr 9056 * Operation attribute. 9057 * @param[in] handle 9058 * Action handle to be destroyed. 9059 * @param[in] user_data 9060 * Pointer to the user_data. 9061 * @param[out] error 9062 * Pointer to error structure. 9063 * 9064 * @return 9065 * 0 on success, negative value otherwise and rte_errno is set. 9066 */ 9067 static int 9068 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9069 const struct rte_flow_op_attr *attr, 9070 struct rte_flow_action_handle *handle, 9071 void *user_data, 9072 struct rte_flow_error *error) 9073 { 9074 const struct mlx5_flow_driver_ops *fops = 9075 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9076 9077 return fops->async_action_destroy(dev, queue, attr, handle, 9078 user_data, error); 9079 } 9080 9081 /** 9082 * Allocate a new memory for the counter values wrapped by all the needed 9083 * management. 9084 * 9085 * @param[in] sh 9086 * Pointer to mlx5_dev_ctx_shared object. 9087 * 9088 * @return 9089 * 0 on success, a negative errno value otherwise. 9090 */ 9091 static int 9092 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9093 { 9094 struct mlx5_counter_stats_mem_mng *mem_mng; 9095 volatile struct flow_counter_stats *raw_data; 9096 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9097 int size = (sizeof(struct flow_counter_stats) * 9098 MLX5_COUNTERS_PER_POOL + 9099 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9100 sizeof(struct mlx5_counter_stats_mem_mng); 9101 size_t pgsize = rte_mem_page_size(); 9102 uint8_t *mem; 9103 int ret; 9104 int i; 9105 9106 if (pgsize == (size_t)-1) { 9107 DRV_LOG(ERR, "Failed to get mem page size"); 9108 rte_errno = ENOMEM; 9109 return -ENOMEM; 9110 } 9111 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9112 if (!mem) { 9113 rte_errno = ENOMEM; 9114 return -ENOMEM; 9115 } 9116 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9117 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9118 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9119 sh->cdev->pdn, mem, size, 9120 &mem_mng->wm); 9121 if (ret) { 9122 rte_errno = errno; 9123 mlx5_free(mem); 9124 return -rte_errno; 9125 } 9126 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9127 raw_data = (volatile struct flow_counter_stats *)mem; 9128 for (i = 0; i < raws_n; ++i) { 9129 mem_mng->raws[i].mem_mng = mem_mng; 9130 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9131 } 9132 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9133 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9134 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9135 next); 9136 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9137 sh->sws_cmng.mem_mng = mem_mng; 9138 return 0; 9139 } 9140 9141 /** 9142 * Set the statistic memory to the new counter pool. 9143 * 9144 * @param[in] sh 9145 * Pointer to mlx5_dev_ctx_shared object. 9146 * @param[in] pool 9147 * Pointer to the pool to set the statistic memory. 9148 * 9149 * @return 9150 * 0 on success, a negative errno value otherwise. 9151 */ 9152 static int 9153 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9154 struct mlx5_flow_counter_pool *pool) 9155 { 9156 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9157 /* Resize statistic memory once used out. */ 9158 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9159 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9160 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9161 return -1; 9162 } 9163 rte_spinlock_lock(&pool->sl); 9164 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9165 rte_spinlock_unlock(&pool->sl); 9166 pool->raw_hw = NULL; 9167 return 0; 9168 } 9169 9170 #define MLX5_POOL_QUERY_FREQ_US 1000000 9171 9172 /** 9173 * Set the periodic procedure for triggering asynchronous batch queries for all 9174 * the counter pools. 9175 * 9176 * @param[in] sh 9177 * Pointer to mlx5_dev_ctx_shared object. 9178 */ 9179 void 9180 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9181 { 9182 uint32_t pools_n, us; 9183 9184 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9185 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9186 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9187 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9188 sh->sws_cmng.query_thread_on = 0; 9189 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9190 } else { 9191 sh->sws_cmng.query_thread_on = 1; 9192 } 9193 } 9194 9195 /** 9196 * The periodic procedure for triggering asynchronous batch queries for all the 9197 * counter pools. This function is probably called by the host thread. 9198 * 9199 * @param[in] arg 9200 * The parameter for the alarm process. 9201 */ 9202 void 9203 mlx5_flow_query_alarm(void *arg) 9204 { 9205 struct mlx5_dev_ctx_shared *sh = arg; 9206 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9207 uint16_t pool_index = cmng->pool_index; 9208 struct mlx5_flow_counter_pool *pool; 9209 uint16_t n_valid; 9210 int ret; 9211 9212 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9213 goto set_alarm; 9214 rte_spinlock_lock(&cmng->pool_update_sl); 9215 pool = cmng->pools[pool_index]; 9216 n_valid = cmng->n_valid; 9217 rte_spinlock_unlock(&cmng->pool_update_sl); 9218 /* Set the statistic memory to the new created pool. */ 9219 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9220 goto set_alarm; 9221 if (pool->raw_hw) 9222 /* There is a pool query in progress. */ 9223 goto set_alarm; 9224 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9225 if (!pool->raw_hw) 9226 /* No free counter statistics raw memory. */ 9227 goto set_alarm; 9228 /* 9229 * Identify the counters released between query trigger and query 9230 * handle more efficiently. The counter released in this gap period 9231 * should wait for a new round of query as the new arrived packets 9232 * will not be taken into account. 9233 */ 9234 pool->query_gen++; 9235 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9236 MLX5_COUNTERS_PER_POOL, 9237 NULL, NULL, 9238 pool->raw_hw->mem_mng->wm.lkey, 9239 (void *)(uintptr_t) 9240 pool->raw_hw->data, 9241 sh->devx_comp, 9242 (uint64_t)(uintptr_t)pool); 9243 if (ret) { 9244 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9245 " %d", pool->min_dcs->id); 9246 pool->raw_hw = NULL; 9247 goto set_alarm; 9248 } 9249 LIST_REMOVE(pool->raw_hw, next); 9250 cmng->pending_queries++; 9251 pool_index++; 9252 if (pool_index >= n_valid) 9253 pool_index = 0; 9254 set_alarm: 9255 cmng->pool_index = pool_index; 9256 mlx5_set_query_alarm(sh); 9257 } 9258 9259 /** 9260 * Check and callback event for new aged flow in the counter pool 9261 * 9262 * @param[in] sh 9263 * Pointer to mlx5_dev_ctx_shared object. 9264 * @param[in] pool 9265 * Pointer to Current counter pool. 9266 */ 9267 static void 9268 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9269 struct mlx5_flow_counter_pool *pool) 9270 { 9271 struct mlx5_priv *priv; 9272 struct mlx5_flow_counter *cnt; 9273 struct mlx5_age_info *age_info; 9274 struct mlx5_age_param *age_param; 9275 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9276 struct mlx5_counter_stats_raw *prev = pool->raw; 9277 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9278 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9279 uint16_t expected = AGE_CANDIDATE; 9280 uint32_t i; 9281 9282 pool->time_of_last_age_check = curr_time; 9283 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9284 cnt = MLX5_POOL_GET_CNT(pool, i); 9285 age_param = MLX5_CNT_TO_AGE(cnt); 9286 if (__atomic_load_n(&age_param->state, 9287 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9288 continue; 9289 if (cur->data[i].hits != prev->data[i].hits) { 9290 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9291 __ATOMIC_RELAXED); 9292 continue; 9293 } 9294 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9295 time_delta, 9296 __ATOMIC_RELAXED) <= age_param->timeout) 9297 continue; 9298 /** 9299 * Hold the lock first, or if between the 9300 * state AGE_TMOUT and tailq operation the 9301 * release happened, the release procedure 9302 * may delete a non-existent tailq node. 9303 */ 9304 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9305 age_info = GET_PORT_AGE_INFO(priv); 9306 rte_spinlock_lock(&age_info->aged_sl); 9307 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9308 AGE_TMOUT, false, 9309 __ATOMIC_RELAXED, 9310 __ATOMIC_RELAXED)) { 9311 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9312 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9313 } 9314 rte_spinlock_unlock(&age_info->aged_sl); 9315 } 9316 mlx5_age_event_prepare(sh); 9317 } 9318 9319 /** 9320 * Handler for the HW respond about ready values from an asynchronous batch 9321 * query. This function is probably called by the host thread. 9322 * 9323 * @param[in] sh 9324 * The pointer to the shared device context. 9325 * @param[in] async_id 9326 * The Devx async ID. 9327 * @param[in] status 9328 * The status of the completion. 9329 */ 9330 void 9331 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9332 uint64_t async_id, int status) 9333 { 9334 struct mlx5_flow_counter_pool *pool = 9335 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9336 struct mlx5_counter_stats_raw *raw_to_free; 9337 uint8_t query_gen = pool->query_gen ^ 1; 9338 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9339 enum mlx5_counter_type cnt_type = 9340 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9341 MLX5_COUNTER_TYPE_ORIGIN; 9342 9343 if (unlikely(status)) { 9344 raw_to_free = pool->raw_hw; 9345 } else { 9346 raw_to_free = pool->raw; 9347 if (pool->is_aged) 9348 mlx5_flow_aging_check(sh, pool); 9349 rte_spinlock_lock(&pool->sl); 9350 pool->raw = pool->raw_hw; 9351 rte_spinlock_unlock(&pool->sl); 9352 /* Be sure the new raw counters data is updated in memory. */ 9353 rte_io_wmb(); 9354 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9355 rte_spinlock_lock(&cmng->csl[cnt_type]); 9356 TAILQ_CONCAT(&cmng->counters[cnt_type], 9357 &pool->counters[query_gen], next); 9358 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9359 } 9360 } 9361 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9362 pool->raw_hw = NULL; 9363 sh->sws_cmng.pending_queries--; 9364 } 9365 9366 static int 9367 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9368 const struct flow_grp_info *grp_info, 9369 struct rte_flow_error *error) 9370 { 9371 if (grp_info->transfer && grp_info->external && 9372 grp_info->fdb_def_rule) { 9373 if (group == UINT32_MAX) 9374 return rte_flow_error_set 9375 (error, EINVAL, 9376 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9377 NULL, 9378 "group index not supported"); 9379 *table = group + 1; 9380 } else { 9381 *table = group; 9382 } 9383 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9384 return 0; 9385 } 9386 9387 /** 9388 * Translate the rte_flow group index to HW table value. 9389 * 9390 * If tunnel offload is disabled, all group ids converted to flow table 9391 * id using the standard method. 9392 * If tunnel offload is enabled, group id can be converted using the 9393 * standard or tunnel conversion method. Group conversion method 9394 * selection depends on flags in `grp_info` parameter: 9395 * - Internal (grp_info.external == 0) groups conversion uses the 9396 * standard method. 9397 * - Group ids in JUMP action converted with the tunnel conversion. 9398 * - Group id in rule attribute conversion depends on a rule type and 9399 * group id value: 9400 * ** non zero group attributes converted with the tunnel method 9401 * ** zero group attribute in non-tunnel rule is converted using the 9402 * standard method - there's only one root table 9403 * ** zero group attribute in steer tunnel rule is converted with the 9404 * standard method - single root table 9405 * ** zero group attribute in match tunnel rule is a special OvS 9406 * case: that value is used for portability reasons. That group 9407 * id is converted with the tunnel conversion method. 9408 * 9409 * @param[in] dev 9410 * Port device 9411 * @param[in] tunnel 9412 * PMD tunnel offload object 9413 * @param[in] group 9414 * rte_flow group index value. 9415 * @param[out] table 9416 * HW table value. 9417 * @param[in] grp_info 9418 * flags used for conversion 9419 * @param[out] error 9420 * Pointer to error structure. 9421 * 9422 * @return 9423 * 0 on success, a negative errno value otherwise and rte_errno is set. 9424 */ 9425 int 9426 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9427 const struct mlx5_flow_tunnel *tunnel, 9428 uint32_t group, uint32_t *table, 9429 const struct flow_grp_info *grp_info, 9430 struct rte_flow_error *error) 9431 { 9432 int ret; 9433 bool standard_translation; 9434 9435 if (!grp_info->skip_scale && grp_info->external && 9436 group < MLX5_MAX_TABLES_EXTERNAL) 9437 group *= MLX5_FLOW_TABLE_FACTOR; 9438 if (is_tunnel_offload_active(dev)) { 9439 standard_translation = !grp_info->external || 9440 grp_info->std_tbl_fix; 9441 } else { 9442 standard_translation = true; 9443 } 9444 DRV_LOG(DEBUG, 9445 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9446 dev->data->port_id, group, grp_info->transfer, 9447 grp_info->external, grp_info->fdb_def_rule, 9448 standard_translation ? "STANDARD" : "TUNNEL"); 9449 if (standard_translation) 9450 ret = flow_group_to_table(dev->data->port_id, group, table, 9451 grp_info, error); 9452 else 9453 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9454 table, error); 9455 9456 return ret; 9457 } 9458 9459 /** 9460 * Discover availability of metadata reg_c's. 9461 * 9462 * Iteratively use test flows to check availability. 9463 * 9464 * @param[in] dev 9465 * Pointer to the Ethernet device structure. 9466 * 9467 * @return 9468 * 0 on success, a negative errno value otherwise and rte_errno is set. 9469 */ 9470 int 9471 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9472 { 9473 struct mlx5_priv *priv = dev->data->dev_private; 9474 enum modify_reg idx; 9475 int n = 0; 9476 9477 /* reg_c[0] and reg_c[1] are reserved. */ 9478 priv->sh->flow_mreg_c[n++] = REG_C_0; 9479 priv->sh->flow_mreg_c[n++] = REG_C_1; 9480 /* Discover availability of other reg_c's. */ 9481 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9482 struct rte_flow_attr attr = { 9483 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9484 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9485 .ingress = 1, 9486 }; 9487 struct rte_flow_item items[] = { 9488 [0] = { 9489 .type = RTE_FLOW_ITEM_TYPE_END, 9490 }, 9491 }; 9492 struct rte_flow_action actions[] = { 9493 [0] = { 9494 .type = (enum rte_flow_action_type) 9495 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9496 .conf = &(struct mlx5_flow_action_copy_mreg){ 9497 .src = REG_C_1, 9498 .dst = idx, 9499 }, 9500 }, 9501 [1] = { 9502 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9503 .conf = &(struct rte_flow_action_jump){ 9504 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9505 }, 9506 }, 9507 [2] = { 9508 .type = RTE_FLOW_ACTION_TYPE_END, 9509 }, 9510 }; 9511 uint32_t flow_idx; 9512 struct rte_flow *flow; 9513 struct rte_flow_error error; 9514 9515 if (!priv->sh->config.dv_flow_en) 9516 break; 9517 /* Create internal flow, validation skips copy action. */ 9518 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9519 items, actions, false, &error); 9520 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9521 flow_idx); 9522 if (!flow) 9523 continue; 9524 priv->sh->flow_mreg_c[n++] = idx; 9525 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9526 } 9527 for (; n < MLX5_MREG_C_NUM; ++n) 9528 priv->sh->flow_mreg_c[n] = REG_NON; 9529 priv->sh->metadata_regc_check_flag = 1; 9530 return 0; 9531 } 9532 9533 int 9534 save_dump_file(const uint8_t *data, uint32_t size, 9535 uint32_t type, uint64_t id, void *arg, FILE *file) 9536 { 9537 char line[BUF_SIZE]; 9538 uint32_t out = 0; 9539 uint32_t k; 9540 uint32_t actions_num; 9541 struct rte_flow_query_count *count; 9542 9543 memset(line, 0, BUF_SIZE); 9544 switch (type) { 9545 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9546 actions_num = *(uint32_t *)(arg); 9547 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9548 type, id, actions_num); 9549 break; 9550 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9551 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9552 type, id); 9553 break; 9554 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9555 count = (struct rte_flow_query_count *)arg; 9556 fprintf(file, 9557 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9558 type, id, count->hits, count->bytes); 9559 return 0; 9560 default: 9561 return -1; 9562 } 9563 9564 for (k = 0; k < size; k++) { 9565 /* Make sure we do not overrun the line buffer length. */ 9566 if (out >= BUF_SIZE - 4) { 9567 line[out] = '\0'; 9568 break; 9569 } 9570 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9571 (data[k]) & 0xff); 9572 } 9573 fprintf(file, "%s\n", line); 9574 return 0; 9575 } 9576 9577 int 9578 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9579 struct rte_flow_query_count *count, struct rte_flow_error *error) 9580 { 9581 struct rte_flow_action action[2]; 9582 enum mlx5_flow_drv_type ftype; 9583 const struct mlx5_flow_driver_ops *fops; 9584 9585 if (!flow) { 9586 return rte_flow_error_set(error, ENOENT, 9587 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9588 NULL, 9589 "invalid flow handle"); 9590 } 9591 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9592 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9593 if (flow->counter) { 9594 memset(count, 0, sizeof(struct rte_flow_query_count)); 9595 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9596 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9597 ftype < MLX5_FLOW_TYPE_MAX); 9598 fops = flow_get_drv_ops(ftype); 9599 return fops->query(dev, flow, action, count, error); 9600 } 9601 return -1; 9602 } 9603 9604 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9605 /** 9606 * Dump flow ipool data to file 9607 * 9608 * @param[in] dev 9609 * The pointer to Ethernet device. 9610 * @param[in] file 9611 * A pointer to a file for output. 9612 * @param[out] error 9613 * Perform verbose error reporting if not NULL. PMDs initialize this 9614 * structure in case of error only. 9615 * @return 9616 * 0 on success, a negative value otherwise. 9617 */ 9618 int 9619 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9620 struct rte_flow *flow, FILE *file, 9621 struct rte_flow_error *error) 9622 { 9623 struct mlx5_priv *priv = dev->data->dev_private; 9624 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9625 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9626 uint32_t handle_idx; 9627 struct mlx5_flow_handle *dh; 9628 struct rte_flow_query_count count; 9629 uint32_t actions_num; 9630 const uint8_t *data; 9631 size_t size; 9632 uint64_t id; 9633 uint32_t type; 9634 void *action = NULL; 9635 9636 if (!flow) { 9637 return rte_flow_error_set(error, ENOENT, 9638 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9639 NULL, 9640 "invalid flow handle"); 9641 } 9642 handle_idx = flow->dev_handles; 9643 /* query counter */ 9644 if (flow->counter && 9645 (!mlx5_counter_query(dev, flow->counter, false, 9646 &count.hits, &count.bytes, &action)) && action) { 9647 id = (uint64_t)(uintptr_t)action; 9648 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9649 save_dump_file(NULL, 0, type, 9650 id, (void *)&count, file); 9651 } 9652 9653 while (handle_idx) { 9654 dh = mlx5_ipool_get(priv->sh->ipool 9655 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9656 if (!dh) 9657 continue; 9658 handle_idx = dh->next.next; 9659 9660 /* Get modify_hdr and encap_decap buf from ipools. */ 9661 encap_decap = NULL; 9662 modify_hdr = dh->dvh.modify_hdr; 9663 9664 if (dh->dvh.rix_encap_decap) { 9665 encap_decap = mlx5_ipool_get(priv->sh->ipool 9666 [MLX5_IPOOL_DECAP_ENCAP], 9667 dh->dvh.rix_encap_decap); 9668 } 9669 if (modify_hdr) { 9670 data = (const uint8_t *)modify_hdr->actions; 9671 size = (size_t)(modify_hdr->actions_num) * 8; 9672 id = (uint64_t)(uintptr_t)modify_hdr->action; 9673 actions_num = modify_hdr->actions_num; 9674 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9675 save_dump_file(data, size, type, id, 9676 (void *)(&actions_num), file); 9677 } 9678 if (encap_decap) { 9679 data = encap_decap->buf; 9680 size = encap_decap->size; 9681 id = (uint64_t)(uintptr_t)encap_decap->action; 9682 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9683 save_dump_file(data, size, type, 9684 id, NULL, file); 9685 } 9686 } 9687 return 0; 9688 } 9689 9690 /** 9691 * Dump all flow's encap_decap/modify_hdr/counter data to file 9692 * 9693 * @param[in] dev 9694 * The pointer to Ethernet device. 9695 * @param[in] file 9696 * A pointer to a file for output. 9697 * @param[out] error 9698 * Perform verbose error reporting if not NULL. PMDs initialize this 9699 * structure in case of error only. 9700 * @return 9701 * 0 on success, a negative value otherwise. 9702 */ 9703 static int 9704 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9705 FILE *file, struct rte_flow_error *error __rte_unused) 9706 { 9707 struct mlx5_priv *priv = dev->data->dev_private; 9708 struct mlx5_dev_ctx_shared *sh = priv->sh; 9709 struct mlx5_hlist *h; 9710 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9711 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9712 struct rte_flow_query_count count; 9713 uint32_t actions_num; 9714 const uint8_t *data; 9715 size_t size; 9716 uint64_t id; 9717 uint32_t type; 9718 uint32_t i; 9719 uint32_t j; 9720 struct mlx5_list_inconst *l_inconst; 9721 struct mlx5_list_entry *e; 9722 int lcore_index; 9723 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9724 uint32_t max; 9725 void *action; 9726 9727 /* encap_decap hlist is lcore_share, get global core cache. */ 9728 i = MLX5_LIST_GLOBAL; 9729 h = sh->encaps_decaps; 9730 if (h) { 9731 for (j = 0; j <= h->mask; j++) { 9732 l_inconst = &h->buckets[j].l; 9733 if (!l_inconst || !l_inconst->cache[i]) 9734 continue; 9735 9736 e = LIST_FIRST(&l_inconst->cache[i]->h); 9737 while (e) { 9738 encap_decap = 9739 (struct mlx5_flow_dv_encap_decap_resource *)e; 9740 data = encap_decap->buf; 9741 size = encap_decap->size; 9742 id = (uint64_t)(uintptr_t)encap_decap->action; 9743 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9744 save_dump_file(data, size, type, 9745 id, NULL, file); 9746 e = LIST_NEXT(e, next); 9747 } 9748 } 9749 } 9750 9751 /* get modify_hdr */ 9752 h = sh->modify_cmds; 9753 if (h) { 9754 lcore_index = rte_lcore_index(rte_lcore_id()); 9755 if (unlikely(lcore_index == -1)) { 9756 lcore_index = MLX5_LIST_NLCORE; 9757 rte_spinlock_lock(&h->l_const.lcore_lock); 9758 } 9759 i = lcore_index; 9760 9761 for (j = 0; j <= h->mask; j++) { 9762 l_inconst = &h->buckets[j].l; 9763 if (!l_inconst || !l_inconst->cache[i]) 9764 continue; 9765 9766 e = LIST_FIRST(&l_inconst->cache[i]->h); 9767 while (e) { 9768 modify_hdr = 9769 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9770 data = (const uint8_t *)modify_hdr->actions; 9771 size = (size_t)(modify_hdr->actions_num) * 8; 9772 actions_num = modify_hdr->actions_num; 9773 id = (uint64_t)(uintptr_t)modify_hdr->action; 9774 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9775 save_dump_file(data, size, type, id, 9776 (void *)(&actions_num), file); 9777 e = LIST_NEXT(e, next); 9778 } 9779 } 9780 9781 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9782 rte_spinlock_unlock(&h->l_const.lcore_lock); 9783 } 9784 9785 /* get counter */ 9786 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 9787 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9788 for (j = 1; j <= max; j++) { 9789 action = NULL; 9790 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9791 &count.bytes, &action)) && action) { 9792 id = (uint64_t)(uintptr_t)action; 9793 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9794 save_dump_file(NULL, 0, type, 9795 id, (void *)&count, file); 9796 } 9797 } 9798 return 0; 9799 } 9800 #endif 9801 9802 /** 9803 * Dump flow raw hw data to file 9804 * 9805 * @param[in] dev 9806 * The pointer to Ethernet device. 9807 * @param[in] file 9808 * A pointer to a file for output. 9809 * @param[out] error 9810 * Perform verbose error reporting if not NULL. PMDs initialize this 9811 * structure in case of error only. 9812 * @return 9813 * 0 on success, a negative value otherwise. 9814 */ 9815 int 9816 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9817 FILE *file, 9818 struct rte_flow_error *error __rte_unused) 9819 { 9820 struct mlx5_priv *priv = dev->data->dev_private; 9821 struct mlx5_dev_ctx_shared *sh = priv->sh; 9822 uint32_t handle_idx; 9823 int ret; 9824 struct mlx5_flow_handle *dh; 9825 struct rte_flow *flow; 9826 9827 if (!sh->config.dv_flow_en) { 9828 if (fputs("device dv flow disabled\n", file) <= 0) 9829 return -errno; 9830 return -ENOTSUP; 9831 } 9832 9833 /* dump all */ 9834 if (!flow_idx) { 9835 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9836 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9837 return -EINVAL; 9838 #endif 9839 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9840 sh->rx_domain, 9841 sh->tx_domain, file); 9842 } 9843 /* dump one */ 9844 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9845 (uintptr_t)(void *)flow_idx); 9846 if (!flow) 9847 return -EINVAL; 9848 9849 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9850 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9851 #endif 9852 handle_idx = flow->dev_handles; 9853 while (handle_idx) { 9854 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9855 handle_idx); 9856 if (!dh) 9857 return -ENOENT; 9858 if (dh->drv_flow) { 9859 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9860 file); 9861 if (ret) 9862 return -ENOENT; 9863 } 9864 handle_idx = dh->next.next; 9865 } 9866 return 0; 9867 } 9868 9869 /** 9870 * Get aged-out flows. 9871 * 9872 * @param[in] dev 9873 * Pointer to the Ethernet device structure. 9874 * @param[in] context 9875 * The address of an array of pointers to the aged-out flows contexts. 9876 * @param[in] nb_countexts 9877 * The length of context array pointers. 9878 * @param[out] error 9879 * Perform verbose error reporting if not NULL. Initialized in case of 9880 * error only. 9881 * 9882 * @return 9883 * how many contexts get in success, otherwise negative errno value. 9884 * if nb_contexts is 0, return the amount of all aged contexts. 9885 * if nb_contexts is not 0 , return the amount of aged flows reported 9886 * in the context array. 9887 */ 9888 int 9889 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9890 uint32_t nb_contexts, struct rte_flow_error *error) 9891 { 9892 const struct mlx5_flow_driver_ops *fops; 9893 struct rte_flow_attr attr = { .transfer = 0 }; 9894 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, &attr); 9895 9896 if (type == MLX5_FLOW_TYPE_DV || type == MLX5_FLOW_TYPE_HW) { 9897 fops = flow_get_drv_ops(type); 9898 return fops->get_aged_flows(dev, contexts, nb_contexts, error); 9899 } 9900 DRV_LOG(ERR, "port %u get aged flows is not supported.", 9901 dev->data->port_id); 9902 return -ENOTSUP; 9903 } 9904 9905 /** 9906 * Get aged-out flows per HWS queue. 9907 * 9908 * @param[in] dev 9909 * Pointer to the Ethernet device structure. 9910 * @param[in] queue_id 9911 * Flow queue to query. 9912 * @param[in] context 9913 * The address of an array of pointers to the aged-out flows contexts. 9914 * @param[in] nb_countexts 9915 * The length of context array pointers. 9916 * @param[out] error 9917 * Perform verbose error reporting if not NULL. Initialized in case of 9918 * error only. 9919 * 9920 * @return 9921 * how many contexts get in success, otherwise negative errno value. 9922 * if nb_contexts is 0, return the amount of all aged contexts. 9923 * if nb_contexts is not 0 , return the amount of aged flows reported 9924 * in the context array. 9925 */ 9926 int 9927 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 9928 void **contexts, uint32_t nb_contexts, 9929 struct rte_flow_error *error) 9930 { 9931 const struct mlx5_flow_driver_ops *fops; 9932 struct rte_flow_attr attr = { 0 }; 9933 9934 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 9935 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9936 return fops->get_q_aged_flows(dev, queue_id, contexts, 9937 nb_contexts, error); 9938 } 9939 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 9940 dev->data->port_id, queue_id); 9941 return rte_flow_error_set(error, ENOTSUP, 9942 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9943 "get Q aged flows with incorrect steering mode"); 9944 } 9945 9946 /* Wrapper for driver action_validate op callback */ 9947 static int 9948 flow_drv_action_validate(struct rte_eth_dev *dev, 9949 const struct rte_flow_indir_action_conf *conf, 9950 const struct rte_flow_action *action, 9951 const struct mlx5_flow_driver_ops *fops, 9952 struct rte_flow_error *error) 9953 { 9954 static const char err_msg[] = "indirect action validation unsupported"; 9955 9956 if (!fops->action_validate) { 9957 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9958 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9959 NULL, err_msg); 9960 return -rte_errno; 9961 } 9962 return fops->action_validate(dev, conf, action, error); 9963 } 9964 9965 /** 9966 * Destroys the shared action by handle. 9967 * 9968 * @param dev 9969 * Pointer to Ethernet device structure. 9970 * @param[in] handle 9971 * Handle for the indirect action object to be destroyed. 9972 * @param[out] error 9973 * Perform verbose error reporting if not NULL. PMDs initialize this 9974 * structure in case of error only. 9975 * 9976 * @return 9977 * 0 on success, a negative errno value otherwise and rte_errno is set. 9978 * 9979 * @note: wrapper for driver action_create op callback. 9980 */ 9981 static int 9982 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 9983 struct rte_flow_action_handle *handle, 9984 struct rte_flow_error *error) 9985 { 9986 static const char err_msg[] = "indirect action destruction unsupported"; 9987 struct rte_flow_attr attr = { .transfer = 0 }; 9988 const struct mlx5_flow_driver_ops *fops = 9989 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 9990 9991 if (!fops->action_destroy) { 9992 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 9993 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 9994 NULL, err_msg); 9995 return -rte_errno; 9996 } 9997 return fops->action_destroy(dev, handle, error); 9998 } 9999 10000 /* Wrapper for driver action_destroy op callback */ 10001 static int 10002 flow_drv_action_update(struct rte_eth_dev *dev, 10003 struct rte_flow_action_handle *handle, 10004 const void *update, 10005 const struct mlx5_flow_driver_ops *fops, 10006 struct rte_flow_error *error) 10007 { 10008 static const char err_msg[] = "indirect action update unsupported"; 10009 10010 if (!fops->action_update) { 10011 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10012 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10013 NULL, err_msg); 10014 return -rte_errno; 10015 } 10016 return fops->action_update(dev, handle, update, error); 10017 } 10018 10019 /* Wrapper for driver action_destroy op callback */ 10020 static int 10021 flow_drv_action_query(struct rte_eth_dev *dev, 10022 const struct rte_flow_action_handle *handle, 10023 void *data, 10024 const struct mlx5_flow_driver_ops *fops, 10025 struct rte_flow_error *error) 10026 { 10027 static const char err_msg[] = "indirect action query unsupported"; 10028 10029 if (!fops->action_query) { 10030 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10031 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10032 NULL, err_msg); 10033 return -rte_errno; 10034 } 10035 return fops->action_query(dev, handle, data, error); 10036 } 10037 10038 /** 10039 * Create indirect action for reuse in multiple flow rules. 10040 * 10041 * @param dev 10042 * Pointer to Ethernet device structure. 10043 * @param conf 10044 * Pointer to indirect action object configuration. 10045 * @param[in] action 10046 * Action configuration for indirect action object creation. 10047 * @param[out] error 10048 * Perform verbose error reporting if not NULL. PMDs initialize this 10049 * structure in case of error only. 10050 * @return 10051 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10052 */ 10053 static struct rte_flow_action_handle * 10054 mlx5_action_handle_create(struct rte_eth_dev *dev, 10055 const struct rte_flow_indir_action_conf *conf, 10056 const struct rte_flow_action *action, 10057 struct rte_flow_error *error) 10058 { 10059 static const char err_msg[] = "indirect action creation unsupported"; 10060 struct rte_flow_attr attr = { .transfer = 0 }; 10061 const struct mlx5_flow_driver_ops *fops = 10062 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10063 10064 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10065 return NULL; 10066 if (!fops->action_create) { 10067 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10068 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10069 NULL, err_msg); 10070 return NULL; 10071 } 10072 return fops->action_create(dev, conf, action, error); 10073 } 10074 10075 /** 10076 * Updates inplace the indirect action configuration pointed by *handle* 10077 * with the configuration provided as *update* argument. 10078 * The update of the indirect action configuration effects all flow rules 10079 * reusing the action via handle. 10080 * 10081 * @param dev 10082 * Pointer to Ethernet device structure. 10083 * @param[in] handle 10084 * Handle for the indirect action to be updated. 10085 * @param[in] update 10086 * Action specification used to modify the action pointed by handle. 10087 * *update* could be of same type with the action pointed by the *handle* 10088 * handle argument, or some other structures like a wrapper, depending on 10089 * the indirect action type. 10090 * @param[out] error 10091 * Perform verbose error reporting if not NULL. PMDs initialize this 10092 * structure in case of error only. 10093 * 10094 * @return 10095 * 0 on success, a negative errno value otherwise and rte_errno is set. 10096 */ 10097 static int 10098 mlx5_action_handle_update(struct rte_eth_dev *dev, 10099 struct rte_flow_action_handle *handle, 10100 const void *update, 10101 struct rte_flow_error *error) 10102 { 10103 struct rte_flow_attr attr = { .transfer = 0 }; 10104 const struct mlx5_flow_driver_ops *fops = 10105 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10106 int ret; 10107 10108 ret = flow_drv_action_validate(dev, NULL, 10109 (const struct rte_flow_action *)update, fops, error); 10110 if (ret) 10111 return ret; 10112 return flow_drv_action_update(dev, handle, update, fops, 10113 error); 10114 } 10115 10116 /** 10117 * Query the indirect action by handle. 10118 * 10119 * This function allows retrieving action-specific data such as counters. 10120 * Data is gathered by special action which may be present/referenced in 10121 * more than one flow rule definition. 10122 * 10123 * see @RTE_FLOW_ACTION_TYPE_COUNT 10124 * 10125 * @param dev 10126 * Pointer to Ethernet device structure. 10127 * @param[in] handle 10128 * Handle for the indirect action to query. 10129 * @param[in, out] data 10130 * Pointer to storage for the associated query data type. 10131 * @param[out] error 10132 * Perform verbose error reporting if not NULL. PMDs initialize this 10133 * structure in case of error only. 10134 * 10135 * @return 10136 * 0 on success, a negative errno value otherwise and rte_errno is set. 10137 */ 10138 static int 10139 mlx5_action_handle_query(struct rte_eth_dev *dev, 10140 const struct rte_flow_action_handle *handle, 10141 void *data, 10142 struct rte_flow_error *error) 10143 { 10144 struct rte_flow_attr attr = { .transfer = 0 }; 10145 const struct mlx5_flow_driver_ops *fops = 10146 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10147 10148 return flow_drv_action_query(dev, handle, data, fops, error); 10149 } 10150 10151 /** 10152 * Destroy all indirect actions (shared RSS). 10153 * 10154 * @param dev 10155 * Pointer to Ethernet device. 10156 * 10157 * @return 10158 * 0 on success, a negative errno value otherwise and rte_errno is set. 10159 */ 10160 int 10161 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10162 { 10163 struct rte_flow_error error; 10164 struct mlx5_priv *priv = dev->data->dev_private; 10165 struct mlx5_shared_action_rss *shared_rss; 10166 int ret = 0; 10167 uint32_t idx; 10168 10169 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10170 priv->rss_shared_actions, idx, shared_rss, next) { 10171 ret |= mlx5_action_handle_destroy(dev, 10172 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10173 } 10174 return ret; 10175 } 10176 10177 /** 10178 * Validate existing indirect actions against current device configuration 10179 * and attach them to device resources. 10180 * 10181 * @param dev 10182 * Pointer to Ethernet device. 10183 * 10184 * @return 10185 * 0 on success, a negative errno value otherwise and rte_errno is set. 10186 */ 10187 int 10188 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10189 { 10190 struct mlx5_priv *priv = dev->data->dev_private; 10191 int ret = 0; 10192 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10193 10194 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10195 const char *message; 10196 uint32_t queue_idx; 10197 10198 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10199 ind_tbl->queues_n, 10200 &message, &queue_idx); 10201 if (ret != 0) { 10202 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10203 dev->data->port_id, ind_tbl->queues[queue_idx], 10204 message); 10205 break; 10206 } 10207 } 10208 if (ret != 0) 10209 return ret; 10210 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10211 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10212 if (ret != 0) { 10213 DRV_LOG(ERR, "Port %u could not attach " 10214 "indirection table obj %p", 10215 dev->data->port_id, (void *)ind_tbl); 10216 goto error; 10217 } 10218 } 10219 10220 return 0; 10221 error: 10222 ind_tbl_last = ind_tbl; 10223 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10224 if (ind_tbl == ind_tbl_last) 10225 break; 10226 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10227 DRV_LOG(CRIT, "Port %u could not detach " 10228 "indirection table obj %p on rollback", 10229 dev->data->port_id, (void *)ind_tbl); 10230 } 10231 return ret; 10232 } 10233 10234 /** 10235 * Detach indirect actions of the device from its resources. 10236 * 10237 * @param dev 10238 * Pointer to Ethernet device. 10239 * 10240 * @return 10241 * 0 on success, a negative errno value otherwise and rte_errno is set. 10242 */ 10243 int 10244 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10245 { 10246 struct mlx5_priv *priv = dev->data->dev_private; 10247 int ret = 0; 10248 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10249 10250 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10251 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10252 if (ret != 0) { 10253 DRV_LOG(ERR, "Port %u could not detach " 10254 "indirection table obj %p", 10255 dev->data->port_id, (void *)ind_tbl); 10256 goto error; 10257 } 10258 } 10259 return 0; 10260 error: 10261 ind_tbl_last = ind_tbl; 10262 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10263 if (ind_tbl == ind_tbl_last) 10264 break; 10265 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10266 DRV_LOG(CRIT, "Port %u could not attach " 10267 "indirection table obj %p on rollback", 10268 dev->data->port_id, (void *)ind_tbl); 10269 } 10270 return ret; 10271 } 10272 10273 #ifndef HAVE_MLX5DV_DR 10274 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10275 #else 10276 #define MLX5_DOMAIN_SYNC_FLOW \ 10277 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10278 #endif 10279 10280 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10281 { 10282 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10283 const struct mlx5_flow_driver_ops *fops; 10284 int ret; 10285 struct rte_flow_attr attr = { .transfer = 0 }; 10286 10287 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10288 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10289 if (ret > 0) 10290 ret = -ret; 10291 return ret; 10292 } 10293 10294 const struct mlx5_flow_tunnel * 10295 mlx5_get_tof(const struct rte_flow_item *item, 10296 const struct rte_flow_action *action, 10297 enum mlx5_tof_rule_type *rule_type) 10298 { 10299 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10300 if (item->type == (typeof(item->type)) 10301 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10302 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10303 return flow_items_to_tunnel(item); 10304 } 10305 } 10306 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10307 if (action->type == (typeof(action->type)) 10308 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10309 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10310 return flow_actions_to_tunnel(action); 10311 } 10312 } 10313 return NULL; 10314 } 10315 10316 /** 10317 * tunnel offload functionality is defined for DV environment only 10318 */ 10319 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10320 __extension__ 10321 union tunnel_offload_mark { 10322 uint32_t val; 10323 struct { 10324 uint32_t app_reserve:8; 10325 uint32_t table_id:15; 10326 uint32_t transfer:1; 10327 uint32_t _unused_:8; 10328 }; 10329 }; 10330 10331 static bool 10332 mlx5_access_tunnel_offload_db 10333 (struct rte_eth_dev *dev, 10334 bool (*match)(struct rte_eth_dev *, 10335 struct mlx5_flow_tunnel *, const void *), 10336 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10337 void (*miss)(struct rte_eth_dev *, void *), 10338 void *ctx, bool lock_op); 10339 10340 static int 10341 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10342 struct rte_flow *flow, 10343 const struct rte_flow_attr *attr, 10344 const struct rte_flow_action *app_actions, 10345 uint32_t flow_idx, 10346 const struct mlx5_flow_tunnel *tunnel, 10347 struct tunnel_default_miss_ctx *ctx, 10348 struct rte_flow_error *error) 10349 { 10350 struct mlx5_priv *priv = dev->data->dev_private; 10351 struct mlx5_flow *dev_flow; 10352 struct rte_flow_attr miss_attr = *attr; 10353 const struct rte_flow_item miss_items[2] = { 10354 { 10355 .type = RTE_FLOW_ITEM_TYPE_ETH, 10356 .spec = NULL, 10357 .last = NULL, 10358 .mask = NULL 10359 }, 10360 { 10361 .type = RTE_FLOW_ITEM_TYPE_END, 10362 .spec = NULL, 10363 .last = NULL, 10364 .mask = NULL 10365 } 10366 }; 10367 union tunnel_offload_mark mark_id; 10368 struct rte_flow_action_mark miss_mark; 10369 struct rte_flow_action miss_actions[3] = { 10370 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10371 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10372 }; 10373 const struct rte_flow_action_jump *jump_data; 10374 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10375 struct flow_grp_info grp_info = { 10376 .external = 1, 10377 .transfer = attr->transfer, 10378 .fdb_def_rule = !!priv->fdb_def_rule, 10379 .std_tbl_fix = 0, 10380 }; 10381 int ret; 10382 10383 if (!attr->transfer) { 10384 uint32_t q_size; 10385 10386 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10387 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10388 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10389 0, SOCKET_ID_ANY); 10390 if (!ctx->queue) 10391 return rte_flow_error_set 10392 (error, ENOMEM, 10393 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10394 NULL, "invalid default miss RSS"); 10395 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10396 ctx->action_rss.level = 0, 10397 ctx->action_rss.types = priv->rss_conf.rss_hf, 10398 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10399 ctx->action_rss.queue_num = priv->reta_idx_n, 10400 ctx->action_rss.key = priv->rss_conf.rss_key, 10401 ctx->action_rss.queue = ctx->queue; 10402 if (!priv->reta_idx_n || !priv->rxqs_n) 10403 return rte_flow_error_set 10404 (error, EINVAL, 10405 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10406 NULL, "invalid port configuration"); 10407 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10408 ctx->action_rss.types = 0; 10409 for (i = 0; i != priv->reta_idx_n; ++i) 10410 ctx->queue[i] = (*priv->reta_idx)[i]; 10411 } else { 10412 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10413 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10414 } 10415 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10416 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10417 jump_data = app_actions->conf; 10418 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10419 miss_attr.group = jump_data->group; 10420 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10421 &flow_table, &grp_info, error); 10422 if (ret) 10423 return rte_flow_error_set(error, EINVAL, 10424 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10425 NULL, "invalid tunnel id"); 10426 mark_id.app_reserve = 0; 10427 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10428 mark_id.transfer = !!attr->transfer; 10429 mark_id._unused_ = 0; 10430 miss_mark.id = mark_id.val; 10431 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10432 miss_items, miss_actions, flow_idx, error); 10433 if (!dev_flow) 10434 return -rte_errno; 10435 dev_flow->flow = flow; 10436 dev_flow->external = true; 10437 dev_flow->tunnel = tunnel; 10438 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10439 /* Subflow object was created, we must include one in the list. */ 10440 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10441 dev_flow->handle, next); 10442 DRV_LOG(DEBUG, 10443 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10444 dev->data->port_id, tunnel->app_tunnel.type, 10445 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10446 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10447 miss_actions, error); 10448 if (!ret) 10449 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10450 error); 10451 10452 return ret; 10453 } 10454 10455 static const struct mlx5_flow_tbl_data_entry * 10456 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10457 { 10458 struct mlx5_priv *priv = dev->data->dev_private; 10459 struct mlx5_dev_ctx_shared *sh = priv->sh; 10460 struct mlx5_list_entry *he; 10461 union tunnel_offload_mark mbits = { .val = mark }; 10462 union mlx5_flow_tbl_key table_key = { 10463 { 10464 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10465 .id = 0, 10466 .reserved = 0, 10467 .dummy = 0, 10468 .is_fdb = !!mbits.transfer, 10469 .is_egress = 0, 10470 } 10471 }; 10472 struct mlx5_flow_cb_ctx ctx = { 10473 .data = &table_key.v64, 10474 }; 10475 10476 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10477 return he ? 10478 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10479 } 10480 10481 static void 10482 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10483 struct mlx5_list_entry *entry) 10484 { 10485 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10486 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10487 10488 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10489 tunnel_flow_tbl_to_id(tte->flow_table)); 10490 mlx5_free(tte); 10491 } 10492 10493 static int 10494 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10495 struct mlx5_list_entry *entry, void *cb_ctx) 10496 { 10497 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10498 union tunnel_tbl_key tbl = { 10499 .val = *(uint64_t *)(ctx->data), 10500 }; 10501 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10502 10503 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10504 } 10505 10506 static struct mlx5_list_entry * 10507 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10508 { 10509 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10510 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10511 struct tunnel_tbl_entry *tte; 10512 union tunnel_tbl_key tbl = { 10513 .val = *(uint64_t *)(ctx->data), 10514 }; 10515 10516 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10517 sizeof(*tte), 0, 10518 SOCKET_ID_ANY); 10519 if (!tte) 10520 goto err; 10521 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10522 &tte->flow_table); 10523 if (tte->flow_table >= MLX5_MAX_TABLES) { 10524 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10525 tte->flow_table); 10526 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10527 tte->flow_table); 10528 goto err; 10529 } else if (!tte->flow_table) { 10530 goto err; 10531 } 10532 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10533 tte->tunnel_id = tbl.tunnel_id; 10534 tte->group = tbl.group; 10535 return &tte->hash; 10536 err: 10537 if (tte) 10538 mlx5_free(tte); 10539 return NULL; 10540 } 10541 10542 static struct mlx5_list_entry * 10543 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10544 struct mlx5_list_entry *oentry, 10545 void *cb_ctx __rte_unused) 10546 { 10547 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10548 0, SOCKET_ID_ANY); 10549 10550 if (!tte) 10551 return NULL; 10552 memcpy(tte, oentry, sizeof(*tte)); 10553 return &tte->hash; 10554 } 10555 10556 static void 10557 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10558 struct mlx5_list_entry *entry) 10559 { 10560 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10561 10562 mlx5_free(tte); 10563 } 10564 10565 static uint32_t 10566 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10567 const struct mlx5_flow_tunnel *tunnel, 10568 uint32_t group, uint32_t *table, 10569 struct rte_flow_error *error) 10570 { 10571 struct mlx5_list_entry *he; 10572 struct tunnel_tbl_entry *tte; 10573 union tunnel_tbl_key key = { 10574 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10575 .group = group 10576 }; 10577 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10578 struct mlx5_hlist *group_hash; 10579 struct mlx5_flow_cb_ctx ctx = { 10580 .data = &key.val, 10581 }; 10582 10583 group_hash = tunnel ? tunnel->groups : thub->groups; 10584 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10585 if (!he) 10586 return rte_flow_error_set(error, EINVAL, 10587 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10588 NULL, 10589 "tunnel group index not supported"); 10590 tte = container_of(he, typeof(*tte), hash); 10591 *table = tte->flow_table; 10592 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10593 dev->data->port_id, key.tunnel_id, group, *table); 10594 return 0; 10595 } 10596 10597 static void 10598 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10599 struct mlx5_flow_tunnel *tunnel) 10600 { 10601 struct mlx5_priv *priv = dev->data->dev_private; 10602 struct mlx5_indexed_pool *ipool; 10603 10604 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10605 dev->data->port_id, tunnel->tunnel_id); 10606 LIST_REMOVE(tunnel, chain); 10607 mlx5_hlist_destroy(tunnel->groups); 10608 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10609 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10610 } 10611 10612 static bool 10613 mlx5_access_tunnel_offload_db 10614 (struct rte_eth_dev *dev, 10615 bool (*match)(struct rte_eth_dev *, 10616 struct mlx5_flow_tunnel *, const void *), 10617 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10618 void (*miss)(struct rte_eth_dev *, void *), 10619 void *ctx, bool lock_op) 10620 { 10621 bool verdict = false; 10622 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10623 struct mlx5_flow_tunnel *tunnel; 10624 10625 rte_spinlock_lock(&thub->sl); 10626 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10627 verdict = match(dev, tunnel, (const void *)ctx); 10628 if (verdict) 10629 break; 10630 } 10631 if (!lock_op) 10632 rte_spinlock_unlock(&thub->sl); 10633 if (verdict && hit) 10634 hit(dev, tunnel, ctx); 10635 if (!verdict && miss) 10636 miss(dev, ctx); 10637 if (lock_op) 10638 rte_spinlock_unlock(&thub->sl); 10639 10640 return verdict; 10641 } 10642 10643 struct tunnel_db_find_tunnel_id_ctx { 10644 uint32_t tunnel_id; 10645 struct mlx5_flow_tunnel *tunnel; 10646 }; 10647 10648 static bool 10649 find_tunnel_id_match(struct rte_eth_dev *dev, 10650 struct mlx5_flow_tunnel *tunnel, const void *x) 10651 { 10652 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10653 10654 RTE_SET_USED(dev); 10655 return tunnel->tunnel_id == ctx->tunnel_id; 10656 } 10657 10658 static void 10659 find_tunnel_id_hit(struct rte_eth_dev *dev, 10660 struct mlx5_flow_tunnel *tunnel, void *x) 10661 { 10662 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10663 RTE_SET_USED(dev); 10664 ctx->tunnel = tunnel; 10665 } 10666 10667 static struct mlx5_flow_tunnel * 10668 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10669 { 10670 struct tunnel_db_find_tunnel_id_ctx ctx = { 10671 .tunnel_id = id, 10672 }; 10673 10674 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10675 find_tunnel_id_hit, NULL, &ctx, true); 10676 10677 return ctx.tunnel; 10678 } 10679 10680 static struct mlx5_flow_tunnel * 10681 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10682 const struct rte_flow_tunnel *app_tunnel) 10683 { 10684 struct mlx5_priv *priv = dev->data->dev_private; 10685 struct mlx5_indexed_pool *ipool; 10686 struct mlx5_flow_tunnel *tunnel; 10687 uint32_t id; 10688 10689 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10690 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10691 if (!tunnel) 10692 return NULL; 10693 if (id >= MLX5_MAX_TUNNELS) { 10694 mlx5_ipool_free(ipool, id); 10695 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10696 return NULL; 10697 } 10698 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10699 priv->sh, 10700 mlx5_flow_tunnel_grp2tbl_create_cb, 10701 mlx5_flow_tunnel_grp2tbl_match_cb, 10702 mlx5_flow_tunnel_grp2tbl_remove_cb, 10703 mlx5_flow_tunnel_grp2tbl_clone_cb, 10704 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10705 if (!tunnel->groups) { 10706 mlx5_ipool_free(ipool, id); 10707 return NULL; 10708 } 10709 /* initiate new PMD tunnel */ 10710 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10711 tunnel->tunnel_id = id; 10712 tunnel->action.type = (typeof(tunnel->action.type)) 10713 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10714 tunnel->action.conf = tunnel; 10715 tunnel->item.type = (typeof(tunnel->item.type)) 10716 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10717 tunnel->item.spec = tunnel; 10718 tunnel->item.last = NULL; 10719 tunnel->item.mask = NULL; 10720 10721 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10722 dev->data->port_id, tunnel->tunnel_id); 10723 10724 return tunnel; 10725 } 10726 10727 struct tunnel_db_get_tunnel_ctx { 10728 const struct rte_flow_tunnel *app_tunnel; 10729 struct mlx5_flow_tunnel *tunnel; 10730 }; 10731 10732 static bool get_tunnel_match(struct rte_eth_dev *dev, 10733 struct mlx5_flow_tunnel *tunnel, const void *x) 10734 { 10735 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10736 10737 RTE_SET_USED(dev); 10738 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10739 sizeof(*ctx->app_tunnel)); 10740 } 10741 10742 static void get_tunnel_hit(struct rte_eth_dev *dev, 10743 struct mlx5_flow_tunnel *tunnel, void *x) 10744 { 10745 /* called under tunnel spinlock protection */ 10746 struct tunnel_db_get_tunnel_ctx *ctx = x; 10747 10748 RTE_SET_USED(dev); 10749 tunnel->refctn++; 10750 ctx->tunnel = tunnel; 10751 } 10752 10753 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10754 { 10755 /* called under tunnel spinlock protection */ 10756 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10757 struct tunnel_db_get_tunnel_ctx *ctx = x; 10758 10759 rte_spinlock_unlock(&thub->sl); 10760 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10761 rte_spinlock_lock(&thub->sl); 10762 if (ctx->tunnel) { 10763 ctx->tunnel->refctn = 1; 10764 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10765 } 10766 } 10767 10768 10769 static int 10770 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10771 const struct rte_flow_tunnel *app_tunnel, 10772 struct mlx5_flow_tunnel **tunnel) 10773 { 10774 struct tunnel_db_get_tunnel_ctx ctx = { 10775 .app_tunnel = app_tunnel, 10776 }; 10777 10778 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10779 get_tunnel_miss, &ctx, true); 10780 *tunnel = ctx.tunnel; 10781 return ctx.tunnel ? 0 : -ENOMEM; 10782 } 10783 10784 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10785 { 10786 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10787 10788 if (!thub) 10789 return; 10790 if (!LIST_EMPTY(&thub->tunnels)) 10791 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10792 mlx5_hlist_destroy(thub->groups); 10793 mlx5_free(thub); 10794 } 10795 10796 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10797 { 10798 int err; 10799 struct mlx5_flow_tunnel_hub *thub; 10800 10801 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10802 0, SOCKET_ID_ANY); 10803 if (!thub) 10804 return -ENOMEM; 10805 LIST_INIT(&thub->tunnels); 10806 rte_spinlock_init(&thub->sl); 10807 thub->groups = mlx5_hlist_create("flow groups", 64, 10808 false, true, sh, 10809 mlx5_flow_tunnel_grp2tbl_create_cb, 10810 mlx5_flow_tunnel_grp2tbl_match_cb, 10811 mlx5_flow_tunnel_grp2tbl_remove_cb, 10812 mlx5_flow_tunnel_grp2tbl_clone_cb, 10813 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10814 if (!thub->groups) { 10815 err = -rte_errno; 10816 goto err; 10817 } 10818 sh->tunnel_hub = thub; 10819 10820 return 0; 10821 10822 err: 10823 if (thub->groups) 10824 mlx5_hlist_destroy(thub->groups); 10825 if (thub) 10826 mlx5_free(thub); 10827 return err; 10828 } 10829 10830 static inline int 10831 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10832 struct rte_flow_tunnel *tunnel, 10833 struct rte_flow_error *error) 10834 { 10835 struct mlx5_priv *priv = dev->data->dev_private; 10836 10837 if (!priv->sh->config.dv_flow_en) 10838 return rte_flow_error_set(error, ENOTSUP, 10839 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10840 "flow DV interface is off"); 10841 if (!is_tunnel_offload_active(dev)) 10842 return rte_flow_error_set(error, ENOTSUP, 10843 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10844 "tunnel offload was not activated"); 10845 if (!tunnel) 10846 return rte_flow_error_set(error, EINVAL, 10847 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10848 "no application tunnel"); 10849 switch (tunnel->type) { 10850 default: 10851 return rte_flow_error_set(error, EINVAL, 10852 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10853 "unsupported tunnel type"); 10854 case RTE_FLOW_ITEM_TYPE_VXLAN: 10855 case RTE_FLOW_ITEM_TYPE_GRE: 10856 case RTE_FLOW_ITEM_TYPE_NVGRE: 10857 case RTE_FLOW_ITEM_TYPE_GENEVE: 10858 break; 10859 } 10860 return 0; 10861 } 10862 10863 static int 10864 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10865 struct rte_flow_tunnel *app_tunnel, 10866 struct rte_flow_action **actions, 10867 uint32_t *num_of_actions, 10868 struct rte_flow_error *error) 10869 { 10870 struct mlx5_flow_tunnel *tunnel; 10871 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10872 10873 if (ret) 10874 return ret; 10875 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10876 if (ret < 0) { 10877 return rte_flow_error_set(error, ret, 10878 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10879 "failed to initialize pmd tunnel"); 10880 } 10881 *actions = &tunnel->action; 10882 *num_of_actions = 1; 10883 return 0; 10884 } 10885 10886 static int 10887 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10888 struct rte_flow_tunnel *app_tunnel, 10889 struct rte_flow_item **items, 10890 uint32_t *num_of_items, 10891 struct rte_flow_error *error) 10892 { 10893 struct mlx5_flow_tunnel *tunnel; 10894 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10895 10896 if (ret) 10897 return ret; 10898 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10899 if (ret < 0) { 10900 return rte_flow_error_set(error, ret, 10901 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10902 "failed to initialize pmd tunnel"); 10903 } 10904 *items = &tunnel->item; 10905 *num_of_items = 1; 10906 return 0; 10907 } 10908 10909 struct tunnel_db_element_release_ctx { 10910 struct rte_flow_item *items; 10911 struct rte_flow_action *actions; 10912 uint32_t num_elements; 10913 struct rte_flow_error *error; 10914 int ret; 10915 }; 10916 10917 static bool 10918 tunnel_element_release_match(struct rte_eth_dev *dev, 10919 struct mlx5_flow_tunnel *tunnel, const void *x) 10920 { 10921 const struct tunnel_db_element_release_ctx *ctx = x; 10922 10923 RTE_SET_USED(dev); 10924 if (ctx->num_elements != 1) 10925 return false; 10926 else if (ctx->items) 10927 return ctx->items == &tunnel->item; 10928 else if (ctx->actions) 10929 return ctx->actions == &tunnel->action; 10930 10931 return false; 10932 } 10933 10934 static void 10935 tunnel_element_release_hit(struct rte_eth_dev *dev, 10936 struct mlx5_flow_tunnel *tunnel, void *x) 10937 { 10938 struct tunnel_db_element_release_ctx *ctx = x; 10939 ctx->ret = 0; 10940 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 10941 mlx5_flow_tunnel_free(dev, tunnel); 10942 } 10943 10944 static void 10945 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 10946 { 10947 struct tunnel_db_element_release_ctx *ctx = x; 10948 RTE_SET_USED(dev); 10949 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 10950 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10951 "invalid argument"); 10952 } 10953 10954 static int 10955 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 10956 struct rte_flow_item *pmd_items, 10957 uint32_t num_items, struct rte_flow_error *err) 10958 { 10959 struct tunnel_db_element_release_ctx ctx = { 10960 .items = pmd_items, 10961 .actions = NULL, 10962 .num_elements = num_items, 10963 .error = err, 10964 }; 10965 10966 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10967 tunnel_element_release_hit, 10968 tunnel_element_release_miss, &ctx, false); 10969 10970 return ctx.ret; 10971 } 10972 10973 static int 10974 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 10975 struct rte_flow_action *pmd_actions, 10976 uint32_t num_actions, struct rte_flow_error *err) 10977 { 10978 struct tunnel_db_element_release_ctx ctx = { 10979 .items = NULL, 10980 .actions = pmd_actions, 10981 .num_elements = num_actions, 10982 .error = err, 10983 }; 10984 10985 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 10986 tunnel_element_release_hit, 10987 tunnel_element_release_miss, &ctx, false); 10988 10989 return ctx.ret; 10990 } 10991 10992 static int 10993 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 10994 struct rte_mbuf *m, 10995 struct rte_flow_restore_info *info, 10996 struct rte_flow_error *err) 10997 { 10998 uint64_t ol_flags = m->ol_flags; 10999 const struct mlx5_flow_tbl_data_entry *tble; 11000 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11001 11002 if (!is_tunnel_offload_active(dev)) { 11003 info->flags = 0; 11004 return 0; 11005 } 11006 11007 if ((ol_flags & mask) != mask) 11008 goto err; 11009 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11010 if (!tble) { 11011 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11012 dev->data->port_id, m->hash.fdir.hi); 11013 goto err; 11014 } 11015 MLX5_ASSERT(tble->tunnel); 11016 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11017 info->group_id = tble->group_id; 11018 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11019 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11020 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11021 11022 return 0; 11023 11024 err: 11025 return rte_flow_error_set(err, EINVAL, 11026 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11027 "failed to get restore info"); 11028 } 11029 11030 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11031 static int 11032 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11033 __rte_unused struct rte_flow_tunnel *app_tunnel, 11034 __rte_unused struct rte_flow_action **actions, 11035 __rte_unused uint32_t *num_of_actions, 11036 __rte_unused struct rte_flow_error *error) 11037 { 11038 return -ENOTSUP; 11039 } 11040 11041 static int 11042 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11043 __rte_unused struct rte_flow_tunnel *app_tunnel, 11044 __rte_unused struct rte_flow_item **items, 11045 __rte_unused uint32_t *num_of_items, 11046 __rte_unused struct rte_flow_error *error) 11047 { 11048 return -ENOTSUP; 11049 } 11050 11051 static int 11052 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11053 __rte_unused struct rte_flow_item *pmd_items, 11054 __rte_unused uint32_t num_items, 11055 __rte_unused struct rte_flow_error *err) 11056 { 11057 return -ENOTSUP; 11058 } 11059 11060 static int 11061 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11062 __rte_unused struct rte_flow_action *pmd_action, 11063 __rte_unused uint32_t num_actions, 11064 __rte_unused struct rte_flow_error *err) 11065 { 11066 return -ENOTSUP; 11067 } 11068 11069 static int 11070 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11071 __rte_unused struct rte_mbuf *m, 11072 __rte_unused struct rte_flow_restore_info *i, 11073 __rte_unused struct rte_flow_error *err) 11074 { 11075 return -ENOTSUP; 11076 } 11077 11078 static int 11079 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11080 __rte_unused struct rte_flow *flow, 11081 __rte_unused const struct rte_flow_attr *attr, 11082 __rte_unused const struct rte_flow_action *actions, 11083 __rte_unused uint32_t flow_idx, 11084 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11085 __rte_unused struct tunnel_default_miss_ctx *ctx, 11086 __rte_unused struct rte_flow_error *error) 11087 { 11088 return -ENOTSUP; 11089 } 11090 11091 static struct mlx5_flow_tunnel * 11092 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11093 __rte_unused uint32_t id) 11094 { 11095 return NULL; 11096 } 11097 11098 static void 11099 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11100 __rte_unused struct mlx5_flow_tunnel *tunnel) 11101 { 11102 } 11103 11104 static uint32_t 11105 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11106 __rte_unused const struct mlx5_flow_tunnel *t, 11107 __rte_unused uint32_t group, 11108 __rte_unused uint32_t *table, 11109 struct rte_flow_error *error) 11110 { 11111 return rte_flow_error_set(error, ENOTSUP, 11112 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11113 "tunnel offload requires DV support"); 11114 } 11115 11116 void 11117 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11118 __rte_unused uint16_t port_id) 11119 { 11120 } 11121 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11122 11123 /* Flex flow item API */ 11124 static struct rte_flow_item_flex_handle * 11125 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11126 const struct rte_flow_item_flex_conf *conf, 11127 struct rte_flow_error *error) 11128 { 11129 static const char err_msg[] = "flex item creation unsupported"; 11130 struct mlx5_priv *priv = dev->data->dev_private; 11131 struct rte_flow_attr attr = { .transfer = 0 }; 11132 const struct mlx5_flow_driver_ops *fops = 11133 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11134 11135 if (!priv->pci_dev) { 11136 rte_flow_error_set(error, ENOTSUP, 11137 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11138 "create flex item on PF only"); 11139 return NULL; 11140 } 11141 switch (priv->pci_dev->id.device_id) { 11142 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11143 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11144 break; 11145 default: 11146 rte_flow_error_set(error, ENOTSUP, 11147 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11148 "flex item available on BlueField ports only"); 11149 return NULL; 11150 } 11151 if (!fops->item_create) { 11152 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11153 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11154 NULL, err_msg); 11155 return NULL; 11156 } 11157 return fops->item_create(dev, conf, error); 11158 } 11159 11160 static int 11161 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11162 const struct rte_flow_item_flex_handle *handle, 11163 struct rte_flow_error *error) 11164 { 11165 static const char err_msg[] = "flex item release unsupported"; 11166 struct rte_flow_attr attr = { .transfer = 0 }; 11167 const struct mlx5_flow_driver_ops *fops = 11168 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11169 11170 if (!fops->item_release) { 11171 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11172 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11173 NULL, err_msg); 11174 return -rte_errno; 11175 } 11176 return fops->item_release(dev, handle, error); 11177 } 11178 11179 static void 11180 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11181 { 11182 int ret; 11183 struct rte_flow_error error; 11184 11185 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11186 char *item_name; 11187 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11188 sizeof(item_name), 11189 (void *)(uintptr_t)item->type, &error); 11190 if (ret > 0) 11191 printf("%s ", item_name); 11192 else 11193 printf("%d\n", (int)item->type); 11194 } 11195 printf("END\n"); 11196 } 11197 11198 static int 11199 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11200 { 11201 const struct rte_flow_item_udp *spec = udp_item->spec; 11202 const struct rte_flow_item_udp *mask = udp_item->mask; 11203 uint16_t udp_dport = 0; 11204 11205 if (spec != NULL) { 11206 if (!mask) 11207 mask = &rte_flow_item_udp_mask; 11208 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11209 mask->hdr.dst_port); 11210 } 11211 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11212 } 11213 11214 static const struct mlx5_flow_expand_node * 11215 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11216 unsigned int item_idx, 11217 const struct mlx5_flow_expand_node graph[], 11218 const struct mlx5_flow_expand_node *node) 11219 { 11220 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11221 11222 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11223 node != NULL && 11224 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11225 /* 11226 * The expansion node is VXLAN and it is also the last 11227 * expandable item in the pattern, so need to continue 11228 * expansion of the inner tunnel. 11229 */ 11230 MLX5_ASSERT(item_idx > 0); 11231 prev_item = pattern + item_idx - 1; 11232 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11233 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11234 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11235 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11236 } 11237 return node; 11238 } 11239 11240 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11241 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11242 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11243 }; 11244 11245 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11246 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11247 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11248 { 9, 10, 11 }, { 12, 13, 14 }, 11249 }; 11250 11251 /** 11252 * Discover the number of available flow priorities. 11253 * 11254 * @param dev 11255 * Ethernet device. 11256 * 11257 * @return 11258 * On success, number of available flow priorities. 11259 * On failure, a negative errno-style code and rte_errno is set. 11260 */ 11261 int 11262 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11263 { 11264 static const uint16_t vprio[] = {8, 16}; 11265 const struct mlx5_priv *priv = dev->data->dev_private; 11266 const struct mlx5_flow_driver_ops *fops; 11267 enum mlx5_flow_drv_type type; 11268 int ret; 11269 11270 type = mlx5_flow_os_get_type(); 11271 if (type == MLX5_FLOW_TYPE_MAX) { 11272 type = MLX5_FLOW_TYPE_VERBS; 11273 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11274 type = MLX5_FLOW_TYPE_DV; 11275 } 11276 fops = flow_get_drv_ops(type); 11277 if (fops->discover_priorities == NULL) { 11278 DRV_LOG(ERR, "Priority discovery not supported"); 11279 rte_errno = ENOTSUP; 11280 return -rte_errno; 11281 } 11282 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11283 if (ret < 0) 11284 return ret; 11285 switch (ret) { 11286 case 8: 11287 ret = RTE_DIM(priority_map_3); 11288 break; 11289 case 16: 11290 ret = RTE_DIM(priority_map_5); 11291 break; 11292 default: 11293 rte_errno = ENOTSUP; 11294 DRV_LOG(ERR, 11295 "port %u maximum priority: %d expected 8/16", 11296 dev->data->port_id, ret); 11297 return -rte_errno; 11298 } 11299 DRV_LOG(INFO, "port %u supported flow priorities:" 11300 " 0-%d for ingress or egress root table," 11301 " 0-%d for non-root table or transfer root table.", 11302 dev->data->port_id, ret - 2, 11303 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11304 return ret; 11305 } 11306 11307 /** 11308 * Adjust flow priority based on the highest layer and the request priority. 11309 * 11310 * @param[in] dev 11311 * Pointer to the Ethernet device structure. 11312 * @param[in] priority 11313 * The rule base priority. 11314 * @param[in] subpriority 11315 * The priority based on the items. 11316 * 11317 * @return 11318 * The new priority. 11319 */ 11320 uint32_t 11321 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11322 uint32_t subpriority) 11323 { 11324 uint32_t res = 0; 11325 struct mlx5_priv *priv = dev->data->dev_private; 11326 11327 switch (priv->sh->flow_max_priority) { 11328 case RTE_DIM(priority_map_3): 11329 res = priority_map_3[priority][subpriority]; 11330 break; 11331 case RTE_DIM(priority_map_5): 11332 res = priority_map_5[priority][subpriority]; 11333 break; 11334 } 11335 return res; 11336 } 11337 11338 /** 11339 * Get the priority for sending traffic to kernel table. 11340 * 11341 * @param[in] dev 11342 * Pointer to the Ethernet device structure. 11343 * 11344 * @return 11345 * On success: the value of priority for sending traffic to kernel table 11346 * On failure: -1 11347 */ 11348 uint32_t 11349 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11350 { 11351 struct mlx5_priv *priv = dev->data->dev_private; 11352 uint32_t res; 11353 11354 switch (priv->sh->flow_max_priority) { 11355 case RTE_DIM(priority_map_5): 11356 res = 15; 11357 break; 11358 case RTE_DIM(priority_map_3): 11359 res = 7; 11360 break; 11361 default: 11362 DRV_LOG(ERR, 11363 "port %u maximum priority: %d expected 8/16", 11364 dev->data->port_id, priv->sh->flow_max_priority); 11365 res = (uint32_t)-1; 11366 } 11367 return res; 11368 } 11369 11370 /** 11371 * Get the E-Switch Manager vport id. 11372 * 11373 * @param[in] dev 11374 * Pointer to the Ethernet device structure. 11375 * 11376 * @return 11377 * The vport id. 11378 */ 11379 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11380 { 11381 struct mlx5_priv *priv = dev->data->dev_private; 11382 struct mlx5_common_device *cdev = priv->sh->cdev; 11383 11384 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11385 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11386 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11387 11388 if (priv->pci_dev == NULL) 11389 return 0; 11390 switch (priv->pci_dev->id.device_id) { 11391 case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF: 11392 case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF: 11393 case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF: 11394 /* 11395 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11396 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11397 * ECPF vport ID is selected and not the host port (0) in any BF case. 11398 */ 11399 return (int16_t)MLX5_ECPF_VPORT_ID; 11400 default: 11401 return MLX5_PF_VPORT_ID; 11402 } 11403 } 11404 11405 /** 11406 * Parse item to get the vport id. 11407 * 11408 * @param[in] dev 11409 * Pointer to the Ethernet device structure. 11410 * @param[in] item 11411 * The src port id match item. 11412 * @param[out] vport_id 11413 * Pointer to put the vport id. 11414 * @param[out] all_ports 11415 * Indicate if the item matches all ports. 11416 * @param[out] error 11417 * Pointer to error structure. 11418 * 11419 * @return 11420 * 0 on success, a negative errno value otherwise and rte_errno is set. 11421 */ 11422 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11423 const struct rte_flow_item *item, 11424 uint16_t *vport_id, 11425 bool *all_ports, 11426 struct rte_flow_error *error) 11427 { 11428 struct mlx5_priv *port_priv; 11429 const struct rte_flow_item_port_id *pid_v = NULL; 11430 const struct rte_flow_item_ethdev *dev_v = NULL; 11431 uint32_t esw_mgr_port; 11432 uint32_t src_port; 11433 11434 if (all_ports) 11435 *all_ports = false; 11436 switch (item->type) { 11437 case RTE_FLOW_ITEM_TYPE_PORT_ID: 11438 pid_v = item->spec; 11439 if (!pid_v) 11440 return 0; 11441 src_port = pid_v->id; 11442 esw_mgr_port = MLX5_PORT_ESW_MGR; 11443 break; 11444 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 11445 dev_v = item->spec; 11446 if (!dev_v) { 11447 if (all_ports) 11448 *all_ports = true; 11449 return 0; 11450 } 11451 src_port = dev_v->port_id; 11452 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11453 break; 11454 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 11455 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11456 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11457 break; 11458 default: 11459 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11460 NULL, "Incorrect item type."); 11461 } 11462 if (src_port == esw_mgr_port) { 11463 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11464 } else { 11465 port_priv = mlx5_port_to_eswitch_info(src_port, false); 11466 if (!port_priv) 11467 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11468 NULL, "Failed to get port info."); 11469 *vport_id = port_priv->representor_id; 11470 } 11471 11472 return 0; 11473 } 11474 11475 int 11476 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11477 uint16_t *proxy_port_id, 11478 struct rte_flow_error *error) 11479 { 11480 const struct mlx5_priv *priv = dev->data->dev_private; 11481 uint16_t port_id; 11482 11483 if (!priv->sh->config.dv_esw_en) 11484 return rte_flow_error_set(error, EINVAL, 11485 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11486 NULL, 11487 "unable to provide a proxy port" 11488 " without E-Switch configured"); 11489 if (!priv->master && !priv->representor) 11490 return rte_flow_error_set(error, EINVAL, 11491 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11492 NULL, 11493 "unable to provide a proxy port" 11494 " for port which is not a master" 11495 " or a representor port"); 11496 if (priv->master) { 11497 *proxy_port_id = dev->data->port_id; 11498 return 0; 11499 } 11500 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11501 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11502 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11503 11504 if (port_priv->master && 11505 port_priv->domain_id == priv->domain_id) { 11506 *proxy_port_id = port_id; 11507 return 0; 11508 } 11509 } 11510 return rte_flow_error_set(error, ENODEV, 11511 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11512 NULL, "unable to find a proxy port"); 11513 } 11514