1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static bool 168 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 169 { 170 switch (item->type) { 171 case RTE_FLOW_ITEM_TYPE_ETH: 172 case RTE_FLOW_ITEM_TYPE_VLAN: 173 case RTE_FLOW_ITEM_TYPE_IPV4: 174 case RTE_FLOW_ITEM_TYPE_IPV6: 175 case RTE_FLOW_ITEM_TYPE_UDP: 176 case RTE_FLOW_ITEM_TYPE_TCP: 177 case RTE_FLOW_ITEM_TYPE_ESP: 178 case RTE_FLOW_ITEM_TYPE_ICMP: 179 case RTE_FLOW_ITEM_TYPE_ICMP6: 180 case RTE_FLOW_ITEM_TYPE_VXLAN: 181 case RTE_FLOW_ITEM_TYPE_NVGRE: 182 case RTE_FLOW_ITEM_TYPE_GRE: 183 case RTE_FLOW_ITEM_TYPE_GENEVE: 184 case RTE_FLOW_ITEM_TYPE_MPLS: 185 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 186 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 187 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 188 case RTE_FLOW_ITEM_TYPE_GTP: 189 return true; 190 default: 191 break; 192 } 193 return false; 194 } 195 196 /** 197 * Network Service Header (NSH) and its next protocol values 198 * are described in RFC-8393. 199 */ 200 static enum rte_flow_item_type 201 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 202 { 203 enum rte_flow_item_type type; 204 205 switch (proto_mask & proto_spec) { 206 case 0: 207 type = RTE_FLOW_ITEM_TYPE_VOID; 208 break; 209 case RTE_VXLAN_GPE_TYPE_IPV4: 210 type = RTE_FLOW_ITEM_TYPE_IPV4; 211 break; 212 case RTE_VXLAN_GPE_TYPE_IPV6: 213 type = RTE_VXLAN_GPE_TYPE_IPV6; 214 break; 215 case RTE_VXLAN_GPE_TYPE_ETH: 216 type = RTE_FLOW_ITEM_TYPE_ETH; 217 break; 218 default: 219 type = RTE_FLOW_ITEM_TYPE_END; 220 } 221 return type; 222 } 223 224 static enum rte_flow_item_type 225 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 226 { 227 enum rte_flow_item_type type; 228 229 switch (proto_mask & proto_spec) { 230 case 0: 231 type = RTE_FLOW_ITEM_TYPE_VOID; 232 break; 233 case IPPROTO_UDP: 234 type = RTE_FLOW_ITEM_TYPE_UDP; 235 break; 236 case IPPROTO_TCP: 237 type = RTE_FLOW_ITEM_TYPE_TCP; 238 break; 239 case IPPROTO_IPIP: 240 type = RTE_FLOW_ITEM_TYPE_IPV4; 241 break; 242 case IPPROTO_IPV6: 243 type = RTE_FLOW_ITEM_TYPE_IPV6; 244 break; 245 case IPPROTO_ESP: 246 type = RTE_FLOW_ITEM_TYPE_ESP; 247 break; 248 default: 249 type = RTE_FLOW_ITEM_TYPE_END; 250 } 251 return type; 252 } 253 254 static enum rte_flow_item_type 255 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 256 rte_be16_t type_mask, bool is_tunnel) 257 { 258 enum rte_flow_item_type type; 259 260 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 261 case 0: 262 type = RTE_FLOW_ITEM_TYPE_VOID; 263 break; 264 case RTE_ETHER_TYPE_TEB: 265 type = is_tunnel ? 266 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 267 break; 268 case RTE_ETHER_TYPE_VLAN: 269 type = !is_tunnel ? 270 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 271 break; 272 case RTE_ETHER_TYPE_IPV4: 273 type = RTE_FLOW_ITEM_TYPE_IPV4; 274 break; 275 case RTE_ETHER_TYPE_IPV6: 276 type = RTE_FLOW_ITEM_TYPE_IPV6; 277 break; 278 default: 279 type = RTE_FLOW_ITEM_TYPE_END; 280 } 281 return type; 282 } 283 284 static enum rte_flow_item_type 285 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 286 { 287 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 288 do { \ 289 const void *m = item->mask; \ 290 const void *s = item->spec; \ 291 mask = m ? \ 292 ((const struct rte_flow_item_##type *)m)->fld : \ 293 rte_flow_item_##type##_mask.fld; \ 294 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 295 } while (0) 296 297 enum rte_flow_item_type ret; 298 uint16_t spec, mask; 299 300 if (item == NULL || item->spec == NULL) 301 return RTE_FLOW_ITEM_TYPE_VOID; 302 switch (item->type) { 303 case RTE_FLOW_ITEM_TYPE_ETH: 304 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 305 if (!mask) 306 return RTE_FLOW_ITEM_TYPE_VOID; 307 ret = mlx5_ethertype_to_item_type(spec, mask, false); 308 break; 309 case RTE_FLOW_ITEM_TYPE_VLAN: 310 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 311 if (!mask) 312 return RTE_FLOW_ITEM_TYPE_VOID; 313 ret = mlx5_ethertype_to_item_type(spec, mask, false); 314 break; 315 case RTE_FLOW_ITEM_TYPE_IPV4: 316 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 317 if (!mask) 318 return RTE_FLOW_ITEM_TYPE_VOID; 319 ret = mlx5_inet_proto_to_item_type(spec, mask); 320 break; 321 case RTE_FLOW_ITEM_TYPE_IPV6: 322 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 323 if (!mask) 324 return RTE_FLOW_ITEM_TYPE_VOID; 325 ret = mlx5_inet_proto_to_item_type(spec, mask); 326 break; 327 case RTE_FLOW_ITEM_TYPE_GENEVE: 328 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 329 ret = mlx5_ethertype_to_item_type(spec, mask, true); 330 break; 331 case RTE_FLOW_ITEM_TYPE_GRE: 332 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 333 ret = mlx5_ethertype_to_item_type(spec, mask, true); 334 break; 335 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 336 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 337 ret = mlx5_nsh_proto_to_item_type(spec, mask); 338 break; 339 default: 340 ret = RTE_FLOW_ITEM_TYPE_VOID; 341 break; 342 } 343 return ret; 344 #undef MLX5_XSET_ITEM_MASK_SPEC 345 } 346 347 static const int * 348 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 349 const int *next_node) 350 { 351 const struct mlx5_flow_expand_node *node = NULL; 352 const int *next = next_node; 353 354 while (next && *next) { 355 /* 356 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 357 * flag set, because they were not found in the flow pattern. 358 */ 359 node = &graph[*next]; 360 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 361 break; 362 next = node->next; 363 } 364 return next; 365 } 366 367 #define MLX5_RSS_EXP_ELT_N 16 368 369 /** 370 * Expand RSS flows into several possible flows according to the RSS hash 371 * fields requested and the driver capabilities. 372 * 373 * @param[out] buf 374 * Buffer to store the result expansion. 375 * @param[in] size 376 * Buffer size in bytes. If 0, @p buf can be NULL. 377 * @param[in] pattern 378 * User flow pattern. 379 * @param[in] types 380 * RSS types to expand (see RTE_ETH_RSS_* definitions). 381 * @param[in] graph 382 * Input graph to expand @p pattern according to @p types. 383 * @param[in] graph_root_index 384 * Index of root node in @p graph, typically 0. 385 * 386 * @return 387 * A positive value representing the size of @p buf in bytes regardless of 388 * @p size on success, a negative errno value otherwise and rte_errno is 389 * set, the following errors are defined: 390 * 391 * -E2BIG: graph-depth @p graph is too deep. 392 * -EINVAL: @p size has not enough space for expanded pattern. 393 */ 394 static int 395 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 396 const struct rte_flow_item *pattern, uint64_t types, 397 const struct mlx5_flow_expand_node graph[], 398 int graph_root_index) 399 { 400 const struct rte_flow_item *item; 401 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 402 const int *next_node; 403 const int *stack[MLX5_RSS_EXP_ELT_N]; 404 int stack_pos = 0; 405 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 406 unsigned int i, item_idx, last_expand_item_idx = 0; 407 size_t lsize; 408 size_t user_pattern_size = 0; 409 void *addr = NULL; 410 const struct mlx5_flow_expand_node *next = NULL; 411 struct rte_flow_item missed_item; 412 int missed = 0; 413 int elt = 0; 414 const struct rte_flow_item *last_expand_item = NULL; 415 416 memset(&missed_item, 0, sizeof(missed_item)); 417 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 418 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 419 if (lsize > size) 420 return -EINVAL; 421 buf->entry[0].priority = 0; 422 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 423 buf->entries = 0; 424 addr = buf->entry[0].pattern; 425 for (item = pattern, item_idx = 0; 426 item->type != RTE_FLOW_ITEM_TYPE_END; 427 item++, item_idx++) { 428 if (!mlx5_flow_is_rss_expandable_item(item)) { 429 user_pattern_size += sizeof(*item); 430 continue; 431 } 432 last_expand_item = item; 433 last_expand_item_idx = item_idx; 434 i = 0; 435 while (node->next && node->next[i]) { 436 next = &graph[node->next[i]]; 437 if (next->type == item->type) 438 break; 439 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 440 node = next; 441 i = 0; 442 } else { 443 ++i; 444 } 445 } 446 if (next) 447 node = next; 448 user_pattern_size += sizeof(*item); 449 } 450 user_pattern_size += sizeof(*item); /* Handle END item. */ 451 lsize += user_pattern_size; 452 if (lsize > size) 453 return -EINVAL; 454 /* Copy the user pattern in the first entry of the buffer. */ 455 rte_memcpy(addr, pattern, user_pattern_size); 456 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 457 buf->entries = 1; 458 /* Start expanding. */ 459 memset(flow_items, 0, sizeof(flow_items)); 460 user_pattern_size -= sizeof(*item); 461 /* 462 * Check if the last valid item has spec set, need complete pattern, 463 * and the pattern can be used for expansion. 464 */ 465 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 466 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 467 /* Item type END indicates expansion is not required. */ 468 return lsize; 469 } 470 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 471 next = NULL; 472 missed = 1; 473 i = 0; 474 while (node->next && node->next[i]) { 475 next = &graph[node->next[i]]; 476 if (next->type == missed_item.type) { 477 flow_items[0].type = missed_item.type; 478 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 479 break; 480 } 481 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 482 node = next; 483 i = 0; 484 } else { 485 ++i; 486 } 487 next = NULL; 488 } 489 } 490 if (next && missed) { 491 elt = 2; /* missed item + item end. */ 492 node = next; 493 lsize += elt * sizeof(*item) + user_pattern_size; 494 if (lsize > size) 495 return -EINVAL; 496 if (node->rss_types & types) { 497 buf->entry[buf->entries].priority = 1; 498 buf->entry[buf->entries].pattern = addr; 499 buf->entries++; 500 rte_memcpy(addr, buf->entry[0].pattern, 501 user_pattern_size); 502 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 503 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 504 addr = (void *)(((uintptr_t)addr) + 505 elt * sizeof(*item)); 506 } 507 } else if (last_expand_item != NULL) { 508 node = mlx5_flow_expand_rss_adjust_node(pattern, 509 last_expand_item_idx, graph, node); 510 } 511 memset(flow_items, 0, sizeof(flow_items)); 512 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 513 node->next); 514 stack[stack_pos] = next_node; 515 node = next_node ? &graph[*next_node] : NULL; 516 while (node) { 517 flow_items[stack_pos].type = node->type; 518 if (node->rss_types & types) { 519 size_t n; 520 /* 521 * compute the number of items to copy from the 522 * expansion and copy it. 523 * When the stack_pos is 0, there are 1 element in it, 524 * plus the addition END item. 525 */ 526 elt = stack_pos + 2; 527 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 528 lsize += elt * sizeof(*item) + user_pattern_size; 529 if (lsize > size) 530 return -EINVAL; 531 n = elt * sizeof(*item); 532 buf->entry[buf->entries].priority = 533 stack_pos + 1 + missed; 534 buf->entry[buf->entries].pattern = addr; 535 buf->entries++; 536 rte_memcpy(addr, buf->entry[0].pattern, 537 user_pattern_size); 538 addr = (void *)(((uintptr_t)addr) + 539 user_pattern_size); 540 rte_memcpy(addr, &missed_item, 541 missed * sizeof(*item)); 542 addr = (void *)(((uintptr_t)addr) + 543 missed * sizeof(*item)); 544 rte_memcpy(addr, flow_items, n); 545 addr = (void *)(((uintptr_t)addr) + n); 546 } 547 /* Go deeper. */ 548 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 549 node->next) { 550 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 551 node->next); 552 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 553 rte_errno = E2BIG; 554 return -rte_errno; 555 } 556 stack[stack_pos] = next_node; 557 } else if (*(next_node + 1)) { 558 /* Follow up with the next possibility. */ 559 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 560 ++next_node); 561 } else if (!stack_pos) { 562 /* 563 * Completing the traverse over the different paths. 564 * The next_node is advanced to the terminator. 565 */ 566 ++next_node; 567 } else { 568 /* Move to the next path. */ 569 while (stack_pos) { 570 next_node = stack[--stack_pos]; 571 next_node++; 572 if (*next_node) 573 break; 574 } 575 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 576 next_node); 577 stack[stack_pos] = next_node; 578 } 579 node = next_node && *next_node ? &graph[*next_node] : NULL; 580 }; 581 return lsize; 582 } 583 584 /** 585 * Expand SQN flows into several possible flows according to the Tx queue 586 * number 587 * 588 * @param[in] buf 589 * Buffer to store the result expansion. 590 * @param[in] size 591 * Buffer size in bytes. If 0, @p buf can be NULL. 592 * @param[in] pattern 593 * User flow pattern. 594 * @param[in] sq_specs 595 * Buffer to store sq spec. 596 * 597 * @return 598 * 0 for success and negative value for failure 599 * 600 */ 601 static int 602 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 603 const struct rte_flow_item *pattern, 604 struct mlx5_rte_flow_item_sq *sq_specs) 605 { 606 const struct rte_flow_item *item; 607 bool port_representor = false; 608 size_t user_pattern_size = 0; 609 struct rte_eth_dev *dev; 610 struct mlx5_priv *priv; 611 void *addr = NULL; 612 uint16_t port_id; 613 size_t lsize; 614 int elt = 2; 615 uint16_t i; 616 617 buf->entries = 0; 618 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 619 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 620 const struct rte_flow_item_ethdev *pid_v = item->spec; 621 622 if (!pid_v) 623 return 0; 624 port_id = pid_v->port_id; 625 port_representor = true; 626 } 627 user_pattern_size += sizeof(*item); 628 } 629 if (!port_representor) 630 return 0; 631 dev = &rte_eth_devices[port_id]; 632 priv = dev->data->dev_private; 633 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 634 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 635 sizeof(buf->entry[0]) * priv->txqs_n; 636 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 637 return -EINVAL; 638 addr = buf->entry[0].pattern; 639 for (i = 0; i != priv->txqs_n; ++i) { 640 struct rte_flow_item pattern_add[] = { 641 { 642 .type = (enum rte_flow_item_type) 643 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 644 .spec = &sq_specs[i], 645 }, 646 { 647 .type = RTE_FLOW_ITEM_TYPE_END, 648 }, 649 }; 650 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 651 652 if (txq == NULL) 653 return -EINVAL; 654 buf->entry[i].pattern = addr; 655 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 656 mlx5_txq_release(dev, i); 657 rte_memcpy(addr, pattern, user_pattern_size); 658 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 659 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 660 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 661 buf->entries++; 662 } 663 return 0; 664 } 665 666 enum mlx5_expansion { 667 MLX5_EXPANSION_ROOT, 668 MLX5_EXPANSION_ROOT_OUTER, 669 MLX5_EXPANSION_OUTER_ETH, 670 MLX5_EXPANSION_OUTER_VLAN, 671 MLX5_EXPANSION_OUTER_IPV4, 672 MLX5_EXPANSION_OUTER_IPV4_UDP, 673 MLX5_EXPANSION_OUTER_IPV4_TCP, 674 MLX5_EXPANSION_OUTER_IPV4_ESP, 675 MLX5_EXPANSION_OUTER_IPV4_ICMP, 676 MLX5_EXPANSION_OUTER_IPV6, 677 MLX5_EXPANSION_OUTER_IPV6_UDP, 678 MLX5_EXPANSION_OUTER_IPV6_TCP, 679 MLX5_EXPANSION_OUTER_IPV6_ESP, 680 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 681 MLX5_EXPANSION_VXLAN, 682 MLX5_EXPANSION_STD_VXLAN, 683 MLX5_EXPANSION_L3_VXLAN, 684 MLX5_EXPANSION_VXLAN_GPE, 685 MLX5_EXPANSION_GRE, 686 MLX5_EXPANSION_NVGRE, 687 MLX5_EXPANSION_GRE_KEY, 688 MLX5_EXPANSION_MPLS, 689 MLX5_EXPANSION_ETH, 690 MLX5_EXPANSION_VLAN, 691 MLX5_EXPANSION_IPV4, 692 MLX5_EXPANSION_IPV4_UDP, 693 MLX5_EXPANSION_IPV4_TCP, 694 MLX5_EXPANSION_IPV4_ESP, 695 MLX5_EXPANSION_IPV4_ICMP, 696 MLX5_EXPANSION_IPV6, 697 MLX5_EXPANSION_IPV6_UDP, 698 MLX5_EXPANSION_IPV6_TCP, 699 MLX5_EXPANSION_IPV6_ESP, 700 MLX5_EXPANSION_IPV6_ICMP6, 701 MLX5_EXPANSION_IPV6_FRAG_EXT, 702 MLX5_EXPANSION_GTP, 703 MLX5_EXPANSION_GENEVE, 704 }; 705 706 /** Supported expansion of items. */ 707 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 708 [MLX5_EXPANSION_ROOT] = { 709 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 710 MLX5_EXPANSION_IPV4, 711 MLX5_EXPANSION_IPV6), 712 .type = RTE_FLOW_ITEM_TYPE_END, 713 }, 714 [MLX5_EXPANSION_ROOT_OUTER] = { 715 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 716 MLX5_EXPANSION_OUTER_IPV4, 717 MLX5_EXPANSION_OUTER_IPV6), 718 .type = RTE_FLOW_ITEM_TYPE_END, 719 }, 720 [MLX5_EXPANSION_OUTER_ETH] = { 721 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 722 .type = RTE_FLOW_ITEM_TYPE_ETH, 723 .rss_types = 0, 724 }, 725 [MLX5_EXPANSION_OUTER_VLAN] = { 726 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 727 MLX5_EXPANSION_OUTER_IPV6), 728 .type = RTE_FLOW_ITEM_TYPE_VLAN, 729 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 730 }, 731 [MLX5_EXPANSION_OUTER_IPV4] = { 732 .next = MLX5_FLOW_EXPAND_RSS_NEXT 733 (MLX5_EXPANSION_OUTER_IPV4_UDP, 734 MLX5_EXPANSION_OUTER_IPV4_TCP, 735 MLX5_EXPANSION_OUTER_IPV4_ESP, 736 MLX5_EXPANSION_OUTER_IPV4_ICMP, 737 MLX5_EXPANSION_GRE, 738 MLX5_EXPANSION_NVGRE, 739 MLX5_EXPANSION_IPV4, 740 MLX5_EXPANSION_IPV6), 741 .type = RTE_FLOW_ITEM_TYPE_IPV4, 742 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 743 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 744 }, 745 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 746 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 747 MLX5_EXPANSION_VXLAN_GPE, 748 MLX5_EXPANSION_MPLS, 749 MLX5_EXPANSION_GENEVE, 750 MLX5_EXPANSION_GTP), 751 .type = RTE_FLOW_ITEM_TYPE_UDP, 752 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 753 }, 754 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 755 .type = RTE_FLOW_ITEM_TYPE_TCP, 756 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 757 }, 758 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 759 .type = RTE_FLOW_ITEM_TYPE_ESP, 760 .rss_types = RTE_ETH_RSS_ESP, 761 }, 762 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 763 .type = RTE_FLOW_ITEM_TYPE_ICMP, 764 }, 765 [MLX5_EXPANSION_OUTER_IPV6] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT 767 (MLX5_EXPANSION_OUTER_IPV6_UDP, 768 MLX5_EXPANSION_OUTER_IPV6_TCP, 769 MLX5_EXPANSION_OUTER_IPV6_ESP, 770 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 771 MLX5_EXPANSION_IPV4, 772 MLX5_EXPANSION_IPV6, 773 MLX5_EXPANSION_GRE, 774 MLX5_EXPANSION_NVGRE), 775 .type = RTE_FLOW_ITEM_TYPE_IPV6, 776 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 777 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 778 }, 779 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 780 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 781 MLX5_EXPANSION_VXLAN_GPE, 782 MLX5_EXPANSION_MPLS, 783 MLX5_EXPANSION_GENEVE, 784 MLX5_EXPANSION_GTP), 785 .type = RTE_FLOW_ITEM_TYPE_UDP, 786 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 787 }, 788 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 789 .type = RTE_FLOW_ITEM_TYPE_TCP, 790 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 791 }, 792 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 793 .type = RTE_FLOW_ITEM_TYPE_ESP, 794 .rss_types = RTE_ETH_RSS_ESP, 795 }, 796 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 797 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 798 }, 799 [MLX5_EXPANSION_VXLAN] = { 800 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 801 MLX5_EXPANSION_IPV4, 802 MLX5_EXPANSION_IPV6), 803 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 804 }, 805 [MLX5_EXPANSION_STD_VXLAN] = { 806 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 807 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 808 }, 809 [MLX5_EXPANSION_L3_VXLAN] = { 810 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 811 MLX5_EXPANSION_IPV6), 812 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 813 }, 814 [MLX5_EXPANSION_VXLAN_GPE] = { 815 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 816 MLX5_EXPANSION_IPV4, 817 MLX5_EXPANSION_IPV6), 818 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 819 }, 820 [MLX5_EXPANSION_GRE] = { 821 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 822 MLX5_EXPANSION_IPV4, 823 MLX5_EXPANSION_IPV6, 824 MLX5_EXPANSION_GRE_KEY, 825 MLX5_EXPANSION_MPLS), 826 .type = RTE_FLOW_ITEM_TYPE_GRE, 827 }, 828 [MLX5_EXPANSION_GRE_KEY] = { 829 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 830 MLX5_EXPANSION_IPV6, 831 MLX5_EXPANSION_MPLS), 832 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 833 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 834 }, 835 [MLX5_EXPANSION_NVGRE] = { 836 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 837 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 838 }, 839 [MLX5_EXPANSION_MPLS] = { 840 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 841 MLX5_EXPANSION_IPV6, 842 MLX5_EXPANSION_ETH), 843 .type = RTE_FLOW_ITEM_TYPE_MPLS, 844 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 845 }, 846 [MLX5_EXPANSION_ETH] = { 847 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 848 .type = RTE_FLOW_ITEM_TYPE_ETH, 849 }, 850 [MLX5_EXPANSION_VLAN] = { 851 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 852 MLX5_EXPANSION_IPV6), 853 .type = RTE_FLOW_ITEM_TYPE_VLAN, 854 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 855 }, 856 [MLX5_EXPANSION_IPV4] = { 857 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 858 MLX5_EXPANSION_IPV4_TCP, 859 MLX5_EXPANSION_IPV4_ESP, 860 MLX5_EXPANSION_IPV4_ICMP), 861 .type = RTE_FLOW_ITEM_TYPE_IPV4, 862 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 863 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 864 }, 865 [MLX5_EXPANSION_IPV4_UDP] = { 866 .type = RTE_FLOW_ITEM_TYPE_UDP, 867 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 868 }, 869 [MLX5_EXPANSION_IPV4_TCP] = { 870 .type = RTE_FLOW_ITEM_TYPE_TCP, 871 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 872 }, 873 [MLX5_EXPANSION_IPV4_ESP] = { 874 .type = RTE_FLOW_ITEM_TYPE_ESP, 875 .rss_types = RTE_ETH_RSS_ESP, 876 }, 877 [MLX5_EXPANSION_IPV4_ICMP] = { 878 .type = RTE_FLOW_ITEM_TYPE_ICMP, 879 }, 880 [MLX5_EXPANSION_IPV6] = { 881 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 882 MLX5_EXPANSION_IPV6_TCP, 883 MLX5_EXPANSION_IPV6_ESP, 884 MLX5_EXPANSION_IPV6_ICMP6, 885 MLX5_EXPANSION_IPV6_FRAG_EXT), 886 .type = RTE_FLOW_ITEM_TYPE_IPV6, 887 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 888 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 889 }, 890 [MLX5_EXPANSION_IPV6_UDP] = { 891 .type = RTE_FLOW_ITEM_TYPE_UDP, 892 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 893 }, 894 [MLX5_EXPANSION_IPV6_TCP] = { 895 .type = RTE_FLOW_ITEM_TYPE_TCP, 896 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 897 }, 898 [MLX5_EXPANSION_IPV6_ESP] = { 899 .type = RTE_FLOW_ITEM_TYPE_ESP, 900 .rss_types = RTE_ETH_RSS_ESP, 901 }, 902 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 903 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 904 }, 905 [MLX5_EXPANSION_IPV6_ICMP6] = { 906 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 907 }, 908 [MLX5_EXPANSION_GTP] = { 909 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 910 MLX5_EXPANSION_IPV6), 911 .type = RTE_FLOW_ITEM_TYPE_GTP, 912 }, 913 [MLX5_EXPANSION_GENEVE] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 915 MLX5_EXPANSION_IPV4, 916 MLX5_EXPANSION_IPV6), 917 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 918 }, 919 }; 920 921 static struct rte_flow_action_handle * 922 mlx5_action_handle_create(struct rte_eth_dev *dev, 923 const struct rte_flow_indir_action_conf *conf, 924 const struct rte_flow_action *action, 925 struct rte_flow_error *error); 926 static int mlx5_action_handle_destroy 927 (struct rte_eth_dev *dev, 928 struct rte_flow_action_handle *handle, 929 struct rte_flow_error *error); 930 static int mlx5_action_handle_update 931 (struct rte_eth_dev *dev, 932 struct rte_flow_action_handle *handle, 933 const void *update, 934 struct rte_flow_error *error); 935 static int mlx5_action_handle_query 936 (struct rte_eth_dev *dev, 937 const struct rte_flow_action_handle *handle, 938 void *data, 939 struct rte_flow_error *error); 940 static int 941 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 942 struct rte_flow_tunnel *app_tunnel, 943 struct rte_flow_action **actions, 944 uint32_t *num_of_actions, 945 struct rte_flow_error *error); 946 static int 947 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 948 struct rte_flow_tunnel *app_tunnel, 949 struct rte_flow_item **items, 950 uint32_t *num_of_items, 951 struct rte_flow_error *error); 952 static int 953 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 954 struct rte_flow_item *pmd_items, 955 uint32_t num_items, struct rte_flow_error *err); 956 static int 957 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 958 struct rte_flow_action *pmd_actions, 959 uint32_t num_actions, 960 struct rte_flow_error *err); 961 static int 962 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 963 struct rte_mbuf *m, 964 struct rte_flow_restore_info *info, 965 struct rte_flow_error *err); 966 static struct rte_flow_item_flex_handle * 967 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 968 const struct rte_flow_item_flex_conf *conf, 969 struct rte_flow_error *error); 970 static int 971 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 972 const struct rte_flow_item_flex_handle *handle, 973 struct rte_flow_error *error); 974 static int 975 mlx5_flow_info_get(struct rte_eth_dev *dev, 976 struct rte_flow_port_info *port_info, 977 struct rte_flow_queue_info *queue_info, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_port_configure(struct rte_eth_dev *dev, 981 const struct rte_flow_port_attr *port_attr, 982 uint16_t nb_queue, 983 const struct rte_flow_queue_attr *queue_attr[], 984 struct rte_flow_error *err); 985 986 static struct rte_flow_pattern_template * 987 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 988 const struct rte_flow_pattern_template_attr *attr, 989 const struct rte_flow_item items[], 990 struct rte_flow_error *error); 991 992 static int 993 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 994 struct rte_flow_pattern_template *template, 995 struct rte_flow_error *error); 996 static struct rte_flow_actions_template * 997 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 998 const struct rte_flow_actions_template_attr *attr, 999 const struct rte_flow_action actions[], 1000 const struct rte_flow_action masks[], 1001 struct rte_flow_error *error); 1002 static int 1003 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1004 struct rte_flow_actions_template *template, 1005 struct rte_flow_error *error); 1006 1007 static struct rte_flow_template_table * 1008 mlx5_flow_table_create(struct rte_eth_dev *dev, 1009 const struct rte_flow_template_table_attr *attr, 1010 struct rte_flow_pattern_template *item_templates[], 1011 uint8_t nb_item_templates, 1012 struct rte_flow_actions_template *action_templates[], 1013 uint8_t nb_action_templates, 1014 struct rte_flow_error *error); 1015 static int 1016 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1017 struct rte_flow_template_table *table, 1018 struct rte_flow_error *error); 1019 static struct rte_flow * 1020 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1021 uint32_t queue, 1022 const struct rte_flow_op_attr *attr, 1023 struct rte_flow_template_table *table, 1024 const struct rte_flow_item items[], 1025 uint8_t pattern_template_index, 1026 const struct rte_flow_action actions[], 1027 uint8_t action_template_index, 1028 void *user_data, 1029 struct rte_flow_error *error); 1030 static int 1031 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1032 uint32_t queue, 1033 const struct rte_flow_op_attr *attr, 1034 struct rte_flow *flow, 1035 void *user_data, 1036 struct rte_flow_error *error); 1037 static int 1038 mlx5_flow_pull(struct rte_eth_dev *dev, 1039 uint32_t queue, 1040 struct rte_flow_op_result res[], 1041 uint16_t n_res, 1042 struct rte_flow_error *error); 1043 static int 1044 mlx5_flow_push(struct rte_eth_dev *dev, 1045 uint32_t queue, 1046 struct rte_flow_error *error); 1047 1048 static struct rte_flow_action_handle * 1049 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1050 const struct rte_flow_op_attr *attr, 1051 const struct rte_flow_indir_action_conf *conf, 1052 const struct rte_flow_action *action, 1053 void *user_data, 1054 struct rte_flow_error *error); 1055 1056 static int 1057 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1058 const struct rte_flow_op_attr *attr, 1059 struct rte_flow_action_handle *handle, 1060 const void *update, 1061 void *user_data, 1062 struct rte_flow_error *error); 1063 1064 static int 1065 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1066 const struct rte_flow_op_attr *attr, 1067 struct rte_flow_action_handle *handle, 1068 void *user_data, 1069 struct rte_flow_error *error); 1070 1071 static int 1072 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1073 const struct rte_flow_op_attr *attr, 1074 const struct rte_flow_action_handle *handle, 1075 void *data, 1076 void *user_data, 1077 struct rte_flow_error *error); 1078 1079 static const struct rte_flow_ops mlx5_flow_ops = { 1080 .validate = mlx5_flow_validate, 1081 .create = mlx5_flow_create, 1082 .destroy = mlx5_flow_destroy, 1083 .flush = mlx5_flow_flush, 1084 .isolate = mlx5_flow_isolate, 1085 .query = mlx5_flow_query, 1086 .dev_dump = mlx5_flow_dev_dump, 1087 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1088 .get_aged_flows = mlx5_flow_get_aged_flows, 1089 .action_handle_create = mlx5_action_handle_create, 1090 .action_handle_destroy = mlx5_action_handle_destroy, 1091 .action_handle_update = mlx5_action_handle_update, 1092 .action_handle_query = mlx5_action_handle_query, 1093 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1094 .tunnel_match = mlx5_flow_tunnel_match, 1095 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1096 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1097 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1098 .flex_item_create = mlx5_flow_flex_item_create, 1099 .flex_item_release = mlx5_flow_flex_item_release, 1100 .info_get = mlx5_flow_info_get, 1101 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1102 .configure = mlx5_flow_port_configure, 1103 .pattern_template_create = mlx5_flow_pattern_template_create, 1104 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1105 .actions_template_create = mlx5_flow_actions_template_create, 1106 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1107 .template_table_create = mlx5_flow_table_create, 1108 .template_table_destroy = mlx5_flow_table_destroy, 1109 .async_create = mlx5_flow_async_flow_create, 1110 .async_destroy = mlx5_flow_async_flow_destroy, 1111 .pull = mlx5_flow_pull, 1112 .push = mlx5_flow_push, 1113 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1114 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1115 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1116 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1117 }; 1118 1119 /* Tunnel information. */ 1120 struct mlx5_flow_tunnel_info { 1121 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1122 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1123 }; 1124 1125 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1126 { 1127 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1128 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1129 }, 1130 { 1131 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1132 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1133 }, 1134 { 1135 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1136 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1137 }, 1138 { 1139 .tunnel = MLX5_FLOW_LAYER_GRE, 1140 .ptype = RTE_PTYPE_TUNNEL_GRE, 1141 }, 1142 { 1143 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1144 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1145 }, 1146 { 1147 .tunnel = MLX5_FLOW_LAYER_MPLS, 1148 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1149 }, 1150 { 1151 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1152 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1153 }, 1154 { 1155 .tunnel = MLX5_FLOW_LAYER_IPIP, 1156 .ptype = RTE_PTYPE_TUNNEL_IP, 1157 }, 1158 { 1159 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1160 .ptype = RTE_PTYPE_TUNNEL_IP, 1161 }, 1162 { 1163 .tunnel = MLX5_FLOW_LAYER_GTP, 1164 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1165 }, 1166 }; 1167 1168 1169 1170 /** 1171 * Translate tag ID to register. 1172 * 1173 * @param[in] dev 1174 * Pointer to the Ethernet device structure. 1175 * @param[in] feature 1176 * The feature that request the register. 1177 * @param[in] id 1178 * The request register ID. 1179 * @param[out] error 1180 * Error description in case of any. 1181 * 1182 * @return 1183 * The request register on success, a negative errno 1184 * value otherwise and rte_errno is set. 1185 */ 1186 int 1187 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1188 enum mlx5_feature_name feature, 1189 uint32_t id, 1190 struct rte_flow_error *error) 1191 { 1192 struct mlx5_priv *priv = dev->data->dev_private; 1193 struct mlx5_sh_config *config = &priv->sh->config; 1194 enum modify_reg start_reg; 1195 bool skip_mtr_reg = false; 1196 1197 switch (feature) { 1198 case MLX5_HAIRPIN_RX: 1199 return REG_B; 1200 case MLX5_HAIRPIN_TX: 1201 return REG_A; 1202 case MLX5_METADATA_RX: 1203 switch (config->dv_xmeta_en) { 1204 case MLX5_XMETA_MODE_LEGACY: 1205 return REG_B; 1206 case MLX5_XMETA_MODE_META16: 1207 return REG_C_0; 1208 case MLX5_XMETA_MODE_META32: 1209 return REG_C_1; 1210 case MLX5_XMETA_MODE_META32_HWS: 1211 return REG_C_1; 1212 } 1213 break; 1214 case MLX5_METADATA_TX: 1215 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1216 return REG_C_1; 1217 } else { 1218 return REG_A; 1219 } 1220 case MLX5_METADATA_FDB: 1221 switch (config->dv_xmeta_en) { 1222 case MLX5_XMETA_MODE_LEGACY: 1223 return REG_NON; 1224 case MLX5_XMETA_MODE_META16: 1225 return REG_C_0; 1226 case MLX5_XMETA_MODE_META32: 1227 return REG_C_1; 1228 case MLX5_XMETA_MODE_META32_HWS: 1229 return REG_C_1; 1230 } 1231 break; 1232 case MLX5_FLOW_MARK: 1233 switch (config->dv_xmeta_en) { 1234 case MLX5_XMETA_MODE_LEGACY: 1235 case MLX5_XMETA_MODE_META32_HWS: 1236 return REG_NON; 1237 case MLX5_XMETA_MODE_META16: 1238 return REG_C_1; 1239 case MLX5_XMETA_MODE_META32: 1240 return REG_C_0; 1241 } 1242 break; 1243 case MLX5_MTR_ID: 1244 /* 1245 * If meter color and meter id share one register, flow match 1246 * should use the meter color register for match. 1247 */ 1248 if (priv->mtr_reg_share) 1249 return priv->mtr_color_reg; 1250 else 1251 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1252 REG_C_3; 1253 case MLX5_MTR_COLOR: 1254 case MLX5_ASO_FLOW_HIT: 1255 case MLX5_ASO_CONNTRACK: 1256 case MLX5_SAMPLE_ID: 1257 /* All features use the same REG_C. */ 1258 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1259 return priv->mtr_color_reg; 1260 case MLX5_COPY_MARK: 1261 /* 1262 * Metadata COPY_MARK register using is in meter suffix sub 1263 * flow while with meter. It's safe to share the same register. 1264 */ 1265 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1266 case MLX5_APP_TAG: 1267 /* 1268 * If meter is enable, it will engage the register for color 1269 * match and flow match. If meter color match is not using the 1270 * REG_C_2, need to skip the REG_C_x be used by meter color 1271 * match. 1272 * If meter is disable, free to use all available registers. 1273 */ 1274 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1275 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1276 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1277 if (id > (uint32_t)(REG_C_7 - start_reg)) 1278 return rte_flow_error_set(error, EINVAL, 1279 RTE_FLOW_ERROR_TYPE_ITEM, 1280 NULL, "invalid tag id"); 1281 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1282 return rte_flow_error_set(error, ENOTSUP, 1283 RTE_FLOW_ERROR_TYPE_ITEM, 1284 NULL, "unsupported tag id"); 1285 /* 1286 * This case means meter is using the REG_C_x great than 2. 1287 * Take care not to conflict with meter color REG_C_x. 1288 * If the available index REG_C_y >= REG_C_x, skip the 1289 * color register. 1290 */ 1291 if (skip_mtr_reg && priv->sh->flow_mreg_c 1292 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1293 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1294 return rte_flow_error_set(error, EINVAL, 1295 RTE_FLOW_ERROR_TYPE_ITEM, 1296 NULL, "invalid tag id"); 1297 if (priv->sh->flow_mreg_c 1298 [id + 1 + start_reg - REG_C_0] != REG_NON) 1299 return priv->sh->flow_mreg_c 1300 [id + 1 + start_reg - REG_C_0]; 1301 return rte_flow_error_set(error, ENOTSUP, 1302 RTE_FLOW_ERROR_TYPE_ITEM, 1303 NULL, "unsupported tag id"); 1304 } 1305 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1306 } 1307 MLX5_ASSERT(false); 1308 return rte_flow_error_set(error, EINVAL, 1309 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1310 NULL, "invalid feature name"); 1311 } 1312 1313 /** 1314 * Check extensive flow metadata register support. 1315 * 1316 * @param dev 1317 * Pointer to rte_eth_dev structure. 1318 * 1319 * @return 1320 * True if device supports extensive flow metadata register, otherwise false. 1321 */ 1322 bool 1323 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1324 { 1325 struct mlx5_priv *priv = dev->data->dev_private; 1326 1327 /* 1328 * Having available reg_c can be regarded inclusively as supporting 1329 * extensive flow metadata register, which could mean, 1330 * - metadata register copy action by modify header. 1331 * - 16 modify header actions is supported. 1332 * - reg_c's are preserved across different domain (FDB and NIC) on 1333 * packet loopback by flow lookup miss. 1334 */ 1335 return priv->sh->flow_mreg_c[2] != REG_NON; 1336 } 1337 1338 /** 1339 * Get the lowest priority. 1340 * 1341 * @param[in] dev 1342 * Pointer to the Ethernet device structure. 1343 * @param[in] attributes 1344 * Pointer to device flow rule attributes. 1345 * 1346 * @return 1347 * The value of lowest priority of flow. 1348 */ 1349 uint32_t 1350 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1351 const struct rte_flow_attr *attr) 1352 { 1353 struct mlx5_priv *priv = dev->data->dev_private; 1354 1355 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1356 return priv->sh->flow_max_priority - 2; 1357 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1358 } 1359 1360 /** 1361 * Calculate matcher priority of the flow. 1362 * 1363 * @param[in] dev 1364 * Pointer to the Ethernet device structure. 1365 * @param[in] attr 1366 * Pointer to device flow rule attributes. 1367 * @param[in] subpriority 1368 * The priority based on the items. 1369 * @param[in] external 1370 * Flow is user flow. 1371 * @return 1372 * The matcher priority of the flow. 1373 */ 1374 uint16_t 1375 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1376 const struct rte_flow_attr *attr, 1377 uint32_t subpriority, bool external) 1378 { 1379 uint16_t priority = (uint16_t)attr->priority; 1380 struct mlx5_priv *priv = dev->data->dev_private; 1381 1382 /* NIC root rules */ 1383 if (!attr->group && !attr->transfer) { 1384 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1385 priority = priv->sh->flow_max_priority - 1; 1386 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1387 /* FDB root rules */ 1388 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1389 attr->group == 0 && 1390 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1391 return (priv->sh->flow_max_priority - 1) * 3; 1392 } 1393 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1394 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1395 return priority * 3 + subpriority; 1396 } 1397 1398 /** 1399 * Verify the @p item specifications (spec, last, mask) are compatible with the 1400 * NIC capabilities. 1401 * 1402 * @param[in] item 1403 * Item specification. 1404 * @param[in] mask 1405 * @p item->mask or flow default bit-masks. 1406 * @param[in] nic_mask 1407 * Bit-masks covering supported fields by the NIC to compare with user mask. 1408 * @param[in] size 1409 * Bit-masks size in bytes. 1410 * @param[in] range_accepted 1411 * True if range of values is accepted for specific fields, false otherwise. 1412 * @param[out] error 1413 * Pointer to error structure. 1414 * 1415 * @return 1416 * 0 on success, a negative errno value otherwise and rte_errno is set. 1417 */ 1418 int 1419 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1420 const uint8_t *mask, 1421 const uint8_t *nic_mask, 1422 unsigned int size, 1423 bool range_accepted, 1424 struct rte_flow_error *error) 1425 { 1426 unsigned int i; 1427 1428 MLX5_ASSERT(nic_mask); 1429 for (i = 0; i < size; ++i) 1430 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1431 return rte_flow_error_set(error, ENOTSUP, 1432 RTE_FLOW_ERROR_TYPE_ITEM, 1433 item, 1434 "mask enables non supported" 1435 " bits"); 1436 if (!item->spec && (item->mask || item->last)) 1437 return rte_flow_error_set(error, EINVAL, 1438 RTE_FLOW_ERROR_TYPE_ITEM, item, 1439 "mask/last without a spec is not" 1440 " supported"); 1441 if (item->spec && item->last && !range_accepted) { 1442 uint8_t spec[size]; 1443 uint8_t last[size]; 1444 unsigned int i; 1445 int ret; 1446 1447 for (i = 0; i < size; ++i) { 1448 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1449 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1450 } 1451 ret = memcmp(spec, last, size); 1452 if (ret != 0) 1453 return rte_flow_error_set(error, EINVAL, 1454 RTE_FLOW_ERROR_TYPE_ITEM, 1455 item, 1456 "range is not valid"); 1457 } 1458 return 0; 1459 } 1460 1461 /** 1462 * Adjust the hash fields according to the @p flow information. 1463 * 1464 * @param[in] dev_flow. 1465 * Pointer to the mlx5_flow. 1466 * @param[in] tunnel 1467 * 1 when the hash field is for a tunnel item. 1468 * @param[in] layer_types 1469 * RTE_ETH_RSS_* types. 1470 * @param[in] hash_fields 1471 * Item hash fields. 1472 * 1473 * @return 1474 * The hash fields that should be used. 1475 */ 1476 uint64_t 1477 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1478 int tunnel __rte_unused, uint64_t layer_types, 1479 uint64_t hash_fields) 1480 { 1481 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1482 int rss_request_inner = rss_desc->level >= 2; 1483 1484 /* Check RSS hash level for tunnel. */ 1485 if (tunnel && rss_request_inner) 1486 hash_fields |= IBV_RX_HASH_INNER; 1487 else if (tunnel || rss_request_inner) 1488 return 0; 1489 #endif 1490 /* Check if requested layer matches RSS hash fields. */ 1491 if (!(rss_desc->types & layer_types)) 1492 return 0; 1493 return hash_fields; 1494 } 1495 1496 /** 1497 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1498 * if several tunnel rules are used on this queue, the tunnel ptype will be 1499 * cleared. 1500 * 1501 * @param rxq_ctrl 1502 * Rx queue to update. 1503 */ 1504 static void 1505 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1506 { 1507 unsigned int i; 1508 uint32_t tunnel_ptype = 0; 1509 1510 /* Look up for the ptype to use. */ 1511 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1512 if (!rxq_ctrl->flow_tunnels_n[i]) 1513 continue; 1514 if (!tunnel_ptype) { 1515 tunnel_ptype = tunnels_info[i].ptype; 1516 } else { 1517 tunnel_ptype = 0; 1518 break; 1519 } 1520 } 1521 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1522 } 1523 1524 /** 1525 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1526 * flow. 1527 * 1528 * @param[in] dev 1529 * Pointer to the Ethernet device structure. 1530 * @param[in] dev_handle 1531 * Pointer to device flow handle structure. 1532 */ 1533 void 1534 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1535 struct mlx5_flow_handle *dev_handle) 1536 { 1537 struct mlx5_priv *priv = dev->data->dev_private; 1538 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1539 struct mlx5_ind_table_obj *ind_tbl = NULL; 1540 unsigned int i; 1541 1542 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1543 struct mlx5_hrxq *hrxq; 1544 1545 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1546 dev_handle->rix_hrxq); 1547 if (hrxq) 1548 ind_tbl = hrxq->ind_table; 1549 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1550 struct mlx5_shared_action_rss *shared_rss; 1551 1552 shared_rss = mlx5_ipool_get 1553 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1554 dev_handle->rix_srss); 1555 if (shared_rss) 1556 ind_tbl = shared_rss->ind_tbl; 1557 } 1558 if (!ind_tbl) 1559 return; 1560 for (i = 0; i != ind_tbl->queues_n; ++i) { 1561 int idx = ind_tbl->queues[i]; 1562 struct mlx5_rxq_ctrl *rxq_ctrl; 1563 1564 if (mlx5_is_external_rxq(dev, idx)) 1565 continue; 1566 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1567 MLX5_ASSERT(rxq_ctrl != NULL); 1568 if (rxq_ctrl == NULL) 1569 continue; 1570 /* 1571 * To support metadata register copy on Tx loopback, 1572 * this must be always enabled (metadata may arive 1573 * from other port - not from local flows only. 1574 */ 1575 if (tunnel) { 1576 unsigned int j; 1577 1578 /* Increase the counter matching the flow. */ 1579 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1580 if ((tunnels_info[j].tunnel & 1581 dev_handle->layers) == 1582 tunnels_info[j].tunnel) { 1583 rxq_ctrl->flow_tunnels_n[j]++; 1584 break; 1585 } 1586 } 1587 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1588 } 1589 } 1590 } 1591 1592 static void 1593 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1594 { 1595 struct mlx5_priv *priv = dev->data->dev_private; 1596 struct mlx5_rxq_ctrl *rxq_ctrl; 1597 uint16_t port_id; 1598 1599 if (priv->sh->shared_mark_enabled) 1600 return; 1601 if (priv->master || priv->representor) { 1602 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1603 struct mlx5_priv *opriv = 1604 rte_eth_devices[port_id].data->dev_private; 1605 1606 if (!opriv || 1607 opriv->sh != priv->sh || 1608 opriv->domain_id != priv->domain_id || 1609 opriv->mark_enabled) 1610 continue; 1611 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1612 rxq_ctrl->rxq.mark = 1; 1613 } 1614 opriv->mark_enabled = 1; 1615 } 1616 } else { 1617 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1618 rxq_ctrl->rxq.mark = 1; 1619 } 1620 priv->mark_enabled = 1; 1621 } 1622 priv->sh->shared_mark_enabled = 1; 1623 } 1624 1625 /** 1626 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1627 * 1628 * @param[in] dev 1629 * Pointer to the Ethernet device structure. 1630 * @param[in] flow 1631 * Pointer to flow structure. 1632 */ 1633 static void 1634 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1635 { 1636 struct mlx5_priv *priv = dev->data->dev_private; 1637 uint32_t handle_idx; 1638 struct mlx5_flow_handle *dev_handle; 1639 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1640 1641 MLX5_ASSERT(wks); 1642 if (wks->mark) 1643 flow_rxq_mark_flag_set(dev); 1644 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1645 handle_idx, dev_handle, next) 1646 flow_drv_rxq_flags_set(dev, dev_handle); 1647 } 1648 1649 /** 1650 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1651 * device flow if no other flow uses it with the same kind of request. 1652 * 1653 * @param dev 1654 * Pointer to Ethernet device. 1655 * @param[in] dev_handle 1656 * Pointer to the device flow handle structure. 1657 */ 1658 static void 1659 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1660 struct mlx5_flow_handle *dev_handle) 1661 { 1662 struct mlx5_priv *priv = dev->data->dev_private; 1663 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1664 struct mlx5_ind_table_obj *ind_tbl = NULL; 1665 unsigned int i; 1666 1667 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1668 struct mlx5_hrxq *hrxq; 1669 1670 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1671 dev_handle->rix_hrxq); 1672 if (hrxq) 1673 ind_tbl = hrxq->ind_table; 1674 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1675 struct mlx5_shared_action_rss *shared_rss; 1676 1677 shared_rss = mlx5_ipool_get 1678 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1679 dev_handle->rix_srss); 1680 if (shared_rss) 1681 ind_tbl = shared_rss->ind_tbl; 1682 } 1683 if (!ind_tbl) 1684 return; 1685 MLX5_ASSERT(dev->data->dev_started); 1686 for (i = 0; i != ind_tbl->queues_n; ++i) { 1687 int idx = ind_tbl->queues[i]; 1688 struct mlx5_rxq_ctrl *rxq_ctrl; 1689 1690 if (mlx5_is_external_rxq(dev, idx)) 1691 continue; 1692 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1693 MLX5_ASSERT(rxq_ctrl != NULL); 1694 if (rxq_ctrl == NULL) 1695 continue; 1696 if (tunnel) { 1697 unsigned int j; 1698 1699 /* Decrease the counter matching the flow. */ 1700 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1701 if ((tunnels_info[j].tunnel & 1702 dev_handle->layers) == 1703 tunnels_info[j].tunnel) { 1704 rxq_ctrl->flow_tunnels_n[j]--; 1705 break; 1706 } 1707 } 1708 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1709 } 1710 } 1711 } 1712 1713 /** 1714 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1715 * @p flow if no other flow uses it with the same kind of request. 1716 * 1717 * @param dev 1718 * Pointer to Ethernet device. 1719 * @param[in] flow 1720 * Pointer to the flow. 1721 */ 1722 static void 1723 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1724 { 1725 struct mlx5_priv *priv = dev->data->dev_private; 1726 uint32_t handle_idx; 1727 struct mlx5_flow_handle *dev_handle; 1728 1729 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1730 handle_idx, dev_handle, next) 1731 flow_drv_rxq_flags_trim(dev, dev_handle); 1732 } 1733 1734 /** 1735 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1736 * 1737 * @param dev 1738 * Pointer to Ethernet device. 1739 */ 1740 static void 1741 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1742 { 1743 struct mlx5_priv *priv = dev->data->dev_private; 1744 unsigned int i; 1745 1746 for (i = 0; i != priv->rxqs_n; ++i) { 1747 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1748 unsigned int j; 1749 1750 if (rxq == NULL || rxq->ctrl == NULL) 1751 continue; 1752 rxq->ctrl->rxq.mark = 0; 1753 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1754 rxq->ctrl->flow_tunnels_n[j] = 0; 1755 rxq->ctrl->rxq.tunnel = 0; 1756 } 1757 priv->mark_enabled = 0; 1758 priv->sh->shared_mark_enabled = 0; 1759 } 1760 1761 /** 1762 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1763 * 1764 * @param[in] dev 1765 * Pointer to the Ethernet device structure. 1766 */ 1767 void 1768 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1769 { 1770 struct mlx5_priv *priv = dev->data->dev_private; 1771 unsigned int i; 1772 1773 for (i = 0; i != priv->rxqs_n; ++i) { 1774 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1775 struct mlx5_rxq_data *data; 1776 1777 if (rxq == NULL || rxq->ctrl == NULL) 1778 continue; 1779 data = &rxq->ctrl->rxq; 1780 if (!rte_flow_dynf_metadata_avail()) { 1781 data->dynf_meta = 0; 1782 data->flow_meta_mask = 0; 1783 data->flow_meta_offset = -1; 1784 data->flow_meta_port_mask = 0; 1785 } else { 1786 data->dynf_meta = 1; 1787 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1788 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1789 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1790 } 1791 } 1792 } 1793 1794 /* 1795 * return a pointer to the desired action in the list of actions. 1796 * 1797 * @param[in] actions 1798 * The list of actions to search the action in. 1799 * @param[in] action 1800 * The action to find. 1801 * 1802 * @return 1803 * Pointer to the action in the list, if found. NULL otherwise. 1804 */ 1805 const struct rte_flow_action * 1806 mlx5_flow_find_action(const struct rte_flow_action *actions, 1807 enum rte_flow_action_type action) 1808 { 1809 if (actions == NULL) 1810 return NULL; 1811 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1812 if (actions->type == action) 1813 return actions; 1814 return NULL; 1815 } 1816 1817 /* 1818 * Validate the flag action. 1819 * 1820 * @param[in] action_flags 1821 * Bit-fields that holds the actions detected until now. 1822 * @param[in] attr 1823 * Attributes of flow that includes this action. 1824 * @param[out] error 1825 * Pointer to error structure. 1826 * 1827 * @return 1828 * 0 on success, a negative errno value otherwise and rte_errno is set. 1829 */ 1830 int 1831 mlx5_flow_validate_action_flag(uint64_t action_flags, 1832 const struct rte_flow_attr *attr, 1833 struct rte_flow_error *error) 1834 { 1835 if (action_flags & MLX5_FLOW_ACTION_MARK) 1836 return rte_flow_error_set(error, EINVAL, 1837 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1838 "can't mark and flag in same flow"); 1839 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1840 return rte_flow_error_set(error, EINVAL, 1841 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1842 "can't have 2 flag" 1843 " actions in same flow"); 1844 if (attr->egress) 1845 return rte_flow_error_set(error, ENOTSUP, 1846 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1847 "flag action not supported for " 1848 "egress"); 1849 return 0; 1850 } 1851 1852 /* 1853 * Validate the mark action. 1854 * 1855 * @param[in] action 1856 * Pointer to the queue action. 1857 * @param[in] action_flags 1858 * Bit-fields that holds the actions detected until now. 1859 * @param[in] attr 1860 * Attributes of flow that includes this action. 1861 * @param[out] error 1862 * Pointer to error structure. 1863 * 1864 * @return 1865 * 0 on success, a negative errno value otherwise and rte_errno is set. 1866 */ 1867 int 1868 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1869 uint64_t action_flags, 1870 const struct rte_flow_attr *attr, 1871 struct rte_flow_error *error) 1872 { 1873 const struct rte_flow_action_mark *mark = action->conf; 1874 1875 if (!mark) 1876 return rte_flow_error_set(error, EINVAL, 1877 RTE_FLOW_ERROR_TYPE_ACTION, 1878 action, 1879 "configuration cannot be null"); 1880 if (mark->id >= MLX5_FLOW_MARK_MAX) 1881 return rte_flow_error_set(error, EINVAL, 1882 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1883 &mark->id, 1884 "mark id must in 0 <= id < " 1885 RTE_STR(MLX5_FLOW_MARK_MAX)); 1886 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1887 return rte_flow_error_set(error, EINVAL, 1888 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1889 "can't flag and mark in same flow"); 1890 if (action_flags & MLX5_FLOW_ACTION_MARK) 1891 return rte_flow_error_set(error, EINVAL, 1892 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1893 "can't have 2 mark actions in same" 1894 " flow"); 1895 if (attr->egress) 1896 return rte_flow_error_set(error, ENOTSUP, 1897 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1898 "mark action not supported for " 1899 "egress"); 1900 return 0; 1901 } 1902 1903 /* 1904 * Validate the drop action. 1905 * 1906 * @param[in] action_flags 1907 * Bit-fields that holds the actions detected until now. 1908 * @param[in] attr 1909 * Attributes of flow that includes this action. 1910 * @param[out] error 1911 * Pointer to error structure. 1912 * 1913 * @return 1914 * 0 on success, a negative errno value otherwise and rte_errno is set. 1915 */ 1916 int 1917 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1918 const struct rte_flow_attr *attr, 1919 struct rte_flow_error *error) 1920 { 1921 if (attr->egress) 1922 return rte_flow_error_set(error, ENOTSUP, 1923 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1924 "drop action not supported for " 1925 "egress"); 1926 return 0; 1927 } 1928 1929 /* 1930 * Validate the queue action. 1931 * 1932 * @param[in] action 1933 * Pointer to the queue action. 1934 * @param[in] action_flags 1935 * Bit-fields that holds the actions detected until now. 1936 * @param[in] dev 1937 * Pointer to the Ethernet device structure. 1938 * @param[in] attr 1939 * Attributes of flow that includes this action. 1940 * @param[out] error 1941 * Pointer to error structure. 1942 * 1943 * @return 1944 * 0 on success, a negative errno value otherwise and rte_errno is set. 1945 */ 1946 int 1947 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1948 uint64_t action_flags, 1949 struct rte_eth_dev *dev, 1950 const struct rte_flow_attr *attr, 1951 struct rte_flow_error *error) 1952 { 1953 struct mlx5_priv *priv = dev->data->dev_private; 1954 const struct rte_flow_action_queue *queue = action->conf; 1955 1956 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1957 return rte_flow_error_set(error, EINVAL, 1958 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1959 "can't have 2 fate actions in" 1960 " same flow"); 1961 if (attr->egress) 1962 return rte_flow_error_set(error, ENOTSUP, 1963 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1964 "queue action not supported for egress."); 1965 if (mlx5_is_external_rxq(dev, queue->index)) 1966 return 0; 1967 if (!priv->rxqs_n) 1968 return rte_flow_error_set(error, EINVAL, 1969 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1970 NULL, "No Rx queues configured"); 1971 if (queue->index >= priv->rxqs_n) 1972 return rte_flow_error_set(error, EINVAL, 1973 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1974 &queue->index, 1975 "queue index out of range"); 1976 if (mlx5_rxq_get(dev, queue->index) == NULL) 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1979 &queue->index, 1980 "queue is not configured"); 1981 return 0; 1982 } 1983 1984 /** 1985 * Validate queue numbers for device RSS. 1986 * 1987 * @param[in] dev 1988 * Configured device. 1989 * @param[in] queues 1990 * Array of queue numbers. 1991 * @param[in] queues_n 1992 * Size of the @p queues array. 1993 * @param[out] error 1994 * On error, filled with a textual error description. 1995 * @param[out] queue_idx 1996 * On error, filled with an offending queue index in @p queues array. 1997 * 1998 * @return 1999 * 0 on success, a negative errno code on error. 2000 */ 2001 static int 2002 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2003 const uint16_t *queues, uint32_t queues_n, 2004 const char **error, uint32_t *queue_idx) 2005 { 2006 const struct mlx5_priv *priv = dev->data->dev_private; 2007 bool is_hairpin = false; 2008 bool is_ext_rss = false; 2009 uint32_t i; 2010 2011 for (i = 0; i != queues_n; ++i) { 2012 struct mlx5_rxq_ctrl *rxq_ctrl; 2013 2014 if (mlx5_is_external_rxq(dev, queues[0])) { 2015 is_ext_rss = true; 2016 continue; 2017 } 2018 if (is_ext_rss) { 2019 *error = "Combining external and regular RSS queues is not supported"; 2020 *queue_idx = i; 2021 return -ENOTSUP; 2022 } 2023 if (queues[i] >= priv->rxqs_n) { 2024 *error = "queue index out of range"; 2025 *queue_idx = i; 2026 return -EINVAL; 2027 } 2028 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2029 if (rxq_ctrl == NULL) { 2030 *error = "queue is not configured"; 2031 *queue_idx = i; 2032 return -EINVAL; 2033 } 2034 if (i == 0 && rxq_ctrl->is_hairpin) 2035 is_hairpin = true; 2036 if (is_hairpin != rxq_ctrl->is_hairpin) { 2037 *error = "combining hairpin and regular RSS queues is not supported"; 2038 *queue_idx = i; 2039 return -ENOTSUP; 2040 } 2041 } 2042 return 0; 2043 } 2044 2045 /* 2046 * Validate the rss action. 2047 * 2048 * @param[in] dev 2049 * Pointer to the Ethernet device structure. 2050 * @param[in] action 2051 * Pointer to the queue action. 2052 * @param[out] error 2053 * Pointer to error structure. 2054 * 2055 * @return 2056 * 0 on success, a negative errno value otherwise and rte_errno is set. 2057 */ 2058 int 2059 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2060 const struct rte_flow_action *action, 2061 struct rte_flow_error *error) 2062 { 2063 struct mlx5_priv *priv = dev->data->dev_private; 2064 const struct rte_flow_action_rss *rss = action->conf; 2065 int ret; 2066 const char *message; 2067 uint32_t queue_idx; 2068 2069 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2070 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2071 return rte_flow_error_set(error, ENOTSUP, 2072 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2073 &rss->func, 2074 "RSS hash function not supported"); 2075 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2076 if (rss->level > 2) 2077 #else 2078 if (rss->level > 1) 2079 #endif 2080 return rte_flow_error_set(error, ENOTSUP, 2081 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2082 &rss->level, 2083 "tunnel RSS is not supported"); 2084 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2085 if (rss->key_len == 0 && rss->key != NULL) 2086 return rte_flow_error_set(error, ENOTSUP, 2087 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2088 &rss->key_len, 2089 "RSS hash key length 0"); 2090 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2091 return rte_flow_error_set(error, ENOTSUP, 2092 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2093 &rss->key_len, 2094 "RSS hash key too small"); 2095 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2096 return rte_flow_error_set(error, ENOTSUP, 2097 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2098 &rss->key_len, 2099 "RSS hash key too large"); 2100 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2101 return rte_flow_error_set(error, ENOTSUP, 2102 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2103 &rss->queue_num, 2104 "number of queues too large"); 2105 if (rss->types & MLX5_RSS_HF_MASK) 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2108 &rss->types, 2109 "some RSS protocols are not" 2110 " supported"); 2111 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2112 !(rss->types & RTE_ETH_RSS_IP)) 2113 return rte_flow_error_set(error, EINVAL, 2114 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2115 "L3 partial RSS requested but L3 RSS" 2116 " type not specified"); 2117 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2118 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2119 return rte_flow_error_set(error, EINVAL, 2120 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2121 "L4 partial RSS requested but L4 RSS" 2122 " type not specified"); 2123 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2124 return rte_flow_error_set(error, EINVAL, 2125 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2126 NULL, "No Rx queues configured"); 2127 if (!rss->queue_num) 2128 return rte_flow_error_set(error, EINVAL, 2129 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2130 NULL, "No queues configured"); 2131 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2132 &message, &queue_idx); 2133 if (ret != 0) { 2134 return rte_flow_error_set(error, -ret, 2135 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2136 &rss->queue[queue_idx], message); 2137 } 2138 return 0; 2139 } 2140 2141 /* 2142 * Validate the rss action. 2143 * 2144 * @param[in] action 2145 * Pointer to the queue action. 2146 * @param[in] action_flags 2147 * Bit-fields that holds the actions detected until now. 2148 * @param[in] dev 2149 * Pointer to the Ethernet device structure. 2150 * @param[in] attr 2151 * Attributes of flow that includes this action. 2152 * @param[in] item_flags 2153 * Items that were detected. 2154 * @param[out] error 2155 * Pointer to error structure. 2156 * 2157 * @return 2158 * 0 on success, a negative errno value otherwise and rte_errno is set. 2159 */ 2160 int 2161 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2162 uint64_t action_flags, 2163 struct rte_eth_dev *dev, 2164 const struct rte_flow_attr *attr, 2165 uint64_t item_flags, 2166 struct rte_flow_error *error) 2167 { 2168 const struct rte_flow_action_rss *rss = action->conf; 2169 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2170 int ret; 2171 2172 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2173 return rte_flow_error_set(error, EINVAL, 2174 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2175 "can't have 2 fate actions" 2176 " in same flow"); 2177 ret = mlx5_validate_action_rss(dev, action, error); 2178 if (ret) 2179 return ret; 2180 if (attr->egress) 2181 return rte_flow_error_set(error, ENOTSUP, 2182 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2183 "rss action not supported for " 2184 "egress"); 2185 if (rss->level > 1 && !tunnel) 2186 return rte_flow_error_set(error, EINVAL, 2187 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2188 "inner RSS is not supported for " 2189 "non-tunnel flows"); 2190 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2191 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2192 return rte_flow_error_set(error, EINVAL, 2193 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2194 "RSS on eCPRI is not supported now"); 2195 } 2196 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2197 !(item_flags & 2198 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2199 rss->level > 1) 2200 return rte_flow_error_set(error, EINVAL, 2201 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2202 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2203 return 0; 2204 } 2205 2206 /* 2207 * Validate the default miss action. 2208 * 2209 * @param[in] action_flags 2210 * Bit-fields that holds the actions detected until now. 2211 * @param[out] error 2212 * Pointer to error structure. 2213 * 2214 * @return 2215 * 0 on success, a negative errno value otherwise and rte_errno is set. 2216 */ 2217 int 2218 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2219 const struct rte_flow_attr *attr, 2220 struct rte_flow_error *error) 2221 { 2222 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2225 "can't have 2 fate actions in" 2226 " same flow"); 2227 if (attr->egress) 2228 return rte_flow_error_set(error, ENOTSUP, 2229 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2230 "default miss action not supported " 2231 "for egress"); 2232 if (attr->group) 2233 return rte_flow_error_set(error, ENOTSUP, 2234 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2235 "only group 0 is supported"); 2236 if (attr->transfer) 2237 return rte_flow_error_set(error, ENOTSUP, 2238 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2239 NULL, "transfer is not supported"); 2240 return 0; 2241 } 2242 2243 /* 2244 * Validate the count action. 2245 * 2246 * @param[in] dev 2247 * Pointer to the Ethernet device structure. 2248 * @param[in] attr 2249 * Attributes of flow that includes this action. 2250 * @param[out] error 2251 * Pointer to error structure. 2252 * 2253 * @return 2254 * 0 on success, a negative errno value otherwise and rte_errno is set. 2255 */ 2256 int 2257 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2258 const struct rte_flow_attr *attr, 2259 struct rte_flow_error *error) 2260 { 2261 if (attr->egress) 2262 return rte_flow_error_set(error, ENOTSUP, 2263 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2264 "count action not supported for " 2265 "egress"); 2266 return 0; 2267 } 2268 2269 /* 2270 * Validate the ASO CT action. 2271 * 2272 * @param[in] dev 2273 * Pointer to the Ethernet device structure. 2274 * @param[in] conntrack 2275 * Pointer to the CT action profile. 2276 * @param[out] error 2277 * Pointer to error structure. 2278 * 2279 * @return 2280 * 0 on success, a negative errno value otherwise and rte_errno is set. 2281 */ 2282 int 2283 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2284 const struct rte_flow_action_conntrack *conntrack, 2285 struct rte_flow_error *error) 2286 { 2287 RTE_SET_USED(dev); 2288 2289 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2290 return rte_flow_error_set(error, EINVAL, 2291 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2292 "Invalid CT state"); 2293 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2296 "Invalid last TCP packet flag"); 2297 return 0; 2298 } 2299 2300 /** 2301 * Validate ICMP6 item. 2302 * 2303 * @param[in] item 2304 * Item specification. 2305 * @param[in] item_flags 2306 * Bit-fields that holds the items detected until now. 2307 * @param[in] ext_vlan_sup 2308 * Whether extended VLAN features are supported or not. 2309 * @param[out] error 2310 * Pointer to error structure. 2311 * 2312 * @return 2313 * 0 on success, a negative errno value otherwise and rte_errno is set. 2314 */ 2315 int 2316 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2317 uint64_t item_flags, 2318 uint8_t target_protocol, 2319 struct rte_flow_error *error) 2320 { 2321 const struct rte_flow_item_icmp6 *mask = item->mask; 2322 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2323 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2324 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2325 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2326 MLX5_FLOW_LAYER_OUTER_L4; 2327 int ret; 2328 2329 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2330 return rte_flow_error_set(error, EINVAL, 2331 RTE_FLOW_ERROR_TYPE_ITEM, item, 2332 "protocol filtering not compatible" 2333 " with ICMP6 layer"); 2334 if (!(item_flags & l3m)) 2335 return rte_flow_error_set(error, EINVAL, 2336 RTE_FLOW_ERROR_TYPE_ITEM, item, 2337 "IPv6 is mandatory to filter on" 2338 " ICMP6"); 2339 if (item_flags & l4m) 2340 return rte_flow_error_set(error, EINVAL, 2341 RTE_FLOW_ERROR_TYPE_ITEM, item, 2342 "multiple L4 layers not supported"); 2343 if (!mask) 2344 mask = &rte_flow_item_icmp6_mask; 2345 ret = mlx5_flow_item_acceptable 2346 (item, (const uint8_t *)mask, 2347 (const uint8_t *)&rte_flow_item_icmp6_mask, 2348 sizeof(struct rte_flow_item_icmp6), 2349 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2350 if (ret < 0) 2351 return ret; 2352 return 0; 2353 } 2354 2355 /** 2356 * Validate ICMP6 echo request/reply item. 2357 * 2358 * @param[in] item 2359 * Item specification. 2360 * @param[in] item_flags 2361 * Bit-fields that holds the items detected until now. 2362 * @param[in] ext_vlan_sup 2363 * Whether extended VLAN features are supported or not. 2364 * @param[out] error 2365 * Pointer to error structure. 2366 * 2367 * @return 2368 * 0 on success, a negative errno value otherwise and rte_errno is set. 2369 */ 2370 int 2371 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2372 uint64_t item_flags, 2373 uint8_t target_protocol, 2374 struct rte_flow_error *error) 2375 { 2376 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2377 const struct rte_flow_item_icmp6_echo nic_mask = { 2378 .hdr.base.type = 0xff, 2379 .hdr.base.code = 0xff, 2380 .hdr.identifier = RTE_BE16(0xffff), 2381 .hdr.sequence = RTE_BE16(0xffff), 2382 }; 2383 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2384 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2385 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2386 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2387 MLX5_FLOW_LAYER_OUTER_L4; 2388 int ret; 2389 2390 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2391 return rte_flow_error_set(error, EINVAL, 2392 RTE_FLOW_ERROR_TYPE_ITEM, item, 2393 "protocol filtering not compatible" 2394 " with ICMP6 layer"); 2395 if (!(item_flags & l3m)) 2396 return rte_flow_error_set(error, EINVAL, 2397 RTE_FLOW_ERROR_TYPE_ITEM, item, 2398 "IPv6 is mandatory to filter on" 2399 " ICMP6"); 2400 if (item_flags & l4m) 2401 return rte_flow_error_set(error, EINVAL, 2402 RTE_FLOW_ERROR_TYPE_ITEM, item, 2403 "multiple L4 layers not supported"); 2404 if (!mask) 2405 mask = &nic_mask; 2406 ret = mlx5_flow_item_acceptable 2407 (item, (const uint8_t *)mask, 2408 (const uint8_t *)&nic_mask, 2409 sizeof(struct rte_flow_item_icmp6_echo), 2410 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2411 if (ret < 0) 2412 return ret; 2413 return 0; 2414 } 2415 2416 /** 2417 * Validate ICMP item. 2418 * 2419 * @param[in] item 2420 * Item specification. 2421 * @param[in] item_flags 2422 * Bit-fields that holds the items detected until now. 2423 * @param[out] error 2424 * Pointer to error structure. 2425 * 2426 * @return 2427 * 0 on success, a negative errno value otherwise and rte_errno is set. 2428 */ 2429 int 2430 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2431 uint64_t item_flags, 2432 uint8_t target_protocol, 2433 struct rte_flow_error *error) 2434 { 2435 const struct rte_flow_item_icmp *mask = item->mask; 2436 const struct rte_flow_item_icmp nic_mask = { 2437 .hdr.icmp_type = 0xff, 2438 .hdr.icmp_code = 0xff, 2439 .hdr.icmp_ident = RTE_BE16(0xffff), 2440 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2441 }; 2442 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2443 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2444 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2445 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2446 MLX5_FLOW_LAYER_OUTER_L4; 2447 int ret; 2448 2449 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2450 return rte_flow_error_set(error, EINVAL, 2451 RTE_FLOW_ERROR_TYPE_ITEM, item, 2452 "protocol filtering not compatible" 2453 " with ICMP layer"); 2454 if (!(item_flags & l3m)) 2455 return rte_flow_error_set(error, EINVAL, 2456 RTE_FLOW_ERROR_TYPE_ITEM, item, 2457 "IPv4 is mandatory to filter" 2458 " on ICMP"); 2459 if (item_flags & l4m) 2460 return rte_flow_error_set(error, EINVAL, 2461 RTE_FLOW_ERROR_TYPE_ITEM, item, 2462 "multiple L4 layers not supported"); 2463 if (!mask) 2464 mask = &nic_mask; 2465 ret = mlx5_flow_item_acceptable 2466 (item, (const uint8_t *)mask, 2467 (const uint8_t *)&nic_mask, 2468 sizeof(struct rte_flow_item_icmp), 2469 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2470 if (ret < 0) 2471 return ret; 2472 return 0; 2473 } 2474 2475 /** 2476 * Validate Ethernet item. 2477 * 2478 * @param[in] item 2479 * Item specification. 2480 * @param[in] item_flags 2481 * Bit-fields that holds the items detected until now. 2482 * @param[out] error 2483 * Pointer to error structure. 2484 * 2485 * @return 2486 * 0 on success, a negative errno value otherwise and rte_errno is set. 2487 */ 2488 int 2489 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2490 uint64_t item_flags, bool ext_vlan_sup, 2491 struct rte_flow_error *error) 2492 { 2493 const struct rte_flow_item_eth *mask = item->mask; 2494 const struct rte_flow_item_eth nic_mask = { 2495 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2496 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2497 .hdr.ether_type = RTE_BE16(0xffff), 2498 .has_vlan = ext_vlan_sup ? 1 : 0, 2499 }; 2500 int ret; 2501 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2502 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2503 MLX5_FLOW_LAYER_OUTER_L2; 2504 2505 if (item_flags & ethm) 2506 return rte_flow_error_set(error, ENOTSUP, 2507 RTE_FLOW_ERROR_TYPE_ITEM, item, 2508 "multiple L2 layers not supported"); 2509 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2510 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2511 return rte_flow_error_set(error, EINVAL, 2512 RTE_FLOW_ERROR_TYPE_ITEM, item, 2513 "L2 layer should not follow " 2514 "L3 layers"); 2515 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2516 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2517 return rte_flow_error_set(error, EINVAL, 2518 RTE_FLOW_ERROR_TYPE_ITEM, item, 2519 "L2 layer should not follow VLAN"); 2520 if (item_flags & MLX5_FLOW_LAYER_GTP) 2521 return rte_flow_error_set(error, EINVAL, 2522 RTE_FLOW_ERROR_TYPE_ITEM, item, 2523 "L2 layer should not follow GTP"); 2524 if (!mask) 2525 mask = &rte_flow_item_eth_mask; 2526 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2527 (const uint8_t *)&nic_mask, 2528 sizeof(struct rte_flow_item_eth), 2529 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2530 return ret; 2531 } 2532 2533 /** 2534 * Validate VLAN item. 2535 * 2536 * @param[in] item 2537 * Item specification. 2538 * @param[in] item_flags 2539 * Bit-fields that holds the items detected until now. 2540 * @param[in] dev 2541 * Ethernet device flow is being created on. 2542 * @param[out] error 2543 * Pointer to error structure. 2544 * 2545 * @return 2546 * 0 on success, a negative errno value otherwise and rte_errno is set. 2547 */ 2548 int 2549 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2550 uint64_t item_flags, 2551 struct rte_eth_dev *dev, 2552 struct rte_flow_error *error) 2553 { 2554 const struct rte_flow_item_vlan *spec = item->spec; 2555 const struct rte_flow_item_vlan *mask = item->mask; 2556 const struct rte_flow_item_vlan nic_mask = { 2557 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2558 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2559 }; 2560 uint16_t vlan_tag = 0; 2561 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2562 int ret; 2563 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2564 MLX5_FLOW_LAYER_INNER_L4) : 2565 (MLX5_FLOW_LAYER_OUTER_L3 | 2566 MLX5_FLOW_LAYER_OUTER_L4); 2567 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2568 MLX5_FLOW_LAYER_OUTER_VLAN; 2569 2570 if (item_flags & vlanm) 2571 return rte_flow_error_set(error, EINVAL, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "multiple VLAN layers not supported"); 2574 else if ((item_flags & l34m) != 0) 2575 return rte_flow_error_set(error, EINVAL, 2576 RTE_FLOW_ERROR_TYPE_ITEM, item, 2577 "VLAN cannot follow L3/L4 layer"); 2578 if (!mask) 2579 mask = &rte_flow_item_vlan_mask; 2580 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2581 (const uint8_t *)&nic_mask, 2582 sizeof(struct rte_flow_item_vlan), 2583 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2584 if (ret) 2585 return ret; 2586 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2587 struct mlx5_priv *priv = dev->data->dev_private; 2588 2589 if (priv->vmwa_context) { 2590 /* 2591 * Non-NULL context means we have a virtual machine 2592 * and SR-IOV enabled, we have to create VLAN interface 2593 * to make hypervisor to setup E-Switch vport 2594 * context correctly. We avoid creating the multiple 2595 * VLAN interfaces, so we cannot support VLAN tag mask. 2596 */ 2597 return rte_flow_error_set(error, EINVAL, 2598 RTE_FLOW_ERROR_TYPE_ITEM, 2599 item, 2600 "VLAN tag mask is not" 2601 " supported in virtual" 2602 " environment"); 2603 } 2604 } 2605 if (spec) { 2606 vlan_tag = spec->hdr.vlan_tci; 2607 vlan_tag &= mask->hdr.vlan_tci; 2608 } 2609 /* 2610 * From verbs perspective an empty VLAN is equivalent 2611 * to a packet without VLAN layer. 2612 */ 2613 if (!vlan_tag) 2614 return rte_flow_error_set(error, EINVAL, 2615 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2616 item->spec, 2617 "VLAN cannot be empty"); 2618 return 0; 2619 } 2620 2621 /** 2622 * Validate IPV4 item. 2623 * 2624 * @param[in] item 2625 * Item specification. 2626 * @param[in] item_flags 2627 * Bit-fields that holds the items detected until now. 2628 * @param[in] last_item 2629 * Previous validated item in the pattern items. 2630 * @param[in] ether_type 2631 * Type in the ethernet layer header (including dot1q). 2632 * @param[in] acc_mask 2633 * Acceptable mask, if NULL default internal default mask 2634 * will be used to check whether item fields are supported. 2635 * @param[in] range_accepted 2636 * True if range of values is accepted for specific fields, false otherwise. 2637 * @param[out] error 2638 * Pointer to error structure. 2639 * 2640 * @return 2641 * 0 on success, a negative errno value otherwise and rte_errno is set. 2642 */ 2643 int 2644 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2645 uint64_t item_flags, 2646 uint64_t last_item, 2647 uint16_t ether_type, 2648 const struct rte_flow_item_ipv4 *acc_mask, 2649 bool range_accepted, 2650 struct rte_flow_error *error) 2651 { 2652 const struct rte_flow_item_ipv4 *mask = item->mask; 2653 const struct rte_flow_item_ipv4 *spec = item->spec; 2654 const struct rte_flow_item_ipv4 nic_mask = { 2655 .hdr = { 2656 .src_addr = RTE_BE32(0xffffffff), 2657 .dst_addr = RTE_BE32(0xffffffff), 2658 .type_of_service = 0xff, 2659 .next_proto_id = 0xff, 2660 }, 2661 }; 2662 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2663 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2664 MLX5_FLOW_LAYER_OUTER_L3; 2665 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2666 MLX5_FLOW_LAYER_OUTER_L4; 2667 int ret; 2668 uint8_t next_proto = 0xFF; 2669 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2670 MLX5_FLOW_LAYER_OUTER_VLAN | 2671 MLX5_FLOW_LAYER_INNER_VLAN); 2672 2673 if ((last_item & l2_vlan) && ether_type && 2674 ether_type != RTE_ETHER_TYPE_IPV4) 2675 return rte_flow_error_set(error, EINVAL, 2676 RTE_FLOW_ERROR_TYPE_ITEM, item, 2677 "IPv4 cannot follow L2/VLAN layer " 2678 "which ether type is not IPv4"); 2679 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2680 if (mask && spec) 2681 next_proto = mask->hdr.next_proto_id & 2682 spec->hdr.next_proto_id; 2683 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2684 return rte_flow_error_set(error, EINVAL, 2685 RTE_FLOW_ERROR_TYPE_ITEM, 2686 item, 2687 "multiple tunnel " 2688 "not supported"); 2689 } 2690 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2691 return rte_flow_error_set(error, EINVAL, 2692 RTE_FLOW_ERROR_TYPE_ITEM, item, 2693 "wrong tunnel type - IPv6 specified " 2694 "but IPv4 item provided"); 2695 if (item_flags & l3m) 2696 return rte_flow_error_set(error, ENOTSUP, 2697 RTE_FLOW_ERROR_TYPE_ITEM, item, 2698 "multiple L3 layers not supported"); 2699 else if (item_flags & l4m) 2700 return rte_flow_error_set(error, EINVAL, 2701 RTE_FLOW_ERROR_TYPE_ITEM, item, 2702 "L3 cannot follow an L4 layer."); 2703 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2704 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2705 return rte_flow_error_set(error, EINVAL, 2706 RTE_FLOW_ERROR_TYPE_ITEM, item, 2707 "L3 cannot follow an NVGRE layer."); 2708 if (!mask) 2709 mask = &rte_flow_item_ipv4_mask; 2710 else if (mask->hdr.next_proto_id != 0 && 2711 mask->hdr.next_proto_id != 0xff) 2712 return rte_flow_error_set(error, EINVAL, 2713 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2714 "partial mask is not supported" 2715 " for protocol"); 2716 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2717 acc_mask ? (const uint8_t *)acc_mask 2718 : (const uint8_t *)&nic_mask, 2719 sizeof(struct rte_flow_item_ipv4), 2720 range_accepted, error); 2721 if (ret < 0) 2722 return ret; 2723 return 0; 2724 } 2725 2726 /** 2727 * Validate IPV6 item. 2728 * 2729 * @param[in] item 2730 * Item specification. 2731 * @param[in] item_flags 2732 * Bit-fields that holds the items detected until now. 2733 * @param[in] last_item 2734 * Previous validated item in the pattern items. 2735 * @param[in] ether_type 2736 * Type in the ethernet layer header (including dot1q). 2737 * @param[in] acc_mask 2738 * Acceptable mask, if NULL default internal default mask 2739 * will be used to check whether item fields are supported. 2740 * @param[out] error 2741 * Pointer to error structure. 2742 * 2743 * @return 2744 * 0 on success, a negative errno value otherwise and rte_errno is set. 2745 */ 2746 int 2747 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2748 uint64_t item_flags, 2749 uint64_t last_item, 2750 uint16_t ether_type, 2751 const struct rte_flow_item_ipv6 *acc_mask, 2752 struct rte_flow_error *error) 2753 { 2754 const struct rte_flow_item_ipv6 *mask = item->mask; 2755 const struct rte_flow_item_ipv6 *spec = item->spec; 2756 const struct rte_flow_item_ipv6 nic_mask = { 2757 .hdr = { 2758 .src_addr = 2759 "\xff\xff\xff\xff\xff\xff\xff\xff" 2760 "\xff\xff\xff\xff\xff\xff\xff\xff", 2761 .dst_addr = 2762 "\xff\xff\xff\xff\xff\xff\xff\xff" 2763 "\xff\xff\xff\xff\xff\xff\xff\xff", 2764 .vtc_flow = RTE_BE32(0xffffffff), 2765 .proto = 0xff, 2766 }, 2767 }; 2768 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2769 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2770 MLX5_FLOW_LAYER_OUTER_L3; 2771 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2772 MLX5_FLOW_LAYER_OUTER_L4; 2773 int ret; 2774 uint8_t next_proto = 0xFF; 2775 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2776 MLX5_FLOW_LAYER_OUTER_VLAN | 2777 MLX5_FLOW_LAYER_INNER_VLAN); 2778 2779 if ((last_item & l2_vlan) && ether_type && 2780 ether_type != RTE_ETHER_TYPE_IPV6) 2781 return rte_flow_error_set(error, EINVAL, 2782 RTE_FLOW_ERROR_TYPE_ITEM, item, 2783 "IPv6 cannot follow L2/VLAN layer " 2784 "which ether type is not IPv6"); 2785 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2786 next_proto = spec->hdr.proto; 2787 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2788 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2789 return rte_flow_error_set(error, EINVAL, 2790 RTE_FLOW_ERROR_TYPE_ITEM, 2791 item, 2792 "multiple tunnel " 2793 "not supported"); 2794 } 2795 if (next_proto == IPPROTO_HOPOPTS || 2796 next_proto == IPPROTO_ROUTING || 2797 next_proto == IPPROTO_FRAGMENT || 2798 next_proto == IPPROTO_ESP || 2799 next_proto == IPPROTO_AH || 2800 next_proto == IPPROTO_DSTOPTS) 2801 return rte_flow_error_set(error, EINVAL, 2802 RTE_FLOW_ERROR_TYPE_ITEM, item, 2803 "IPv6 proto (next header) should " 2804 "not be set as extension header"); 2805 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2806 return rte_flow_error_set(error, EINVAL, 2807 RTE_FLOW_ERROR_TYPE_ITEM, item, 2808 "wrong tunnel type - IPv4 specified " 2809 "but IPv6 item provided"); 2810 if (item_flags & l3m) 2811 return rte_flow_error_set(error, ENOTSUP, 2812 RTE_FLOW_ERROR_TYPE_ITEM, item, 2813 "multiple L3 layers not supported"); 2814 else if (item_flags & l4m) 2815 return rte_flow_error_set(error, EINVAL, 2816 RTE_FLOW_ERROR_TYPE_ITEM, item, 2817 "L3 cannot follow an L4 layer."); 2818 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2819 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2820 return rte_flow_error_set(error, EINVAL, 2821 RTE_FLOW_ERROR_TYPE_ITEM, item, 2822 "L3 cannot follow an NVGRE layer."); 2823 if (!mask) 2824 mask = &rte_flow_item_ipv6_mask; 2825 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2826 acc_mask ? (const uint8_t *)acc_mask 2827 : (const uint8_t *)&nic_mask, 2828 sizeof(struct rte_flow_item_ipv6), 2829 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2830 if (ret < 0) 2831 return ret; 2832 return 0; 2833 } 2834 2835 /** 2836 * Validate UDP item. 2837 * 2838 * @param[in] item 2839 * Item specification. 2840 * @param[in] item_flags 2841 * Bit-fields that holds the items detected until now. 2842 * @param[in] target_protocol 2843 * The next protocol in the previous item. 2844 * @param[in] flow_mask 2845 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2846 * @param[out] error 2847 * Pointer to error structure. 2848 * 2849 * @return 2850 * 0 on success, a negative errno value otherwise and rte_errno is set. 2851 */ 2852 int 2853 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2854 uint64_t item_flags, 2855 uint8_t target_protocol, 2856 struct rte_flow_error *error) 2857 { 2858 const struct rte_flow_item_udp *mask = item->mask; 2859 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2860 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2861 MLX5_FLOW_LAYER_OUTER_L3; 2862 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2863 MLX5_FLOW_LAYER_OUTER_L4; 2864 int ret; 2865 2866 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2867 return rte_flow_error_set(error, EINVAL, 2868 RTE_FLOW_ERROR_TYPE_ITEM, item, 2869 "protocol filtering not compatible" 2870 " with UDP layer"); 2871 if (!(item_flags & l3m)) 2872 return rte_flow_error_set(error, EINVAL, 2873 RTE_FLOW_ERROR_TYPE_ITEM, item, 2874 "L3 is mandatory to filter on L4"); 2875 if (item_flags & l4m) 2876 return rte_flow_error_set(error, EINVAL, 2877 RTE_FLOW_ERROR_TYPE_ITEM, item, 2878 "multiple L4 layers not supported"); 2879 if (!mask) 2880 mask = &rte_flow_item_udp_mask; 2881 ret = mlx5_flow_item_acceptable 2882 (item, (const uint8_t *)mask, 2883 (const uint8_t *)&rte_flow_item_udp_mask, 2884 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2885 error); 2886 if (ret < 0) 2887 return ret; 2888 return 0; 2889 } 2890 2891 /** 2892 * Validate TCP item. 2893 * 2894 * @param[in] item 2895 * Item specification. 2896 * @param[in] item_flags 2897 * Bit-fields that holds the items detected until now. 2898 * @param[in] target_protocol 2899 * The next protocol in the previous item. 2900 * @param[out] error 2901 * Pointer to error structure. 2902 * 2903 * @return 2904 * 0 on success, a negative errno value otherwise and rte_errno is set. 2905 */ 2906 int 2907 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2908 uint64_t item_flags, 2909 uint8_t target_protocol, 2910 const struct rte_flow_item_tcp *flow_mask, 2911 struct rte_flow_error *error) 2912 { 2913 const struct rte_flow_item_tcp *mask = item->mask; 2914 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2915 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2916 MLX5_FLOW_LAYER_OUTER_L3; 2917 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2918 MLX5_FLOW_LAYER_OUTER_L4; 2919 int ret; 2920 2921 MLX5_ASSERT(flow_mask); 2922 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2923 return rte_flow_error_set(error, EINVAL, 2924 RTE_FLOW_ERROR_TYPE_ITEM, item, 2925 "protocol filtering not compatible" 2926 " with TCP layer"); 2927 if (!(item_flags & l3m)) 2928 return rte_flow_error_set(error, EINVAL, 2929 RTE_FLOW_ERROR_TYPE_ITEM, item, 2930 "L3 is mandatory to filter on L4"); 2931 if (item_flags & l4m) 2932 return rte_flow_error_set(error, EINVAL, 2933 RTE_FLOW_ERROR_TYPE_ITEM, item, 2934 "multiple L4 layers not supported"); 2935 if (!mask) 2936 mask = &rte_flow_item_tcp_mask; 2937 ret = mlx5_flow_item_acceptable 2938 (item, (const uint8_t *)mask, 2939 (const uint8_t *)flow_mask, 2940 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2941 error); 2942 if (ret < 0) 2943 return ret; 2944 return 0; 2945 } 2946 2947 /** 2948 * Validate VXLAN item. 2949 * 2950 * @param[in] dev 2951 * Pointer to the Ethernet device structure. 2952 * @param[in] udp_dport 2953 * UDP destination port 2954 * @param[in] item 2955 * Item specification. 2956 * @param[in] item_flags 2957 * Bit-fields that holds the items detected until now. 2958 * @param root 2959 * Whether action is on root table. 2960 * @param[out] error 2961 * Pointer to error structure. 2962 * 2963 * @return 2964 * 0 on success, a negative errno value otherwise and rte_errno is set. 2965 */ 2966 int 2967 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2968 uint16_t udp_dport, 2969 const struct rte_flow_item *item, 2970 uint64_t item_flags, 2971 bool root, 2972 struct rte_flow_error *error) 2973 { 2974 const struct rte_flow_item_vxlan *spec = item->spec; 2975 const struct rte_flow_item_vxlan *mask = item->mask; 2976 int ret; 2977 struct mlx5_priv *priv = dev->data->dev_private; 2978 union vni { 2979 uint32_t vlan_id; 2980 uint8_t vni[4]; 2981 } id = { .vlan_id = 0, }; 2982 const struct rte_flow_item_vxlan nic_mask = { 2983 .hdr.vni = "\xff\xff\xff", 2984 .hdr.rsvd1 = 0xff, 2985 }; 2986 const struct rte_flow_item_vxlan *valid_mask; 2987 2988 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 2989 return rte_flow_error_set(error, ENOTSUP, 2990 RTE_FLOW_ERROR_TYPE_ITEM, item, 2991 "multiple tunnel layers not" 2992 " supported"); 2993 valid_mask = &rte_flow_item_vxlan_mask; 2994 /* 2995 * Verify only UDPv4 is present as defined in 2996 * https://tools.ietf.org/html/rfc7348 2997 */ 2998 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 2999 return rte_flow_error_set(error, EINVAL, 3000 RTE_FLOW_ERROR_TYPE_ITEM, item, 3001 "no outer UDP layer found"); 3002 if (!mask) 3003 mask = &rte_flow_item_vxlan_mask; 3004 3005 if (priv->sh->steering_format_version != 3006 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3007 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3008 /* non-root table */ 3009 if (!root && priv->sh->misc5_cap) 3010 valid_mask = &nic_mask; 3011 /* Group zero in NIC domain */ 3012 if (!root && priv->sh->tunnel_header_0_1) 3013 valid_mask = &nic_mask; 3014 } 3015 ret = mlx5_flow_item_acceptable 3016 (item, (const uint8_t *)mask, 3017 (const uint8_t *)valid_mask, 3018 sizeof(struct rte_flow_item_vxlan), 3019 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3020 if (ret < 0) 3021 return ret; 3022 if (spec) { 3023 memcpy(&id.vni[1], spec->hdr.vni, 3); 3024 memcpy(&id.vni[1], mask->hdr.vni, 3); 3025 } 3026 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3027 return rte_flow_error_set(error, ENOTSUP, 3028 RTE_FLOW_ERROR_TYPE_ITEM, item, 3029 "VXLAN tunnel must be fully defined"); 3030 return 0; 3031 } 3032 3033 /** 3034 * Validate VXLAN_GPE item. 3035 * 3036 * @param[in] item 3037 * Item specification. 3038 * @param[in] item_flags 3039 * Bit-fields that holds the items detected until now. 3040 * @param[in] priv 3041 * Pointer to the private data structure. 3042 * @param[in] target_protocol 3043 * The next protocol in the previous item. 3044 * @param[out] error 3045 * Pointer to error structure. 3046 * 3047 * @return 3048 * 0 on success, a negative errno value otherwise and rte_errno is set. 3049 */ 3050 int 3051 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3052 uint64_t item_flags, 3053 struct rte_eth_dev *dev, 3054 struct rte_flow_error *error) 3055 { 3056 struct mlx5_priv *priv = dev->data->dev_private; 3057 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3058 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3059 int ret; 3060 union vni { 3061 uint32_t vlan_id; 3062 uint8_t vni[4]; 3063 } id = { .vlan_id = 0, }; 3064 3065 if (!priv->sh->config.l3_vxlan_en) 3066 return rte_flow_error_set(error, ENOTSUP, 3067 RTE_FLOW_ERROR_TYPE_ITEM, item, 3068 "L3 VXLAN is not enabled by device" 3069 " parameter and/or not configured in" 3070 " firmware"); 3071 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3072 return rte_flow_error_set(error, ENOTSUP, 3073 RTE_FLOW_ERROR_TYPE_ITEM, item, 3074 "multiple tunnel layers not" 3075 " supported"); 3076 /* 3077 * Verify only UDPv4 is present as defined in 3078 * https://tools.ietf.org/html/rfc7348 3079 */ 3080 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3081 return rte_flow_error_set(error, EINVAL, 3082 RTE_FLOW_ERROR_TYPE_ITEM, item, 3083 "no outer UDP layer found"); 3084 if (!mask) 3085 mask = &rte_flow_item_vxlan_gpe_mask; 3086 ret = mlx5_flow_item_acceptable 3087 (item, (const uint8_t *)mask, 3088 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 3089 sizeof(struct rte_flow_item_vxlan_gpe), 3090 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3091 if (ret < 0) 3092 return ret; 3093 if (spec) { 3094 if (spec->hdr.proto) 3095 return rte_flow_error_set(error, ENOTSUP, 3096 RTE_FLOW_ERROR_TYPE_ITEM, 3097 item, 3098 "VxLAN-GPE protocol" 3099 " not supported"); 3100 memcpy(&id.vni[1], spec->hdr.vni, 3); 3101 memcpy(&id.vni[1], mask->hdr.vni, 3); 3102 } 3103 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3104 return rte_flow_error_set(error, ENOTSUP, 3105 RTE_FLOW_ERROR_TYPE_ITEM, item, 3106 "VXLAN-GPE tunnel must be fully" 3107 " defined"); 3108 return 0; 3109 } 3110 /** 3111 * Validate GRE Key item. 3112 * 3113 * @param[in] item 3114 * Item specification. 3115 * @param[in] item_flags 3116 * Bit flags to mark detected items. 3117 * @param[in] gre_item 3118 * Pointer to gre_item 3119 * @param[out] error 3120 * Pointer to error structure. 3121 * 3122 * @return 3123 * 0 on success, a negative errno value otherwise and rte_errno is set. 3124 */ 3125 int 3126 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3127 uint64_t item_flags, 3128 const struct rte_flow_item *gre_item, 3129 struct rte_flow_error *error) 3130 { 3131 const rte_be32_t *mask = item->mask; 3132 int ret = 0; 3133 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3134 const struct rte_flow_item_gre *gre_spec; 3135 const struct rte_flow_item_gre *gre_mask; 3136 3137 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3138 return rte_flow_error_set(error, ENOTSUP, 3139 RTE_FLOW_ERROR_TYPE_ITEM, item, 3140 "Multiple GRE key not support"); 3141 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3142 return rte_flow_error_set(error, ENOTSUP, 3143 RTE_FLOW_ERROR_TYPE_ITEM, item, 3144 "No preceding GRE header"); 3145 if (item_flags & MLX5_FLOW_LAYER_INNER) 3146 return rte_flow_error_set(error, ENOTSUP, 3147 RTE_FLOW_ERROR_TYPE_ITEM, item, 3148 "GRE key following a wrong item"); 3149 gre_mask = gre_item->mask; 3150 if (!gre_mask) 3151 gre_mask = &rte_flow_item_gre_mask; 3152 gre_spec = gre_item->spec; 3153 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3154 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3155 return rte_flow_error_set(error, EINVAL, 3156 RTE_FLOW_ERROR_TYPE_ITEM, item, 3157 "Key bit must be on"); 3158 3159 if (!mask) 3160 mask = &gre_key_default_mask; 3161 ret = mlx5_flow_item_acceptable 3162 (item, (const uint8_t *)mask, 3163 (const uint8_t *)&gre_key_default_mask, 3164 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3165 return ret; 3166 } 3167 3168 /** 3169 * Validate GRE optional item. 3170 * 3171 * @param[in] dev 3172 * Pointer to the Ethernet device structure. 3173 * @param[in] item 3174 * Item specification. 3175 * @param[in] item_flags 3176 * Bit flags to mark detected items. 3177 * @param[in] attr 3178 * Flow rule attributes. 3179 * @param[in] gre_item 3180 * Pointer to gre_item 3181 * @param[out] error 3182 * Pointer to error structure. 3183 * 3184 * @return 3185 * 0 on success, a negative errno value otherwise and rte_errno is set. 3186 */ 3187 int 3188 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3189 const struct rte_flow_item *item, 3190 uint64_t item_flags, 3191 const struct rte_flow_attr *attr, 3192 const struct rte_flow_item *gre_item, 3193 struct rte_flow_error *error) 3194 { 3195 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3196 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3197 const struct rte_flow_item_gre_opt *spec = item->spec; 3198 const struct rte_flow_item_gre_opt *mask = item->mask; 3199 struct mlx5_priv *priv = dev->data->dev_private; 3200 int ret = 0; 3201 struct rte_flow_item_gre_opt nic_mask = { 3202 .checksum_rsvd = { 3203 .checksum = RTE_BE16(UINT16_MAX), 3204 .reserved1 = 0x0, 3205 }, 3206 .key = { 3207 .key = RTE_BE32(UINT32_MAX), 3208 }, 3209 .sequence = { 3210 .sequence = RTE_BE32(UINT32_MAX), 3211 }, 3212 }; 3213 3214 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3215 return rte_flow_error_set(error, ENOTSUP, 3216 RTE_FLOW_ERROR_TYPE_ITEM, item, 3217 "No preceding GRE header"); 3218 if (item_flags & MLX5_FLOW_LAYER_INNER) 3219 return rte_flow_error_set(error, ENOTSUP, 3220 RTE_FLOW_ERROR_TYPE_ITEM, item, 3221 "GRE option following a wrong item"); 3222 if (!spec || !mask) 3223 return rte_flow_error_set(error, EINVAL, 3224 RTE_FLOW_ERROR_TYPE_ITEM, item, 3225 "At least one field gre_option(checksum/key/sequence) must be specified"); 3226 if (!gre_mask) 3227 gre_mask = &rte_flow_item_gre_mask; 3228 if (mask->checksum_rsvd.checksum) 3229 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3230 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3231 return rte_flow_error_set(error, EINVAL, 3232 RTE_FLOW_ERROR_TYPE_ITEM, 3233 item, 3234 "Checksum bit must be on"); 3235 if (mask->key.key) 3236 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3237 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3238 return rte_flow_error_set(error, EINVAL, 3239 RTE_FLOW_ERROR_TYPE_ITEM, 3240 item, "Key bit must be on"); 3241 if (mask->sequence.sequence) 3242 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3243 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3244 return rte_flow_error_set(error, EINVAL, 3245 RTE_FLOW_ERROR_TYPE_ITEM, 3246 item, 3247 "Sequence bit must be on"); 3248 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3249 if (priv->sh->steering_format_version == 3250 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3251 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3252 !priv->sh->misc5_cap) || 3253 (!(priv->sh->tunnel_header_0_1 && 3254 priv->sh->tunnel_header_2_3) && 3255 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3256 return rte_flow_error_set(error, EINVAL, 3257 RTE_FLOW_ERROR_TYPE_ITEM, 3258 item, 3259 "Checksum/Sequence not supported"); 3260 } 3261 ret = mlx5_flow_item_acceptable 3262 (item, (const uint8_t *)mask, 3263 (const uint8_t *)&nic_mask, 3264 sizeof(struct rte_flow_item_gre_opt), 3265 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3266 return ret; 3267 } 3268 3269 /** 3270 * Validate GRE item. 3271 * 3272 * @param[in] item 3273 * Item specification. 3274 * @param[in] item_flags 3275 * Bit flags to mark detected items. 3276 * @param[in] target_protocol 3277 * The next protocol in the previous item. 3278 * @param[out] error 3279 * Pointer to error structure. 3280 * 3281 * @return 3282 * 0 on success, a negative errno value otherwise and rte_errno is set. 3283 */ 3284 int 3285 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3286 uint64_t item_flags, 3287 uint8_t target_protocol, 3288 struct rte_flow_error *error) 3289 { 3290 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3291 const struct rte_flow_item_gre *mask = item->mask; 3292 int ret; 3293 const struct rte_flow_item_gre nic_mask = { 3294 .c_rsvd0_ver = RTE_BE16(0xB000), 3295 .protocol = RTE_BE16(UINT16_MAX), 3296 }; 3297 3298 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3299 return rte_flow_error_set(error, EINVAL, 3300 RTE_FLOW_ERROR_TYPE_ITEM, item, 3301 "protocol filtering not compatible" 3302 " with this GRE layer"); 3303 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3304 return rte_flow_error_set(error, ENOTSUP, 3305 RTE_FLOW_ERROR_TYPE_ITEM, item, 3306 "multiple tunnel layers not" 3307 " supported"); 3308 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3309 return rte_flow_error_set(error, ENOTSUP, 3310 RTE_FLOW_ERROR_TYPE_ITEM, item, 3311 "L3 Layer is missing"); 3312 if (!mask) 3313 mask = &rte_flow_item_gre_mask; 3314 ret = mlx5_flow_item_acceptable 3315 (item, (const uint8_t *)mask, 3316 (const uint8_t *)&nic_mask, 3317 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3318 error); 3319 if (ret < 0) 3320 return ret; 3321 #ifndef HAVE_MLX5DV_DR 3322 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3323 if (spec && (spec->protocol & mask->protocol)) 3324 return rte_flow_error_set(error, ENOTSUP, 3325 RTE_FLOW_ERROR_TYPE_ITEM, item, 3326 "without MPLS support the" 3327 " specification cannot be used for" 3328 " filtering"); 3329 #endif 3330 #endif 3331 return 0; 3332 } 3333 3334 /** 3335 * Validate Geneve item. 3336 * 3337 * @param[in] item 3338 * Item specification. 3339 * @param[in] itemFlags 3340 * Bit-fields that holds the items detected until now. 3341 * @param[in] enPriv 3342 * Pointer to the private data structure. 3343 * @param[out] error 3344 * Pointer to error structure. 3345 * 3346 * @return 3347 * 0 on success, a negative errno value otherwise and rte_errno is set. 3348 */ 3349 3350 int 3351 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3352 uint64_t item_flags, 3353 struct rte_eth_dev *dev, 3354 struct rte_flow_error *error) 3355 { 3356 struct mlx5_priv *priv = dev->data->dev_private; 3357 const struct rte_flow_item_geneve *spec = item->spec; 3358 const struct rte_flow_item_geneve *mask = item->mask; 3359 int ret; 3360 uint16_t gbhdr; 3361 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3362 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3363 const struct rte_flow_item_geneve nic_mask = { 3364 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3365 .vni = "\xff\xff\xff", 3366 .protocol = RTE_BE16(UINT16_MAX), 3367 }; 3368 3369 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3370 return rte_flow_error_set(error, ENOTSUP, 3371 RTE_FLOW_ERROR_TYPE_ITEM, item, 3372 "L3 Geneve is not enabled by device" 3373 " parameter and/or not configured in" 3374 " firmware"); 3375 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3376 return rte_flow_error_set(error, ENOTSUP, 3377 RTE_FLOW_ERROR_TYPE_ITEM, item, 3378 "multiple tunnel layers not" 3379 " supported"); 3380 /* 3381 * Verify only UDPv4 is present as defined in 3382 * https://tools.ietf.org/html/rfc7348 3383 */ 3384 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3385 return rte_flow_error_set(error, EINVAL, 3386 RTE_FLOW_ERROR_TYPE_ITEM, item, 3387 "no outer UDP layer found"); 3388 if (!mask) 3389 mask = &rte_flow_item_geneve_mask; 3390 ret = mlx5_flow_item_acceptable 3391 (item, (const uint8_t *)mask, 3392 (const uint8_t *)&nic_mask, 3393 sizeof(struct rte_flow_item_geneve), 3394 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3395 if (ret) 3396 return ret; 3397 if (spec) { 3398 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3399 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3400 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3401 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3402 return rte_flow_error_set(error, ENOTSUP, 3403 RTE_FLOW_ERROR_TYPE_ITEM, 3404 item, 3405 "Geneve protocol unsupported" 3406 " fields are being used"); 3407 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3408 return rte_flow_error_set 3409 (error, ENOTSUP, 3410 RTE_FLOW_ERROR_TYPE_ITEM, 3411 item, 3412 "Unsupported Geneve options length"); 3413 } 3414 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3415 return rte_flow_error_set 3416 (error, ENOTSUP, 3417 RTE_FLOW_ERROR_TYPE_ITEM, item, 3418 "Geneve tunnel must be fully defined"); 3419 return 0; 3420 } 3421 3422 /** 3423 * Validate Geneve TLV option item. 3424 * 3425 * @param[in] item 3426 * Item specification. 3427 * @param[in] last_item 3428 * Previous validated item in the pattern items. 3429 * @param[in] geneve_item 3430 * Previous GENEVE item specification. 3431 * @param[in] dev 3432 * Pointer to the rte_eth_dev structure. 3433 * @param[out] error 3434 * Pointer to error structure. 3435 * 3436 * @return 3437 * 0 on success, a negative errno value otherwise and rte_errno is set. 3438 */ 3439 int 3440 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3441 uint64_t last_item, 3442 const struct rte_flow_item *geneve_item, 3443 struct rte_eth_dev *dev, 3444 struct rte_flow_error *error) 3445 { 3446 struct mlx5_priv *priv = dev->data->dev_private; 3447 struct mlx5_dev_ctx_shared *sh = priv->sh; 3448 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3449 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3450 uint8_t data_max_supported = 3451 hca_attr->max_geneve_tlv_option_data_len * 4; 3452 const struct rte_flow_item_geneve *geneve_spec; 3453 const struct rte_flow_item_geneve *geneve_mask; 3454 const struct rte_flow_item_geneve_opt *spec = item->spec; 3455 const struct rte_flow_item_geneve_opt *mask = item->mask; 3456 unsigned int i; 3457 unsigned int data_len; 3458 uint8_t tlv_option_len; 3459 uint16_t optlen_m, optlen_v; 3460 const struct rte_flow_item_geneve_opt full_mask = { 3461 .option_class = RTE_BE16(0xffff), 3462 .option_type = 0xff, 3463 .option_len = 0x1f, 3464 }; 3465 3466 if (!mask) 3467 mask = &rte_flow_item_geneve_opt_mask; 3468 if (!spec) 3469 return rte_flow_error_set 3470 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3471 "Geneve TLV opt class/type/length must be specified"); 3472 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3473 return rte_flow_error_set 3474 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3475 "Geneve TLV opt length exceeds the limit (31)"); 3476 /* Check if class type and length masks are full. */ 3477 if (full_mask.option_class != mask->option_class || 3478 full_mask.option_type != mask->option_type || 3479 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3480 return rte_flow_error_set 3481 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3482 "Geneve TLV opt class/type/length masks must be full"); 3483 /* Check if length is supported */ 3484 if ((uint32_t)spec->option_len > 3485 hca_attr->max_geneve_tlv_option_data_len) 3486 return rte_flow_error_set 3487 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3488 "Geneve TLV opt length not supported"); 3489 if (hca_attr->max_geneve_tlv_options > 1) 3490 DRV_LOG(DEBUG, 3491 "max_geneve_tlv_options supports more than 1 option"); 3492 /* Check GENEVE item preceding. */ 3493 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3494 return rte_flow_error_set 3495 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3496 "Geneve opt item must be preceded with Geneve item"); 3497 geneve_spec = geneve_item->spec; 3498 geneve_mask = geneve_item->mask ? geneve_item->mask : 3499 &rte_flow_item_geneve_mask; 3500 /* Check if GENEVE TLV option size doesn't exceed option length */ 3501 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3502 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3503 tlv_option_len = spec->option_len & mask->option_len; 3504 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3505 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3506 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3507 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3508 if ((optlen_v & optlen_m) <= tlv_option_len) 3509 return rte_flow_error_set 3510 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3511 "GENEVE TLV option length exceeds optlen"); 3512 } 3513 /* Check if length is 0 or data is 0. */ 3514 if (spec->data == NULL || spec->option_len == 0) 3515 return rte_flow_error_set 3516 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3517 "Geneve TLV opt with zero data/length not supported"); 3518 /* Check not all data & mask are 0. */ 3519 data_len = spec->option_len * 4; 3520 if (mask->data == NULL) { 3521 for (i = 0; i < data_len; i++) 3522 if (spec->data[i]) 3523 break; 3524 if (i == data_len) 3525 return rte_flow_error_set(error, ENOTSUP, 3526 RTE_FLOW_ERROR_TYPE_ITEM, item, 3527 "Can't match on Geneve option data 0"); 3528 } else { 3529 for (i = 0; i < data_len; i++) 3530 if (spec->data[i] & mask->data[i]) 3531 break; 3532 if (i == data_len) 3533 return rte_flow_error_set(error, ENOTSUP, 3534 RTE_FLOW_ERROR_TYPE_ITEM, item, 3535 "Can't match on Geneve option data and mask 0"); 3536 /* Check data mask supported. */ 3537 for (i = data_max_supported; i < data_len ; i++) 3538 if (mask->data[i]) 3539 return rte_flow_error_set(error, ENOTSUP, 3540 RTE_FLOW_ERROR_TYPE_ITEM, item, 3541 "Data mask is of unsupported size"); 3542 } 3543 /* Check GENEVE option is supported in NIC. */ 3544 if (!hca_attr->geneve_tlv_opt) 3545 return rte_flow_error_set 3546 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3547 "Geneve TLV opt not supported"); 3548 /* Check if we already have geneve option with different type/class. */ 3549 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3550 geneve_opt_resource = sh->geneve_tlv_option_resource; 3551 if (geneve_opt_resource != NULL) 3552 if (geneve_opt_resource->option_class != spec->option_class || 3553 geneve_opt_resource->option_type != spec->option_type || 3554 geneve_opt_resource->length != spec->option_len) { 3555 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3556 return rte_flow_error_set(error, ENOTSUP, 3557 RTE_FLOW_ERROR_TYPE_ITEM, item, 3558 "Only one Geneve TLV option supported"); 3559 } 3560 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3561 return 0; 3562 } 3563 3564 /** 3565 * Validate MPLS item. 3566 * 3567 * @param[in] dev 3568 * Pointer to the rte_eth_dev structure. 3569 * @param[in] item 3570 * Item specification. 3571 * @param[in] item_flags 3572 * Bit-fields that holds the items detected until now. 3573 * @param[in] prev_layer 3574 * The protocol layer indicated in previous item. 3575 * @param[out] error 3576 * Pointer to error structure. 3577 * 3578 * @return 3579 * 0 on success, a negative errno value otherwise and rte_errno is set. 3580 */ 3581 int 3582 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3583 const struct rte_flow_item *item __rte_unused, 3584 uint64_t item_flags __rte_unused, 3585 uint64_t prev_layer __rte_unused, 3586 struct rte_flow_error *error) 3587 { 3588 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3589 const struct rte_flow_item_mpls *mask = item->mask; 3590 struct mlx5_priv *priv = dev->data->dev_private; 3591 int ret; 3592 3593 if (!priv->sh->dev_cap.mpls_en) 3594 return rte_flow_error_set(error, ENOTSUP, 3595 RTE_FLOW_ERROR_TYPE_ITEM, item, 3596 "MPLS not supported or" 3597 " disabled in firmware" 3598 " configuration."); 3599 /* MPLS over UDP, GRE is allowed */ 3600 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3601 MLX5_FLOW_LAYER_GRE | 3602 MLX5_FLOW_LAYER_GRE_KEY))) 3603 return rte_flow_error_set(error, EINVAL, 3604 RTE_FLOW_ERROR_TYPE_ITEM, item, 3605 "protocol filtering not compatible" 3606 " with MPLS layer"); 3607 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3608 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3609 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3610 return rte_flow_error_set(error, ENOTSUP, 3611 RTE_FLOW_ERROR_TYPE_ITEM, item, 3612 "multiple tunnel layers not" 3613 " supported"); 3614 if (!mask) 3615 mask = &rte_flow_item_mpls_mask; 3616 ret = mlx5_flow_item_acceptable 3617 (item, (const uint8_t *)mask, 3618 (const uint8_t *)&rte_flow_item_mpls_mask, 3619 sizeof(struct rte_flow_item_mpls), 3620 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3621 if (ret < 0) 3622 return ret; 3623 return 0; 3624 #else 3625 return rte_flow_error_set(error, ENOTSUP, 3626 RTE_FLOW_ERROR_TYPE_ITEM, item, 3627 "MPLS is not supported by Verbs, please" 3628 " update."); 3629 #endif 3630 } 3631 3632 /** 3633 * Validate NVGRE item. 3634 * 3635 * @param[in] item 3636 * Item specification. 3637 * @param[in] item_flags 3638 * Bit flags to mark detected items. 3639 * @param[in] target_protocol 3640 * The next protocol in the previous item. 3641 * @param[out] error 3642 * Pointer to error structure. 3643 * 3644 * @return 3645 * 0 on success, a negative errno value otherwise and rte_errno is set. 3646 */ 3647 int 3648 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3649 uint64_t item_flags, 3650 uint8_t target_protocol, 3651 struct rte_flow_error *error) 3652 { 3653 const struct rte_flow_item_nvgre *mask = item->mask; 3654 int ret; 3655 3656 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3657 return rte_flow_error_set(error, EINVAL, 3658 RTE_FLOW_ERROR_TYPE_ITEM, item, 3659 "protocol filtering not compatible" 3660 " with this GRE layer"); 3661 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3662 return rte_flow_error_set(error, ENOTSUP, 3663 RTE_FLOW_ERROR_TYPE_ITEM, item, 3664 "multiple tunnel layers not" 3665 " supported"); 3666 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3667 return rte_flow_error_set(error, ENOTSUP, 3668 RTE_FLOW_ERROR_TYPE_ITEM, item, 3669 "L3 Layer is missing"); 3670 if (!mask) 3671 mask = &rte_flow_item_nvgre_mask; 3672 ret = mlx5_flow_item_acceptable 3673 (item, (const uint8_t *)mask, 3674 (const uint8_t *)&rte_flow_item_nvgre_mask, 3675 sizeof(struct rte_flow_item_nvgre), 3676 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3677 if (ret < 0) 3678 return ret; 3679 return 0; 3680 } 3681 3682 /** 3683 * Validate eCPRI item. 3684 * 3685 * @param[in] item 3686 * Item specification. 3687 * @param[in] item_flags 3688 * Bit-fields that holds the items detected until now. 3689 * @param[in] last_item 3690 * Previous validated item in the pattern items. 3691 * @param[in] ether_type 3692 * Type in the ethernet layer header (including dot1q). 3693 * @param[in] acc_mask 3694 * Acceptable mask, if NULL default internal default mask 3695 * will be used to check whether item fields are supported. 3696 * @param[out] error 3697 * Pointer to error structure. 3698 * 3699 * @return 3700 * 0 on success, a negative errno value otherwise and rte_errno is set. 3701 */ 3702 int 3703 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3704 uint64_t item_flags, 3705 uint64_t last_item, 3706 uint16_t ether_type, 3707 const struct rte_flow_item_ecpri *acc_mask, 3708 struct rte_flow_error *error) 3709 { 3710 const struct rte_flow_item_ecpri *mask = item->mask; 3711 const struct rte_flow_item_ecpri nic_mask = { 3712 .hdr = { 3713 .common = { 3714 .u32 = 3715 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3716 .type = 0xFF, 3717 }).u32), 3718 }, 3719 .dummy[0] = 0xFFFFFFFF, 3720 }, 3721 }; 3722 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3723 MLX5_FLOW_LAYER_OUTER_VLAN); 3724 struct rte_flow_item_ecpri mask_lo; 3725 3726 if (!(last_item & outer_l2_vlan) && 3727 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3728 return rte_flow_error_set(error, EINVAL, 3729 RTE_FLOW_ERROR_TYPE_ITEM, item, 3730 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3731 if ((last_item & outer_l2_vlan) && ether_type && 3732 ether_type != RTE_ETHER_TYPE_ECPRI) 3733 return rte_flow_error_set(error, EINVAL, 3734 RTE_FLOW_ERROR_TYPE_ITEM, item, 3735 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3736 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3737 return rte_flow_error_set(error, EINVAL, 3738 RTE_FLOW_ERROR_TYPE_ITEM, item, 3739 "eCPRI with tunnel is not supported right now"); 3740 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3741 return rte_flow_error_set(error, ENOTSUP, 3742 RTE_FLOW_ERROR_TYPE_ITEM, item, 3743 "multiple L3 layers not supported"); 3744 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3745 return rte_flow_error_set(error, EINVAL, 3746 RTE_FLOW_ERROR_TYPE_ITEM, item, 3747 "eCPRI cannot coexist with a TCP layer"); 3748 /* In specification, eCPRI could be over UDP layer. */ 3749 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3750 return rte_flow_error_set(error, EINVAL, 3751 RTE_FLOW_ERROR_TYPE_ITEM, item, 3752 "eCPRI over UDP layer is not yet supported right now"); 3753 /* Mask for type field in common header could be zero. */ 3754 if (!mask) 3755 mask = &rte_flow_item_ecpri_mask; 3756 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3757 /* Input mask is in big-endian format. */ 3758 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3759 return rte_flow_error_set(error, EINVAL, 3760 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3761 "partial mask is not supported for protocol"); 3762 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3763 return rte_flow_error_set(error, EINVAL, 3764 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3765 "message header mask must be after a type mask"); 3766 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3767 acc_mask ? (const uint8_t *)acc_mask 3768 : (const uint8_t *)&nic_mask, 3769 sizeof(struct rte_flow_item_ecpri), 3770 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3771 } 3772 3773 static int 3774 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3775 const struct rte_flow_attr *attr __rte_unused, 3776 const struct rte_flow_item items[] __rte_unused, 3777 const struct rte_flow_action actions[] __rte_unused, 3778 bool external __rte_unused, 3779 int hairpin __rte_unused, 3780 struct rte_flow_error *error) 3781 { 3782 return rte_flow_error_set(error, ENOTSUP, 3783 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3784 } 3785 3786 static struct mlx5_flow * 3787 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3788 const struct rte_flow_attr *attr __rte_unused, 3789 const struct rte_flow_item items[] __rte_unused, 3790 const struct rte_flow_action actions[] __rte_unused, 3791 struct rte_flow_error *error) 3792 { 3793 rte_flow_error_set(error, ENOTSUP, 3794 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3795 return NULL; 3796 } 3797 3798 static int 3799 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3800 struct mlx5_flow *dev_flow __rte_unused, 3801 const struct rte_flow_attr *attr __rte_unused, 3802 const struct rte_flow_item items[] __rte_unused, 3803 const struct rte_flow_action actions[] __rte_unused, 3804 struct rte_flow_error *error) 3805 { 3806 return rte_flow_error_set(error, ENOTSUP, 3807 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3808 } 3809 3810 static int 3811 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3812 struct rte_flow *flow __rte_unused, 3813 struct rte_flow_error *error) 3814 { 3815 return rte_flow_error_set(error, ENOTSUP, 3816 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3817 } 3818 3819 static void 3820 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3821 struct rte_flow *flow __rte_unused) 3822 { 3823 } 3824 3825 static void 3826 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3827 struct rte_flow *flow __rte_unused) 3828 { 3829 } 3830 3831 static int 3832 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3833 struct rte_flow *flow __rte_unused, 3834 const struct rte_flow_action *actions __rte_unused, 3835 void *data __rte_unused, 3836 struct rte_flow_error *error) 3837 { 3838 return rte_flow_error_set(error, ENOTSUP, 3839 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3840 } 3841 3842 static int 3843 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3844 uint32_t domains __rte_unused, 3845 uint32_t flags __rte_unused) 3846 { 3847 return 0; 3848 } 3849 3850 int 3851 flow_null_get_aged_flows(struct rte_eth_dev *dev, 3852 void **context __rte_unused, 3853 uint32_t nb_contexts __rte_unused, 3854 struct rte_flow_error *error __rte_unused) 3855 { 3856 DRV_LOG(ERR, "port %u get aged flows is not supported.", 3857 dev->data->port_id); 3858 return -ENOTSUP; 3859 } 3860 3861 uint32_t 3862 flow_null_counter_allocate(struct rte_eth_dev *dev) 3863 { 3864 DRV_LOG(ERR, "port %u counter allocate is not supported.", 3865 dev->data->port_id); 3866 return 0; 3867 } 3868 3869 void 3870 flow_null_counter_free(struct rte_eth_dev *dev, 3871 uint32_t counter __rte_unused) 3872 { 3873 DRV_LOG(ERR, "port %u counter free is not supported.", 3874 dev->data->port_id); 3875 } 3876 3877 int 3878 flow_null_counter_query(struct rte_eth_dev *dev, 3879 uint32_t counter __rte_unused, 3880 bool clear __rte_unused, 3881 uint64_t *pkts __rte_unused, 3882 uint64_t *bytes __rte_unused, 3883 void **action __rte_unused) 3884 { 3885 DRV_LOG(ERR, "port %u counter query is not supported.", 3886 dev->data->port_id); 3887 return -ENOTSUP; 3888 } 3889 3890 /* Void driver to protect from null pointer reference. */ 3891 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3892 .validate = flow_null_validate, 3893 .prepare = flow_null_prepare, 3894 .translate = flow_null_translate, 3895 .apply = flow_null_apply, 3896 .remove = flow_null_remove, 3897 .destroy = flow_null_destroy, 3898 .query = flow_null_query, 3899 .sync_domain = flow_null_sync_domain, 3900 .get_aged_flows = flow_null_get_aged_flows, 3901 .counter_alloc = flow_null_counter_allocate, 3902 .counter_free = flow_null_counter_free, 3903 .counter_query = flow_null_counter_query 3904 }; 3905 3906 /** 3907 * Select flow driver type according to flow attributes and device 3908 * configuration. 3909 * 3910 * @param[in] dev 3911 * Pointer to the dev structure. 3912 * @param[in] attr 3913 * Pointer to the flow attributes. 3914 * 3915 * @return 3916 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3917 */ 3918 static enum mlx5_flow_drv_type 3919 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3920 { 3921 struct mlx5_priv *priv = dev->data->dev_private; 3922 /* The OS can determine first a specific flow type (DV, VERBS) */ 3923 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3924 3925 if (type != MLX5_FLOW_TYPE_MAX) 3926 return type; 3927 /* 3928 * Currently when dv_flow_en == 2, only HW steering engine is 3929 * supported. New engines can also be chosen here if ready. 3930 */ 3931 if (priv->sh->config.dv_flow_en == 2) 3932 return MLX5_FLOW_TYPE_HW; 3933 if (!attr) 3934 return MLX5_FLOW_TYPE_MIN; 3935 /* If no OS specific type - continue with DV/VERBS selection */ 3936 if (attr->transfer && priv->sh->config.dv_esw_en) 3937 type = MLX5_FLOW_TYPE_DV; 3938 if (!attr->transfer) 3939 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3940 MLX5_FLOW_TYPE_VERBS; 3941 return type; 3942 } 3943 3944 #define flow_get_drv_ops(type) flow_drv_ops[type] 3945 3946 /** 3947 * Flow driver validation API. This abstracts calling driver specific functions. 3948 * The type of flow driver is determined according to flow attributes. 3949 * 3950 * @param[in] dev 3951 * Pointer to the dev structure. 3952 * @param[in] attr 3953 * Pointer to the flow attributes. 3954 * @param[in] items 3955 * Pointer to the list of items. 3956 * @param[in] actions 3957 * Pointer to the list of actions. 3958 * @param[in] external 3959 * This flow rule is created by request external to PMD. 3960 * @param[in] hairpin 3961 * Number of hairpin TX actions, 0 means classic flow. 3962 * @param[out] error 3963 * Pointer to the error structure. 3964 * 3965 * @return 3966 * 0 on success, a negative errno value otherwise and rte_errno is set. 3967 */ 3968 static inline int 3969 flow_drv_validate(struct rte_eth_dev *dev, 3970 const struct rte_flow_attr *attr, 3971 const struct rte_flow_item items[], 3972 const struct rte_flow_action actions[], 3973 bool external, int hairpin, struct rte_flow_error *error) 3974 { 3975 const struct mlx5_flow_driver_ops *fops; 3976 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3977 3978 fops = flow_get_drv_ops(type); 3979 return fops->validate(dev, attr, items, actions, external, 3980 hairpin, error); 3981 } 3982 3983 /** 3984 * Flow driver preparation API. This abstracts calling driver specific 3985 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 3986 * calculates the size of memory required for device flow, allocates the memory, 3987 * initializes the device flow and returns the pointer. 3988 * 3989 * @note 3990 * This function initializes device flow structure such as dv or verbs in 3991 * struct mlx5_flow. However, it is caller's responsibility to initialize the 3992 * rest. For example, adding returning device flow to flow->dev_flow list and 3993 * setting backward reference to the flow should be done out of this function. 3994 * layers field is not filled either. 3995 * 3996 * @param[in] dev 3997 * Pointer to the dev structure. 3998 * @param[in] attr 3999 * Pointer to the flow attributes. 4000 * @param[in] items 4001 * Pointer to the list of items. 4002 * @param[in] actions 4003 * Pointer to the list of actions. 4004 * @param[in] flow_idx 4005 * This memory pool index to the flow. 4006 * @param[out] error 4007 * Pointer to the error structure. 4008 * 4009 * @return 4010 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4011 */ 4012 static inline struct mlx5_flow * 4013 flow_drv_prepare(struct rte_eth_dev *dev, 4014 const struct rte_flow *flow, 4015 const struct rte_flow_attr *attr, 4016 const struct rte_flow_item items[], 4017 const struct rte_flow_action actions[], 4018 uint32_t flow_idx, 4019 struct rte_flow_error *error) 4020 { 4021 const struct mlx5_flow_driver_ops *fops; 4022 enum mlx5_flow_drv_type type = flow->drv_type; 4023 struct mlx5_flow *mlx5_flow = NULL; 4024 4025 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4026 fops = flow_get_drv_ops(type); 4027 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4028 if (mlx5_flow) 4029 mlx5_flow->flow_idx = flow_idx; 4030 return mlx5_flow; 4031 } 4032 4033 /** 4034 * Flow driver translation API. This abstracts calling driver specific 4035 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4036 * translates a generic flow into a driver flow. flow_drv_prepare() must 4037 * precede. 4038 * 4039 * @note 4040 * dev_flow->layers could be filled as a result of parsing during translation 4041 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4042 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4043 * flow->actions could be overwritten even though all the expanded dev_flows 4044 * have the same actions. 4045 * 4046 * @param[in] dev 4047 * Pointer to the rte dev structure. 4048 * @param[in, out] dev_flow 4049 * Pointer to the mlx5 flow. 4050 * @param[in] attr 4051 * Pointer to the flow attributes. 4052 * @param[in] items 4053 * Pointer to the list of items. 4054 * @param[in] actions 4055 * Pointer to the list of actions. 4056 * @param[out] error 4057 * Pointer to the error structure. 4058 * 4059 * @return 4060 * 0 on success, a negative errno value otherwise and rte_errno is set. 4061 */ 4062 static inline int 4063 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4064 const struct rte_flow_attr *attr, 4065 const struct rte_flow_item items[], 4066 const struct rte_flow_action actions[], 4067 struct rte_flow_error *error) 4068 { 4069 const struct mlx5_flow_driver_ops *fops; 4070 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4071 4072 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4073 fops = flow_get_drv_ops(type); 4074 return fops->translate(dev, dev_flow, attr, items, actions, error); 4075 } 4076 4077 /** 4078 * Flow driver apply API. This abstracts calling driver specific functions. 4079 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4080 * translated driver flows on to device. flow_drv_translate() must precede. 4081 * 4082 * @param[in] dev 4083 * Pointer to Ethernet device structure. 4084 * @param[in, out] flow 4085 * Pointer to flow structure. 4086 * @param[out] error 4087 * Pointer to error structure. 4088 * 4089 * @return 4090 * 0 on success, a negative errno value otherwise and rte_errno is set. 4091 */ 4092 static inline int 4093 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4094 struct rte_flow_error *error) 4095 { 4096 const struct mlx5_flow_driver_ops *fops; 4097 enum mlx5_flow_drv_type type = flow->drv_type; 4098 4099 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4100 fops = flow_get_drv_ops(type); 4101 return fops->apply(dev, flow, error); 4102 } 4103 4104 /** 4105 * Flow driver destroy API. This abstracts calling driver specific functions. 4106 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4107 * on device and releases resources of the flow. 4108 * 4109 * @param[in] dev 4110 * Pointer to Ethernet device. 4111 * @param[in, out] flow 4112 * Pointer to flow structure. 4113 */ 4114 static inline void 4115 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4116 { 4117 const struct mlx5_flow_driver_ops *fops; 4118 enum mlx5_flow_drv_type type = flow->drv_type; 4119 4120 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4121 fops = flow_get_drv_ops(type); 4122 fops->destroy(dev, flow); 4123 } 4124 4125 /** 4126 * Flow driver find RSS policy tbl API. This abstracts calling driver 4127 * specific functions. Parent flow (rte_flow) should have driver 4128 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4129 * 4130 * @param[in] dev 4131 * Pointer to Ethernet device. 4132 * @param[in, out] flow 4133 * Pointer to flow structure. 4134 * @param[in] policy 4135 * Pointer to meter policy table. 4136 * @param[in] rss_desc 4137 * Pointer to rss_desc 4138 */ 4139 static struct mlx5_flow_meter_sub_policy * 4140 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4141 struct rte_flow *flow, 4142 struct mlx5_flow_meter_policy *policy, 4143 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4144 { 4145 const struct mlx5_flow_driver_ops *fops; 4146 enum mlx5_flow_drv_type type = flow->drv_type; 4147 4148 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4149 fops = flow_get_drv_ops(type); 4150 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4151 } 4152 4153 /** 4154 * Flow driver color tag rule API. This abstracts calling driver 4155 * specific functions. Parent flow (rte_flow) should have driver 4156 * type (drv_type). It will create the color tag rules in hierarchy meter. 4157 * 4158 * @param[in] dev 4159 * Pointer to Ethernet device. 4160 * @param[in, out] flow 4161 * Pointer to flow structure. 4162 * @param[in] fm 4163 * Pointer to flow meter structure. 4164 * @param[in] src_port 4165 * The src port this extra rule should use. 4166 * @param[in] item 4167 * The src port id match item. 4168 * @param[out] error 4169 * Pointer to error structure. 4170 */ 4171 static int 4172 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4173 struct rte_flow *flow, 4174 struct mlx5_flow_meter_info *fm, 4175 int32_t src_port, 4176 const struct rte_flow_item *item, 4177 struct rte_flow_error *error) 4178 { 4179 const struct mlx5_flow_driver_ops *fops; 4180 enum mlx5_flow_drv_type type = flow->drv_type; 4181 4182 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4183 fops = flow_get_drv_ops(type); 4184 return fops->meter_hierarchy_rule_create(dev, fm, 4185 src_port, item, error); 4186 } 4187 4188 /** 4189 * Get RSS action from the action list. 4190 * 4191 * @param[in] dev 4192 * Pointer to Ethernet device. 4193 * @param[in] actions 4194 * Pointer to the list of actions. 4195 * @param[in] flow 4196 * Parent flow structure pointer. 4197 * 4198 * @return 4199 * Pointer to the RSS action if exist, else return NULL. 4200 */ 4201 static const struct rte_flow_action_rss* 4202 flow_get_rss_action(struct rte_eth_dev *dev, 4203 const struct rte_flow_action actions[]) 4204 { 4205 struct mlx5_priv *priv = dev->data->dev_private; 4206 const struct rte_flow_action_rss *rss = NULL; 4207 struct mlx5_meter_policy_action_container *acg; 4208 struct mlx5_meter_policy_action_container *acy; 4209 4210 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4211 switch (actions->type) { 4212 case RTE_FLOW_ACTION_TYPE_RSS: 4213 rss = actions->conf; 4214 break; 4215 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4216 { 4217 const struct rte_flow_action_sample *sample = 4218 actions->conf; 4219 const struct rte_flow_action *act = sample->actions; 4220 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4221 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4222 rss = act->conf; 4223 break; 4224 } 4225 case RTE_FLOW_ACTION_TYPE_METER: 4226 { 4227 uint32_t mtr_idx; 4228 struct mlx5_flow_meter_info *fm; 4229 struct mlx5_flow_meter_policy *policy; 4230 const struct rte_flow_action_meter *mtr = actions->conf; 4231 4232 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4233 if (fm && !fm->def_policy) { 4234 policy = mlx5_flow_meter_policy_find(dev, 4235 fm->policy_id, NULL); 4236 MLX5_ASSERT(policy); 4237 if (policy->is_hierarchy) { 4238 policy = 4239 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4240 policy); 4241 if (!policy) 4242 return NULL; 4243 } 4244 if (policy->is_rss) { 4245 acg = 4246 &policy->act_cnt[RTE_COLOR_GREEN]; 4247 acy = 4248 &policy->act_cnt[RTE_COLOR_YELLOW]; 4249 if (acg->fate_action == 4250 MLX5_FLOW_FATE_SHARED_RSS) 4251 rss = acg->rss->conf; 4252 else if (acy->fate_action == 4253 MLX5_FLOW_FATE_SHARED_RSS) 4254 rss = acy->rss->conf; 4255 } 4256 } 4257 break; 4258 } 4259 default: 4260 break; 4261 } 4262 } 4263 return rss; 4264 } 4265 4266 /** 4267 * Get ASO age action by index. 4268 * 4269 * @param[in] dev 4270 * Pointer to the Ethernet device structure. 4271 * @param[in] age_idx 4272 * Index to the ASO age action. 4273 * 4274 * @return 4275 * The specified ASO age action. 4276 */ 4277 struct mlx5_aso_age_action* 4278 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4279 { 4280 uint16_t pool_idx = age_idx & UINT16_MAX; 4281 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4282 struct mlx5_priv *priv = dev->data->dev_private; 4283 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4284 struct mlx5_aso_age_pool *pool; 4285 4286 rte_rwlock_read_lock(&mng->resize_rwl); 4287 pool = mng->pools[pool_idx]; 4288 rte_rwlock_read_unlock(&mng->resize_rwl); 4289 return &pool->actions[offset - 1]; 4290 } 4291 4292 /* maps indirect action to translated direct in some actions array */ 4293 struct mlx5_translated_action_handle { 4294 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4295 int index; /**< Index in related array of rte_flow_action. */ 4296 }; 4297 4298 /** 4299 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4300 * direct action if translation possible. 4301 * This functionality used to run same execution path for both direct and 4302 * indirect actions on flow create. All necessary preparations for indirect 4303 * action handling should be performed on *handle* actions list returned 4304 * from this call. 4305 * 4306 * @param[in] dev 4307 * Pointer to Ethernet device. 4308 * @param[in] actions 4309 * List of actions to translate. 4310 * @param[out] handle 4311 * List to store translated indirect action object handles. 4312 * @param[in, out] indir_n 4313 * Size of *handle* array. On return should be updated with number of 4314 * indirect actions retrieved from the *actions* list. 4315 * @param[out] translated_actions 4316 * List of actions where all indirect actions were translated to direct 4317 * if possible. NULL if no translation took place. 4318 * @param[out] error 4319 * Pointer to the error structure. 4320 * 4321 * @return 4322 * 0 on success, a negative errno value otherwise and rte_errno is set. 4323 */ 4324 static int 4325 flow_action_handles_translate(struct rte_eth_dev *dev, 4326 const struct rte_flow_action actions[], 4327 struct mlx5_translated_action_handle *handle, 4328 int *indir_n, 4329 struct rte_flow_action **translated_actions, 4330 struct rte_flow_error *error) 4331 { 4332 struct mlx5_priv *priv = dev->data->dev_private; 4333 struct rte_flow_action *translated = NULL; 4334 size_t actions_size; 4335 int n; 4336 int copied_n = 0; 4337 struct mlx5_translated_action_handle *handle_end = NULL; 4338 4339 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4340 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4341 continue; 4342 if (copied_n == *indir_n) { 4343 return rte_flow_error_set 4344 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4345 NULL, "too many shared actions"); 4346 } 4347 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4348 sizeof(actions[n].conf)); 4349 handle[copied_n].index = n; 4350 copied_n++; 4351 } 4352 n++; 4353 *indir_n = copied_n; 4354 if (!copied_n) 4355 return 0; 4356 actions_size = sizeof(struct rte_flow_action) * n; 4357 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4358 if (!translated) { 4359 rte_errno = ENOMEM; 4360 return -ENOMEM; 4361 } 4362 memcpy(translated, actions, actions_size); 4363 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4364 struct mlx5_shared_action_rss *shared_rss; 4365 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4366 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4367 uint32_t idx = act_idx & 4368 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4369 4370 switch (type) { 4371 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4372 shared_rss = mlx5_ipool_get 4373 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4374 translated[handle->index].type = 4375 RTE_FLOW_ACTION_TYPE_RSS; 4376 translated[handle->index].conf = 4377 &shared_rss->origin; 4378 break; 4379 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4380 translated[handle->index].type = 4381 (enum rte_flow_action_type) 4382 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4383 translated[handle->index].conf = (void *)(uintptr_t)idx; 4384 break; 4385 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4386 translated[handle->index].type = 4387 (enum rte_flow_action_type) 4388 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4389 translated[handle->index].conf = (void *)(uintptr_t)idx; 4390 break; 4391 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4392 if (priv->sh->flow_hit_aso_en) { 4393 translated[handle->index].type = 4394 (enum rte_flow_action_type) 4395 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4396 translated[handle->index].conf = 4397 (void *)(uintptr_t)idx; 4398 break; 4399 } 4400 /* Fall-through */ 4401 case MLX5_INDIRECT_ACTION_TYPE_CT: 4402 if (priv->sh->ct_aso_en) { 4403 translated[handle->index].type = 4404 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4405 translated[handle->index].conf = 4406 (void *)(uintptr_t)idx; 4407 break; 4408 } 4409 /* Fall-through */ 4410 default: 4411 mlx5_free(translated); 4412 return rte_flow_error_set 4413 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4414 NULL, "invalid indirect action type"); 4415 } 4416 } 4417 *translated_actions = translated; 4418 return 0; 4419 } 4420 4421 /** 4422 * Get Shared RSS action from the action list. 4423 * 4424 * @param[in] dev 4425 * Pointer to Ethernet device. 4426 * @param[in] shared 4427 * Pointer to the list of actions. 4428 * @param[in] shared_n 4429 * Actions list length. 4430 * 4431 * @return 4432 * The MLX5 RSS action ID if exists, otherwise return 0. 4433 */ 4434 static uint32_t 4435 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4436 struct mlx5_translated_action_handle *handle, 4437 int shared_n) 4438 { 4439 struct mlx5_translated_action_handle *handle_end; 4440 struct mlx5_priv *priv = dev->data->dev_private; 4441 struct mlx5_shared_action_rss *shared_rss; 4442 4443 4444 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4445 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4446 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4447 uint32_t idx = act_idx & 4448 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4449 switch (type) { 4450 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4451 shared_rss = mlx5_ipool_get 4452 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4453 idx); 4454 __atomic_add_fetch(&shared_rss->refcnt, 1, 4455 __ATOMIC_RELAXED); 4456 return idx; 4457 default: 4458 break; 4459 } 4460 } 4461 return 0; 4462 } 4463 4464 static unsigned int 4465 find_graph_root(uint32_t rss_level) 4466 { 4467 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4468 MLX5_EXPANSION_ROOT_OUTER; 4469 } 4470 4471 /** 4472 * Get layer flags from the prefix flow. 4473 * 4474 * Some flows may be split to several subflows, the prefix subflow gets the 4475 * match items and the suffix sub flow gets the actions. 4476 * Some actions need the user defined match item flags to get the detail for 4477 * the action. 4478 * This function helps the suffix flow to get the item layer flags from prefix 4479 * subflow. 4480 * 4481 * @param[in] dev_flow 4482 * Pointer the created prefix subflow. 4483 * 4484 * @return 4485 * The layers get from prefix subflow. 4486 */ 4487 static inline uint64_t 4488 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4489 { 4490 uint64_t layers = 0; 4491 4492 /* 4493 * Layers bits could be localization, but usually the compiler will 4494 * help to do the optimization work for source code. 4495 * If no decap actions, use the layers directly. 4496 */ 4497 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4498 return dev_flow->handle->layers; 4499 /* Convert L3 layers with decap action. */ 4500 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4501 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4502 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4503 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4504 /* Convert L4 layers with decap action. */ 4505 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4506 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4507 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4508 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4509 return layers; 4510 } 4511 4512 /** 4513 * Get metadata split action information. 4514 * 4515 * @param[in] actions 4516 * Pointer to the list of actions. 4517 * @param[out] qrss 4518 * Pointer to the return pointer. 4519 * @param[out] qrss_type 4520 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4521 * if no QUEUE/RSS is found. 4522 * @param[out] encap_idx 4523 * Pointer to the index of the encap action if exists, otherwise the last 4524 * action index. 4525 * 4526 * @return 4527 * Total number of actions. 4528 */ 4529 static int 4530 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4531 const struct rte_flow_action **qrss, 4532 int *encap_idx) 4533 { 4534 const struct rte_flow_action_raw_encap *raw_encap; 4535 int actions_n = 0; 4536 int raw_decap_idx = -1; 4537 4538 *encap_idx = -1; 4539 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4540 switch (actions->type) { 4541 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4542 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4543 *encap_idx = actions_n; 4544 break; 4545 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4546 raw_decap_idx = actions_n; 4547 break; 4548 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4549 raw_encap = actions->conf; 4550 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4551 *encap_idx = raw_decap_idx != -1 ? 4552 raw_decap_idx : actions_n; 4553 break; 4554 case RTE_FLOW_ACTION_TYPE_QUEUE: 4555 case RTE_FLOW_ACTION_TYPE_RSS: 4556 *qrss = actions; 4557 break; 4558 default: 4559 break; 4560 } 4561 actions_n++; 4562 } 4563 if (*encap_idx == -1) 4564 *encap_idx = actions_n; 4565 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4566 return actions_n + 1; 4567 } 4568 4569 /** 4570 * Check if the action will change packet. 4571 * 4572 * @param dev 4573 * Pointer to Ethernet device. 4574 * @param[in] type 4575 * action type. 4576 * 4577 * @return 4578 * true if action will change packet, false otherwise. 4579 */ 4580 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4581 enum rte_flow_action_type type) 4582 { 4583 struct mlx5_priv *priv = dev->data->dev_private; 4584 4585 switch (type) { 4586 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4587 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4588 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4589 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4590 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4591 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4592 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4593 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4594 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4595 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4596 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4597 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4598 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4599 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4600 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4601 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4602 case RTE_FLOW_ACTION_TYPE_SET_META: 4603 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4604 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4605 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4606 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4607 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4608 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4609 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4610 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4611 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4612 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4613 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4614 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4615 return true; 4616 case RTE_FLOW_ACTION_TYPE_FLAG: 4617 case RTE_FLOW_ACTION_TYPE_MARK: 4618 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4619 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4620 return true; 4621 else 4622 return false; 4623 default: 4624 return false; 4625 } 4626 } 4627 4628 /** 4629 * Check meter action from the action list. 4630 * 4631 * @param dev 4632 * Pointer to Ethernet device. 4633 * @param[in] actions 4634 * Pointer to the list of actions. 4635 * @param[out] has_mtr 4636 * Pointer to the meter exist flag. 4637 * @param[out] has_modify 4638 * Pointer to the flag showing there's packet change action. 4639 * @param[out] meter_id 4640 * Pointer to the meter id. 4641 * 4642 * @return 4643 * Total number of actions. 4644 */ 4645 static int 4646 flow_check_meter_action(struct rte_eth_dev *dev, 4647 const struct rte_flow_action actions[], 4648 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4649 { 4650 const struct rte_flow_action_meter *mtr = NULL; 4651 int actions_n = 0; 4652 4653 MLX5_ASSERT(has_mtr); 4654 *has_mtr = false; 4655 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4656 switch (actions->type) { 4657 case RTE_FLOW_ACTION_TYPE_METER: 4658 mtr = actions->conf; 4659 *meter_id = mtr->mtr_id; 4660 *has_mtr = true; 4661 break; 4662 default: 4663 break; 4664 } 4665 if (!*has_mtr) 4666 *has_modify |= flow_check_modify_action_type(dev, 4667 actions->type); 4668 actions_n++; 4669 } 4670 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4671 return actions_n + 1; 4672 } 4673 4674 /** 4675 * Check if the flow should be split due to hairpin. 4676 * The reason for the split is that in current HW we can't 4677 * support encap and push-vlan on Rx, so if a flow contains 4678 * these actions we move it to Tx. 4679 * 4680 * @param dev 4681 * Pointer to Ethernet device. 4682 * @param[in] attr 4683 * Flow rule attributes. 4684 * @param[in] actions 4685 * Associated actions (list terminated by the END action). 4686 * 4687 * @return 4688 * > 0 the number of actions and the flow should be split, 4689 * 0 when no split required. 4690 */ 4691 static int 4692 flow_check_hairpin_split(struct rte_eth_dev *dev, 4693 const struct rte_flow_attr *attr, 4694 const struct rte_flow_action actions[]) 4695 { 4696 int queue_action = 0; 4697 int action_n = 0; 4698 int split = 0; 4699 int push_vlan = 0; 4700 const struct rte_flow_action_queue *queue; 4701 const struct rte_flow_action_rss *rss; 4702 const struct rte_flow_action_raw_encap *raw_encap; 4703 const struct rte_eth_hairpin_conf *conf; 4704 4705 if (!attr->ingress) 4706 return 0; 4707 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4708 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4709 push_vlan = 1; 4710 switch (actions->type) { 4711 case RTE_FLOW_ACTION_TYPE_QUEUE: 4712 queue = actions->conf; 4713 if (queue == NULL) 4714 return 0; 4715 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4716 if (conf == NULL || conf->tx_explicit != 0) 4717 return 0; 4718 queue_action = 1; 4719 action_n++; 4720 break; 4721 case RTE_FLOW_ACTION_TYPE_RSS: 4722 rss = actions->conf; 4723 if (rss == NULL || rss->queue_num == 0) 4724 return 0; 4725 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4726 if (conf == NULL || conf->tx_explicit != 0) 4727 return 0; 4728 queue_action = 1; 4729 action_n++; 4730 break; 4731 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4732 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4733 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4734 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4735 split++; 4736 action_n++; 4737 break; 4738 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4739 if (push_vlan) 4740 split++; 4741 action_n++; 4742 break; 4743 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4744 raw_encap = actions->conf; 4745 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4746 split++; 4747 action_n++; 4748 break; 4749 default: 4750 action_n++; 4751 break; 4752 } 4753 } 4754 if (split && queue_action) 4755 return action_n; 4756 return 0; 4757 } 4758 4759 /* Declare flow create/destroy prototype in advance. */ 4760 static uint32_t 4761 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4762 const struct rte_flow_attr *attr, 4763 const struct rte_flow_item items[], 4764 const struct rte_flow_action actions[], 4765 bool external, struct rte_flow_error *error); 4766 4767 static void 4768 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4769 uint32_t flow_idx); 4770 4771 int 4772 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4773 struct mlx5_list_entry *entry, void *cb_ctx) 4774 { 4775 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4776 struct mlx5_flow_mreg_copy_resource *mcp_res = 4777 container_of(entry, typeof(*mcp_res), hlist_ent); 4778 4779 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4780 } 4781 4782 struct mlx5_list_entry * 4783 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4784 { 4785 struct rte_eth_dev *dev = tool_ctx; 4786 struct mlx5_priv *priv = dev->data->dev_private; 4787 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4788 struct mlx5_flow_mreg_copy_resource *mcp_res; 4789 struct rte_flow_error *error = ctx->error; 4790 uint32_t idx = 0; 4791 int ret; 4792 uint32_t mark_id = *(uint32_t *)(ctx->data); 4793 struct rte_flow_attr attr = { 4794 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4795 .ingress = 1, 4796 }; 4797 struct mlx5_rte_flow_item_tag tag_spec = { 4798 .data = mark_id, 4799 }; 4800 struct rte_flow_item items[] = { 4801 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4802 }; 4803 struct rte_flow_action_mark ftag = { 4804 .id = mark_id, 4805 }; 4806 struct mlx5_flow_action_copy_mreg cp_mreg = { 4807 .dst = REG_B, 4808 .src = REG_NON, 4809 }; 4810 struct rte_flow_action_jump jump = { 4811 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4812 }; 4813 struct rte_flow_action actions[] = { 4814 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4815 }; 4816 4817 /* Fill the register fields in the flow. */ 4818 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4819 if (ret < 0) 4820 return NULL; 4821 tag_spec.id = ret; 4822 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4823 if (ret < 0) 4824 return NULL; 4825 cp_mreg.src = ret; 4826 /* Provide the full width of FLAG specific value. */ 4827 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4828 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4829 /* Build a new flow. */ 4830 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4831 items[0] = (struct rte_flow_item){ 4832 .type = (enum rte_flow_item_type) 4833 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4834 .spec = &tag_spec, 4835 }; 4836 items[1] = (struct rte_flow_item){ 4837 .type = RTE_FLOW_ITEM_TYPE_END, 4838 }; 4839 actions[0] = (struct rte_flow_action){ 4840 .type = (enum rte_flow_action_type) 4841 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4842 .conf = &ftag, 4843 }; 4844 actions[1] = (struct rte_flow_action){ 4845 .type = (enum rte_flow_action_type) 4846 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4847 .conf = &cp_mreg, 4848 }; 4849 actions[2] = (struct rte_flow_action){ 4850 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4851 .conf = &jump, 4852 }; 4853 actions[3] = (struct rte_flow_action){ 4854 .type = RTE_FLOW_ACTION_TYPE_END, 4855 }; 4856 } else { 4857 /* Default rule, wildcard match. */ 4858 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4859 items[0] = (struct rte_flow_item){ 4860 .type = RTE_FLOW_ITEM_TYPE_END, 4861 }; 4862 actions[0] = (struct rte_flow_action){ 4863 .type = (enum rte_flow_action_type) 4864 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4865 .conf = &cp_mreg, 4866 }; 4867 actions[1] = (struct rte_flow_action){ 4868 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4869 .conf = &jump, 4870 }; 4871 actions[2] = (struct rte_flow_action){ 4872 .type = RTE_FLOW_ACTION_TYPE_END, 4873 }; 4874 } 4875 /* Build a new entry. */ 4876 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4877 if (!mcp_res) { 4878 rte_errno = ENOMEM; 4879 return NULL; 4880 } 4881 mcp_res->idx = idx; 4882 mcp_res->mark_id = mark_id; 4883 /* 4884 * The copy Flows are not included in any list. There 4885 * ones are referenced from other Flows and can not 4886 * be applied, removed, deleted in arbitrary order 4887 * by list traversing. 4888 */ 4889 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4890 &attr, items, actions, false, error); 4891 if (!mcp_res->rix_flow) { 4892 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4893 return NULL; 4894 } 4895 return &mcp_res->hlist_ent; 4896 } 4897 4898 struct mlx5_list_entry * 4899 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4900 void *cb_ctx __rte_unused) 4901 { 4902 struct rte_eth_dev *dev = tool_ctx; 4903 struct mlx5_priv *priv = dev->data->dev_private; 4904 struct mlx5_flow_mreg_copy_resource *mcp_res; 4905 uint32_t idx = 0; 4906 4907 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4908 if (!mcp_res) { 4909 rte_errno = ENOMEM; 4910 return NULL; 4911 } 4912 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4913 mcp_res->idx = idx; 4914 return &mcp_res->hlist_ent; 4915 } 4916 4917 void 4918 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4919 { 4920 struct mlx5_flow_mreg_copy_resource *mcp_res = 4921 container_of(entry, typeof(*mcp_res), hlist_ent); 4922 struct rte_eth_dev *dev = tool_ctx; 4923 struct mlx5_priv *priv = dev->data->dev_private; 4924 4925 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4926 } 4927 4928 /** 4929 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4930 * 4931 * As mark_id is unique, if there's already a registered flow for the mark_id, 4932 * return by increasing the reference counter of the resource. Otherwise, create 4933 * the resource (mcp_res) and flow. 4934 * 4935 * Flow looks like, 4936 * - If ingress port is ANY and reg_c[1] is mark_id, 4937 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4938 * 4939 * For default flow (zero mark_id), flow is like, 4940 * - If ingress port is ANY, 4941 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4942 * 4943 * @param dev 4944 * Pointer to Ethernet device. 4945 * @param mark_id 4946 * ID of MARK action, zero means default flow for META. 4947 * @param[out] error 4948 * Perform verbose error reporting if not NULL. 4949 * 4950 * @return 4951 * Associated resource on success, NULL otherwise and rte_errno is set. 4952 */ 4953 static struct mlx5_flow_mreg_copy_resource * 4954 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4955 struct rte_flow_error *error) 4956 { 4957 struct mlx5_priv *priv = dev->data->dev_private; 4958 struct mlx5_list_entry *entry; 4959 struct mlx5_flow_cb_ctx ctx = { 4960 .dev = dev, 4961 .error = error, 4962 .data = &mark_id, 4963 }; 4964 4965 /* Check if already registered. */ 4966 MLX5_ASSERT(priv->mreg_cp_tbl); 4967 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4968 if (!entry) 4969 return NULL; 4970 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4971 hlist_ent); 4972 } 4973 4974 void 4975 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4976 { 4977 struct mlx5_flow_mreg_copy_resource *mcp_res = 4978 container_of(entry, typeof(*mcp_res), hlist_ent); 4979 struct rte_eth_dev *dev = tool_ctx; 4980 struct mlx5_priv *priv = dev->data->dev_private; 4981 4982 MLX5_ASSERT(mcp_res->rix_flow); 4983 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 4984 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4985 } 4986 4987 /** 4988 * Release flow in RX_CP_TBL. 4989 * 4990 * @param dev 4991 * Pointer to Ethernet device. 4992 * @flow 4993 * Parent flow for wich copying is provided. 4994 */ 4995 static void 4996 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 4997 struct rte_flow *flow) 4998 { 4999 struct mlx5_flow_mreg_copy_resource *mcp_res; 5000 struct mlx5_priv *priv = dev->data->dev_private; 5001 5002 if (!flow->rix_mreg_copy) 5003 return; 5004 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5005 flow->rix_mreg_copy); 5006 if (!mcp_res || !priv->mreg_cp_tbl) 5007 return; 5008 MLX5_ASSERT(mcp_res->rix_flow); 5009 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5010 flow->rix_mreg_copy = 0; 5011 } 5012 5013 /** 5014 * Remove the default copy action from RX_CP_TBL. 5015 * 5016 * This functions is called in the mlx5_dev_start(). No thread safe 5017 * is guaranteed. 5018 * 5019 * @param dev 5020 * Pointer to Ethernet device. 5021 */ 5022 static void 5023 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5024 { 5025 struct mlx5_list_entry *entry; 5026 struct mlx5_priv *priv = dev->data->dev_private; 5027 struct mlx5_flow_cb_ctx ctx; 5028 uint32_t mark_id; 5029 5030 /* Check if default flow is registered. */ 5031 if (!priv->mreg_cp_tbl) 5032 return; 5033 mark_id = MLX5_DEFAULT_COPY_ID; 5034 ctx.data = &mark_id; 5035 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5036 if (!entry) 5037 return; 5038 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5039 } 5040 5041 /** 5042 * Add the default copy action in in RX_CP_TBL. 5043 * 5044 * This functions is called in the mlx5_dev_start(). No thread safe 5045 * is guaranteed. 5046 * 5047 * @param dev 5048 * Pointer to Ethernet device. 5049 * @param[out] error 5050 * Perform verbose error reporting if not NULL. 5051 * 5052 * @return 5053 * 0 for success, negative value otherwise and rte_errno is set. 5054 */ 5055 static int 5056 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5057 struct rte_flow_error *error) 5058 { 5059 struct mlx5_priv *priv = dev->data->dev_private; 5060 struct mlx5_flow_mreg_copy_resource *mcp_res; 5061 struct mlx5_flow_cb_ctx ctx; 5062 uint32_t mark_id; 5063 5064 /* Check whether extensive metadata feature is engaged. */ 5065 if (!priv->sh->config.dv_flow_en || 5066 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5067 !mlx5_flow_ext_mreg_supported(dev) || 5068 !priv->sh->dv_regc0_mask) 5069 return 0; 5070 /* 5071 * Add default mreg copy flow may be called multiple time, but 5072 * only be called once in stop. Avoid register it twice. 5073 */ 5074 mark_id = MLX5_DEFAULT_COPY_ID; 5075 ctx.data = &mark_id; 5076 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5077 return 0; 5078 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5079 if (!mcp_res) 5080 return -rte_errno; 5081 return 0; 5082 } 5083 5084 /** 5085 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5086 * 5087 * All the flow having Q/RSS action should be split by 5088 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5089 * performs the following, 5090 * - CQE->flow_tag := reg_c[1] (MARK) 5091 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5092 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5093 * but there should be a flow per each MARK ID set by MARK action. 5094 * 5095 * For the aforementioned reason, if there's a MARK action in flow's action 5096 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5097 * the MARK ID to CQE's flow_tag like, 5098 * - If reg_c[1] is mark_id, 5099 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5100 * 5101 * For SET_META action which stores value in reg_c[0], as the destination is 5102 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5103 * MARK ID means the default flow. The default flow looks like, 5104 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5105 * 5106 * @param dev 5107 * Pointer to Ethernet device. 5108 * @param flow 5109 * Pointer to flow structure. 5110 * @param[in] actions 5111 * Pointer to the list of actions. 5112 * @param[out] error 5113 * Perform verbose error reporting if not NULL. 5114 * 5115 * @return 5116 * 0 on success, negative value otherwise and rte_errno is set. 5117 */ 5118 static int 5119 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5120 struct rte_flow *flow, 5121 const struct rte_flow_action *actions, 5122 struct rte_flow_error *error) 5123 { 5124 struct mlx5_priv *priv = dev->data->dev_private; 5125 struct mlx5_sh_config *config = &priv->sh->config; 5126 struct mlx5_flow_mreg_copy_resource *mcp_res; 5127 const struct rte_flow_action_mark *mark; 5128 5129 /* Check whether extensive metadata feature is engaged. */ 5130 if (!config->dv_flow_en || 5131 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5132 !mlx5_flow_ext_mreg_supported(dev) || 5133 !priv->sh->dv_regc0_mask) 5134 return 0; 5135 /* Find MARK action. */ 5136 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5137 switch (actions->type) { 5138 case RTE_FLOW_ACTION_TYPE_FLAG: 5139 mcp_res = flow_mreg_add_copy_action 5140 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5141 if (!mcp_res) 5142 return -rte_errno; 5143 flow->rix_mreg_copy = mcp_res->idx; 5144 return 0; 5145 case RTE_FLOW_ACTION_TYPE_MARK: 5146 mark = (const struct rte_flow_action_mark *) 5147 actions->conf; 5148 mcp_res = 5149 flow_mreg_add_copy_action(dev, mark->id, error); 5150 if (!mcp_res) 5151 return -rte_errno; 5152 flow->rix_mreg_copy = mcp_res->idx; 5153 return 0; 5154 default: 5155 break; 5156 } 5157 } 5158 return 0; 5159 } 5160 5161 #define MLX5_MAX_SPLIT_ACTIONS 24 5162 #define MLX5_MAX_SPLIT_ITEMS 24 5163 5164 /** 5165 * Split the hairpin flow. 5166 * Since HW can't support encap and push-vlan on Rx, we move these 5167 * actions to Tx. 5168 * If the count action is after the encap then we also 5169 * move the count action. in this case the count will also measure 5170 * the outer bytes. 5171 * 5172 * @param dev 5173 * Pointer to Ethernet device. 5174 * @param[in] actions 5175 * Associated actions (list terminated by the END action). 5176 * @param[out] actions_rx 5177 * Rx flow actions. 5178 * @param[out] actions_tx 5179 * Tx flow actions.. 5180 * @param[out] pattern_tx 5181 * The pattern items for the Tx flow. 5182 * @param[out] flow_id 5183 * The flow ID connected to this flow. 5184 * 5185 * @return 5186 * 0 on success. 5187 */ 5188 static int 5189 flow_hairpin_split(struct rte_eth_dev *dev, 5190 const struct rte_flow_action actions[], 5191 struct rte_flow_action actions_rx[], 5192 struct rte_flow_action actions_tx[], 5193 struct rte_flow_item pattern_tx[], 5194 uint32_t flow_id) 5195 { 5196 const struct rte_flow_action_raw_encap *raw_encap; 5197 const struct rte_flow_action_raw_decap *raw_decap; 5198 struct mlx5_rte_flow_action_set_tag *set_tag; 5199 struct rte_flow_action *tag_action; 5200 struct mlx5_rte_flow_item_tag *tag_item; 5201 struct rte_flow_item *item; 5202 char *addr; 5203 int push_vlan = 0; 5204 int encap = 0; 5205 5206 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5207 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5208 push_vlan = 1; 5209 switch (actions->type) { 5210 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5211 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5212 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5213 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5214 rte_memcpy(actions_tx, actions, 5215 sizeof(struct rte_flow_action)); 5216 actions_tx++; 5217 break; 5218 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5219 if (push_vlan) { 5220 rte_memcpy(actions_tx, actions, 5221 sizeof(struct rte_flow_action)); 5222 actions_tx++; 5223 } else { 5224 rte_memcpy(actions_rx, actions, 5225 sizeof(struct rte_flow_action)); 5226 actions_rx++; 5227 } 5228 break; 5229 case RTE_FLOW_ACTION_TYPE_COUNT: 5230 if (encap) { 5231 rte_memcpy(actions_tx, actions, 5232 sizeof(struct rte_flow_action)); 5233 actions_tx++; 5234 } else { 5235 rte_memcpy(actions_rx, actions, 5236 sizeof(struct rte_flow_action)); 5237 actions_rx++; 5238 } 5239 break; 5240 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5241 raw_encap = actions->conf; 5242 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5243 memcpy(actions_tx, actions, 5244 sizeof(struct rte_flow_action)); 5245 actions_tx++; 5246 encap = 1; 5247 } else { 5248 rte_memcpy(actions_rx, actions, 5249 sizeof(struct rte_flow_action)); 5250 actions_rx++; 5251 } 5252 break; 5253 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5254 raw_decap = actions->conf; 5255 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5256 memcpy(actions_tx, actions, 5257 sizeof(struct rte_flow_action)); 5258 actions_tx++; 5259 } else { 5260 rte_memcpy(actions_rx, actions, 5261 sizeof(struct rte_flow_action)); 5262 actions_rx++; 5263 } 5264 break; 5265 default: 5266 rte_memcpy(actions_rx, actions, 5267 sizeof(struct rte_flow_action)); 5268 actions_rx++; 5269 break; 5270 } 5271 } 5272 /* Add set meta action and end action for the Rx flow. */ 5273 tag_action = actions_rx; 5274 tag_action->type = (enum rte_flow_action_type) 5275 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5276 actions_rx++; 5277 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5278 actions_rx++; 5279 set_tag = (void *)actions_rx; 5280 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5281 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5282 .data = flow_id, 5283 }; 5284 MLX5_ASSERT(set_tag->id > REG_NON); 5285 tag_action->conf = set_tag; 5286 /* Create Tx item list. */ 5287 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5288 addr = (void *)&pattern_tx[2]; 5289 item = pattern_tx; 5290 item->type = (enum rte_flow_item_type) 5291 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5292 tag_item = (void *)addr; 5293 tag_item->data = flow_id; 5294 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5295 MLX5_ASSERT(set_tag->id > REG_NON); 5296 item->spec = tag_item; 5297 addr += sizeof(struct mlx5_rte_flow_item_tag); 5298 tag_item = (void *)addr; 5299 tag_item->data = UINT32_MAX; 5300 tag_item->id = UINT16_MAX; 5301 item->mask = tag_item; 5302 item->last = NULL; 5303 item++; 5304 item->type = RTE_FLOW_ITEM_TYPE_END; 5305 return 0; 5306 } 5307 5308 /** 5309 * The last stage of splitting chain, just creates the subflow 5310 * without any modification. 5311 * 5312 * @param[in] dev 5313 * Pointer to Ethernet device. 5314 * @param[in] flow 5315 * Parent flow structure pointer. 5316 * @param[in, out] sub_flow 5317 * Pointer to return the created subflow, may be NULL. 5318 * @param[in] attr 5319 * Flow rule attributes. 5320 * @param[in] items 5321 * Pattern specification (list terminated by the END pattern item). 5322 * @param[in] actions 5323 * Associated actions (list terminated by the END action). 5324 * @param[in] flow_split_info 5325 * Pointer to flow split info structure. 5326 * @param[out] error 5327 * Perform verbose error reporting if not NULL. 5328 * @return 5329 * 0 on success, negative value otherwise 5330 */ 5331 static int 5332 flow_create_split_inner(struct rte_eth_dev *dev, 5333 struct rte_flow *flow, 5334 struct mlx5_flow **sub_flow, 5335 const struct rte_flow_attr *attr, 5336 const struct rte_flow_item items[], 5337 const struct rte_flow_action actions[], 5338 struct mlx5_flow_split_info *flow_split_info, 5339 struct rte_flow_error *error) 5340 { 5341 struct mlx5_flow *dev_flow; 5342 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5343 5344 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5345 flow_split_info->flow_idx, error); 5346 if (!dev_flow) 5347 return -rte_errno; 5348 dev_flow->flow = flow; 5349 dev_flow->external = flow_split_info->external; 5350 dev_flow->skip_scale = flow_split_info->skip_scale; 5351 /* Subflow object was created, we must include one in the list. */ 5352 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5353 dev_flow->handle, next); 5354 /* 5355 * If dev_flow is as one of the suffix flow, some actions in suffix 5356 * flow may need some user defined item layer flags, and pass the 5357 * Metadata rxq mark flag to suffix flow as well. 5358 */ 5359 if (flow_split_info->prefix_layers) 5360 dev_flow->handle->layers = flow_split_info->prefix_layers; 5361 if (flow_split_info->prefix_mark) { 5362 MLX5_ASSERT(wks); 5363 wks->mark = 1; 5364 } 5365 if (sub_flow) 5366 *sub_flow = dev_flow; 5367 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5368 dev_flow->dv.table_id = flow_split_info->table_id; 5369 #endif 5370 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5371 } 5372 5373 /** 5374 * Get the sub policy of a meter. 5375 * 5376 * @param[in] dev 5377 * Pointer to Ethernet device. 5378 * @param[in] flow 5379 * Parent flow structure pointer. 5380 * @param wks 5381 * Pointer to thread flow work space. 5382 * @param[in] attr 5383 * Flow rule attributes. 5384 * @param[in] items 5385 * Pattern specification (list terminated by the END pattern item). 5386 * @param[out] error 5387 * Perform verbose error reporting if not NULL. 5388 * 5389 * @return 5390 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5391 */ 5392 static struct mlx5_flow_meter_sub_policy * 5393 get_meter_sub_policy(struct rte_eth_dev *dev, 5394 struct rte_flow *flow, 5395 struct mlx5_flow_workspace *wks, 5396 const struct rte_flow_attr *attr, 5397 const struct rte_flow_item items[], 5398 struct rte_flow_error *error) 5399 { 5400 struct mlx5_flow_meter_policy *policy; 5401 struct mlx5_flow_meter_policy *final_policy; 5402 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5403 5404 policy = wks->policy; 5405 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5406 if (final_policy->is_rss || final_policy->is_queue) { 5407 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5408 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5409 uint32_t i; 5410 5411 /* 5412 * This is a tmp dev_flow, 5413 * no need to register any matcher for it in translate. 5414 */ 5415 wks->skip_matcher_reg = 1; 5416 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5417 struct mlx5_flow dev_flow = {0}; 5418 struct mlx5_flow_handle dev_handle = { {0} }; 5419 uint8_t fate = final_policy->act_cnt[i].fate_action; 5420 5421 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5422 const struct rte_flow_action_rss *rss_act = 5423 final_policy->act_cnt[i].rss->conf; 5424 struct rte_flow_action rss_actions[2] = { 5425 [0] = { 5426 .type = RTE_FLOW_ACTION_TYPE_RSS, 5427 .conf = rss_act, 5428 }, 5429 [1] = { 5430 .type = RTE_FLOW_ACTION_TYPE_END, 5431 .conf = NULL, 5432 } 5433 }; 5434 5435 dev_flow.handle = &dev_handle; 5436 dev_flow.ingress = attr->ingress; 5437 dev_flow.flow = flow; 5438 dev_flow.external = 0; 5439 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5440 dev_flow.dv.transfer = attr->transfer; 5441 #endif 5442 /** 5443 * Translate RSS action to get rss hash fields. 5444 */ 5445 if (flow_drv_translate(dev, &dev_flow, attr, 5446 items, rss_actions, error)) 5447 goto exit; 5448 rss_desc_v[i] = wks->rss_desc; 5449 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5450 rss_desc_v[i].hash_fields = 5451 dev_flow.hash_fields; 5452 rss_desc_v[i].queue_num = 5453 rss_desc_v[i].hash_fields ? 5454 rss_desc_v[i].queue_num : 1; 5455 rss_desc_v[i].tunnel = 5456 !!(dev_flow.handle->layers & 5457 MLX5_FLOW_LAYER_TUNNEL); 5458 /* Use the RSS queues in the containers. */ 5459 rss_desc_v[i].queue = 5460 (uint16_t *)(uintptr_t)rss_act->queue; 5461 rss_desc[i] = &rss_desc_v[i]; 5462 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5463 /* This is queue action. */ 5464 rss_desc_v[i] = wks->rss_desc; 5465 rss_desc_v[i].key_len = 0; 5466 rss_desc_v[i].hash_fields = 0; 5467 rss_desc_v[i].queue = 5468 &final_policy->act_cnt[i].queue; 5469 rss_desc_v[i].queue_num = 1; 5470 rss_desc[i] = &rss_desc_v[i]; 5471 } else { 5472 rss_desc[i] = NULL; 5473 } 5474 } 5475 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5476 flow, policy, rss_desc); 5477 } else { 5478 enum mlx5_meter_domain mtr_domain = 5479 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5480 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5481 MLX5_MTR_DOMAIN_INGRESS); 5482 sub_policy = policy->sub_policys[mtr_domain][0]; 5483 } 5484 if (!sub_policy) 5485 rte_flow_error_set(error, EINVAL, 5486 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5487 "Failed to get meter sub-policy."); 5488 exit: 5489 return sub_policy; 5490 } 5491 5492 /** 5493 * Split the meter flow. 5494 * 5495 * As meter flow will split to three sub flow, other than meter 5496 * action, the other actions make sense to only meter accepts 5497 * the packet. If it need to be dropped, no other additional 5498 * actions should be take. 5499 * 5500 * One kind of special action which decapsulates the L3 tunnel 5501 * header will be in the prefix sub flow, as not to take the 5502 * L3 tunnel header into account. 5503 * 5504 * @param[in] dev 5505 * Pointer to Ethernet device. 5506 * @param[in] flow 5507 * Parent flow structure pointer. 5508 * @param wks 5509 * Pointer to thread flow work space. 5510 * @param[in] attr 5511 * Flow rule attributes. 5512 * @param[in] items 5513 * Pattern specification (list terminated by the END pattern item). 5514 * @param[out] sfx_items 5515 * Suffix flow match items (list terminated by the END pattern item). 5516 * @param[in] actions 5517 * Associated actions (list terminated by the END action). 5518 * @param[out] actions_sfx 5519 * Suffix flow actions. 5520 * @param[out] actions_pre 5521 * Prefix flow actions. 5522 * @param[out] mtr_flow_id 5523 * Pointer to meter flow id. 5524 * @param[out] error 5525 * Perform verbose error reporting if not NULL. 5526 * 5527 * @return 5528 * 0 on success, a negative errno value otherwise and rte_errno is set. 5529 */ 5530 static int 5531 flow_meter_split_prep(struct rte_eth_dev *dev, 5532 struct rte_flow *flow, 5533 struct mlx5_flow_workspace *wks, 5534 const struct rte_flow_attr *attr, 5535 const struct rte_flow_item items[], 5536 struct rte_flow_item sfx_items[], 5537 const struct rte_flow_action actions[], 5538 struct rte_flow_action actions_sfx[], 5539 struct rte_flow_action actions_pre[], 5540 uint32_t *mtr_flow_id, 5541 struct rte_flow_error *error) 5542 { 5543 struct mlx5_priv *priv = dev->data->dev_private; 5544 struct mlx5_flow_meter_info *fm = wks->fm; 5545 struct rte_flow_action *tag_action = NULL; 5546 struct rte_flow_item *tag_item; 5547 struct mlx5_rte_flow_action_set_tag *set_tag; 5548 const struct rte_flow_action_raw_encap *raw_encap; 5549 const struct rte_flow_action_raw_decap *raw_decap; 5550 struct mlx5_rte_flow_item_tag *tag_item_spec; 5551 struct mlx5_rte_flow_item_tag *tag_item_mask; 5552 uint32_t tag_id = 0; 5553 struct rte_flow_item *vlan_item_dst = NULL; 5554 const struct rte_flow_item *vlan_item_src = NULL; 5555 const struct rte_flow_item *orig_items = items; 5556 struct rte_flow_action *hw_mtr_action; 5557 struct rte_flow_action *action_pre_head = NULL; 5558 uint16_t flow_src_port = priv->representor_id; 5559 bool mtr_first; 5560 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5561 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5562 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5563 uint32_t flow_id = 0; 5564 uint32_t flow_id_reversed = 0; 5565 uint8_t flow_id_bits = 0; 5566 bool after_meter = false; 5567 int shift; 5568 5569 /* Prepare the suffix subflow items. */ 5570 tag_item = sfx_items++; 5571 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5572 int item_type = items->type; 5573 5574 switch (item_type) { 5575 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5576 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5577 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5578 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5579 return -rte_errno; 5580 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5581 flow_src_port != priv->representor_id) { 5582 if (flow_drv_mtr_hierarchy_rule_create(dev, 5583 flow, fm, 5584 flow_src_port, 5585 items, 5586 error)) 5587 return -rte_errno; 5588 } 5589 memcpy(sfx_items, items, sizeof(*sfx_items)); 5590 sfx_items++; 5591 break; 5592 case RTE_FLOW_ITEM_TYPE_VLAN: 5593 /* Determine if copy vlan item below. */ 5594 vlan_item_src = items; 5595 vlan_item_dst = sfx_items++; 5596 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5597 break; 5598 default: 5599 break; 5600 } 5601 } 5602 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5603 sfx_items++; 5604 mtr_first = priv->sh->meter_aso_en && 5605 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5606 /* For ASO meter, meter must be before tag in TX direction. */ 5607 if (mtr_first) { 5608 action_pre_head = actions_pre++; 5609 /* Leave space for tag action. */ 5610 tag_action = actions_pre++; 5611 } 5612 /* Prepare the actions for prefix and suffix flow. */ 5613 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5614 struct rte_flow_action *action_cur = NULL; 5615 5616 switch (actions->type) { 5617 case RTE_FLOW_ACTION_TYPE_METER: 5618 if (mtr_first) { 5619 action_cur = action_pre_head; 5620 } else { 5621 /* Leave space for tag action. */ 5622 tag_action = actions_pre++; 5623 action_cur = actions_pre++; 5624 } 5625 after_meter = true; 5626 break; 5627 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5628 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5629 action_cur = actions_pre++; 5630 break; 5631 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5632 raw_encap = actions->conf; 5633 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5634 action_cur = actions_pre++; 5635 break; 5636 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5637 raw_decap = actions->conf; 5638 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5639 action_cur = actions_pre++; 5640 break; 5641 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5642 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5643 if (vlan_item_dst && vlan_item_src) { 5644 memcpy(vlan_item_dst, vlan_item_src, 5645 sizeof(*vlan_item_dst)); 5646 /* 5647 * Convert to internal match item, it is used 5648 * for vlan push and set vid. 5649 */ 5650 vlan_item_dst->type = (enum rte_flow_item_type) 5651 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5652 } 5653 break; 5654 case RTE_FLOW_ACTION_TYPE_COUNT: 5655 if (fm->def_policy) 5656 action_cur = after_meter ? 5657 actions_sfx++ : actions_pre++; 5658 break; 5659 default: 5660 break; 5661 } 5662 if (!action_cur) 5663 action_cur = (fm->def_policy) ? 5664 actions_sfx++ : actions_pre++; 5665 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5666 } 5667 /* Add end action to the actions. */ 5668 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5669 if (priv->sh->meter_aso_en) { 5670 /** 5671 * For ASO meter, need to add an extra jump action explicitly, 5672 * to jump from meter to policer table. 5673 */ 5674 struct mlx5_flow_meter_sub_policy *sub_policy; 5675 struct mlx5_flow_tbl_data_entry *tbl_data; 5676 5677 if (!fm->def_policy) { 5678 sub_policy = get_meter_sub_policy(dev, flow, wks, 5679 attr, orig_items, 5680 error); 5681 if (!sub_policy) 5682 return -rte_errno; 5683 } else { 5684 enum mlx5_meter_domain mtr_domain = 5685 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5686 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5687 MLX5_MTR_DOMAIN_INGRESS); 5688 5689 sub_policy = 5690 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5691 } 5692 tbl_data = container_of(sub_policy->tbl_rsc, 5693 struct mlx5_flow_tbl_data_entry, tbl); 5694 hw_mtr_action = actions_pre++; 5695 hw_mtr_action->type = (enum rte_flow_action_type) 5696 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5697 hw_mtr_action->conf = tbl_data->jump.action; 5698 } 5699 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5700 actions_pre++; 5701 if (!tag_action) 5702 return rte_flow_error_set(error, ENOMEM, 5703 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5704 NULL, "No tag action space."); 5705 if (!mtr_flow_id) { 5706 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5707 goto exit; 5708 } 5709 /* Only default-policy Meter creates mtr flow id. */ 5710 if (fm->def_policy) { 5711 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5712 if (!tag_id) 5713 return rte_flow_error_set(error, ENOMEM, 5714 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5715 "Failed to allocate meter flow id."); 5716 flow_id = tag_id - 1; 5717 flow_id_bits = (!flow_id) ? 1 : 5718 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5719 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5720 mtr_reg_bits) { 5721 mlx5_ipool_free(fm->flow_ipool, tag_id); 5722 return rte_flow_error_set(error, EINVAL, 5723 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5724 "Meter flow id exceeds max limit."); 5725 } 5726 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5727 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5728 } 5729 /* Build tag actions and items for meter_id/meter flow_id. */ 5730 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5731 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5732 tag_item_mask = tag_item_spec + 1; 5733 /* Both flow_id and meter_id share the same register. */ 5734 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5735 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5736 0, error), 5737 .offset = mtr_id_offset, 5738 .length = mtr_reg_bits, 5739 .data = flow->meter, 5740 }; 5741 /* 5742 * The color Reg bits used by flow_id are growing from 5743 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5744 */ 5745 for (shift = 0; shift < flow_id_bits; shift++) 5746 flow_id_reversed = (flow_id_reversed << 1) | 5747 ((flow_id >> shift) & 0x1); 5748 set_tag->data |= 5749 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5750 tag_item_spec->id = set_tag->id; 5751 tag_item_spec->data = set_tag->data << mtr_id_offset; 5752 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5753 tag_action->type = (enum rte_flow_action_type) 5754 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5755 tag_action->conf = set_tag; 5756 tag_item->type = (enum rte_flow_item_type) 5757 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5758 tag_item->spec = tag_item_spec; 5759 tag_item->last = NULL; 5760 tag_item->mask = tag_item_mask; 5761 exit: 5762 if (mtr_flow_id) 5763 *mtr_flow_id = tag_id; 5764 return 0; 5765 } 5766 5767 /** 5768 * Split action list having QUEUE/RSS for metadata register copy. 5769 * 5770 * Once Q/RSS action is detected in user's action list, the flow action 5771 * should be split in order to copy metadata registers, which will happen in 5772 * RX_CP_TBL like, 5773 * - CQE->flow_tag := reg_c[1] (MARK) 5774 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5775 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5776 * This is because the last action of each flow must be a terminal action 5777 * (QUEUE, RSS or DROP). 5778 * 5779 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5780 * stored and kept in the mlx5_flow structure per each sub_flow. 5781 * 5782 * The Q/RSS action is replaced with, 5783 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5784 * And the following JUMP action is added at the end, 5785 * - JUMP, to RX_CP_TBL. 5786 * 5787 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5788 * flow_create_split_metadata() routine. The flow will look like, 5789 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5790 * 5791 * @param dev 5792 * Pointer to Ethernet device. 5793 * @param[out] split_actions 5794 * Pointer to store split actions to jump to CP_TBL. 5795 * @param[in] actions 5796 * Pointer to the list of original flow actions. 5797 * @param[in] qrss 5798 * Pointer to the Q/RSS action. 5799 * @param[in] actions_n 5800 * Number of original actions. 5801 * @param[in] mtr_sfx 5802 * Check if it is in meter suffix table. 5803 * @param[out] error 5804 * Perform verbose error reporting if not NULL. 5805 * 5806 * @return 5807 * non-zero unique flow_id on success, otherwise 0 and 5808 * error/rte_error are set. 5809 */ 5810 static uint32_t 5811 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5812 struct rte_flow_action *split_actions, 5813 const struct rte_flow_action *actions, 5814 const struct rte_flow_action *qrss, 5815 int actions_n, int mtr_sfx, 5816 struct rte_flow_error *error) 5817 { 5818 struct mlx5_priv *priv = dev->data->dev_private; 5819 struct mlx5_rte_flow_action_set_tag *set_tag; 5820 struct rte_flow_action_jump *jump; 5821 const int qrss_idx = qrss - actions; 5822 uint32_t flow_id = 0; 5823 int ret = 0; 5824 5825 /* 5826 * Given actions will be split 5827 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5828 * - Add jump to mreg CP_TBL. 5829 * As a result, there will be one more action. 5830 */ 5831 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5832 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5833 ++actions_n; 5834 set_tag = (void *)(split_actions + actions_n); 5835 /* 5836 * If we are not the meter suffix flow, add the tag action. 5837 * Since meter suffix flow already has the tag added. 5838 */ 5839 if (!mtr_sfx) { 5840 /* 5841 * Allocate the new subflow ID. This one is unique within 5842 * device and not shared with representors. Otherwise, 5843 * we would have to resolve multi-thread access synch 5844 * issue. Each flow on the shared device is appended 5845 * with source vport identifier, so the resulting 5846 * flows will be unique in the shared (by master and 5847 * representors) domain even if they have coinciding 5848 * IDs. 5849 */ 5850 mlx5_ipool_malloc(priv->sh->ipool 5851 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5852 if (!flow_id) 5853 return rte_flow_error_set(error, ENOMEM, 5854 RTE_FLOW_ERROR_TYPE_ACTION, 5855 NULL, "can't allocate id " 5856 "for split Q/RSS subflow"); 5857 /* Internal SET_TAG action to set flow ID. */ 5858 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5859 .data = flow_id, 5860 }; 5861 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5862 if (ret < 0) 5863 return ret; 5864 set_tag->id = ret; 5865 /* Construct new actions array. */ 5866 /* Replace QUEUE/RSS action. */ 5867 split_actions[qrss_idx] = (struct rte_flow_action){ 5868 .type = (enum rte_flow_action_type) 5869 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5870 .conf = set_tag, 5871 }; 5872 } else { 5873 /* 5874 * If we are the suffix flow of meter, tag already exist. 5875 * Set the QUEUE/RSS action to void. 5876 */ 5877 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5878 } 5879 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5880 jump = (void *)(set_tag + 1); 5881 *jump = (struct rte_flow_action_jump){ 5882 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5883 }; 5884 split_actions[actions_n - 2] = (struct rte_flow_action){ 5885 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5886 .conf = jump, 5887 }; 5888 split_actions[actions_n - 1] = (struct rte_flow_action){ 5889 .type = RTE_FLOW_ACTION_TYPE_END, 5890 }; 5891 return flow_id; 5892 } 5893 5894 /** 5895 * Extend the given action list for Tx metadata copy. 5896 * 5897 * Copy the given action list to the ext_actions and add flow metadata register 5898 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5899 * 5900 * @param[out] ext_actions 5901 * Pointer to the extended action list. 5902 * @param[in] actions 5903 * Pointer to the list of actions. 5904 * @param[in] actions_n 5905 * Number of actions in the list. 5906 * @param[out] error 5907 * Perform verbose error reporting if not NULL. 5908 * @param[in] encap_idx 5909 * The encap action index. 5910 * 5911 * @return 5912 * 0 on success, negative value otherwise 5913 */ 5914 static int 5915 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5916 struct rte_flow_action *ext_actions, 5917 const struct rte_flow_action *actions, 5918 int actions_n, struct rte_flow_error *error, 5919 int encap_idx) 5920 { 5921 struct mlx5_flow_action_copy_mreg *cp_mreg = 5922 (struct mlx5_flow_action_copy_mreg *) 5923 (ext_actions + actions_n + 1); 5924 int ret; 5925 5926 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5927 if (ret < 0) 5928 return ret; 5929 cp_mreg->dst = ret; 5930 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5931 if (ret < 0) 5932 return ret; 5933 cp_mreg->src = ret; 5934 if (encap_idx != 0) 5935 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5936 if (encap_idx == actions_n - 1) { 5937 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5938 .type = (enum rte_flow_action_type) 5939 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5940 .conf = cp_mreg, 5941 }; 5942 ext_actions[actions_n] = (struct rte_flow_action){ 5943 .type = RTE_FLOW_ACTION_TYPE_END, 5944 }; 5945 } else { 5946 ext_actions[encap_idx] = (struct rte_flow_action){ 5947 .type = (enum rte_flow_action_type) 5948 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5949 .conf = cp_mreg, 5950 }; 5951 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5952 sizeof(*ext_actions) * (actions_n - encap_idx)); 5953 } 5954 return 0; 5955 } 5956 5957 /** 5958 * Check the match action from the action list. 5959 * 5960 * @param[in] actions 5961 * Pointer to the list of actions. 5962 * @param[in] attr 5963 * Flow rule attributes. 5964 * @param[in] action 5965 * The action to be check if exist. 5966 * @param[out] match_action_pos 5967 * Pointer to the position of the matched action if exists, otherwise is -1. 5968 * @param[out] qrss_action_pos 5969 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5970 * @param[out] modify_after_mirror 5971 * Pointer to the flag of modify action after FDB mirroring. 5972 * 5973 * @return 5974 * > 0 the total number of actions. 5975 * 0 if not found match action in action list. 5976 */ 5977 static int 5978 flow_check_match_action(const struct rte_flow_action actions[], 5979 const struct rte_flow_attr *attr, 5980 enum rte_flow_action_type action, 5981 int *match_action_pos, int *qrss_action_pos, 5982 int *modify_after_mirror) 5983 { 5984 const struct rte_flow_action_sample *sample; 5985 const struct rte_flow_action_raw_decap *decap; 5986 int actions_n = 0; 5987 uint32_t ratio = 0; 5988 int sub_type = 0; 5989 int flag = 0; 5990 int fdb_mirror = 0; 5991 5992 *match_action_pos = -1; 5993 *qrss_action_pos = -1; 5994 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5995 if (actions->type == action) { 5996 flag = 1; 5997 *match_action_pos = actions_n; 5998 } 5999 switch (actions->type) { 6000 case RTE_FLOW_ACTION_TYPE_QUEUE: 6001 case RTE_FLOW_ACTION_TYPE_RSS: 6002 *qrss_action_pos = actions_n; 6003 break; 6004 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6005 sample = actions->conf; 6006 ratio = sample->ratio; 6007 sub_type = ((const struct rte_flow_action *) 6008 (sample->actions))->type; 6009 if (ratio == 1 && attr->transfer && 6010 sub_type != RTE_FLOW_ACTION_TYPE_END) 6011 fdb_mirror = 1; 6012 break; 6013 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6014 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6015 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6016 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6017 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6018 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6019 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6020 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6021 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6022 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6023 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6024 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6025 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6026 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6027 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6028 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6029 case RTE_FLOW_ACTION_TYPE_FLAG: 6030 case RTE_FLOW_ACTION_TYPE_MARK: 6031 case RTE_FLOW_ACTION_TYPE_SET_META: 6032 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6033 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6034 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6035 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6036 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6037 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6038 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6039 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6040 case RTE_FLOW_ACTION_TYPE_METER: 6041 if (fdb_mirror) 6042 *modify_after_mirror = 1; 6043 break; 6044 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6045 decap = actions->conf; 6046 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 6047 ; 6048 actions_n++; 6049 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6050 const struct rte_flow_action_raw_encap *encap = 6051 actions->conf; 6052 if (decap->size <= 6053 MLX5_ENCAPSULATION_DECISION_SIZE && 6054 encap->size > 6055 MLX5_ENCAPSULATION_DECISION_SIZE) 6056 /* L3 encap. */ 6057 break; 6058 } 6059 if (fdb_mirror) 6060 *modify_after_mirror = 1; 6061 break; 6062 default: 6063 break; 6064 } 6065 actions_n++; 6066 } 6067 if (flag && fdb_mirror && !*modify_after_mirror) { 6068 /* FDB mirroring uses the destination array to implement 6069 * instead of FLOW_SAMPLER object. 6070 */ 6071 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6072 flag = 0; 6073 } 6074 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6075 return flag ? actions_n + 1 : 0; 6076 } 6077 6078 #define SAMPLE_SUFFIX_ITEM 3 6079 6080 /** 6081 * Split the sample flow. 6082 * 6083 * As sample flow will split to two sub flow, sample flow with 6084 * sample action, the other actions will move to new suffix flow. 6085 * 6086 * Also add unique tag id with tag action in the sample flow, 6087 * the same tag id will be as match in the suffix flow. 6088 * 6089 * @param dev 6090 * Pointer to Ethernet device. 6091 * @param[in] add_tag 6092 * Add extra tag action flag. 6093 * @param[out] sfx_items 6094 * Suffix flow match items (list terminated by the END pattern item). 6095 * @param[in] actions 6096 * Associated actions (list terminated by the END action). 6097 * @param[out] actions_sfx 6098 * Suffix flow actions. 6099 * @param[out] actions_pre 6100 * Prefix flow actions. 6101 * @param[in] actions_n 6102 * The total number of actions. 6103 * @param[in] sample_action_pos 6104 * The sample action position. 6105 * @param[in] qrss_action_pos 6106 * The Queue/RSS action position. 6107 * @param[in] jump_table 6108 * Add extra jump action flag. 6109 * @param[out] error 6110 * Perform verbose error reporting if not NULL. 6111 * 6112 * @return 6113 * 0 on success, or unique flow_id, a negative errno value 6114 * otherwise and rte_errno is set. 6115 */ 6116 static int 6117 flow_sample_split_prep(struct rte_eth_dev *dev, 6118 int add_tag, 6119 const struct rte_flow_item items[], 6120 struct rte_flow_item sfx_items[], 6121 const struct rte_flow_action actions[], 6122 struct rte_flow_action actions_sfx[], 6123 struct rte_flow_action actions_pre[], 6124 int actions_n, 6125 int sample_action_pos, 6126 int qrss_action_pos, 6127 int jump_table, 6128 struct rte_flow_error *error) 6129 { 6130 struct mlx5_priv *priv = dev->data->dev_private; 6131 struct mlx5_rte_flow_action_set_tag *set_tag; 6132 struct mlx5_rte_flow_item_tag *tag_spec; 6133 struct mlx5_rte_flow_item_tag *tag_mask; 6134 struct rte_flow_action_jump *jump_action; 6135 uint32_t tag_id = 0; 6136 int append_index = 0; 6137 int set_tag_idx = -1; 6138 int index; 6139 int ret; 6140 6141 if (sample_action_pos < 0) 6142 return rte_flow_error_set(error, EINVAL, 6143 RTE_FLOW_ERROR_TYPE_ACTION, 6144 NULL, "invalid position of sample " 6145 "action in list"); 6146 /* Prepare the actions for prefix and suffix flow. */ 6147 if (add_tag) { 6148 /* Update the new added tag action index preceding 6149 * the PUSH_VLAN or ENCAP action. 6150 */ 6151 const struct rte_flow_action_raw_encap *raw_encap; 6152 const struct rte_flow_action *action = actions; 6153 int encap_idx; 6154 int action_idx = 0; 6155 int raw_decap_idx = -1; 6156 int push_vlan_idx = -1; 6157 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6158 switch (action->type) { 6159 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6160 raw_decap_idx = action_idx; 6161 break; 6162 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6163 raw_encap = action->conf; 6164 if (raw_encap->size > 6165 MLX5_ENCAPSULATION_DECISION_SIZE) { 6166 encap_idx = raw_decap_idx != -1 ? 6167 raw_decap_idx : action_idx; 6168 if (encap_idx < sample_action_pos && 6169 push_vlan_idx == -1) 6170 set_tag_idx = encap_idx; 6171 } 6172 break; 6173 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6174 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6175 encap_idx = action_idx; 6176 if (encap_idx < sample_action_pos && 6177 push_vlan_idx == -1) 6178 set_tag_idx = encap_idx; 6179 break; 6180 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6181 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6182 if (action_idx < sample_action_pos && 6183 push_vlan_idx == -1) { 6184 set_tag_idx = action_idx; 6185 push_vlan_idx = action_idx; 6186 } 6187 break; 6188 default: 6189 break; 6190 } 6191 action_idx++; 6192 } 6193 } 6194 /* Prepare the actions for prefix and suffix flow. */ 6195 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6196 index = qrss_action_pos; 6197 /* Put the preceding the Queue/RSS action into prefix flow. */ 6198 if (index != 0) 6199 memcpy(actions_pre, actions, 6200 sizeof(struct rte_flow_action) * index); 6201 /* Put others preceding the sample action into prefix flow. */ 6202 if (sample_action_pos > index + 1) 6203 memcpy(actions_pre + index, actions + index + 1, 6204 sizeof(struct rte_flow_action) * 6205 (sample_action_pos - index - 1)); 6206 index = sample_action_pos - 1; 6207 /* Put Queue/RSS action into Suffix flow. */ 6208 memcpy(actions_sfx, actions + qrss_action_pos, 6209 sizeof(struct rte_flow_action)); 6210 actions_sfx++; 6211 } else if (add_tag && set_tag_idx >= 0) { 6212 if (set_tag_idx > 0) 6213 memcpy(actions_pre, actions, 6214 sizeof(struct rte_flow_action) * set_tag_idx); 6215 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6216 sizeof(struct rte_flow_action) * 6217 (sample_action_pos - set_tag_idx)); 6218 index = sample_action_pos; 6219 } else { 6220 index = sample_action_pos; 6221 if (index != 0) 6222 memcpy(actions_pre, actions, 6223 sizeof(struct rte_flow_action) * index); 6224 } 6225 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6226 * For CX6DX and above, metadata registers Cx preserve their value, 6227 * add an extra tag action for NIC-RX and E-Switch Domain. 6228 */ 6229 if (add_tag) { 6230 /* Prepare the prefix tag action. */ 6231 append_index++; 6232 set_tag = (void *)(actions_pre + actions_n + append_index); 6233 /* Trust VF/SF on CX5 not supported meter so that the reserved 6234 * metadata regC is REG_NON, back to use application tag 6235 * index 0. 6236 */ 6237 if (unlikely(priv->mtr_color_reg == REG_NON)) 6238 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6239 else 6240 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6241 if (ret < 0) 6242 return ret; 6243 mlx5_ipool_malloc(priv->sh->ipool 6244 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6245 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6246 .id = ret, 6247 .data = tag_id, 6248 }; 6249 /* Prepare the suffix subflow items. */ 6250 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6251 tag_spec->data = tag_id; 6252 tag_spec->id = set_tag->id; 6253 tag_mask = tag_spec + 1; 6254 tag_mask->data = UINT32_MAX; 6255 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6256 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6257 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6258 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6259 memcpy(sfx_items, items, sizeof(*sfx_items)); 6260 sfx_items++; 6261 break; 6262 } 6263 } 6264 sfx_items[0] = (struct rte_flow_item){ 6265 .type = (enum rte_flow_item_type) 6266 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6267 .spec = tag_spec, 6268 .last = NULL, 6269 .mask = tag_mask, 6270 }; 6271 sfx_items[1] = (struct rte_flow_item){ 6272 .type = (enum rte_flow_item_type) 6273 RTE_FLOW_ITEM_TYPE_END, 6274 }; 6275 /* Prepare the tag action in prefix subflow. */ 6276 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6277 actions_pre[set_tag_idx] = 6278 (struct rte_flow_action){ 6279 .type = (enum rte_flow_action_type) 6280 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6281 .conf = set_tag, 6282 }; 6283 /* Update next sample position due to add one tag action */ 6284 index += 1; 6285 } 6286 /* Copy the sample action into prefix flow. */ 6287 memcpy(actions_pre + index, actions + sample_action_pos, 6288 sizeof(struct rte_flow_action)); 6289 index += 1; 6290 /* For the modify action after the sample action in E-Switch mirroring, 6291 * Add the extra jump action in prefix subflow and jump into the next 6292 * table, then do the modify action in the new table. 6293 */ 6294 if (jump_table) { 6295 /* Prepare the prefix jump action. */ 6296 append_index++; 6297 jump_action = (void *)(actions_pre + actions_n + append_index); 6298 jump_action->group = jump_table; 6299 actions_pre[index++] = 6300 (struct rte_flow_action){ 6301 .type = (enum rte_flow_action_type) 6302 RTE_FLOW_ACTION_TYPE_JUMP, 6303 .conf = jump_action, 6304 }; 6305 } 6306 actions_pre[index] = (struct rte_flow_action){ 6307 .type = (enum rte_flow_action_type) 6308 RTE_FLOW_ACTION_TYPE_END, 6309 }; 6310 /* Put the actions after sample into Suffix flow. */ 6311 memcpy(actions_sfx, actions + sample_action_pos + 1, 6312 sizeof(struct rte_flow_action) * 6313 (actions_n - sample_action_pos - 1)); 6314 return tag_id; 6315 } 6316 6317 /** 6318 * The splitting for metadata feature. 6319 * 6320 * - Q/RSS action on NIC Rx should be split in order to pass by 6321 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6322 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6323 * 6324 * - All the actions on NIC Tx should have a mreg copy action to 6325 * copy reg_a from WQE to reg_c[0]. 6326 * 6327 * @param dev 6328 * Pointer to Ethernet device. 6329 * @param[in] flow 6330 * Parent flow structure pointer. 6331 * @param[in] attr 6332 * Flow rule attributes. 6333 * @param[in] items 6334 * Pattern specification (list terminated by the END pattern item). 6335 * @param[in] actions 6336 * Associated actions (list terminated by the END action). 6337 * @param[in] flow_split_info 6338 * Pointer to flow split info structure. 6339 * @param[out] error 6340 * Perform verbose error reporting if not NULL. 6341 * @return 6342 * 0 on success, negative value otherwise 6343 */ 6344 static int 6345 flow_create_split_metadata(struct rte_eth_dev *dev, 6346 struct rte_flow *flow, 6347 const struct rte_flow_attr *attr, 6348 const struct rte_flow_item items[], 6349 const struct rte_flow_action actions[], 6350 struct mlx5_flow_split_info *flow_split_info, 6351 struct rte_flow_error *error) 6352 { 6353 struct mlx5_priv *priv = dev->data->dev_private; 6354 struct mlx5_sh_config *config = &priv->sh->config; 6355 const struct rte_flow_action *qrss = NULL; 6356 struct rte_flow_action *ext_actions = NULL; 6357 struct mlx5_flow *dev_flow = NULL; 6358 uint32_t qrss_id = 0; 6359 int mtr_sfx = 0; 6360 size_t act_size; 6361 int actions_n; 6362 int encap_idx; 6363 int ret; 6364 6365 /* Check whether extensive metadata feature is engaged. */ 6366 if (!config->dv_flow_en || 6367 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6368 !mlx5_flow_ext_mreg_supported(dev)) 6369 return flow_create_split_inner(dev, flow, NULL, attr, items, 6370 actions, flow_split_info, error); 6371 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6372 &encap_idx); 6373 if (qrss) { 6374 /* Exclude hairpin flows from splitting. */ 6375 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6376 const struct rte_flow_action_queue *queue; 6377 6378 queue = qrss->conf; 6379 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6380 qrss = NULL; 6381 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6382 const struct rte_flow_action_rss *rss; 6383 6384 rss = qrss->conf; 6385 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6386 qrss = NULL; 6387 } 6388 } 6389 if (qrss) { 6390 /* Check if it is in meter suffix table. */ 6391 mtr_sfx = attr->group == 6392 ((attr->transfer && priv->fdb_def_rule) ? 6393 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6394 MLX5_FLOW_TABLE_LEVEL_METER); 6395 /* 6396 * Q/RSS action on NIC Rx should be split in order to pass by 6397 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6398 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6399 */ 6400 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6401 sizeof(struct rte_flow_action_set_tag) + 6402 sizeof(struct rte_flow_action_jump); 6403 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6404 SOCKET_ID_ANY); 6405 if (!ext_actions) 6406 return rte_flow_error_set(error, ENOMEM, 6407 RTE_FLOW_ERROR_TYPE_ACTION, 6408 NULL, "no memory to split " 6409 "metadata flow"); 6410 /* 6411 * Create the new actions list with removed Q/RSS action 6412 * and appended set tag and jump to register copy table 6413 * (RX_CP_TBL). We should preallocate unique tag ID here 6414 * in advance, because it is needed for set tag action. 6415 */ 6416 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6417 qrss, actions_n, 6418 mtr_sfx, error); 6419 if (!mtr_sfx && !qrss_id) { 6420 ret = -rte_errno; 6421 goto exit; 6422 } 6423 } else if (attr->egress) { 6424 /* 6425 * All the actions on NIC Tx should have a metadata register 6426 * copy action to copy reg_a from WQE to reg_c[meta] 6427 */ 6428 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6429 sizeof(struct mlx5_flow_action_copy_mreg); 6430 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6431 SOCKET_ID_ANY); 6432 if (!ext_actions) 6433 return rte_flow_error_set(error, ENOMEM, 6434 RTE_FLOW_ERROR_TYPE_ACTION, 6435 NULL, "no memory to split " 6436 "metadata flow"); 6437 /* Create the action list appended with copy register. */ 6438 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6439 actions_n, error, encap_idx); 6440 if (ret < 0) 6441 goto exit; 6442 } 6443 /* Add the unmodified original or prefix subflow. */ 6444 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6445 items, ext_actions ? ext_actions : 6446 actions, flow_split_info, error); 6447 if (ret < 0) 6448 goto exit; 6449 MLX5_ASSERT(dev_flow); 6450 if (qrss) { 6451 const struct rte_flow_attr q_attr = { 6452 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6453 .ingress = 1, 6454 }; 6455 /* Internal PMD action to set register. */ 6456 struct mlx5_rte_flow_item_tag q_tag_spec = { 6457 .data = qrss_id, 6458 .id = REG_NON, 6459 }; 6460 struct rte_flow_item q_items[] = { 6461 { 6462 .type = (enum rte_flow_item_type) 6463 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6464 .spec = &q_tag_spec, 6465 .last = NULL, 6466 .mask = NULL, 6467 }, 6468 { 6469 .type = RTE_FLOW_ITEM_TYPE_END, 6470 }, 6471 }; 6472 struct rte_flow_action q_actions[] = { 6473 { 6474 .type = qrss->type, 6475 .conf = qrss->conf, 6476 }, 6477 { 6478 .type = RTE_FLOW_ACTION_TYPE_END, 6479 }, 6480 }; 6481 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6482 6483 /* 6484 * Configure the tag item only if there is no meter subflow. 6485 * Since tag is already marked in the meter suffix subflow 6486 * we can just use the meter suffix items as is. 6487 */ 6488 if (qrss_id) { 6489 /* Not meter subflow. */ 6490 MLX5_ASSERT(!mtr_sfx); 6491 /* 6492 * Put unique id in prefix flow due to it is destroyed 6493 * after suffix flow and id will be freed after there 6494 * is no actual flows with this id and identifier 6495 * reallocation becomes possible (for example, for 6496 * other flows in other threads). 6497 */ 6498 dev_flow->handle->split_flow_id = qrss_id; 6499 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6500 error); 6501 if (ret < 0) 6502 goto exit; 6503 q_tag_spec.id = ret; 6504 } 6505 dev_flow = NULL; 6506 /* Add suffix subflow to execute Q/RSS. */ 6507 flow_split_info->prefix_layers = layers; 6508 flow_split_info->prefix_mark = 0; 6509 flow_split_info->table_id = 0; 6510 ret = flow_create_split_inner(dev, flow, &dev_flow, 6511 &q_attr, mtr_sfx ? items : 6512 q_items, q_actions, 6513 flow_split_info, error); 6514 if (ret < 0) 6515 goto exit; 6516 /* qrss ID should be freed if failed. */ 6517 qrss_id = 0; 6518 MLX5_ASSERT(dev_flow); 6519 } 6520 6521 exit: 6522 /* 6523 * We do not destroy the partially created sub_flows in case of error. 6524 * These ones are included into parent flow list and will be destroyed 6525 * by flow_drv_destroy. 6526 */ 6527 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6528 qrss_id); 6529 mlx5_free(ext_actions); 6530 return ret; 6531 } 6532 6533 /** 6534 * Create meter internal drop flow with the original pattern. 6535 * 6536 * @param dev 6537 * Pointer to Ethernet device. 6538 * @param[in] flow 6539 * Parent flow structure pointer. 6540 * @param[in] attr 6541 * Flow rule attributes. 6542 * @param[in] items 6543 * Pattern specification (list terminated by the END pattern item). 6544 * @param[in] flow_split_info 6545 * Pointer to flow split info structure. 6546 * @param[in] fm 6547 * Pointer to flow meter structure. 6548 * @param[out] error 6549 * Perform verbose error reporting if not NULL. 6550 * @return 6551 * 0 on success, negative value otherwise 6552 */ 6553 static uint32_t 6554 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6555 struct rte_flow *flow, 6556 const struct rte_flow_attr *attr, 6557 const struct rte_flow_item items[], 6558 struct mlx5_flow_split_info *flow_split_info, 6559 struct mlx5_flow_meter_info *fm, 6560 struct rte_flow_error *error) 6561 { 6562 struct mlx5_flow *dev_flow = NULL; 6563 struct rte_flow_attr drop_attr = *attr; 6564 struct rte_flow_action drop_actions[3]; 6565 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6566 6567 MLX5_ASSERT(fm->drop_cnt); 6568 drop_actions[0].type = 6569 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6570 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6571 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6572 drop_actions[1].conf = NULL; 6573 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6574 drop_actions[2].conf = NULL; 6575 drop_split_info.external = false; 6576 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6577 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6578 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6579 return flow_create_split_inner(dev, flow, &dev_flow, 6580 &drop_attr, items, drop_actions, 6581 &drop_split_info, error); 6582 } 6583 6584 /** 6585 * The splitting for meter feature. 6586 * 6587 * - The meter flow will be split to two flows as prefix and 6588 * suffix flow. The packets make sense only it pass the prefix 6589 * meter action. 6590 * 6591 * - Reg_C_5 is used for the packet to match betweend prefix and 6592 * suffix flow. 6593 * 6594 * @param dev 6595 * Pointer to Ethernet device. 6596 * @param[in] flow 6597 * Parent flow structure pointer. 6598 * @param[in] attr 6599 * Flow rule attributes. 6600 * @param[in] items 6601 * Pattern specification (list terminated by the END pattern item). 6602 * @param[in] actions 6603 * Associated actions (list terminated by the END action). 6604 * @param[in] flow_split_info 6605 * Pointer to flow split info structure. 6606 * @param[out] error 6607 * Perform verbose error reporting if not NULL. 6608 * @return 6609 * 0 on success, negative value otherwise 6610 */ 6611 static int 6612 flow_create_split_meter(struct rte_eth_dev *dev, 6613 struct rte_flow *flow, 6614 const struct rte_flow_attr *attr, 6615 const struct rte_flow_item items[], 6616 const struct rte_flow_action actions[], 6617 struct mlx5_flow_split_info *flow_split_info, 6618 struct rte_flow_error *error) 6619 { 6620 struct mlx5_priv *priv = dev->data->dev_private; 6621 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6622 struct rte_flow_action *sfx_actions = NULL; 6623 struct rte_flow_action *pre_actions = NULL; 6624 struct rte_flow_item *sfx_items = NULL; 6625 struct mlx5_flow *dev_flow = NULL; 6626 struct rte_flow_attr sfx_attr = *attr; 6627 struct mlx5_flow_meter_info *fm = NULL; 6628 uint8_t skip_scale_restore; 6629 bool has_mtr = false; 6630 bool has_modify = false; 6631 bool set_mtr_reg = true; 6632 bool is_mtr_hierarchy = false; 6633 uint32_t meter_id = 0; 6634 uint32_t mtr_idx = 0; 6635 uint32_t mtr_flow_id = 0; 6636 size_t act_size; 6637 size_t item_size; 6638 int actions_n = 0; 6639 int ret = 0; 6640 6641 if (priv->mtr_en) 6642 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6643 &has_modify, &meter_id); 6644 if (has_mtr) { 6645 if (flow->meter) { 6646 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6647 if (!fm) 6648 return rte_flow_error_set(error, EINVAL, 6649 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6650 NULL, "Meter not found."); 6651 } else { 6652 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6653 if (!fm) 6654 return rte_flow_error_set(error, EINVAL, 6655 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6656 NULL, "Meter not found."); 6657 ret = mlx5_flow_meter_attach(priv, fm, 6658 &sfx_attr, error); 6659 if (ret) 6660 return -rte_errno; 6661 flow->meter = mtr_idx; 6662 } 6663 MLX5_ASSERT(wks); 6664 wks->fm = fm; 6665 if (!fm->def_policy) { 6666 wks->policy = mlx5_flow_meter_policy_find(dev, 6667 fm->policy_id, 6668 NULL); 6669 MLX5_ASSERT(wks->policy); 6670 if (wks->policy->mark) 6671 wks->mark = 1; 6672 if (wks->policy->is_hierarchy) { 6673 wks->final_policy = 6674 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6675 wks->policy); 6676 if (!wks->final_policy) 6677 return rte_flow_error_set(error, 6678 EINVAL, 6679 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6680 "Failed to find terminal policy of hierarchy."); 6681 is_mtr_hierarchy = true; 6682 } 6683 } 6684 /* 6685 * If it isn't default-policy Meter, and 6686 * 1. Not meter hierarchy and there's no action in flow to change 6687 * packet (modify/encap/decap etc.), OR 6688 * 2. No drop count needed for this meter. 6689 * Then no need to use regC to save meter id anymore. 6690 */ 6691 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6692 set_mtr_reg = false; 6693 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6694 #define METER_PREFIX_ACTION 7 6695 act_size = (sizeof(struct rte_flow_action) * 6696 (actions_n + METER_PREFIX_ACTION)) + 6697 sizeof(struct mlx5_rte_flow_action_set_tag); 6698 /* Suffix items: tag, vlan, port id, end. */ 6699 #define METER_SUFFIX_ITEM 4 6700 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6701 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6702 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6703 0, SOCKET_ID_ANY); 6704 if (!sfx_actions) 6705 return rte_flow_error_set(error, ENOMEM, 6706 RTE_FLOW_ERROR_TYPE_ACTION, 6707 NULL, "no memory to split " 6708 "meter flow"); 6709 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6710 act_size); 6711 /* There's no suffix flow for meter of non-default policy. */ 6712 if (!fm->def_policy) 6713 pre_actions = sfx_actions + 1; 6714 else 6715 pre_actions = sfx_actions + actions_n; 6716 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6717 items, sfx_items, actions, 6718 sfx_actions, pre_actions, 6719 (set_mtr_reg ? &mtr_flow_id : NULL), 6720 error); 6721 if (ret) { 6722 ret = -rte_errno; 6723 goto exit; 6724 } 6725 /* Add the prefix subflow. */ 6726 skip_scale_restore = flow_split_info->skip_scale; 6727 flow_split_info->skip_scale |= 6728 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6729 ret = flow_create_split_inner(dev, flow, &dev_flow, 6730 attr, items, pre_actions, 6731 flow_split_info, error); 6732 flow_split_info->skip_scale = skip_scale_restore; 6733 if (ret) { 6734 if (mtr_flow_id) 6735 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6736 ret = -rte_errno; 6737 goto exit; 6738 } 6739 if (mtr_flow_id) { 6740 dev_flow->handle->split_flow_id = mtr_flow_id; 6741 dev_flow->handle->is_meter_flow_id = 1; 6742 } 6743 if (!fm->def_policy) { 6744 if (!set_mtr_reg && fm->drop_cnt) 6745 ret = 6746 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6747 &sfx_attr, items, 6748 flow_split_info, 6749 fm, error); 6750 goto exit; 6751 } 6752 /* Setting the sfx group atrr. */ 6753 sfx_attr.group = sfx_attr.transfer ? 6754 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6755 MLX5_FLOW_TABLE_LEVEL_METER; 6756 flow_split_info->prefix_layers = 6757 flow_get_prefix_layer_flags(dev_flow); 6758 flow_split_info->prefix_mark |= wks->mark; 6759 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6760 } 6761 /* Add the prefix subflow. */ 6762 ret = flow_create_split_metadata(dev, flow, 6763 &sfx_attr, sfx_items ? 6764 sfx_items : items, 6765 sfx_actions ? sfx_actions : actions, 6766 flow_split_info, error); 6767 exit: 6768 if (sfx_actions) 6769 mlx5_free(sfx_actions); 6770 return ret; 6771 } 6772 6773 /** 6774 * The splitting for sample feature. 6775 * 6776 * Once Sample action is detected in the action list, the flow actions should 6777 * be split into prefix sub flow and suffix sub flow. 6778 * 6779 * The original items remain in the prefix sub flow, all actions preceding the 6780 * sample action and the sample action itself will be copied to the prefix 6781 * sub flow, the actions following the sample action will be copied to the 6782 * suffix sub flow, Queue action always be located in the suffix sub flow. 6783 * 6784 * In order to make the packet from prefix sub flow matches with suffix sub 6785 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6786 * flow uses tag item with the unique flow id. 6787 * 6788 * @param dev 6789 * Pointer to Ethernet device. 6790 * @param[in] flow 6791 * Parent flow structure pointer. 6792 * @param[in] attr 6793 * Flow rule attributes. 6794 * @param[in] items 6795 * Pattern specification (list terminated by the END pattern item). 6796 * @param[in] actions 6797 * Associated actions (list terminated by the END action). 6798 * @param[in] flow_split_info 6799 * Pointer to flow split info structure. 6800 * @param[out] error 6801 * Perform verbose error reporting if not NULL. 6802 * @return 6803 * 0 on success, negative value otherwise 6804 */ 6805 static int 6806 flow_create_split_sample(struct rte_eth_dev *dev, 6807 struct rte_flow *flow, 6808 const struct rte_flow_attr *attr, 6809 const struct rte_flow_item items[], 6810 const struct rte_flow_action actions[], 6811 struct mlx5_flow_split_info *flow_split_info, 6812 struct rte_flow_error *error) 6813 { 6814 struct mlx5_priv *priv = dev->data->dev_private; 6815 struct rte_flow_action *sfx_actions = NULL; 6816 struct rte_flow_action *pre_actions = NULL; 6817 struct rte_flow_item *sfx_items = NULL; 6818 struct mlx5_flow *dev_flow = NULL; 6819 struct rte_flow_attr sfx_attr = *attr; 6820 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6821 struct mlx5_flow_dv_sample_resource *sample_res; 6822 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6823 struct mlx5_flow_tbl_resource *sfx_tbl; 6824 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6825 #endif 6826 size_t act_size; 6827 size_t item_size; 6828 uint32_t fdb_tx = 0; 6829 int32_t tag_id = 0; 6830 int actions_n = 0; 6831 int sample_action_pos; 6832 int qrss_action_pos; 6833 int add_tag = 0; 6834 int modify_after_mirror = 0; 6835 uint16_t jump_table = 0; 6836 const uint32_t next_ft_step = 1; 6837 int ret = 0; 6838 struct mlx5_priv *item_port_priv = NULL; 6839 const struct rte_flow_item *item; 6840 6841 if (priv->sampler_en) 6842 actions_n = flow_check_match_action(actions, attr, 6843 RTE_FLOW_ACTION_TYPE_SAMPLE, 6844 &sample_action_pos, &qrss_action_pos, 6845 &modify_after_mirror); 6846 if (actions_n) { 6847 /* The prefix actions must includes sample, tag, end. */ 6848 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6849 + sizeof(struct mlx5_rte_flow_action_set_tag); 6850 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6851 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6852 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6853 item_size), 0, SOCKET_ID_ANY); 6854 if (!sfx_actions) 6855 return rte_flow_error_set(error, ENOMEM, 6856 RTE_FLOW_ERROR_TYPE_ACTION, 6857 NULL, "no memory to split " 6858 "sample flow"); 6859 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 6860 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6861 const struct rte_flow_item_port_id *spec; 6862 6863 spec = (const struct rte_flow_item_port_id *)item->spec; 6864 if (spec) 6865 item_port_priv = 6866 mlx5_port_to_eswitch_info(spec->id, true); 6867 break; 6868 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6869 const struct rte_flow_item_ethdev *spec; 6870 6871 spec = (const struct rte_flow_item_ethdev *)item->spec; 6872 if (spec) 6873 item_port_priv = 6874 mlx5_port_to_eswitch_info(spec->port_id, true); 6875 break; 6876 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 6877 const struct rte_flow_item_ethdev *spec; 6878 6879 spec = (const struct rte_flow_item_ethdev *)item->spec; 6880 if (spec) 6881 item_port_priv = 6882 mlx5_port_to_eswitch_info(spec->port_id, true); 6883 break; 6884 } 6885 } 6886 /* The representor_id is UINT16_MAX for uplink. */ 6887 fdb_tx = (attr->transfer && 6888 flow_source_vport_representor(priv, item_port_priv)); 6889 /* 6890 * When reg_c_preserve is set, metadata registers Cx preserve 6891 * their value even through packet duplication. 6892 */ 6893 add_tag = (!fdb_tx || 6894 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6895 if (add_tag) 6896 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6897 + act_size); 6898 if (modify_after_mirror) 6899 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6900 next_ft_step; 6901 pre_actions = sfx_actions + actions_n; 6902 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6903 actions, sfx_actions, 6904 pre_actions, actions_n, 6905 sample_action_pos, 6906 qrss_action_pos, jump_table, 6907 error); 6908 if (tag_id < 0 || (add_tag && !tag_id)) { 6909 ret = -rte_errno; 6910 goto exit; 6911 } 6912 if (modify_after_mirror) 6913 flow_split_info->skip_scale = 6914 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6915 /* Add the prefix subflow. */ 6916 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6917 items, pre_actions, 6918 flow_split_info, error); 6919 if (ret) { 6920 ret = -rte_errno; 6921 goto exit; 6922 } 6923 dev_flow->handle->split_flow_id = tag_id; 6924 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6925 if (!modify_after_mirror) { 6926 /* Set the sfx group attr. */ 6927 sample_res = (struct mlx5_flow_dv_sample_resource *) 6928 dev_flow->dv.sample_res; 6929 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6930 sample_res->normal_path_tbl; 6931 sfx_tbl_data = container_of(sfx_tbl, 6932 struct mlx5_flow_tbl_data_entry, 6933 tbl); 6934 sfx_attr.group = sfx_attr.transfer ? 6935 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6936 } else { 6937 MLX5_ASSERT(attr->transfer); 6938 sfx_attr.group = jump_table; 6939 } 6940 flow_split_info->prefix_layers = 6941 flow_get_prefix_layer_flags(dev_flow); 6942 MLX5_ASSERT(wks); 6943 flow_split_info->prefix_mark |= wks->mark; 6944 /* Suffix group level already be scaled with factor, set 6945 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6946 * again in translation. 6947 */ 6948 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6949 #endif 6950 } 6951 /* Add the suffix subflow. */ 6952 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6953 sfx_items ? sfx_items : items, 6954 sfx_actions ? sfx_actions : actions, 6955 flow_split_info, error); 6956 exit: 6957 if (sfx_actions) 6958 mlx5_free(sfx_actions); 6959 return ret; 6960 } 6961 6962 /** 6963 * Split the flow to subflow set. The splitters might be linked 6964 * in the chain, like this: 6965 * flow_create_split_outer() calls: 6966 * flow_create_split_meter() calls: 6967 * flow_create_split_metadata(meter_subflow_0) calls: 6968 * flow_create_split_inner(metadata_subflow_0) 6969 * flow_create_split_inner(metadata_subflow_1) 6970 * flow_create_split_inner(metadata_subflow_2) 6971 * flow_create_split_metadata(meter_subflow_1) calls: 6972 * flow_create_split_inner(metadata_subflow_0) 6973 * flow_create_split_inner(metadata_subflow_1) 6974 * flow_create_split_inner(metadata_subflow_2) 6975 * 6976 * This provide flexible way to add new levels of flow splitting. 6977 * The all of successfully created subflows are included to the 6978 * parent flow dev_flow list. 6979 * 6980 * @param dev 6981 * Pointer to Ethernet device. 6982 * @param[in] flow 6983 * Parent flow structure pointer. 6984 * @param[in] attr 6985 * Flow rule attributes. 6986 * @param[in] items 6987 * Pattern specification (list terminated by the END pattern item). 6988 * @param[in] actions 6989 * Associated actions (list terminated by the END action). 6990 * @param[in] flow_split_info 6991 * Pointer to flow split info structure. 6992 * @param[out] error 6993 * Perform verbose error reporting if not NULL. 6994 * @return 6995 * 0 on success, negative value otherwise 6996 */ 6997 static int 6998 flow_create_split_outer(struct rte_eth_dev *dev, 6999 struct rte_flow *flow, 7000 const struct rte_flow_attr *attr, 7001 const struct rte_flow_item items[], 7002 const struct rte_flow_action actions[], 7003 struct mlx5_flow_split_info *flow_split_info, 7004 struct rte_flow_error *error) 7005 { 7006 int ret; 7007 7008 ret = flow_create_split_sample(dev, flow, attr, items, 7009 actions, flow_split_info, error); 7010 MLX5_ASSERT(ret <= 0); 7011 return ret; 7012 } 7013 7014 static inline struct mlx5_flow_tunnel * 7015 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7016 { 7017 struct mlx5_flow_tunnel *tunnel; 7018 7019 #pragma GCC diagnostic push 7020 #pragma GCC diagnostic ignored "-Wcast-qual" 7021 tunnel = (typeof(tunnel))flow->tunnel; 7022 #pragma GCC diagnostic pop 7023 7024 return tunnel; 7025 } 7026 7027 /** 7028 * Adjust flow RSS workspace if needed. 7029 * 7030 * @param wks 7031 * Pointer to thread flow work space. 7032 * @param rss_desc 7033 * Pointer to RSS descriptor. 7034 * @param[in] nrssq_num 7035 * New RSS queue number. 7036 * 7037 * @return 7038 * 0 on success, -1 otherwise and rte_errno is set. 7039 */ 7040 static int 7041 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 7042 struct mlx5_flow_rss_desc *rss_desc, 7043 uint32_t nrssq_num) 7044 { 7045 if (likely(nrssq_num <= wks->rssq_num)) 7046 return 0; 7047 rss_desc->queue = realloc(rss_desc->queue, 7048 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 7049 if (!rss_desc->queue) { 7050 rte_errno = ENOMEM; 7051 return -1; 7052 } 7053 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 7054 return 0; 7055 } 7056 7057 /** 7058 * Create a flow and add it to @p list. 7059 * 7060 * @param dev 7061 * Pointer to Ethernet device. 7062 * @param list 7063 * Pointer to a TAILQ flow list. If this parameter NULL, 7064 * no list insertion occurred, flow is just created, 7065 * this is caller's responsibility to track the 7066 * created flow. 7067 * @param[in] attr 7068 * Flow rule attributes. 7069 * @param[in] items 7070 * Pattern specification (list terminated by the END pattern item). 7071 * @param[in] actions 7072 * Associated actions (list terminated by the END action). 7073 * @param[in] external 7074 * This flow rule is created by request external to PMD. 7075 * @param[out] error 7076 * Perform verbose error reporting if not NULL. 7077 * 7078 * @return 7079 * A flow index on success, 0 otherwise and rte_errno is set. 7080 */ 7081 static uint32_t 7082 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7083 const struct rte_flow_attr *attr, 7084 const struct rte_flow_item items[], 7085 const struct rte_flow_action original_actions[], 7086 bool external, struct rte_flow_error *error) 7087 { 7088 struct mlx5_priv *priv = dev->data->dev_private; 7089 struct rte_flow *flow = NULL; 7090 struct mlx5_flow *dev_flow; 7091 const struct rte_flow_action_rss *rss = NULL; 7092 struct mlx5_translated_action_handle 7093 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7094 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7095 union { 7096 struct mlx5_flow_expand_rss buf; 7097 uint8_t buffer[8192]; 7098 } expand_buffer; 7099 union { 7100 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7101 uint8_t buffer[2048]; 7102 } actions_rx; 7103 union { 7104 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7105 uint8_t buffer[2048]; 7106 } actions_hairpin_tx; 7107 union { 7108 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7109 uint8_t buffer[2048]; 7110 } items_tx; 7111 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7112 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7113 struct mlx5_flow_rss_desc *rss_desc; 7114 const struct rte_flow_action *p_actions_rx; 7115 uint32_t i; 7116 uint32_t idx = 0; 7117 int hairpin_flow; 7118 struct rte_flow_attr attr_tx = { .priority = 0 }; 7119 const struct rte_flow_action *actions; 7120 struct rte_flow_action *translated_actions = NULL; 7121 struct mlx5_flow_tunnel *tunnel; 7122 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7123 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7124 struct mlx5_flow_split_info flow_split_info = { 7125 .external = !!external, 7126 .skip_scale = 0, 7127 .flow_idx = 0, 7128 .prefix_mark = 0, 7129 .prefix_layers = 0, 7130 .table_id = 0 7131 }; 7132 int ret; 7133 7134 MLX5_ASSERT(wks); 7135 rss_desc = &wks->rss_desc; 7136 ret = flow_action_handles_translate(dev, original_actions, 7137 indir_actions, 7138 &indir_actions_n, 7139 &translated_actions, error); 7140 if (ret < 0) { 7141 MLX5_ASSERT(translated_actions == NULL); 7142 return 0; 7143 } 7144 actions = translated_actions ? translated_actions : original_actions; 7145 p_actions_rx = actions; 7146 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7147 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7148 external, hairpin_flow, error); 7149 if (ret < 0) 7150 goto error_before_hairpin_split; 7151 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7152 if (!flow) { 7153 rte_errno = ENOMEM; 7154 goto error_before_hairpin_split; 7155 } 7156 if (hairpin_flow > 0) { 7157 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7158 rte_errno = EINVAL; 7159 goto error_before_hairpin_split; 7160 } 7161 flow_hairpin_split(dev, actions, actions_rx.actions, 7162 actions_hairpin_tx.actions, items_tx.items, 7163 idx); 7164 p_actions_rx = actions_rx.actions; 7165 } 7166 flow_split_info.flow_idx = idx; 7167 flow->drv_type = flow_get_drv_type(dev, attr); 7168 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7169 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7170 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7171 /* RSS Action only works on NIC RX domain */ 7172 if (attr->ingress) 7173 rss = flow_get_rss_action(dev, p_actions_rx); 7174 if (rss) { 7175 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 7176 return 0; 7177 /* 7178 * The following information is required by 7179 * mlx5_flow_hashfields_adjust() in advance. 7180 */ 7181 rss_desc->level = rss->level; 7182 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7183 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7184 } 7185 flow->dev_handles = 0; 7186 if (rss && rss->types) { 7187 unsigned int graph_root; 7188 7189 graph_root = find_graph_root(rss->level); 7190 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7191 items, rss->types, 7192 mlx5_support_expansion, graph_root); 7193 MLX5_ASSERT(ret > 0 && 7194 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7195 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7196 for (i = 0; i < buf->entries; ++i) 7197 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7198 } 7199 } else { 7200 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7201 sizeof(expand_buffer.buffer), 7202 items, sq_specs); 7203 if (ret) { 7204 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7205 NULL, "not enough memory for rte_flow"); 7206 goto error; 7207 } 7208 if (buf->entries == 0) { 7209 buf->entries = 1; 7210 buf->entry[0].pattern = (void *)(uintptr_t)items; 7211 } 7212 } 7213 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7214 indir_actions_n); 7215 for (i = 0; i < buf->entries; ++i) { 7216 /* Initialize flow split data. */ 7217 flow_split_info.prefix_layers = 0; 7218 flow_split_info.prefix_mark = 0; 7219 flow_split_info.skip_scale = 0; 7220 /* 7221 * The splitter may create multiple dev_flows, 7222 * depending on configuration. In the simplest 7223 * case it just creates unmodified original flow. 7224 */ 7225 ret = flow_create_split_outer(dev, flow, attr, 7226 buf->entry[i].pattern, 7227 p_actions_rx, &flow_split_info, 7228 error); 7229 if (ret < 0) 7230 goto error; 7231 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7232 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7233 p_actions_rx, 7234 idx, 7235 wks->flows[0].tunnel, 7236 &default_miss_ctx, 7237 error); 7238 if (ret < 0) { 7239 mlx5_free(default_miss_ctx.queue); 7240 goto error; 7241 } 7242 } 7243 } 7244 /* Create the tx flow. */ 7245 if (hairpin_flow) { 7246 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7247 attr_tx.ingress = 0; 7248 attr_tx.egress = 1; 7249 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7250 actions_hairpin_tx.actions, 7251 idx, error); 7252 if (!dev_flow) 7253 goto error; 7254 dev_flow->flow = flow; 7255 dev_flow->external = 0; 7256 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7257 dev_flow->handle, next); 7258 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7259 items_tx.items, 7260 actions_hairpin_tx.actions, error); 7261 if (ret < 0) 7262 goto error; 7263 } 7264 /* 7265 * Update the metadata register copy table. If extensive 7266 * metadata feature is enabled and registers are supported 7267 * we might create the extra rte_flow for each unique 7268 * MARK/FLAG action ID. 7269 * 7270 * The table is updated for ingress and transfer flows only, because 7271 * the egress Flows belong to the different device and 7272 * copy table should be updated in peer NIC Rx domain. 7273 */ 7274 if ((attr->ingress || attr->transfer) && 7275 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7276 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7277 if (ret) 7278 goto error; 7279 } 7280 /* 7281 * If the flow is external (from application) OR device is started, 7282 * OR mreg discover, then apply immediately. 7283 */ 7284 if (external || dev->data->dev_started || 7285 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7286 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7287 ret = flow_drv_apply(dev, flow, error); 7288 if (ret < 0) 7289 goto error; 7290 } 7291 flow->type = type; 7292 flow_rxq_flags_set(dev, flow); 7293 rte_free(translated_actions); 7294 tunnel = flow_tunnel_from_rule(wks->flows); 7295 if (tunnel) { 7296 flow->tunnel = 1; 7297 flow->tunnel_id = tunnel->tunnel_id; 7298 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7299 mlx5_free(default_miss_ctx.queue); 7300 } 7301 mlx5_flow_pop_thread_workspace(); 7302 return idx; 7303 error: 7304 MLX5_ASSERT(flow); 7305 ret = rte_errno; /* Save rte_errno before cleanup. */ 7306 flow_mreg_del_copy_action(dev, flow); 7307 flow_drv_destroy(dev, flow); 7308 if (rss_desc->shared_rss) 7309 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7310 mlx5_ipool_get 7311 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7312 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7313 mlx5_ipool_free(priv->flows[type], idx); 7314 rte_errno = ret; /* Restore rte_errno. */ 7315 ret = rte_errno; 7316 rte_errno = ret; 7317 error_before_hairpin_split: 7318 mlx5_flow_pop_thread_workspace(); 7319 rte_free(translated_actions); 7320 return 0; 7321 } 7322 7323 /** 7324 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7325 * incoming packets to table 1. 7326 * 7327 * Other flow rules, requested for group n, will be created in 7328 * e-switch table n+1. 7329 * Jump action to e-switch group n will be created to group n+1. 7330 * 7331 * Used when working in switchdev mode, to utilise advantages of table 1 7332 * and above. 7333 * 7334 * @param dev 7335 * Pointer to Ethernet device. 7336 * 7337 * @return 7338 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7339 */ 7340 struct rte_flow * 7341 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7342 { 7343 const struct rte_flow_attr attr = { 7344 .group = 0, 7345 .priority = 0, 7346 .ingress = 0, 7347 .egress = 0, 7348 .transfer = 1, 7349 }; 7350 const struct rte_flow_item pattern = { 7351 .type = RTE_FLOW_ITEM_TYPE_END, 7352 }; 7353 struct rte_flow_action_jump jump = { 7354 .group = 1, 7355 }; 7356 const struct rte_flow_action actions[] = { 7357 { 7358 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7359 .conf = &jump, 7360 }, 7361 { 7362 .type = RTE_FLOW_ACTION_TYPE_END, 7363 }, 7364 }; 7365 struct rte_flow_error error; 7366 7367 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7368 &attr, &pattern, 7369 actions, false, &error); 7370 } 7371 7372 /** 7373 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7374 * and sq number, directs all packets to peer vport. 7375 * 7376 * @param dev 7377 * Pointer to Ethernet device. 7378 * @param sq_num 7379 * SQ number. 7380 * 7381 * @return 7382 * Flow ID on success, 0 otherwise and rte_errno is set. 7383 */ 7384 uint32_t 7385 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7386 { 7387 struct rte_flow_attr attr = { 7388 .group = 0, 7389 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7390 .ingress = 0, 7391 .egress = 0, 7392 .transfer = 1, 7393 }; 7394 struct rte_flow_item_port_id port_spec = { 7395 .id = MLX5_PORT_ESW_MGR, 7396 }; 7397 struct mlx5_rte_flow_item_sq sq_spec = { 7398 .queue = sq_num, 7399 }; 7400 struct rte_flow_item pattern[] = { 7401 { 7402 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7403 .spec = &port_spec, 7404 }, 7405 { 7406 .type = (enum rte_flow_item_type) 7407 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7408 .spec = &sq_spec, 7409 }, 7410 { 7411 .type = RTE_FLOW_ITEM_TYPE_END, 7412 }, 7413 }; 7414 struct rte_flow_action_jump jump = { 7415 .group = 1, 7416 }; 7417 struct rte_flow_action_port_id port = { 7418 .id = dev->data->port_id, 7419 }; 7420 struct rte_flow_action actions[] = { 7421 { 7422 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7423 .conf = &jump, 7424 }, 7425 { 7426 .type = RTE_FLOW_ACTION_TYPE_END, 7427 }, 7428 }; 7429 struct rte_flow_error error; 7430 7431 /* 7432 * Creates group 0, highest priority jump flow. 7433 * Matches txq to bypass kernel packets. 7434 */ 7435 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7436 false, &error) == 0) 7437 return 0; 7438 /* Create group 1, lowest priority redirect flow for txq. */ 7439 attr.group = 1; 7440 actions[0].conf = &port; 7441 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7442 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7443 actions, false, &error); 7444 } 7445 7446 /** 7447 * Validate a flow supported by the NIC. 7448 * 7449 * @see rte_flow_validate() 7450 * @see rte_flow_ops 7451 */ 7452 int 7453 mlx5_flow_validate(struct rte_eth_dev *dev, 7454 const struct rte_flow_attr *attr, 7455 const struct rte_flow_item items[], 7456 const struct rte_flow_action original_actions[], 7457 struct rte_flow_error *error) 7458 { 7459 int hairpin_flow; 7460 struct mlx5_translated_action_handle 7461 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7462 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7463 const struct rte_flow_action *actions; 7464 struct rte_flow_action *translated_actions = NULL; 7465 int ret = flow_action_handles_translate(dev, original_actions, 7466 indir_actions, 7467 &indir_actions_n, 7468 &translated_actions, error); 7469 7470 if (ret) 7471 return ret; 7472 actions = translated_actions ? translated_actions : original_actions; 7473 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7474 ret = flow_drv_validate(dev, attr, items, actions, 7475 true, hairpin_flow, error); 7476 rte_free(translated_actions); 7477 return ret; 7478 } 7479 7480 /** 7481 * Create a flow. 7482 * 7483 * @see rte_flow_create() 7484 * @see rte_flow_ops 7485 */ 7486 struct rte_flow * 7487 mlx5_flow_create(struct rte_eth_dev *dev, 7488 const struct rte_flow_attr *attr, 7489 const struct rte_flow_item items[], 7490 const struct rte_flow_action actions[], 7491 struct rte_flow_error *error) 7492 { 7493 struct mlx5_priv *priv = dev->data->dev_private; 7494 7495 if (priv->sh->config.dv_flow_en == 2) { 7496 rte_flow_error_set(error, ENOTSUP, 7497 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7498 NULL, 7499 "Flow non-Q creation not supported"); 7500 return NULL; 7501 } 7502 /* 7503 * If the device is not started yet, it is not allowed to created a 7504 * flow from application. PMD default flows and traffic control flows 7505 * are not affected. 7506 */ 7507 if (unlikely(!dev->data->dev_started)) { 7508 DRV_LOG(DEBUG, "port %u is not started when " 7509 "inserting a flow", dev->data->port_id); 7510 rte_flow_error_set(error, ENODEV, 7511 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7512 NULL, 7513 "port not started"); 7514 return NULL; 7515 } 7516 7517 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7518 attr, items, actions, 7519 true, error); 7520 } 7521 7522 /** 7523 * Destroy a flow in a list. 7524 * 7525 * @param dev 7526 * Pointer to Ethernet device. 7527 * @param[in] flow_idx 7528 * Index of flow to destroy. 7529 */ 7530 static void 7531 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7532 uint32_t flow_idx) 7533 { 7534 struct mlx5_priv *priv = dev->data->dev_private; 7535 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7536 7537 if (!flow) 7538 return; 7539 MLX5_ASSERT(flow->type == type); 7540 /* 7541 * Update RX queue flags only if port is started, otherwise it is 7542 * already clean. 7543 */ 7544 if (dev->data->dev_started) 7545 flow_rxq_flags_trim(dev, flow); 7546 flow_drv_destroy(dev, flow); 7547 if (flow->tunnel) { 7548 struct mlx5_flow_tunnel *tunnel; 7549 7550 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7551 RTE_VERIFY(tunnel); 7552 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7553 mlx5_flow_tunnel_free(dev, tunnel); 7554 } 7555 flow_mreg_del_copy_action(dev, flow); 7556 mlx5_ipool_free(priv->flows[type], flow_idx); 7557 } 7558 7559 /** 7560 * Destroy all flows. 7561 * 7562 * @param dev 7563 * Pointer to Ethernet device. 7564 * @param type 7565 * Flow type to be flushed. 7566 * @param active 7567 * If flushing is called actively. 7568 */ 7569 void 7570 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7571 bool active) 7572 { 7573 struct mlx5_priv *priv = dev->data->dev_private; 7574 uint32_t num_flushed = 0, fidx = 1; 7575 struct rte_flow *flow; 7576 7577 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7578 if (priv->sh->config.dv_flow_en == 2 && 7579 type == MLX5_FLOW_TYPE_GEN) { 7580 flow_hw_q_flow_flush(dev, NULL); 7581 return; 7582 } 7583 #endif 7584 7585 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7586 flow_list_destroy(dev, type, fidx); 7587 num_flushed++; 7588 } 7589 if (active) { 7590 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7591 dev->data->port_id, num_flushed); 7592 } 7593 } 7594 7595 /** 7596 * Stop all default actions for flows. 7597 * 7598 * @param dev 7599 * Pointer to Ethernet device. 7600 */ 7601 void 7602 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7603 { 7604 flow_mreg_del_default_copy_action(dev); 7605 flow_rxq_flags_clear(dev); 7606 } 7607 7608 /** 7609 * Set rxq flag. 7610 * 7611 * @param[in] dev 7612 * Pointer to the rte_eth_dev structure. 7613 * @param[in] enable 7614 * Flag to enable or not. 7615 */ 7616 void 7617 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 7618 { 7619 struct mlx5_priv *priv = dev->data->dev_private; 7620 unsigned int i; 7621 7622 if ((!priv->mark_enabled && !enable) || 7623 (priv->mark_enabled && enable)) 7624 return; 7625 for (i = 0; i < priv->rxqs_n; ++i) { 7626 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 7627 7628 /* With RXQ start/stop feature, RXQ might be stopped. */ 7629 if (!rxq_ctrl) 7630 continue; 7631 rxq_ctrl->rxq.mark = enable; 7632 } 7633 priv->mark_enabled = enable; 7634 } 7635 7636 /** 7637 * Start all default actions for flows. 7638 * 7639 * @param dev 7640 * Pointer to Ethernet device. 7641 * @return 7642 * 0 on success, a negative errno value otherwise and rte_errno is set. 7643 */ 7644 int 7645 mlx5_flow_start_default(struct rte_eth_dev *dev) 7646 { 7647 struct rte_flow_error error; 7648 7649 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7650 return flow_mreg_add_default_copy_action(dev, &error); 7651 } 7652 7653 /** 7654 * Release key of thread specific flow workspace data. 7655 */ 7656 void 7657 flow_release_workspace(void *data) 7658 { 7659 struct mlx5_flow_workspace *wks = data; 7660 struct mlx5_flow_workspace *next; 7661 7662 while (wks) { 7663 next = wks->next; 7664 free(wks->rss_desc.queue); 7665 free(wks); 7666 wks = next; 7667 } 7668 } 7669 7670 /** 7671 * Get thread specific current flow workspace. 7672 * 7673 * @return pointer to thread specific flow workspace data, NULL on error. 7674 */ 7675 struct mlx5_flow_workspace* 7676 mlx5_flow_get_thread_workspace(void) 7677 { 7678 struct mlx5_flow_workspace *data; 7679 7680 data = mlx5_flow_os_get_specific_workspace(); 7681 MLX5_ASSERT(data && data->inuse); 7682 if (!data || !data->inuse) 7683 DRV_LOG(ERR, "flow workspace not initialized."); 7684 return data; 7685 } 7686 7687 /** 7688 * Allocate and init new flow workspace. 7689 * 7690 * @return pointer to flow workspace data, NULL on error. 7691 */ 7692 static struct mlx5_flow_workspace* 7693 flow_alloc_thread_workspace(void) 7694 { 7695 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7696 7697 if (!data) { 7698 DRV_LOG(ERR, "Failed to allocate flow workspace " 7699 "memory."); 7700 return NULL; 7701 } 7702 data->rss_desc.queue = calloc(1, 7703 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 7704 if (!data->rss_desc.queue) 7705 goto err; 7706 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 7707 return data; 7708 err: 7709 free(data->rss_desc.queue); 7710 free(data); 7711 return NULL; 7712 } 7713 7714 /** 7715 * Get new thread specific flow workspace. 7716 * 7717 * If current workspace inuse, create new one and set as current. 7718 * 7719 * @return pointer to thread specific flow workspace data, NULL on error. 7720 */ 7721 struct mlx5_flow_workspace* 7722 mlx5_flow_push_thread_workspace(void) 7723 { 7724 struct mlx5_flow_workspace *curr; 7725 struct mlx5_flow_workspace *data; 7726 7727 curr = mlx5_flow_os_get_specific_workspace(); 7728 if (!curr) { 7729 data = flow_alloc_thread_workspace(); 7730 if (!data) 7731 return NULL; 7732 } else if (!curr->inuse) { 7733 data = curr; 7734 } else if (curr->next) { 7735 data = curr->next; 7736 } else { 7737 data = flow_alloc_thread_workspace(); 7738 if (!data) 7739 return NULL; 7740 curr->next = data; 7741 data->prev = curr; 7742 } 7743 data->inuse = 1; 7744 data->flow_idx = 0; 7745 /* Set as current workspace */ 7746 if (mlx5_flow_os_set_specific_workspace(data)) 7747 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7748 return data; 7749 } 7750 7751 /** 7752 * Close current thread specific flow workspace. 7753 * 7754 * If previous workspace available, set it as current. 7755 * 7756 * @return pointer to thread specific flow workspace data, NULL on error. 7757 */ 7758 void 7759 mlx5_flow_pop_thread_workspace(void) 7760 { 7761 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 7762 7763 if (!data) 7764 return; 7765 if (!data->inuse) { 7766 DRV_LOG(ERR, "Failed to close unused flow workspace."); 7767 return; 7768 } 7769 data->inuse = 0; 7770 if (!data->prev) 7771 return; 7772 if (mlx5_flow_os_set_specific_workspace(data->prev)) 7773 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 7774 } 7775 7776 /** 7777 * Verify the flow list is empty 7778 * 7779 * @param dev 7780 * Pointer to Ethernet device. 7781 * 7782 * @return the number of flows not released. 7783 */ 7784 int 7785 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 7786 { 7787 struct mlx5_priv *priv = dev->data->dev_private; 7788 struct rte_flow *flow; 7789 uint32_t idx = 0; 7790 int ret = 0, i; 7791 7792 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 7793 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 7794 DRV_LOG(DEBUG, "port %u flow %p still referenced", 7795 dev->data->port_id, (void *)flow); 7796 ret++; 7797 } 7798 } 7799 return ret; 7800 } 7801 7802 /** 7803 * Enable default hairpin egress flow. 7804 * 7805 * @param dev 7806 * Pointer to Ethernet device. 7807 * @param sq_num 7808 * The SQ hw number. 7809 * 7810 * @return 7811 * 0 on success, a negative errno value otherwise and rte_errno is set. 7812 */ 7813 int 7814 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 7815 uint32_t sq_num) 7816 { 7817 const struct rte_flow_attr attr = { 7818 .egress = 1, 7819 .priority = 0, 7820 }; 7821 struct mlx5_rte_flow_item_sq queue_spec = { 7822 .queue = sq_num, 7823 }; 7824 struct mlx5_rte_flow_item_sq queue_mask = { 7825 .queue = UINT32_MAX, 7826 }; 7827 struct rte_flow_item items[] = { 7828 { 7829 .type = (enum rte_flow_item_type) 7830 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7831 .spec = &queue_spec, 7832 .last = NULL, 7833 .mask = &queue_mask, 7834 }, 7835 { 7836 .type = RTE_FLOW_ITEM_TYPE_END, 7837 }, 7838 }; 7839 struct rte_flow_action_jump jump = { 7840 .group = MLX5_HAIRPIN_TX_TABLE, 7841 }; 7842 struct rte_flow_action actions[2]; 7843 uint32_t flow_idx; 7844 struct rte_flow_error error; 7845 7846 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 7847 actions[0].conf = &jump; 7848 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 7849 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7850 &attr, items, actions, false, &error); 7851 if (!flow_idx) { 7852 DRV_LOG(DEBUG, 7853 "Failed to create ctrl flow: rte_errno(%d)," 7854 " type(%d), message(%s)", 7855 rte_errno, error.type, 7856 error.message ? error.message : " (no stated reason)"); 7857 return -rte_errno; 7858 } 7859 return 0; 7860 } 7861 7862 /** 7863 * Enable a control flow configured from the control plane. 7864 * 7865 * @param dev 7866 * Pointer to Ethernet device. 7867 * @param eth_spec 7868 * An Ethernet flow spec to apply. 7869 * @param eth_mask 7870 * An Ethernet flow mask to apply. 7871 * @param vlan_spec 7872 * A VLAN flow spec to apply. 7873 * @param vlan_mask 7874 * A VLAN flow mask to apply. 7875 * 7876 * @return 7877 * 0 on success, a negative errno value otherwise and rte_errno is set. 7878 */ 7879 int 7880 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 7881 struct rte_flow_item_eth *eth_spec, 7882 struct rte_flow_item_eth *eth_mask, 7883 struct rte_flow_item_vlan *vlan_spec, 7884 struct rte_flow_item_vlan *vlan_mask) 7885 { 7886 struct mlx5_priv *priv = dev->data->dev_private; 7887 const struct rte_flow_attr attr = { 7888 .ingress = 1, 7889 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7890 }; 7891 struct rte_flow_item items[] = { 7892 { 7893 .type = RTE_FLOW_ITEM_TYPE_ETH, 7894 .spec = eth_spec, 7895 .last = NULL, 7896 .mask = eth_mask, 7897 }, 7898 { 7899 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 7900 RTE_FLOW_ITEM_TYPE_END, 7901 .spec = vlan_spec, 7902 .last = NULL, 7903 .mask = vlan_mask, 7904 }, 7905 { 7906 .type = RTE_FLOW_ITEM_TYPE_END, 7907 }, 7908 }; 7909 uint16_t queue[priv->reta_idx_n]; 7910 struct rte_flow_action_rss action_rss = { 7911 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 7912 .level = 0, 7913 .types = priv->rss_conf.rss_hf, 7914 .key_len = priv->rss_conf.rss_key_len, 7915 .queue_num = priv->reta_idx_n, 7916 .key = priv->rss_conf.rss_key, 7917 .queue = queue, 7918 }; 7919 struct rte_flow_action actions[] = { 7920 { 7921 .type = RTE_FLOW_ACTION_TYPE_RSS, 7922 .conf = &action_rss, 7923 }, 7924 { 7925 .type = RTE_FLOW_ACTION_TYPE_END, 7926 }, 7927 }; 7928 uint32_t flow_idx; 7929 struct rte_flow_error error; 7930 unsigned int i; 7931 7932 if (!priv->reta_idx_n || !priv->rxqs_n) { 7933 return 0; 7934 } 7935 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 7936 action_rss.types = 0; 7937 for (i = 0; i != priv->reta_idx_n; ++i) 7938 queue[i] = (*priv->reta_idx)[i]; 7939 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7940 &attr, items, actions, false, &error); 7941 if (!flow_idx) 7942 return -rte_errno; 7943 return 0; 7944 } 7945 7946 /** 7947 * Enable a flow control configured from the control plane. 7948 * 7949 * @param dev 7950 * Pointer to Ethernet device. 7951 * @param eth_spec 7952 * An Ethernet flow spec to apply. 7953 * @param eth_mask 7954 * An Ethernet flow mask to apply. 7955 * 7956 * @return 7957 * 0 on success, a negative errno value otherwise and rte_errno is set. 7958 */ 7959 int 7960 mlx5_ctrl_flow(struct rte_eth_dev *dev, 7961 struct rte_flow_item_eth *eth_spec, 7962 struct rte_flow_item_eth *eth_mask) 7963 { 7964 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 7965 } 7966 7967 /** 7968 * Create default miss flow rule matching lacp traffic 7969 * 7970 * @param dev 7971 * Pointer to Ethernet device. 7972 * @param eth_spec 7973 * An Ethernet flow spec to apply. 7974 * 7975 * @return 7976 * 0 on success, a negative errno value otherwise and rte_errno is set. 7977 */ 7978 int 7979 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 7980 { 7981 /* 7982 * The LACP matching is done by only using ether type since using 7983 * a multicast dst mac causes kernel to give low priority to this flow. 7984 */ 7985 static const struct rte_flow_item_eth lacp_spec = { 7986 .hdr.ether_type = RTE_BE16(0x8809), 7987 }; 7988 static const struct rte_flow_item_eth lacp_mask = { 7989 .hdr.ether_type = 0xffff, 7990 }; 7991 const struct rte_flow_attr attr = { 7992 .ingress = 1, 7993 }; 7994 struct rte_flow_item items[] = { 7995 { 7996 .type = RTE_FLOW_ITEM_TYPE_ETH, 7997 .spec = &lacp_spec, 7998 .mask = &lacp_mask, 7999 }, 8000 { 8001 .type = RTE_FLOW_ITEM_TYPE_END, 8002 }, 8003 }; 8004 struct rte_flow_action actions[] = { 8005 { 8006 .type = (enum rte_flow_action_type) 8007 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8008 }, 8009 { 8010 .type = RTE_FLOW_ACTION_TYPE_END, 8011 }, 8012 }; 8013 struct rte_flow_error error; 8014 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8015 &attr, items, actions, 8016 false, &error); 8017 8018 if (!flow_idx) 8019 return -rte_errno; 8020 return 0; 8021 } 8022 8023 /** 8024 * Destroy a flow. 8025 * 8026 * @see rte_flow_destroy() 8027 * @see rte_flow_ops 8028 */ 8029 int 8030 mlx5_flow_destroy(struct rte_eth_dev *dev, 8031 struct rte_flow *flow, 8032 struct rte_flow_error *error __rte_unused) 8033 { 8034 struct mlx5_priv *priv = dev->data->dev_private; 8035 8036 if (priv->sh->config.dv_flow_en == 2) 8037 return rte_flow_error_set(error, ENOTSUP, 8038 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8039 NULL, 8040 "Flow non-Q destruction not supported"); 8041 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8042 (uintptr_t)(void *)flow); 8043 return 0; 8044 } 8045 8046 /** 8047 * Destroy all flows. 8048 * 8049 * @see rte_flow_flush() 8050 * @see rte_flow_ops 8051 */ 8052 int 8053 mlx5_flow_flush(struct rte_eth_dev *dev, 8054 struct rte_flow_error *error __rte_unused) 8055 { 8056 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8057 return 0; 8058 } 8059 8060 /** 8061 * Isolated mode. 8062 * 8063 * @see rte_flow_isolate() 8064 * @see rte_flow_ops 8065 */ 8066 int 8067 mlx5_flow_isolate(struct rte_eth_dev *dev, 8068 int enable, 8069 struct rte_flow_error *error) 8070 { 8071 struct mlx5_priv *priv = dev->data->dev_private; 8072 8073 if (dev->data->dev_started) { 8074 rte_flow_error_set(error, EBUSY, 8075 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8076 NULL, 8077 "port must be stopped first"); 8078 return -rte_errno; 8079 } 8080 priv->isolated = !!enable; 8081 if (enable) 8082 dev->dev_ops = &mlx5_dev_ops_isolate; 8083 else 8084 dev->dev_ops = &mlx5_dev_ops; 8085 8086 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8087 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8088 8089 return 0; 8090 } 8091 8092 /** 8093 * Query a flow. 8094 * 8095 * @see rte_flow_query() 8096 * @see rte_flow_ops 8097 */ 8098 static int 8099 flow_drv_query(struct rte_eth_dev *dev, 8100 struct rte_flow *eflow, 8101 const struct rte_flow_action *actions, 8102 void *data, 8103 struct rte_flow_error *error) 8104 { 8105 struct mlx5_priv *priv = dev->data->dev_private; 8106 const struct mlx5_flow_driver_ops *fops; 8107 struct rte_flow *flow = NULL; 8108 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8109 8110 if (priv->sh->config.dv_flow_en == 2) { 8111 #ifdef HAVE_MLX5_HWS_SUPPORT 8112 flow = eflow; 8113 ftype = MLX5_FLOW_TYPE_HW; 8114 #endif 8115 } else { 8116 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8117 (uintptr_t)(void *)eflow); 8118 } 8119 if (!flow) { 8120 return rte_flow_error_set(error, ENOENT, 8121 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8122 NULL, 8123 "invalid flow handle"); 8124 } 8125 if (ftype == MLX5_FLOW_TYPE_MIN) 8126 ftype = flow->drv_type; 8127 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8128 fops = flow_get_drv_ops(ftype); 8129 8130 return fops->query(dev, flow, actions, data, error); 8131 } 8132 8133 /** 8134 * Query a flow. 8135 * 8136 * @see rte_flow_query() 8137 * @see rte_flow_ops 8138 */ 8139 int 8140 mlx5_flow_query(struct rte_eth_dev *dev, 8141 struct rte_flow *flow, 8142 const struct rte_flow_action *actions, 8143 void *data, 8144 struct rte_flow_error *error) 8145 { 8146 int ret; 8147 8148 ret = flow_drv_query(dev, flow, actions, data, 8149 error); 8150 if (ret < 0) 8151 return ret; 8152 return 0; 8153 } 8154 8155 /** 8156 * Get rte_flow callbacks. 8157 * 8158 * @param dev 8159 * Pointer to Ethernet device structure. 8160 * @param ops 8161 * Pointer to operation-specific structure. 8162 * 8163 * @return 0 8164 */ 8165 int 8166 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8167 const struct rte_flow_ops **ops) 8168 { 8169 *ops = &mlx5_flow_ops; 8170 return 0; 8171 } 8172 8173 /** 8174 * Validate meter policy actions. 8175 * Dispatcher for action type specific validation. 8176 * 8177 * @param[in] dev 8178 * Pointer to the Ethernet device structure. 8179 * @param[in] action 8180 * The meter policy action object to validate. 8181 * @param[in] attr 8182 * Attributes of flow to determine steering domain. 8183 * @param[out] is_rss 8184 * Is RSS or not. 8185 * @param[out] domain_bitmap 8186 * Domain bitmap. 8187 * @param[out] is_def_policy 8188 * Is default policy or not. 8189 * @param[out] error 8190 * Perform verbose error reporting if not NULL. Initialized in case of 8191 * error only. 8192 * 8193 * @return 8194 * 0 on success, otherwise negative errno value. 8195 */ 8196 int 8197 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8198 const struct rte_flow_action *actions[RTE_COLORS], 8199 struct rte_flow_attr *attr, 8200 bool *is_rss, 8201 uint8_t *domain_bitmap, 8202 uint8_t *policy_mode, 8203 struct rte_mtr_error *error) 8204 { 8205 const struct mlx5_flow_driver_ops *fops; 8206 8207 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8208 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8209 domain_bitmap, policy_mode, error); 8210 } 8211 8212 /** 8213 * Destroy the meter table set. 8214 * 8215 * @param[in] dev 8216 * Pointer to Ethernet device. 8217 * @param[in] mtr_policy 8218 * Meter policy struct. 8219 */ 8220 void 8221 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8222 struct mlx5_flow_meter_policy *mtr_policy) 8223 { 8224 const struct mlx5_flow_driver_ops *fops; 8225 8226 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8227 fops->destroy_mtr_acts(dev, mtr_policy); 8228 } 8229 8230 /** 8231 * Create policy action, lock free, 8232 * (mutex should be acquired by caller). 8233 * Dispatcher for action type specific call. 8234 * 8235 * @param[in] dev 8236 * Pointer to the Ethernet device structure. 8237 * @param[in] mtr_policy 8238 * Meter policy struct. 8239 * @param[in] action 8240 * Action specification used to create meter actions. 8241 * @param[in] attr 8242 * Flow rule attributes. 8243 * @param[out] error 8244 * Perform verbose error reporting if not NULL. Initialized in case of 8245 * error only. 8246 * 8247 * @return 8248 * 0 on success, otherwise negative errno value. 8249 */ 8250 int 8251 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8252 struct mlx5_flow_meter_policy *mtr_policy, 8253 const struct rte_flow_action *actions[RTE_COLORS], 8254 struct rte_flow_attr *attr, 8255 struct rte_mtr_error *error) 8256 { 8257 const struct mlx5_flow_driver_ops *fops; 8258 8259 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8260 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8261 } 8262 8263 /** 8264 * Create policy rules, lock free, 8265 * (mutex should be acquired by caller). 8266 * Dispatcher for action type specific call. 8267 * 8268 * @param[in] dev 8269 * Pointer to the Ethernet device structure. 8270 * @param[in] mtr_policy 8271 * Meter policy struct. 8272 * 8273 * @return 8274 * 0 on success, -1 otherwise. 8275 */ 8276 int 8277 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8278 struct mlx5_flow_meter_policy *mtr_policy) 8279 { 8280 const struct mlx5_flow_driver_ops *fops; 8281 8282 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8283 return fops->create_policy_rules(dev, mtr_policy); 8284 } 8285 8286 /** 8287 * Destroy policy rules, lock free, 8288 * (mutex should be acquired by caller). 8289 * Dispatcher for action type specific call. 8290 * 8291 * @param[in] dev 8292 * Pointer to the Ethernet device structure. 8293 * @param[in] mtr_policy 8294 * Meter policy struct. 8295 */ 8296 void 8297 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8298 struct mlx5_flow_meter_policy *mtr_policy) 8299 { 8300 const struct mlx5_flow_driver_ops *fops; 8301 8302 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8303 fops->destroy_policy_rules(dev, mtr_policy); 8304 } 8305 8306 /** 8307 * Destroy the default policy table set. 8308 * 8309 * @param[in] dev 8310 * Pointer to Ethernet device. 8311 */ 8312 void 8313 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8314 { 8315 const struct mlx5_flow_driver_ops *fops; 8316 8317 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8318 fops->destroy_def_policy(dev); 8319 } 8320 8321 /** 8322 * Destroy the default policy table set. 8323 * 8324 * @param[in] dev 8325 * Pointer to Ethernet device. 8326 * 8327 * @return 8328 * 0 on success, -1 otherwise. 8329 */ 8330 int 8331 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8332 { 8333 const struct mlx5_flow_driver_ops *fops; 8334 8335 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8336 return fops->create_def_policy(dev); 8337 } 8338 8339 /** 8340 * Create the needed meter and suffix tables. 8341 * 8342 * @param[in] dev 8343 * Pointer to Ethernet device. 8344 * 8345 * @return 8346 * 0 on success, -1 otherwise. 8347 */ 8348 int 8349 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8350 struct mlx5_flow_meter_info *fm, 8351 uint32_t mtr_idx, 8352 uint8_t domain_bitmap) 8353 { 8354 const struct mlx5_flow_driver_ops *fops; 8355 8356 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8357 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8358 } 8359 8360 /** 8361 * Destroy the meter table set. 8362 * 8363 * @param[in] dev 8364 * Pointer to Ethernet device. 8365 * @param[in] tbl 8366 * Pointer to the meter table set. 8367 */ 8368 void 8369 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8370 struct mlx5_flow_meter_info *fm) 8371 { 8372 const struct mlx5_flow_driver_ops *fops; 8373 8374 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8375 fops->destroy_mtr_tbls(dev, fm); 8376 } 8377 8378 /** 8379 * Destroy the global meter drop table. 8380 * 8381 * @param[in] dev 8382 * Pointer to Ethernet device. 8383 */ 8384 void 8385 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8386 { 8387 const struct mlx5_flow_driver_ops *fops; 8388 8389 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8390 fops->destroy_mtr_drop_tbls(dev); 8391 } 8392 8393 /** 8394 * Destroy the sub policy table with RX queue. 8395 * 8396 * @param[in] dev 8397 * Pointer to Ethernet device. 8398 * @param[in] mtr_policy 8399 * Pointer to meter policy table. 8400 */ 8401 void 8402 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8403 struct mlx5_flow_meter_policy *mtr_policy) 8404 { 8405 const struct mlx5_flow_driver_ops *fops; 8406 8407 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8408 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8409 } 8410 8411 /** 8412 * Allocate the needed aso flow meter id. 8413 * 8414 * @param[in] dev 8415 * Pointer to Ethernet device. 8416 * 8417 * @return 8418 * Index to aso flow meter on success, NULL otherwise. 8419 */ 8420 uint32_t 8421 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8422 { 8423 const struct mlx5_flow_driver_ops *fops; 8424 8425 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8426 return fops->create_meter(dev); 8427 } 8428 8429 /** 8430 * Free the aso flow meter id. 8431 * 8432 * @param[in] dev 8433 * Pointer to Ethernet device. 8434 * @param[in] mtr_idx 8435 * Index to aso flow meter to be free. 8436 * 8437 * @return 8438 * 0 on success. 8439 */ 8440 void 8441 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8442 { 8443 const struct mlx5_flow_driver_ops *fops; 8444 8445 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8446 fops->free_meter(dev, mtr_idx); 8447 } 8448 8449 /** 8450 * Allocate a counter. 8451 * 8452 * @param[in] dev 8453 * Pointer to Ethernet device structure. 8454 * 8455 * @return 8456 * Index to allocated counter on success, 0 otherwise. 8457 */ 8458 uint32_t 8459 mlx5_counter_alloc(struct rte_eth_dev *dev) 8460 { 8461 struct rte_flow_attr attr = { .transfer = 0 }; 8462 8463 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8464 (dev); 8465 } 8466 8467 /** 8468 * Free a counter. 8469 * 8470 * @param[in] dev 8471 * Pointer to Ethernet device structure. 8472 * @param[in] cnt 8473 * Index to counter to be free. 8474 */ 8475 void 8476 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8477 { 8478 struct rte_flow_attr attr = { .transfer = 0 }; 8479 8480 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8481 } 8482 8483 /** 8484 * Query counter statistics. 8485 * 8486 * @param[in] dev 8487 * Pointer to Ethernet device structure. 8488 * @param[in] cnt 8489 * Index to counter to query. 8490 * @param[in] clear 8491 * Set to clear counter statistics. 8492 * @param[out] pkts 8493 * The counter hits packets number to save. 8494 * @param[out] bytes 8495 * The counter hits bytes number to save. 8496 * 8497 * @return 8498 * 0 on success, a negative errno value otherwise. 8499 */ 8500 int 8501 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8502 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8503 { 8504 struct rte_flow_attr attr = { .transfer = 0 }; 8505 8506 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8507 (dev, cnt, clear, pkts, bytes, action); 8508 } 8509 8510 /** 8511 * Get information about HWS pre-configurable resources. 8512 * 8513 * @param[in] dev 8514 * Pointer to the rte_eth_dev structure. 8515 * @param[out] port_info 8516 * Pointer to port information. 8517 * @param[out] queue_info 8518 * Pointer to queue information. 8519 * @param[out] error 8520 * Pointer to error structure. 8521 * 8522 * @return 8523 * 0 on success, a negative errno value otherwise and rte_errno is set. 8524 */ 8525 static int 8526 mlx5_flow_info_get(struct rte_eth_dev *dev, 8527 struct rte_flow_port_info *port_info, 8528 struct rte_flow_queue_info *queue_info, 8529 struct rte_flow_error *error) 8530 { 8531 const struct mlx5_flow_driver_ops *fops; 8532 struct rte_flow_attr attr = {0}; 8533 8534 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8535 return rte_flow_error_set(error, ENOTSUP, 8536 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8537 NULL, 8538 "info get with incorrect steering mode"); 8539 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8540 return fops->info_get(dev, port_info, queue_info, error); 8541 } 8542 8543 /** 8544 * Configure port HWS resources. 8545 * 8546 * @param[in] dev 8547 * Pointer to the rte_eth_dev structure. 8548 * @param[in] port_attr 8549 * Port configuration attributes. 8550 * @param[in] nb_queue 8551 * Number of queue. 8552 * @param[in] queue_attr 8553 * Array that holds attributes for each flow queue. 8554 * @param[out] error 8555 * Pointer to error structure. 8556 * 8557 * @return 8558 * 0 on success, a negative errno value otherwise and rte_errno is set. 8559 */ 8560 static int 8561 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8562 const struct rte_flow_port_attr *port_attr, 8563 uint16_t nb_queue, 8564 const struct rte_flow_queue_attr *queue_attr[], 8565 struct rte_flow_error *error) 8566 { 8567 const struct mlx5_flow_driver_ops *fops; 8568 struct rte_flow_attr attr = {0}; 8569 8570 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8571 return rte_flow_error_set(error, ENOTSUP, 8572 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8573 NULL, 8574 "port configure with incorrect steering mode"); 8575 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8576 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8577 } 8578 8579 /** 8580 * Validate item template. 8581 * 8582 * @param[in] dev 8583 * Pointer to the rte_eth_dev structure. 8584 * @param[in] attr 8585 * Pointer to the item template attributes. 8586 * @param[in] items 8587 * The template item pattern. 8588 * @param[out] error 8589 * Pointer to error structure. 8590 * 8591 * @return 8592 * 0 on success, a negative errno value otherwise and rte_errno is set. 8593 */ 8594 int 8595 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8596 const struct rte_flow_pattern_template_attr *attr, 8597 const struct rte_flow_item items[], 8598 struct rte_flow_error *error) 8599 { 8600 const struct mlx5_flow_driver_ops *fops; 8601 struct rte_flow_attr fattr = {0}; 8602 8603 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8604 rte_flow_error_set(error, ENOTSUP, 8605 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8606 "pattern validate with incorrect steering mode"); 8607 return -ENOTSUP; 8608 } 8609 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8610 return fops->pattern_validate(dev, attr, items, error); 8611 } 8612 8613 /** 8614 * Create flow item template. 8615 * 8616 * @param[in] dev 8617 * Pointer to the rte_eth_dev structure. 8618 * @param[in] attr 8619 * Pointer to the item template attributes. 8620 * @param[in] items 8621 * The template item pattern. 8622 * @param[out] error 8623 * Pointer to error structure. 8624 * 8625 * @return 8626 * 0 on success, a negative errno value otherwise and rte_errno is set. 8627 */ 8628 static struct rte_flow_pattern_template * 8629 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8630 const struct rte_flow_pattern_template_attr *attr, 8631 const struct rte_flow_item items[], 8632 struct rte_flow_error *error) 8633 { 8634 const struct mlx5_flow_driver_ops *fops; 8635 struct rte_flow_attr fattr = {0}; 8636 8637 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8638 rte_flow_error_set(error, ENOTSUP, 8639 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8640 NULL, 8641 "pattern create with incorrect steering mode"); 8642 return NULL; 8643 } 8644 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8645 return fops->pattern_template_create(dev, attr, items, error); 8646 } 8647 8648 /** 8649 * Destroy flow item template. 8650 * 8651 * @param[in] dev 8652 * Pointer to the rte_eth_dev structure. 8653 * @param[in] template 8654 * Pointer to the item template to be destroyed. 8655 * @param[out] error 8656 * Pointer to error structure. 8657 * 8658 * @return 8659 * 0 on success, a negative errno value otherwise and rte_errno is set. 8660 */ 8661 static int 8662 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8663 struct rte_flow_pattern_template *template, 8664 struct rte_flow_error *error) 8665 { 8666 const struct mlx5_flow_driver_ops *fops; 8667 struct rte_flow_attr attr = {0}; 8668 8669 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8670 return rte_flow_error_set(error, ENOTSUP, 8671 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8672 NULL, 8673 "pattern destroy with incorrect steering mode"); 8674 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8675 return fops->pattern_template_destroy(dev, template, error); 8676 } 8677 8678 /** 8679 * Validate flow actions template. 8680 * 8681 * @param[in] dev 8682 * Pointer to the rte_eth_dev structure. 8683 * @param[in] attr 8684 * Pointer to the action template attributes. 8685 * @param[in] actions 8686 * Associated actions (list terminated by the END action). 8687 * @param[in] masks 8688 * List of actions that marks which of the action's member is constant. 8689 * @param[out] error 8690 * Pointer to error structure. 8691 * 8692 * @return 8693 * 0 on success, a negative errno value otherwise and rte_errno is set. 8694 */ 8695 int 8696 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 8697 const struct rte_flow_actions_template_attr *attr, 8698 const struct rte_flow_action actions[], 8699 const struct rte_flow_action masks[], 8700 struct rte_flow_error *error) 8701 { 8702 const struct mlx5_flow_driver_ops *fops; 8703 struct rte_flow_attr fattr = {0}; 8704 8705 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8706 rte_flow_error_set(error, ENOTSUP, 8707 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8708 "actions validate with incorrect steering mode"); 8709 return -ENOTSUP; 8710 } 8711 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8712 return fops->actions_validate(dev, attr, actions, masks, error); 8713 } 8714 8715 /** 8716 * Create flow item template. 8717 * 8718 * @param[in] dev 8719 * Pointer to the rte_eth_dev structure. 8720 * @param[in] attr 8721 * Pointer to the action template attributes. 8722 * @param[in] actions 8723 * Associated actions (list terminated by the END action). 8724 * @param[in] masks 8725 * List of actions that marks which of the action's member is constant. 8726 * @param[out] error 8727 * Pointer to error structure. 8728 * 8729 * @return 8730 * 0 on success, a negative errno value otherwise and rte_errno is set. 8731 */ 8732 static struct rte_flow_actions_template * 8733 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 8734 const struct rte_flow_actions_template_attr *attr, 8735 const struct rte_flow_action actions[], 8736 const struct rte_flow_action masks[], 8737 struct rte_flow_error *error) 8738 { 8739 const struct mlx5_flow_driver_ops *fops; 8740 struct rte_flow_attr fattr = {0}; 8741 8742 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8743 rte_flow_error_set(error, ENOTSUP, 8744 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8745 NULL, 8746 "action create with incorrect steering mode"); 8747 return NULL; 8748 } 8749 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8750 return fops->actions_template_create(dev, attr, actions, masks, error); 8751 } 8752 8753 /** 8754 * Destroy flow action template. 8755 * 8756 * @param[in] dev 8757 * Pointer to the rte_eth_dev structure. 8758 * @param[in] template 8759 * Pointer to the action template to be destroyed. 8760 * @param[out] error 8761 * Pointer to error structure. 8762 * 8763 * @return 8764 * 0 on success, a negative errno value otherwise and rte_errno is set. 8765 */ 8766 static int 8767 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 8768 struct rte_flow_actions_template *template, 8769 struct rte_flow_error *error) 8770 { 8771 const struct mlx5_flow_driver_ops *fops; 8772 struct rte_flow_attr attr = {0}; 8773 8774 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8775 return rte_flow_error_set(error, ENOTSUP, 8776 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8777 NULL, 8778 "action destroy with incorrect steering mode"); 8779 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8780 return fops->actions_template_destroy(dev, template, error); 8781 } 8782 8783 /** 8784 * Create flow table. 8785 * 8786 * @param[in] dev 8787 * Pointer to the rte_eth_dev structure. 8788 * @param[in] attr 8789 * Pointer to the table attributes. 8790 * @param[in] item_templates 8791 * Item template array to be binded to the table. 8792 * @param[in] nb_item_templates 8793 * Number of item template. 8794 * @param[in] action_templates 8795 * Action template array to be binded to the table. 8796 * @param[in] nb_action_templates 8797 * Number of action template. 8798 * @param[out] error 8799 * Pointer to error structure. 8800 * 8801 * @return 8802 * Table on success, NULL otherwise and rte_errno is set. 8803 */ 8804 static struct rte_flow_template_table * 8805 mlx5_flow_table_create(struct rte_eth_dev *dev, 8806 const struct rte_flow_template_table_attr *attr, 8807 struct rte_flow_pattern_template *item_templates[], 8808 uint8_t nb_item_templates, 8809 struct rte_flow_actions_template *action_templates[], 8810 uint8_t nb_action_templates, 8811 struct rte_flow_error *error) 8812 { 8813 const struct mlx5_flow_driver_ops *fops; 8814 struct rte_flow_attr fattr = {0}; 8815 8816 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8817 rte_flow_error_set(error, ENOTSUP, 8818 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8819 NULL, 8820 "table create with incorrect steering mode"); 8821 return NULL; 8822 } 8823 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8824 return fops->template_table_create(dev, 8825 attr, 8826 item_templates, 8827 nb_item_templates, 8828 action_templates, 8829 nb_action_templates, 8830 error); 8831 } 8832 8833 /** 8834 * PMD destroy flow table. 8835 * 8836 * @param[in] dev 8837 * Pointer to the rte_eth_dev structure. 8838 * @param[in] table 8839 * Pointer to the table to be destroyed. 8840 * @param[out] error 8841 * Pointer to error structure. 8842 * 8843 * @return 8844 * 0 on success, a negative errno value otherwise and rte_errno is set. 8845 */ 8846 static int 8847 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 8848 struct rte_flow_template_table *table, 8849 struct rte_flow_error *error) 8850 { 8851 const struct mlx5_flow_driver_ops *fops; 8852 struct rte_flow_attr attr = {0}; 8853 8854 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8855 return rte_flow_error_set(error, ENOTSUP, 8856 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8857 NULL, 8858 "table destroy with incorrect steering mode"); 8859 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8860 return fops->template_table_destroy(dev, table, error); 8861 } 8862 8863 /** 8864 * Enqueue flow creation. 8865 * 8866 * @param[in] dev 8867 * Pointer to the rte_eth_dev structure. 8868 * @param[in] queue_id 8869 * The queue to create the flow. 8870 * @param[in] attr 8871 * Pointer to the flow operation attributes. 8872 * @param[in] items 8873 * Items with flow spec value. 8874 * @param[in] pattern_template_index 8875 * The item pattern flow follows from the table. 8876 * @param[in] actions 8877 * Action with flow spec value. 8878 * @param[in] action_template_index 8879 * The action pattern flow follows from the table. 8880 * @param[in] user_data 8881 * Pointer to the user_data. 8882 * @param[out] error 8883 * Pointer to error structure. 8884 * 8885 * @return 8886 * Flow pointer on success, NULL otherwise and rte_errno is set. 8887 */ 8888 static struct rte_flow * 8889 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 8890 uint32_t queue_id, 8891 const struct rte_flow_op_attr *attr, 8892 struct rte_flow_template_table *table, 8893 const struct rte_flow_item items[], 8894 uint8_t pattern_template_index, 8895 const struct rte_flow_action actions[], 8896 uint8_t action_template_index, 8897 void *user_data, 8898 struct rte_flow_error *error) 8899 { 8900 const struct mlx5_flow_driver_ops *fops; 8901 struct rte_flow_attr fattr = {0}; 8902 8903 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8904 rte_flow_error_set(error, ENOTSUP, 8905 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8906 NULL, 8907 "flow_q create with incorrect steering mode"); 8908 return NULL; 8909 } 8910 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8911 return fops->async_flow_create(dev, queue_id, attr, table, 8912 items, pattern_template_index, 8913 actions, action_template_index, 8914 user_data, error); 8915 } 8916 8917 /** 8918 * Enqueue flow destruction. 8919 * 8920 * @param[in] dev 8921 * Pointer to the rte_eth_dev structure. 8922 * @param[in] queue 8923 * The queue to destroy the flow. 8924 * @param[in] attr 8925 * Pointer to the flow operation attributes. 8926 * @param[in] flow 8927 * Pointer to the flow to be destroyed. 8928 * @param[in] user_data 8929 * Pointer to the user_data. 8930 * @param[out] error 8931 * Pointer to error structure. 8932 * 8933 * @return 8934 * 0 on success, negative value otherwise and rte_errno is set. 8935 */ 8936 static int 8937 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 8938 uint32_t queue, 8939 const struct rte_flow_op_attr *attr, 8940 struct rte_flow *flow, 8941 void *user_data, 8942 struct rte_flow_error *error) 8943 { 8944 const struct mlx5_flow_driver_ops *fops; 8945 struct rte_flow_attr fattr = {0}; 8946 8947 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 8948 return rte_flow_error_set(error, ENOTSUP, 8949 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8950 NULL, 8951 "flow_q destroy with incorrect steering mode"); 8952 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8953 return fops->async_flow_destroy(dev, queue, attr, flow, 8954 user_data, error); 8955 } 8956 8957 /** 8958 * Pull the enqueued flows. 8959 * 8960 * @param[in] dev 8961 * Pointer to the rte_eth_dev structure. 8962 * @param[in] queue 8963 * The queue to pull the result. 8964 * @param[in/out] res 8965 * Array to save the results. 8966 * @param[in] n_res 8967 * Available result with the array. 8968 * @param[out] error 8969 * Pointer to error structure. 8970 * 8971 * @return 8972 * Result number on success, negative value otherwise and rte_errno is set. 8973 */ 8974 static int 8975 mlx5_flow_pull(struct rte_eth_dev *dev, 8976 uint32_t queue, 8977 struct rte_flow_op_result res[], 8978 uint16_t n_res, 8979 struct rte_flow_error *error) 8980 { 8981 const struct mlx5_flow_driver_ops *fops; 8982 struct rte_flow_attr attr = {0}; 8983 8984 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8985 return rte_flow_error_set(error, ENOTSUP, 8986 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8987 NULL, 8988 "flow_q pull with incorrect steering mode"); 8989 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8990 return fops->pull(dev, queue, res, n_res, error); 8991 } 8992 8993 /** 8994 * Push the enqueued flows. 8995 * 8996 * @param[in] dev 8997 * Pointer to the rte_eth_dev structure. 8998 * @param[in] queue 8999 * The queue to push the flows. 9000 * @param[out] error 9001 * Pointer to error structure. 9002 * 9003 * @return 9004 * 0 on success, negative value otherwise and rte_errno is set. 9005 */ 9006 static int 9007 mlx5_flow_push(struct rte_eth_dev *dev, 9008 uint32_t queue, 9009 struct rte_flow_error *error) 9010 { 9011 const struct mlx5_flow_driver_ops *fops; 9012 struct rte_flow_attr attr = {0}; 9013 9014 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9015 return rte_flow_error_set(error, ENOTSUP, 9016 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9017 NULL, 9018 "flow_q push with incorrect steering mode"); 9019 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9020 return fops->push(dev, queue, error); 9021 } 9022 9023 /** 9024 * Create shared action. 9025 * 9026 * @param[in] dev 9027 * Pointer to the rte_eth_dev structure. 9028 * @param[in] queue 9029 * Which queue to be used.. 9030 * @param[in] attr 9031 * Operation attribute. 9032 * @param[in] conf 9033 * Indirect action configuration. 9034 * @param[in] action 9035 * rte_flow action detail. 9036 * @param[in] user_data 9037 * Pointer to the user_data. 9038 * @param[out] error 9039 * Pointer to error structure. 9040 * 9041 * @return 9042 * Action handle on success, NULL otherwise and rte_errno is set. 9043 */ 9044 static struct rte_flow_action_handle * 9045 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 9046 const struct rte_flow_op_attr *attr, 9047 const struct rte_flow_indir_action_conf *conf, 9048 const struct rte_flow_action *action, 9049 void *user_data, 9050 struct rte_flow_error *error) 9051 { 9052 const struct mlx5_flow_driver_ops *fops = 9053 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9054 9055 return fops->async_action_create(dev, queue, attr, conf, action, 9056 user_data, error); 9057 } 9058 9059 /** 9060 * Update shared action. 9061 * 9062 * @param[in] dev 9063 * Pointer to the rte_eth_dev structure. 9064 * @param[in] queue 9065 * Which queue to be used.. 9066 * @param[in] attr 9067 * Operation attribute. 9068 * @param[in] handle 9069 * Action handle to be updated. 9070 * @param[in] update 9071 * Update value. 9072 * @param[in] user_data 9073 * Pointer to the user_data. 9074 * @param[out] error 9075 * Pointer to error structure. 9076 * 9077 * @return 9078 * 0 on success, negative value otherwise and rte_errno is set. 9079 */ 9080 static int 9081 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 9082 const struct rte_flow_op_attr *attr, 9083 struct rte_flow_action_handle *handle, 9084 const void *update, 9085 void *user_data, 9086 struct rte_flow_error *error) 9087 { 9088 const struct mlx5_flow_driver_ops *fops = 9089 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9090 9091 return fops->async_action_update(dev, queue, attr, handle, 9092 update, user_data, error); 9093 } 9094 9095 /** 9096 * Query shared action. 9097 * 9098 * @param[in] dev 9099 * Pointer to the rte_eth_dev structure. 9100 * @param[in] queue 9101 * Which queue to be used.. 9102 * @param[in] attr 9103 * Operation attribute. 9104 * @param[in] handle 9105 * Action handle to be updated. 9106 * @param[in] data 9107 * Pointer query result data. 9108 * @param[in] user_data 9109 * Pointer to the user_data. 9110 * @param[out] error 9111 * Pointer to error structure. 9112 * 9113 * @return 9114 * 0 on success, negative value otherwise and rte_errno is set. 9115 */ 9116 static int 9117 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9118 const struct rte_flow_op_attr *attr, 9119 const struct rte_flow_action_handle *handle, 9120 void *data, 9121 void *user_data, 9122 struct rte_flow_error *error) 9123 { 9124 const struct mlx5_flow_driver_ops *fops = 9125 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9126 9127 return fops->async_action_query(dev, queue, attr, handle, 9128 data, user_data, error); 9129 } 9130 9131 /** 9132 * Destroy shared action. 9133 * 9134 * @param[in] dev 9135 * Pointer to the rte_eth_dev structure. 9136 * @param[in] queue 9137 * Which queue to be used.. 9138 * @param[in] attr 9139 * Operation attribute. 9140 * @param[in] handle 9141 * Action handle to be destroyed. 9142 * @param[in] user_data 9143 * Pointer to the user_data. 9144 * @param[out] error 9145 * Pointer to error structure. 9146 * 9147 * @return 9148 * 0 on success, negative value otherwise and rte_errno is set. 9149 */ 9150 static int 9151 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9152 const struct rte_flow_op_attr *attr, 9153 struct rte_flow_action_handle *handle, 9154 void *user_data, 9155 struct rte_flow_error *error) 9156 { 9157 const struct mlx5_flow_driver_ops *fops = 9158 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9159 9160 return fops->async_action_destroy(dev, queue, attr, handle, 9161 user_data, error); 9162 } 9163 9164 /** 9165 * Allocate a new memory for the counter values wrapped by all the needed 9166 * management. 9167 * 9168 * @param[in] sh 9169 * Pointer to mlx5_dev_ctx_shared object. 9170 * 9171 * @return 9172 * 0 on success, a negative errno value otherwise. 9173 */ 9174 static int 9175 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9176 { 9177 struct mlx5_counter_stats_mem_mng *mem_mng; 9178 volatile struct flow_counter_stats *raw_data; 9179 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9180 int size = (sizeof(struct flow_counter_stats) * 9181 MLX5_COUNTERS_PER_POOL + 9182 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9183 sizeof(struct mlx5_counter_stats_mem_mng); 9184 size_t pgsize = rte_mem_page_size(); 9185 uint8_t *mem; 9186 int ret; 9187 int i; 9188 9189 if (pgsize == (size_t)-1) { 9190 DRV_LOG(ERR, "Failed to get mem page size"); 9191 rte_errno = ENOMEM; 9192 return -ENOMEM; 9193 } 9194 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9195 if (!mem) { 9196 rte_errno = ENOMEM; 9197 return -ENOMEM; 9198 } 9199 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9200 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9201 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9202 sh->cdev->pdn, mem, size, 9203 &mem_mng->wm); 9204 if (ret) { 9205 rte_errno = errno; 9206 mlx5_free(mem); 9207 return -rte_errno; 9208 } 9209 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9210 raw_data = (volatile struct flow_counter_stats *)mem; 9211 for (i = 0; i < raws_n; ++i) { 9212 mem_mng->raws[i].mem_mng = mem_mng; 9213 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9214 } 9215 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9216 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9217 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9218 next); 9219 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9220 sh->sws_cmng.mem_mng = mem_mng; 9221 return 0; 9222 } 9223 9224 /** 9225 * Set the statistic memory to the new counter pool. 9226 * 9227 * @param[in] sh 9228 * Pointer to mlx5_dev_ctx_shared object. 9229 * @param[in] pool 9230 * Pointer to the pool to set the statistic memory. 9231 * 9232 * @return 9233 * 0 on success, a negative errno value otherwise. 9234 */ 9235 static int 9236 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9237 struct mlx5_flow_counter_pool *pool) 9238 { 9239 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9240 /* Resize statistic memory once used out. */ 9241 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9242 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9243 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9244 return -1; 9245 } 9246 rte_spinlock_lock(&pool->sl); 9247 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9248 rte_spinlock_unlock(&pool->sl); 9249 pool->raw_hw = NULL; 9250 return 0; 9251 } 9252 9253 #define MLX5_POOL_QUERY_FREQ_US 1000000 9254 9255 /** 9256 * Set the periodic procedure for triggering asynchronous batch queries for all 9257 * the counter pools. 9258 * 9259 * @param[in] sh 9260 * Pointer to mlx5_dev_ctx_shared object. 9261 */ 9262 void 9263 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9264 { 9265 uint32_t pools_n, us; 9266 9267 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9268 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9269 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9270 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9271 sh->sws_cmng.query_thread_on = 0; 9272 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9273 } else { 9274 sh->sws_cmng.query_thread_on = 1; 9275 } 9276 } 9277 9278 /** 9279 * The periodic procedure for triggering asynchronous batch queries for all the 9280 * counter pools. This function is probably called by the host thread. 9281 * 9282 * @param[in] arg 9283 * The parameter for the alarm process. 9284 */ 9285 void 9286 mlx5_flow_query_alarm(void *arg) 9287 { 9288 struct mlx5_dev_ctx_shared *sh = arg; 9289 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9290 uint16_t pool_index = cmng->pool_index; 9291 struct mlx5_flow_counter_pool *pool; 9292 uint16_t n_valid; 9293 int ret; 9294 9295 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9296 goto set_alarm; 9297 rte_spinlock_lock(&cmng->pool_update_sl); 9298 pool = cmng->pools[pool_index]; 9299 n_valid = cmng->n_valid; 9300 rte_spinlock_unlock(&cmng->pool_update_sl); 9301 /* Set the statistic memory to the new created pool. */ 9302 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9303 goto set_alarm; 9304 if (pool->raw_hw) 9305 /* There is a pool query in progress. */ 9306 goto set_alarm; 9307 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9308 if (!pool->raw_hw) 9309 /* No free counter statistics raw memory. */ 9310 goto set_alarm; 9311 /* 9312 * Identify the counters released between query trigger and query 9313 * handle more efficiently. The counter released in this gap period 9314 * should wait for a new round of query as the new arrived packets 9315 * will not be taken into account. 9316 */ 9317 pool->query_gen++; 9318 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9319 MLX5_COUNTERS_PER_POOL, 9320 NULL, NULL, 9321 pool->raw_hw->mem_mng->wm.lkey, 9322 (void *)(uintptr_t) 9323 pool->raw_hw->data, 9324 sh->devx_comp, 9325 (uint64_t)(uintptr_t)pool); 9326 if (ret) { 9327 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9328 " %d", pool->min_dcs->id); 9329 pool->raw_hw = NULL; 9330 goto set_alarm; 9331 } 9332 LIST_REMOVE(pool->raw_hw, next); 9333 cmng->pending_queries++; 9334 pool_index++; 9335 if (pool_index >= n_valid) 9336 pool_index = 0; 9337 set_alarm: 9338 cmng->pool_index = pool_index; 9339 mlx5_set_query_alarm(sh); 9340 } 9341 9342 /** 9343 * Check and callback event for new aged flow in the counter pool 9344 * 9345 * @param[in] sh 9346 * Pointer to mlx5_dev_ctx_shared object. 9347 * @param[in] pool 9348 * Pointer to Current counter pool. 9349 */ 9350 static void 9351 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9352 struct mlx5_flow_counter_pool *pool) 9353 { 9354 struct mlx5_priv *priv; 9355 struct mlx5_flow_counter *cnt; 9356 struct mlx5_age_info *age_info; 9357 struct mlx5_age_param *age_param; 9358 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9359 struct mlx5_counter_stats_raw *prev = pool->raw; 9360 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9361 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9362 uint16_t expected = AGE_CANDIDATE; 9363 uint32_t i; 9364 9365 pool->time_of_last_age_check = curr_time; 9366 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9367 cnt = MLX5_POOL_GET_CNT(pool, i); 9368 age_param = MLX5_CNT_TO_AGE(cnt); 9369 if (__atomic_load_n(&age_param->state, 9370 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9371 continue; 9372 if (cur->data[i].hits != prev->data[i].hits) { 9373 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9374 __ATOMIC_RELAXED); 9375 continue; 9376 } 9377 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9378 time_delta, 9379 __ATOMIC_RELAXED) <= age_param->timeout) 9380 continue; 9381 /** 9382 * Hold the lock first, or if between the 9383 * state AGE_TMOUT and tailq operation the 9384 * release happened, the release procedure 9385 * may delete a non-existent tailq node. 9386 */ 9387 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9388 age_info = GET_PORT_AGE_INFO(priv); 9389 rte_spinlock_lock(&age_info->aged_sl); 9390 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9391 AGE_TMOUT, false, 9392 __ATOMIC_RELAXED, 9393 __ATOMIC_RELAXED)) { 9394 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9395 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9396 } 9397 rte_spinlock_unlock(&age_info->aged_sl); 9398 } 9399 mlx5_age_event_prepare(sh); 9400 } 9401 9402 /** 9403 * Handler for the HW respond about ready values from an asynchronous batch 9404 * query. This function is probably called by the host thread. 9405 * 9406 * @param[in] sh 9407 * The pointer to the shared device context. 9408 * @param[in] async_id 9409 * The Devx async ID. 9410 * @param[in] status 9411 * The status of the completion. 9412 */ 9413 void 9414 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9415 uint64_t async_id, int status) 9416 { 9417 struct mlx5_flow_counter_pool *pool = 9418 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9419 struct mlx5_counter_stats_raw *raw_to_free; 9420 uint8_t query_gen = pool->query_gen ^ 1; 9421 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9422 enum mlx5_counter_type cnt_type = 9423 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9424 MLX5_COUNTER_TYPE_ORIGIN; 9425 9426 if (unlikely(status)) { 9427 raw_to_free = pool->raw_hw; 9428 } else { 9429 raw_to_free = pool->raw; 9430 if (pool->is_aged) 9431 mlx5_flow_aging_check(sh, pool); 9432 rte_spinlock_lock(&pool->sl); 9433 pool->raw = pool->raw_hw; 9434 rte_spinlock_unlock(&pool->sl); 9435 /* Be sure the new raw counters data is updated in memory. */ 9436 rte_io_wmb(); 9437 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9438 rte_spinlock_lock(&cmng->csl[cnt_type]); 9439 TAILQ_CONCAT(&cmng->counters[cnt_type], 9440 &pool->counters[query_gen], next); 9441 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9442 } 9443 } 9444 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9445 pool->raw_hw = NULL; 9446 sh->sws_cmng.pending_queries--; 9447 } 9448 9449 static int 9450 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9451 const struct flow_grp_info *grp_info, 9452 struct rte_flow_error *error) 9453 { 9454 if (grp_info->transfer && grp_info->external && 9455 grp_info->fdb_def_rule) { 9456 if (group == UINT32_MAX) 9457 return rte_flow_error_set 9458 (error, EINVAL, 9459 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9460 NULL, 9461 "group index not supported"); 9462 *table = group + 1; 9463 } else { 9464 *table = group; 9465 } 9466 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9467 return 0; 9468 } 9469 9470 /** 9471 * Translate the rte_flow group index to HW table value. 9472 * 9473 * If tunnel offload is disabled, all group ids converted to flow table 9474 * id using the standard method. 9475 * If tunnel offload is enabled, group id can be converted using the 9476 * standard or tunnel conversion method. Group conversion method 9477 * selection depends on flags in `grp_info` parameter: 9478 * - Internal (grp_info.external == 0) groups conversion uses the 9479 * standard method. 9480 * - Group ids in JUMP action converted with the tunnel conversion. 9481 * - Group id in rule attribute conversion depends on a rule type and 9482 * group id value: 9483 * ** non zero group attributes converted with the tunnel method 9484 * ** zero group attribute in non-tunnel rule is converted using the 9485 * standard method - there's only one root table 9486 * ** zero group attribute in steer tunnel rule is converted with the 9487 * standard method - single root table 9488 * ** zero group attribute in match tunnel rule is a special OvS 9489 * case: that value is used for portability reasons. That group 9490 * id is converted with the tunnel conversion method. 9491 * 9492 * @param[in] dev 9493 * Port device 9494 * @param[in] tunnel 9495 * PMD tunnel offload object 9496 * @param[in] group 9497 * rte_flow group index value. 9498 * @param[out] table 9499 * HW table value. 9500 * @param[in] grp_info 9501 * flags used for conversion 9502 * @param[out] error 9503 * Pointer to error structure. 9504 * 9505 * @return 9506 * 0 on success, a negative errno value otherwise and rte_errno is set. 9507 */ 9508 int 9509 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9510 const struct mlx5_flow_tunnel *tunnel, 9511 uint32_t group, uint32_t *table, 9512 const struct flow_grp_info *grp_info, 9513 struct rte_flow_error *error) 9514 { 9515 int ret; 9516 bool standard_translation; 9517 9518 if (!grp_info->skip_scale && grp_info->external && 9519 group < MLX5_MAX_TABLES_EXTERNAL) 9520 group *= MLX5_FLOW_TABLE_FACTOR; 9521 if (is_tunnel_offload_active(dev)) { 9522 standard_translation = !grp_info->external || 9523 grp_info->std_tbl_fix; 9524 } else { 9525 standard_translation = true; 9526 } 9527 DRV_LOG(DEBUG, 9528 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9529 dev->data->port_id, group, grp_info->transfer, 9530 grp_info->external, grp_info->fdb_def_rule, 9531 standard_translation ? "STANDARD" : "TUNNEL"); 9532 if (standard_translation) 9533 ret = flow_group_to_table(dev->data->port_id, group, table, 9534 grp_info, error); 9535 else 9536 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9537 table, error); 9538 9539 return ret; 9540 } 9541 9542 /** 9543 * Discover availability of metadata reg_c's. 9544 * 9545 * Iteratively use test flows to check availability. 9546 * 9547 * @param[in] dev 9548 * Pointer to the Ethernet device structure. 9549 * 9550 * @return 9551 * 0 on success, a negative errno value otherwise and rte_errno is set. 9552 */ 9553 int 9554 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9555 { 9556 struct mlx5_priv *priv = dev->data->dev_private; 9557 enum modify_reg idx; 9558 int n = 0; 9559 9560 /* reg_c[0] and reg_c[1] are reserved. */ 9561 priv->sh->flow_mreg_c[n++] = REG_C_0; 9562 priv->sh->flow_mreg_c[n++] = REG_C_1; 9563 /* Discover availability of other reg_c's. */ 9564 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9565 struct rte_flow_attr attr = { 9566 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9567 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9568 .ingress = 1, 9569 }; 9570 struct rte_flow_item items[] = { 9571 [0] = { 9572 .type = RTE_FLOW_ITEM_TYPE_END, 9573 }, 9574 }; 9575 struct rte_flow_action actions[] = { 9576 [0] = { 9577 .type = (enum rte_flow_action_type) 9578 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9579 .conf = &(struct mlx5_flow_action_copy_mreg){ 9580 .src = REG_C_1, 9581 .dst = idx, 9582 }, 9583 }, 9584 [1] = { 9585 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9586 .conf = &(struct rte_flow_action_jump){ 9587 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9588 }, 9589 }, 9590 [2] = { 9591 .type = RTE_FLOW_ACTION_TYPE_END, 9592 }, 9593 }; 9594 uint32_t flow_idx; 9595 struct rte_flow *flow; 9596 struct rte_flow_error error; 9597 9598 if (!priv->sh->config.dv_flow_en) 9599 break; 9600 /* Create internal flow, validation skips copy action. */ 9601 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9602 items, actions, false, &error); 9603 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9604 flow_idx); 9605 if (!flow) 9606 continue; 9607 priv->sh->flow_mreg_c[n++] = idx; 9608 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9609 } 9610 for (; n < MLX5_MREG_C_NUM; ++n) 9611 priv->sh->flow_mreg_c[n] = REG_NON; 9612 priv->sh->metadata_regc_check_flag = 1; 9613 return 0; 9614 } 9615 9616 int 9617 save_dump_file(const uint8_t *data, uint32_t size, 9618 uint32_t type, uint64_t id, void *arg, FILE *file) 9619 { 9620 char line[BUF_SIZE]; 9621 uint32_t out = 0; 9622 uint32_t k; 9623 uint32_t actions_num; 9624 struct rte_flow_query_count *count; 9625 9626 memset(line, 0, BUF_SIZE); 9627 switch (type) { 9628 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9629 actions_num = *(uint32_t *)(arg); 9630 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9631 type, id, actions_num); 9632 break; 9633 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9634 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9635 type, id); 9636 break; 9637 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9638 count = (struct rte_flow_query_count *)arg; 9639 fprintf(file, 9640 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9641 type, id, count->hits, count->bytes); 9642 return 0; 9643 default: 9644 return -1; 9645 } 9646 9647 for (k = 0; k < size; k++) { 9648 /* Make sure we do not overrun the line buffer length. */ 9649 if (out >= BUF_SIZE - 4) { 9650 line[out] = '\0'; 9651 break; 9652 } 9653 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9654 (data[k]) & 0xff); 9655 } 9656 fprintf(file, "%s\n", line); 9657 return 0; 9658 } 9659 9660 int 9661 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9662 struct rte_flow_query_count *count, struct rte_flow_error *error) 9663 { 9664 struct rte_flow_action action[2]; 9665 enum mlx5_flow_drv_type ftype; 9666 const struct mlx5_flow_driver_ops *fops; 9667 9668 if (!flow) { 9669 return rte_flow_error_set(error, ENOENT, 9670 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9671 NULL, 9672 "invalid flow handle"); 9673 } 9674 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9675 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9676 if (flow->counter) { 9677 memset(count, 0, sizeof(struct rte_flow_query_count)); 9678 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9679 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9680 ftype < MLX5_FLOW_TYPE_MAX); 9681 fops = flow_get_drv_ops(ftype); 9682 return fops->query(dev, flow, action, count, error); 9683 } 9684 return -1; 9685 } 9686 9687 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9688 /** 9689 * Dump flow ipool data to file 9690 * 9691 * @param[in] dev 9692 * The pointer to Ethernet device. 9693 * @param[in] file 9694 * A pointer to a file for output. 9695 * @param[out] error 9696 * Perform verbose error reporting if not NULL. PMDs initialize this 9697 * structure in case of error only. 9698 * @return 9699 * 0 on success, a negative value otherwise. 9700 */ 9701 int 9702 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9703 struct rte_flow *flow, FILE *file, 9704 struct rte_flow_error *error) 9705 { 9706 struct mlx5_priv *priv = dev->data->dev_private; 9707 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9708 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9709 uint32_t handle_idx; 9710 struct mlx5_flow_handle *dh; 9711 struct rte_flow_query_count count; 9712 uint32_t actions_num; 9713 const uint8_t *data; 9714 size_t size; 9715 uint64_t id; 9716 uint32_t type; 9717 void *action = NULL; 9718 9719 if (!flow) { 9720 return rte_flow_error_set(error, ENOENT, 9721 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9722 NULL, 9723 "invalid flow handle"); 9724 } 9725 handle_idx = flow->dev_handles; 9726 /* query counter */ 9727 if (flow->counter && 9728 (!mlx5_counter_query(dev, flow->counter, false, 9729 &count.hits, &count.bytes, &action)) && action) { 9730 id = (uint64_t)(uintptr_t)action; 9731 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9732 save_dump_file(NULL, 0, type, 9733 id, (void *)&count, file); 9734 } 9735 9736 while (handle_idx) { 9737 dh = mlx5_ipool_get(priv->sh->ipool 9738 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9739 if (!dh) 9740 continue; 9741 handle_idx = dh->next.next; 9742 9743 /* Get modify_hdr and encap_decap buf from ipools. */ 9744 encap_decap = NULL; 9745 modify_hdr = dh->dvh.modify_hdr; 9746 9747 if (dh->dvh.rix_encap_decap) { 9748 encap_decap = mlx5_ipool_get(priv->sh->ipool 9749 [MLX5_IPOOL_DECAP_ENCAP], 9750 dh->dvh.rix_encap_decap); 9751 } 9752 if (modify_hdr) { 9753 data = (const uint8_t *)modify_hdr->actions; 9754 size = (size_t)(modify_hdr->actions_num) * 8; 9755 id = (uint64_t)(uintptr_t)modify_hdr->action; 9756 actions_num = modify_hdr->actions_num; 9757 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9758 save_dump_file(data, size, type, id, 9759 (void *)(&actions_num), file); 9760 } 9761 if (encap_decap) { 9762 data = encap_decap->buf; 9763 size = encap_decap->size; 9764 id = (uint64_t)(uintptr_t)encap_decap->action; 9765 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9766 save_dump_file(data, size, type, 9767 id, NULL, file); 9768 } 9769 } 9770 return 0; 9771 } 9772 9773 /** 9774 * Dump all flow's encap_decap/modify_hdr/counter data to file 9775 * 9776 * @param[in] dev 9777 * The pointer to Ethernet device. 9778 * @param[in] file 9779 * A pointer to a file for output. 9780 * @param[out] error 9781 * Perform verbose error reporting if not NULL. PMDs initialize this 9782 * structure in case of error only. 9783 * @return 9784 * 0 on success, a negative value otherwise. 9785 */ 9786 static int 9787 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9788 FILE *file, struct rte_flow_error *error __rte_unused) 9789 { 9790 struct mlx5_priv *priv = dev->data->dev_private; 9791 struct mlx5_dev_ctx_shared *sh = priv->sh; 9792 struct mlx5_hlist *h; 9793 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9794 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9795 struct rte_flow_query_count count; 9796 uint32_t actions_num; 9797 const uint8_t *data; 9798 size_t size; 9799 uint64_t id; 9800 uint32_t type; 9801 uint32_t i; 9802 uint32_t j; 9803 struct mlx5_list_inconst *l_inconst; 9804 struct mlx5_list_entry *e; 9805 int lcore_index; 9806 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9807 uint32_t max; 9808 void *action; 9809 9810 /* encap_decap hlist is lcore_share, get global core cache. */ 9811 i = MLX5_LIST_GLOBAL; 9812 h = sh->encaps_decaps; 9813 if (h) { 9814 for (j = 0; j <= h->mask; j++) { 9815 l_inconst = &h->buckets[j].l; 9816 if (!l_inconst || !l_inconst->cache[i]) 9817 continue; 9818 9819 e = LIST_FIRST(&l_inconst->cache[i]->h); 9820 while (e) { 9821 encap_decap = 9822 (struct mlx5_flow_dv_encap_decap_resource *)e; 9823 data = encap_decap->buf; 9824 size = encap_decap->size; 9825 id = (uint64_t)(uintptr_t)encap_decap->action; 9826 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9827 save_dump_file(data, size, type, 9828 id, NULL, file); 9829 e = LIST_NEXT(e, next); 9830 } 9831 } 9832 } 9833 9834 /* get modify_hdr */ 9835 h = sh->modify_cmds; 9836 if (h) { 9837 lcore_index = rte_lcore_index(rte_lcore_id()); 9838 if (unlikely(lcore_index == -1)) { 9839 lcore_index = MLX5_LIST_NLCORE; 9840 rte_spinlock_lock(&h->l_const.lcore_lock); 9841 } 9842 i = lcore_index; 9843 9844 for (j = 0; j <= h->mask; j++) { 9845 l_inconst = &h->buckets[j].l; 9846 if (!l_inconst || !l_inconst->cache[i]) 9847 continue; 9848 9849 e = LIST_FIRST(&l_inconst->cache[i]->h); 9850 while (e) { 9851 modify_hdr = 9852 (struct mlx5_flow_dv_modify_hdr_resource *)e; 9853 data = (const uint8_t *)modify_hdr->actions; 9854 size = (size_t)(modify_hdr->actions_num) * 8; 9855 actions_num = modify_hdr->actions_num; 9856 id = (uint64_t)(uintptr_t)modify_hdr->action; 9857 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9858 save_dump_file(data, size, type, id, 9859 (void *)(&actions_num), file); 9860 e = LIST_NEXT(e, next); 9861 } 9862 } 9863 9864 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 9865 rte_spinlock_unlock(&h->l_const.lcore_lock); 9866 } 9867 9868 /* get counter */ 9869 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 9870 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 9871 for (j = 1; j <= max; j++) { 9872 action = NULL; 9873 if ((!mlx5_counter_query(dev, j, false, &count.hits, 9874 &count.bytes, &action)) && action) { 9875 id = (uint64_t)(uintptr_t)action; 9876 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9877 save_dump_file(NULL, 0, type, 9878 id, (void *)&count, file); 9879 } 9880 } 9881 return 0; 9882 } 9883 #endif 9884 9885 /** 9886 * Dump flow raw hw data to file 9887 * 9888 * @param[in] dev 9889 * The pointer to Ethernet device. 9890 * @param[in] file 9891 * A pointer to a file for output. 9892 * @param[out] error 9893 * Perform verbose error reporting if not NULL. PMDs initialize this 9894 * structure in case of error only. 9895 * @return 9896 * 0 on success, a negative value otherwise. 9897 */ 9898 int 9899 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 9900 FILE *file, 9901 struct rte_flow_error *error __rte_unused) 9902 { 9903 struct mlx5_priv *priv = dev->data->dev_private; 9904 struct mlx5_dev_ctx_shared *sh = priv->sh; 9905 uint32_t handle_idx; 9906 int ret; 9907 struct mlx5_flow_handle *dh; 9908 struct rte_flow *flow; 9909 9910 if (!sh->config.dv_flow_en) { 9911 if (fputs("device dv flow disabled\n", file) <= 0) 9912 return -errno; 9913 return -ENOTSUP; 9914 } 9915 9916 /* dump all */ 9917 if (!flow_idx) { 9918 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9919 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 9920 return -EINVAL; 9921 9922 if (sh->config.dv_flow_en == 2) 9923 return mlx5dr_debug_dump(priv->dr_ctx, file); 9924 #endif 9925 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 9926 sh->rx_domain, 9927 sh->tx_domain, file); 9928 } 9929 /* dump one */ 9930 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9931 (uintptr_t)(void *)flow_idx); 9932 if (!flow) 9933 return -EINVAL; 9934 9935 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9936 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 9937 #endif 9938 handle_idx = flow->dev_handles; 9939 while (handle_idx) { 9940 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 9941 handle_idx); 9942 if (!dh) 9943 return -ENOENT; 9944 if (dh->drv_flow) { 9945 if (sh->config.dv_flow_en == 2) 9946 return -ENOTSUP; 9947 9948 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 9949 file); 9950 if (ret) 9951 return -ENOENT; 9952 } 9953 handle_idx = dh->next.next; 9954 } 9955 return 0; 9956 } 9957 9958 /** 9959 * Get aged-out flows. 9960 * 9961 * @param[in] dev 9962 * Pointer to the Ethernet device structure. 9963 * @param[in] context 9964 * The address of an array of pointers to the aged-out flows contexts. 9965 * @param[in] nb_countexts 9966 * The length of context array pointers. 9967 * @param[out] error 9968 * Perform verbose error reporting if not NULL. Initialized in case of 9969 * error only. 9970 * 9971 * @return 9972 * how many contexts get in success, otherwise negative errno value. 9973 * if nb_contexts is 0, return the amount of all aged contexts. 9974 * if nb_contexts is not 0 , return the amount of aged flows reported 9975 * in the context array. 9976 */ 9977 int 9978 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 9979 uint32_t nb_contexts, struct rte_flow_error *error) 9980 { 9981 struct rte_flow_attr attr = { .transfer = 0 }; 9982 9983 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 9984 (dev, contexts, nb_contexts, error); 9985 } 9986 9987 /** 9988 * Get aged-out flows per HWS queue. 9989 * 9990 * @param[in] dev 9991 * Pointer to the Ethernet device structure. 9992 * @param[in] queue_id 9993 * Flow queue to query. 9994 * @param[in] context 9995 * The address of an array of pointers to the aged-out flows contexts. 9996 * @param[in] nb_countexts 9997 * The length of context array pointers. 9998 * @param[out] error 9999 * Perform verbose error reporting if not NULL. Initialized in case of 10000 * error only. 10001 * 10002 * @return 10003 * how many contexts get in success, otherwise negative errno value. 10004 * if nb_contexts is 0, return the amount of all aged contexts. 10005 * if nb_contexts is not 0 , return the amount of aged flows reported 10006 * in the context array. 10007 */ 10008 int 10009 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10010 void **contexts, uint32_t nb_contexts, 10011 struct rte_flow_error *error) 10012 { 10013 const struct mlx5_flow_driver_ops *fops; 10014 struct rte_flow_attr attr = { 0 }; 10015 10016 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10017 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10018 return fops->get_q_aged_flows(dev, queue_id, contexts, 10019 nb_contexts, error); 10020 } 10021 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10022 dev->data->port_id, queue_id); 10023 return rte_flow_error_set(error, ENOTSUP, 10024 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10025 "get Q aged flows with incorrect steering mode"); 10026 } 10027 10028 /* Wrapper for driver action_validate op callback */ 10029 static int 10030 flow_drv_action_validate(struct rte_eth_dev *dev, 10031 const struct rte_flow_indir_action_conf *conf, 10032 const struct rte_flow_action *action, 10033 const struct mlx5_flow_driver_ops *fops, 10034 struct rte_flow_error *error) 10035 { 10036 static const char err_msg[] = "indirect action validation unsupported"; 10037 10038 if (!fops->action_validate) { 10039 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10040 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10041 NULL, err_msg); 10042 return -rte_errno; 10043 } 10044 return fops->action_validate(dev, conf, action, error); 10045 } 10046 10047 /** 10048 * Destroys the shared action by handle. 10049 * 10050 * @param dev 10051 * Pointer to Ethernet device structure. 10052 * @param[in] handle 10053 * Handle for the indirect action object to be destroyed. 10054 * @param[out] error 10055 * Perform verbose error reporting if not NULL. PMDs initialize this 10056 * structure in case of error only. 10057 * 10058 * @return 10059 * 0 on success, a negative errno value otherwise and rte_errno is set. 10060 * 10061 * @note: wrapper for driver action_create op callback. 10062 */ 10063 static int 10064 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10065 struct rte_flow_action_handle *handle, 10066 struct rte_flow_error *error) 10067 { 10068 static const char err_msg[] = "indirect action destruction unsupported"; 10069 struct rte_flow_attr attr = { .transfer = 0 }; 10070 const struct mlx5_flow_driver_ops *fops = 10071 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10072 10073 if (!fops->action_destroy) { 10074 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10075 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10076 NULL, err_msg); 10077 return -rte_errno; 10078 } 10079 return fops->action_destroy(dev, handle, error); 10080 } 10081 10082 /* Wrapper for driver action_destroy op callback */ 10083 static int 10084 flow_drv_action_update(struct rte_eth_dev *dev, 10085 struct rte_flow_action_handle *handle, 10086 const void *update, 10087 const struct mlx5_flow_driver_ops *fops, 10088 struct rte_flow_error *error) 10089 { 10090 static const char err_msg[] = "indirect action update unsupported"; 10091 10092 if (!fops->action_update) { 10093 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10094 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10095 NULL, err_msg); 10096 return -rte_errno; 10097 } 10098 return fops->action_update(dev, handle, update, error); 10099 } 10100 10101 /* Wrapper for driver action_destroy op callback */ 10102 static int 10103 flow_drv_action_query(struct rte_eth_dev *dev, 10104 const struct rte_flow_action_handle *handle, 10105 void *data, 10106 const struct mlx5_flow_driver_ops *fops, 10107 struct rte_flow_error *error) 10108 { 10109 static const char err_msg[] = "indirect action query unsupported"; 10110 10111 if (!fops->action_query) { 10112 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10113 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10114 NULL, err_msg); 10115 return -rte_errno; 10116 } 10117 return fops->action_query(dev, handle, data, error); 10118 } 10119 10120 /** 10121 * Create indirect action for reuse in multiple flow rules. 10122 * 10123 * @param dev 10124 * Pointer to Ethernet device structure. 10125 * @param conf 10126 * Pointer to indirect action object configuration. 10127 * @param[in] action 10128 * Action configuration for indirect action object creation. 10129 * @param[out] error 10130 * Perform verbose error reporting if not NULL. PMDs initialize this 10131 * structure in case of error only. 10132 * @return 10133 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10134 */ 10135 static struct rte_flow_action_handle * 10136 mlx5_action_handle_create(struct rte_eth_dev *dev, 10137 const struct rte_flow_indir_action_conf *conf, 10138 const struct rte_flow_action *action, 10139 struct rte_flow_error *error) 10140 { 10141 static const char err_msg[] = "indirect action creation unsupported"; 10142 struct rte_flow_attr attr = { .transfer = 0 }; 10143 const struct mlx5_flow_driver_ops *fops = 10144 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10145 10146 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10147 return NULL; 10148 if (!fops->action_create) { 10149 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10150 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10151 NULL, err_msg); 10152 return NULL; 10153 } 10154 return fops->action_create(dev, conf, action, error); 10155 } 10156 10157 /** 10158 * Updates inplace the indirect action configuration pointed by *handle* 10159 * with the configuration provided as *update* argument. 10160 * The update of the indirect action configuration effects all flow rules 10161 * reusing the action via handle. 10162 * 10163 * @param dev 10164 * Pointer to Ethernet device structure. 10165 * @param[in] handle 10166 * Handle for the indirect action to be updated. 10167 * @param[in] update 10168 * Action specification used to modify the action pointed by handle. 10169 * *update* could be of same type with the action pointed by the *handle* 10170 * handle argument, or some other structures like a wrapper, depending on 10171 * the indirect action type. 10172 * @param[out] error 10173 * Perform verbose error reporting if not NULL. PMDs initialize this 10174 * structure in case of error only. 10175 * 10176 * @return 10177 * 0 on success, a negative errno value otherwise and rte_errno is set. 10178 */ 10179 static int 10180 mlx5_action_handle_update(struct rte_eth_dev *dev, 10181 struct rte_flow_action_handle *handle, 10182 const void *update, 10183 struct rte_flow_error *error) 10184 { 10185 struct rte_flow_attr attr = { .transfer = 0 }; 10186 const struct mlx5_flow_driver_ops *fops = 10187 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10188 int ret; 10189 10190 ret = flow_drv_action_validate(dev, NULL, 10191 (const struct rte_flow_action *)update, fops, error); 10192 if (ret) 10193 return ret; 10194 return flow_drv_action_update(dev, handle, update, fops, 10195 error); 10196 } 10197 10198 /** 10199 * Query the indirect action by handle. 10200 * 10201 * This function allows retrieving action-specific data such as counters. 10202 * Data is gathered by special action which may be present/referenced in 10203 * more than one flow rule definition. 10204 * 10205 * see @RTE_FLOW_ACTION_TYPE_COUNT 10206 * 10207 * @param dev 10208 * Pointer to Ethernet device structure. 10209 * @param[in] handle 10210 * Handle for the indirect action to query. 10211 * @param[in, out] data 10212 * Pointer to storage for the associated query data type. 10213 * @param[out] error 10214 * Perform verbose error reporting if not NULL. PMDs initialize this 10215 * structure in case of error only. 10216 * 10217 * @return 10218 * 0 on success, a negative errno value otherwise and rte_errno is set. 10219 */ 10220 static int 10221 mlx5_action_handle_query(struct rte_eth_dev *dev, 10222 const struct rte_flow_action_handle *handle, 10223 void *data, 10224 struct rte_flow_error *error) 10225 { 10226 struct rte_flow_attr attr = { .transfer = 0 }; 10227 const struct mlx5_flow_driver_ops *fops = 10228 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10229 10230 return flow_drv_action_query(dev, handle, data, fops, error); 10231 } 10232 10233 /** 10234 * Destroy all indirect actions (shared RSS). 10235 * 10236 * @param dev 10237 * Pointer to Ethernet device. 10238 * 10239 * @return 10240 * 0 on success, a negative errno value otherwise and rte_errno is set. 10241 */ 10242 int 10243 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10244 { 10245 struct rte_flow_error error; 10246 struct mlx5_priv *priv = dev->data->dev_private; 10247 struct mlx5_shared_action_rss *shared_rss; 10248 int ret = 0; 10249 uint32_t idx; 10250 10251 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10252 priv->rss_shared_actions, idx, shared_rss, next) { 10253 ret |= mlx5_action_handle_destroy(dev, 10254 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10255 } 10256 return ret; 10257 } 10258 10259 /** 10260 * Validate existing indirect actions against current device configuration 10261 * and attach them to device resources. 10262 * 10263 * @param dev 10264 * Pointer to Ethernet device. 10265 * 10266 * @return 10267 * 0 on success, a negative errno value otherwise and rte_errno is set. 10268 */ 10269 int 10270 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10271 { 10272 struct mlx5_priv *priv = dev->data->dev_private; 10273 int ret = 0; 10274 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10275 10276 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10277 const char *message; 10278 uint32_t queue_idx; 10279 10280 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10281 ind_tbl->queues_n, 10282 &message, &queue_idx); 10283 if (ret != 0) { 10284 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10285 dev->data->port_id, ind_tbl->queues[queue_idx], 10286 message); 10287 break; 10288 } 10289 } 10290 if (ret != 0) 10291 return ret; 10292 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10293 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10294 if (ret != 0) { 10295 DRV_LOG(ERR, "Port %u could not attach " 10296 "indirection table obj %p", 10297 dev->data->port_id, (void *)ind_tbl); 10298 goto error; 10299 } 10300 } 10301 10302 return 0; 10303 error: 10304 ind_tbl_last = ind_tbl; 10305 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10306 if (ind_tbl == ind_tbl_last) 10307 break; 10308 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10309 DRV_LOG(CRIT, "Port %u could not detach " 10310 "indirection table obj %p on rollback", 10311 dev->data->port_id, (void *)ind_tbl); 10312 } 10313 return ret; 10314 } 10315 10316 /** 10317 * Detach indirect actions of the device from its resources. 10318 * 10319 * @param dev 10320 * Pointer to Ethernet device. 10321 * 10322 * @return 10323 * 0 on success, a negative errno value otherwise and rte_errno is set. 10324 */ 10325 int 10326 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10327 { 10328 struct mlx5_priv *priv = dev->data->dev_private; 10329 int ret = 0; 10330 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10331 10332 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10333 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10334 if (ret != 0) { 10335 DRV_LOG(ERR, "Port %u could not detach " 10336 "indirection table obj %p", 10337 dev->data->port_id, (void *)ind_tbl); 10338 goto error; 10339 } 10340 } 10341 return 0; 10342 error: 10343 ind_tbl_last = ind_tbl; 10344 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10345 if (ind_tbl == ind_tbl_last) 10346 break; 10347 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10348 DRV_LOG(CRIT, "Port %u could not attach " 10349 "indirection table obj %p on rollback", 10350 dev->data->port_id, (void *)ind_tbl); 10351 } 10352 return ret; 10353 } 10354 10355 #ifndef HAVE_MLX5DV_DR 10356 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10357 #else 10358 #define MLX5_DOMAIN_SYNC_FLOW \ 10359 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10360 #endif 10361 10362 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10363 { 10364 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10365 const struct mlx5_flow_driver_ops *fops; 10366 int ret; 10367 struct rte_flow_attr attr = { .transfer = 0 }; 10368 10369 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10370 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10371 if (ret > 0) 10372 ret = -ret; 10373 return ret; 10374 } 10375 10376 const struct mlx5_flow_tunnel * 10377 mlx5_get_tof(const struct rte_flow_item *item, 10378 const struct rte_flow_action *action, 10379 enum mlx5_tof_rule_type *rule_type) 10380 { 10381 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10382 if (item->type == (typeof(item->type)) 10383 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10384 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10385 return flow_items_to_tunnel(item); 10386 } 10387 } 10388 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10389 if (action->type == (typeof(action->type)) 10390 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10391 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10392 return flow_actions_to_tunnel(action); 10393 } 10394 } 10395 return NULL; 10396 } 10397 10398 /** 10399 * tunnel offload functionality is defined for DV environment only 10400 */ 10401 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10402 __extension__ 10403 union tunnel_offload_mark { 10404 uint32_t val; 10405 struct { 10406 uint32_t app_reserve:8; 10407 uint32_t table_id:15; 10408 uint32_t transfer:1; 10409 uint32_t _unused_:8; 10410 }; 10411 }; 10412 10413 static bool 10414 mlx5_access_tunnel_offload_db 10415 (struct rte_eth_dev *dev, 10416 bool (*match)(struct rte_eth_dev *, 10417 struct mlx5_flow_tunnel *, const void *), 10418 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10419 void (*miss)(struct rte_eth_dev *, void *), 10420 void *ctx, bool lock_op); 10421 10422 static int 10423 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10424 struct rte_flow *flow, 10425 const struct rte_flow_attr *attr, 10426 const struct rte_flow_action *app_actions, 10427 uint32_t flow_idx, 10428 const struct mlx5_flow_tunnel *tunnel, 10429 struct tunnel_default_miss_ctx *ctx, 10430 struct rte_flow_error *error) 10431 { 10432 struct mlx5_priv *priv = dev->data->dev_private; 10433 struct mlx5_flow *dev_flow; 10434 struct rte_flow_attr miss_attr = *attr; 10435 const struct rte_flow_item miss_items[2] = { 10436 { 10437 .type = RTE_FLOW_ITEM_TYPE_ETH, 10438 .spec = NULL, 10439 .last = NULL, 10440 .mask = NULL 10441 }, 10442 { 10443 .type = RTE_FLOW_ITEM_TYPE_END, 10444 .spec = NULL, 10445 .last = NULL, 10446 .mask = NULL 10447 } 10448 }; 10449 union tunnel_offload_mark mark_id; 10450 struct rte_flow_action_mark miss_mark; 10451 struct rte_flow_action miss_actions[3] = { 10452 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10453 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10454 }; 10455 const struct rte_flow_action_jump *jump_data; 10456 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10457 struct flow_grp_info grp_info = { 10458 .external = 1, 10459 .transfer = attr->transfer, 10460 .fdb_def_rule = !!priv->fdb_def_rule, 10461 .std_tbl_fix = 0, 10462 }; 10463 int ret; 10464 10465 if (!attr->transfer) { 10466 uint32_t q_size; 10467 10468 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10469 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10470 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10471 0, SOCKET_ID_ANY); 10472 if (!ctx->queue) 10473 return rte_flow_error_set 10474 (error, ENOMEM, 10475 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10476 NULL, "invalid default miss RSS"); 10477 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10478 ctx->action_rss.level = 0, 10479 ctx->action_rss.types = priv->rss_conf.rss_hf, 10480 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10481 ctx->action_rss.queue_num = priv->reta_idx_n, 10482 ctx->action_rss.key = priv->rss_conf.rss_key, 10483 ctx->action_rss.queue = ctx->queue; 10484 if (!priv->reta_idx_n || !priv->rxqs_n) 10485 return rte_flow_error_set 10486 (error, EINVAL, 10487 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10488 NULL, "invalid port configuration"); 10489 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10490 ctx->action_rss.types = 0; 10491 for (i = 0; i != priv->reta_idx_n; ++i) 10492 ctx->queue[i] = (*priv->reta_idx)[i]; 10493 } else { 10494 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10495 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10496 } 10497 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10498 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10499 jump_data = app_actions->conf; 10500 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10501 miss_attr.group = jump_data->group; 10502 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10503 &flow_table, &grp_info, error); 10504 if (ret) 10505 return rte_flow_error_set(error, EINVAL, 10506 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10507 NULL, "invalid tunnel id"); 10508 mark_id.app_reserve = 0; 10509 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10510 mark_id.transfer = !!attr->transfer; 10511 mark_id._unused_ = 0; 10512 miss_mark.id = mark_id.val; 10513 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10514 miss_items, miss_actions, flow_idx, error); 10515 if (!dev_flow) 10516 return -rte_errno; 10517 dev_flow->flow = flow; 10518 dev_flow->external = true; 10519 dev_flow->tunnel = tunnel; 10520 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10521 /* Subflow object was created, we must include one in the list. */ 10522 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10523 dev_flow->handle, next); 10524 DRV_LOG(DEBUG, 10525 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10526 dev->data->port_id, tunnel->app_tunnel.type, 10527 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10528 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10529 miss_actions, error); 10530 if (!ret) 10531 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10532 error); 10533 10534 return ret; 10535 } 10536 10537 static const struct mlx5_flow_tbl_data_entry * 10538 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10539 { 10540 struct mlx5_priv *priv = dev->data->dev_private; 10541 struct mlx5_dev_ctx_shared *sh = priv->sh; 10542 struct mlx5_list_entry *he; 10543 union tunnel_offload_mark mbits = { .val = mark }; 10544 union mlx5_flow_tbl_key table_key = { 10545 { 10546 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10547 .id = 0, 10548 .reserved = 0, 10549 .dummy = 0, 10550 .is_fdb = !!mbits.transfer, 10551 .is_egress = 0, 10552 } 10553 }; 10554 struct mlx5_flow_cb_ctx ctx = { 10555 .data = &table_key.v64, 10556 }; 10557 10558 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10559 return he ? 10560 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10561 } 10562 10563 static void 10564 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10565 struct mlx5_list_entry *entry) 10566 { 10567 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10568 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10569 10570 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10571 tunnel_flow_tbl_to_id(tte->flow_table)); 10572 mlx5_free(tte); 10573 } 10574 10575 static int 10576 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10577 struct mlx5_list_entry *entry, void *cb_ctx) 10578 { 10579 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10580 union tunnel_tbl_key tbl = { 10581 .val = *(uint64_t *)(ctx->data), 10582 }; 10583 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10584 10585 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10586 } 10587 10588 static struct mlx5_list_entry * 10589 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10590 { 10591 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10592 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10593 struct tunnel_tbl_entry *tte; 10594 union tunnel_tbl_key tbl = { 10595 .val = *(uint64_t *)(ctx->data), 10596 }; 10597 10598 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10599 sizeof(*tte), 0, 10600 SOCKET_ID_ANY); 10601 if (!tte) 10602 goto err; 10603 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10604 &tte->flow_table); 10605 if (tte->flow_table >= MLX5_MAX_TABLES) { 10606 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10607 tte->flow_table); 10608 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10609 tte->flow_table); 10610 goto err; 10611 } else if (!tte->flow_table) { 10612 goto err; 10613 } 10614 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10615 tte->tunnel_id = tbl.tunnel_id; 10616 tte->group = tbl.group; 10617 return &tte->hash; 10618 err: 10619 if (tte) 10620 mlx5_free(tte); 10621 return NULL; 10622 } 10623 10624 static struct mlx5_list_entry * 10625 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10626 struct mlx5_list_entry *oentry, 10627 void *cb_ctx __rte_unused) 10628 { 10629 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10630 0, SOCKET_ID_ANY); 10631 10632 if (!tte) 10633 return NULL; 10634 memcpy(tte, oentry, sizeof(*tte)); 10635 return &tte->hash; 10636 } 10637 10638 static void 10639 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10640 struct mlx5_list_entry *entry) 10641 { 10642 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10643 10644 mlx5_free(tte); 10645 } 10646 10647 static uint32_t 10648 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10649 const struct mlx5_flow_tunnel *tunnel, 10650 uint32_t group, uint32_t *table, 10651 struct rte_flow_error *error) 10652 { 10653 struct mlx5_list_entry *he; 10654 struct tunnel_tbl_entry *tte; 10655 union tunnel_tbl_key key = { 10656 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10657 .group = group 10658 }; 10659 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10660 struct mlx5_hlist *group_hash; 10661 struct mlx5_flow_cb_ctx ctx = { 10662 .data = &key.val, 10663 }; 10664 10665 group_hash = tunnel ? tunnel->groups : thub->groups; 10666 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10667 if (!he) 10668 return rte_flow_error_set(error, EINVAL, 10669 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10670 NULL, 10671 "tunnel group index not supported"); 10672 tte = container_of(he, typeof(*tte), hash); 10673 *table = tte->flow_table; 10674 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10675 dev->data->port_id, key.tunnel_id, group, *table); 10676 return 0; 10677 } 10678 10679 static void 10680 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10681 struct mlx5_flow_tunnel *tunnel) 10682 { 10683 struct mlx5_priv *priv = dev->data->dev_private; 10684 struct mlx5_indexed_pool *ipool; 10685 10686 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10687 dev->data->port_id, tunnel->tunnel_id); 10688 LIST_REMOVE(tunnel, chain); 10689 mlx5_hlist_destroy(tunnel->groups); 10690 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10691 mlx5_ipool_free(ipool, tunnel->tunnel_id); 10692 } 10693 10694 static bool 10695 mlx5_access_tunnel_offload_db 10696 (struct rte_eth_dev *dev, 10697 bool (*match)(struct rte_eth_dev *, 10698 struct mlx5_flow_tunnel *, const void *), 10699 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10700 void (*miss)(struct rte_eth_dev *, void *), 10701 void *ctx, bool lock_op) 10702 { 10703 bool verdict = false; 10704 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10705 struct mlx5_flow_tunnel *tunnel; 10706 10707 rte_spinlock_lock(&thub->sl); 10708 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 10709 verdict = match(dev, tunnel, (const void *)ctx); 10710 if (verdict) 10711 break; 10712 } 10713 if (!lock_op) 10714 rte_spinlock_unlock(&thub->sl); 10715 if (verdict && hit) 10716 hit(dev, tunnel, ctx); 10717 if (!verdict && miss) 10718 miss(dev, ctx); 10719 if (lock_op) 10720 rte_spinlock_unlock(&thub->sl); 10721 10722 return verdict; 10723 } 10724 10725 struct tunnel_db_find_tunnel_id_ctx { 10726 uint32_t tunnel_id; 10727 struct mlx5_flow_tunnel *tunnel; 10728 }; 10729 10730 static bool 10731 find_tunnel_id_match(struct rte_eth_dev *dev, 10732 struct mlx5_flow_tunnel *tunnel, const void *x) 10733 { 10734 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10735 10736 RTE_SET_USED(dev); 10737 return tunnel->tunnel_id == ctx->tunnel_id; 10738 } 10739 10740 static void 10741 find_tunnel_id_hit(struct rte_eth_dev *dev, 10742 struct mlx5_flow_tunnel *tunnel, void *x) 10743 { 10744 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 10745 RTE_SET_USED(dev); 10746 ctx->tunnel = tunnel; 10747 } 10748 10749 static struct mlx5_flow_tunnel * 10750 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 10751 { 10752 struct tunnel_db_find_tunnel_id_ctx ctx = { 10753 .tunnel_id = id, 10754 }; 10755 10756 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 10757 find_tunnel_id_hit, NULL, &ctx, true); 10758 10759 return ctx.tunnel; 10760 } 10761 10762 static struct mlx5_flow_tunnel * 10763 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 10764 const struct rte_flow_tunnel *app_tunnel) 10765 { 10766 struct mlx5_priv *priv = dev->data->dev_private; 10767 struct mlx5_indexed_pool *ipool; 10768 struct mlx5_flow_tunnel *tunnel; 10769 uint32_t id; 10770 10771 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 10772 tunnel = mlx5_ipool_zmalloc(ipool, &id); 10773 if (!tunnel) 10774 return NULL; 10775 if (id >= MLX5_MAX_TUNNELS) { 10776 mlx5_ipool_free(ipool, id); 10777 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 10778 return NULL; 10779 } 10780 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 10781 priv->sh, 10782 mlx5_flow_tunnel_grp2tbl_create_cb, 10783 mlx5_flow_tunnel_grp2tbl_match_cb, 10784 mlx5_flow_tunnel_grp2tbl_remove_cb, 10785 mlx5_flow_tunnel_grp2tbl_clone_cb, 10786 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10787 if (!tunnel->groups) { 10788 mlx5_ipool_free(ipool, id); 10789 return NULL; 10790 } 10791 /* initiate new PMD tunnel */ 10792 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 10793 tunnel->tunnel_id = id; 10794 tunnel->action.type = (typeof(tunnel->action.type)) 10795 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 10796 tunnel->action.conf = tunnel; 10797 tunnel->item.type = (typeof(tunnel->item.type)) 10798 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 10799 tunnel->item.spec = tunnel; 10800 tunnel->item.last = NULL; 10801 tunnel->item.mask = NULL; 10802 10803 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 10804 dev->data->port_id, tunnel->tunnel_id); 10805 10806 return tunnel; 10807 } 10808 10809 struct tunnel_db_get_tunnel_ctx { 10810 const struct rte_flow_tunnel *app_tunnel; 10811 struct mlx5_flow_tunnel *tunnel; 10812 }; 10813 10814 static bool get_tunnel_match(struct rte_eth_dev *dev, 10815 struct mlx5_flow_tunnel *tunnel, const void *x) 10816 { 10817 const struct tunnel_db_get_tunnel_ctx *ctx = x; 10818 10819 RTE_SET_USED(dev); 10820 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 10821 sizeof(*ctx->app_tunnel)); 10822 } 10823 10824 static void get_tunnel_hit(struct rte_eth_dev *dev, 10825 struct mlx5_flow_tunnel *tunnel, void *x) 10826 { 10827 /* called under tunnel spinlock protection */ 10828 struct tunnel_db_get_tunnel_ctx *ctx = x; 10829 10830 RTE_SET_USED(dev); 10831 tunnel->refctn++; 10832 ctx->tunnel = tunnel; 10833 } 10834 10835 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 10836 { 10837 /* called under tunnel spinlock protection */ 10838 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10839 struct tunnel_db_get_tunnel_ctx *ctx = x; 10840 10841 rte_spinlock_unlock(&thub->sl); 10842 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 10843 rte_spinlock_lock(&thub->sl); 10844 if (ctx->tunnel) { 10845 ctx->tunnel->refctn = 1; 10846 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 10847 } 10848 } 10849 10850 10851 static int 10852 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 10853 const struct rte_flow_tunnel *app_tunnel, 10854 struct mlx5_flow_tunnel **tunnel) 10855 { 10856 struct tunnel_db_get_tunnel_ctx ctx = { 10857 .app_tunnel = app_tunnel, 10858 }; 10859 10860 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 10861 get_tunnel_miss, &ctx, true); 10862 *tunnel = ctx.tunnel; 10863 return ctx.tunnel ? 0 : -ENOMEM; 10864 } 10865 10866 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 10867 { 10868 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 10869 10870 if (!thub) 10871 return; 10872 if (!LIST_EMPTY(&thub->tunnels)) 10873 DRV_LOG(WARNING, "port %u tunnels present", port_id); 10874 mlx5_hlist_destroy(thub->groups); 10875 mlx5_free(thub); 10876 } 10877 10878 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 10879 { 10880 int err; 10881 struct mlx5_flow_tunnel_hub *thub; 10882 10883 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 10884 0, SOCKET_ID_ANY); 10885 if (!thub) 10886 return -ENOMEM; 10887 LIST_INIT(&thub->tunnels); 10888 rte_spinlock_init(&thub->sl); 10889 thub->groups = mlx5_hlist_create("flow groups", 64, 10890 false, true, sh, 10891 mlx5_flow_tunnel_grp2tbl_create_cb, 10892 mlx5_flow_tunnel_grp2tbl_match_cb, 10893 mlx5_flow_tunnel_grp2tbl_remove_cb, 10894 mlx5_flow_tunnel_grp2tbl_clone_cb, 10895 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 10896 if (!thub->groups) { 10897 err = -rte_errno; 10898 goto err; 10899 } 10900 sh->tunnel_hub = thub; 10901 10902 return 0; 10903 10904 err: 10905 if (thub->groups) 10906 mlx5_hlist_destroy(thub->groups); 10907 if (thub) 10908 mlx5_free(thub); 10909 return err; 10910 } 10911 10912 static inline int 10913 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 10914 struct rte_flow_tunnel *tunnel, 10915 struct rte_flow_error *error) 10916 { 10917 struct mlx5_priv *priv = dev->data->dev_private; 10918 10919 if (!priv->sh->config.dv_flow_en) 10920 return rte_flow_error_set(error, ENOTSUP, 10921 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10922 "flow DV interface is off"); 10923 if (!is_tunnel_offload_active(dev)) 10924 return rte_flow_error_set(error, ENOTSUP, 10925 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10926 "tunnel offload was not activated"); 10927 if (!tunnel) 10928 return rte_flow_error_set(error, EINVAL, 10929 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10930 "no application tunnel"); 10931 switch (tunnel->type) { 10932 default: 10933 return rte_flow_error_set(error, EINVAL, 10934 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10935 "unsupported tunnel type"); 10936 case RTE_FLOW_ITEM_TYPE_VXLAN: 10937 case RTE_FLOW_ITEM_TYPE_GRE: 10938 case RTE_FLOW_ITEM_TYPE_NVGRE: 10939 case RTE_FLOW_ITEM_TYPE_GENEVE: 10940 break; 10941 } 10942 return 0; 10943 } 10944 10945 static int 10946 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 10947 struct rte_flow_tunnel *app_tunnel, 10948 struct rte_flow_action **actions, 10949 uint32_t *num_of_actions, 10950 struct rte_flow_error *error) 10951 { 10952 struct mlx5_flow_tunnel *tunnel; 10953 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10954 10955 if (ret) 10956 return ret; 10957 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10958 if (ret < 0) { 10959 return rte_flow_error_set(error, ret, 10960 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 10961 "failed to initialize pmd tunnel"); 10962 } 10963 *actions = &tunnel->action; 10964 *num_of_actions = 1; 10965 return 0; 10966 } 10967 10968 static int 10969 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 10970 struct rte_flow_tunnel *app_tunnel, 10971 struct rte_flow_item **items, 10972 uint32_t *num_of_items, 10973 struct rte_flow_error *error) 10974 { 10975 struct mlx5_flow_tunnel *tunnel; 10976 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 10977 10978 if (ret) 10979 return ret; 10980 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 10981 if (ret < 0) { 10982 return rte_flow_error_set(error, ret, 10983 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 10984 "failed to initialize pmd tunnel"); 10985 } 10986 *items = &tunnel->item; 10987 *num_of_items = 1; 10988 return 0; 10989 } 10990 10991 struct tunnel_db_element_release_ctx { 10992 struct rte_flow_item *items; 10993 struct rte_flow_action *actions; 10994 uint32_t num_elements; 10995 struct rte_flow_error *error; 10996 int ret; 10997 }; 10998 10999 static bool 11000 tunnel_element_release_match(struct rte_eth_dev *dev, 11001 struct mlx5_flow_tunnel *tunnel, const void *x) 11002 { 11003 const struct tunnel_db_element_release_ctx *ctx = x; 11004 11005 RTE_SET_USED(dev); 11006 if (ctx->num_elements != 1) 11007 return false; 11008 else if (ctx->items) 11009 return ctx->items == &tunnel->item; 11010 else if (ctx->actions) 11011 return ctx->actions == &tunnel->action; 11012 11013 return false; 11014 } 11015 11016 static void 11017 tunnel_element_release_hit(struct rte_eth_dev *dev, 11018 struct mlx5_flow_tunnel *tunnel, void *x) 11019 { 11020 struct tunnel_db_element_release_ctx *ctx = x; 11021 ctx->ret = 0; 11022 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 11023 mlx5_flow_tunnel_free(dev, tunnel); 11024 } 11025 11026 static void 11027 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11028 { 11029 struct tunnel_db_element_release_ctx *ctx = x; 11030 RTE_SET_USED(dev); 11031 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11032 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11033 "invalid argument"); 11034 } 11035 11036 static int 11037 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11038 struct rte_flow_item *pmd_items, 11039 uint32_t num_items, struct rte_flow_error *err) 11040 { 11041 struct tunnel_db_element_release_ctx ctx = { 11042 .items = pmd_items, 11043 .actions = NULL, 11044 .num_elements = num_items, 11045 .error = err, 11046 }; 11047 11048 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11049 tunnel_element_release_hit, 11050 tunnel_element_release_miss, &ctx, false); 11051 11052 return ctx.ret; 11053 } 11054 11055 static int 11056 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11057 struct rte_flow_action *pmd_actions, 11058 uint32_t num_actions, struct rte_flow_error *err) 11059 { 11060 struct tunnel_db_element_release_ctx ctx = { 11061 .items = NULL, 11062 .actions = pmd_actions, 11063 .num_elements = num_actions, 11064 .error = err, 11065 }; 11066 11067 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11068 tunnel_element_release_hit, 11069 tunnel_element_release_miss, &ctx, false); 11070 11071 return ctx.ret; 11072 } 11073 11074 static int 11075 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11076 struct rte_mbuf *m, 11077 struct rte_flow_restore_info *info, 11078 struct rte_flow_error *err) 11079 { 11080 uint64_t ol_flags = m->ol_flags; 11081 const struct mlx5_flow_tbl_data_entry *tble; 11082 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11083 11084 if (!is_tunnel_offload_active(dev)) { 11085 info->flags = 0; 11086 return 0; 11087 } 11088 11089 if ((ol_flags & mask) != mask) 11090 goto err; 11091 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11092 if (!tble) { 11093 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11094 dev->data->port_id, m->hash.fdir.hi); 11095 goto err; 11096 } 11097 MLX5_ASSERT(tble->tunnel); 11098 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11099 info->group_id = tble->group_id; 11100 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11101 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11102 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11103 11104 return 0; 11105 11106 err: 11107 return rte_flow_error_set(err, EINVAL, 11108 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11109 "failed to get restore info"); 11110 } 11111 11112 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11113 static int 11114 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11115 __rte_unused struct rte_flow_tunnel *app_tunnel, 11116 __rte_unused struct rte_flow_action **actions, 11117 __rte_unused uint32_t *num_of_actions, 11118 __rte_unused struct rte_flow_error *error) 11119 { 11120 return -ENOTSUP; 11121 } 11122 11123 static int 11124 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11125 __rte_unused struct rte_flow_tunnel *app_tunnel, 11126 __rte_unused struct rte_flow_item **items, 11127 __rte_unused uint32_t *num_of_items, 11128 __rte_unused struct rte_flow_error *error) 11129 { 11130 return -ENOTSUP; 11131 } 11132 11133 static int 11134 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11135 __rte_unused struct rte_flow_item *pmd_items, 11136 __rte_unused uint32_t num_items, 11137 __rte_unused struct rte_flow_error *err) 11138 { 11139 return -ENOTSUP; 11140 } 11141 11142 static int 11143 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11144 __rte_unused struct rte_flow_action *pmd_action, 11145 __rte_unused uint32_t num_actions, 11146 __rte_unused struct rte_flow_error *err) 11147 { 11148 return -ENOTSUP; 11149 } 11150 11151 static int 11152 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11153 __rte_unused struct rte_mbuf *m, 11154 __rte_unused struct rte_flow_restore_info *i, 11155 __rte_unused struct rte_flow_error *err) 11156 { 11157 return -ENOTSUP; 11158 } 11159 11160 static int 11161 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11162 __rte_unused struct rte_flow *flow, 11163 __rte_unused const struct rte_flow_attr *attr, 11164 __rte_unused const struct rte_flow_action *actions, 11165 __rte_unused uint32_t flow_idx, 11166 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11167 __rte_unused struct tunnel_default_miss_ctx *ctx, 11168 __rte_unused struct rte_flow_error *error) 11169 { 11170 return -ENOTSUP; 11171 } 11172 11173 static struct mlx5_flow_tunnel * 11174 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11175 __rte_unused uint32_t id) 11176 { 11177 return NULL; 11178 } 11179 11180 static void 11181 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11182 __rte_unused struct mlx5_flow_tunnel *tunnel) 11183 { 11184 } 11185 11186 static uint32_t 11187 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11188 __rte_unused const struct mlx5_flow_tunnel *t, 11189 __rte_unused uint32_t group, 11190 __rte_unused uint32_t *table, 11191 struct rte_flow_error *error) 11192 { 11193 return rte_flow_error_set(error, ENOTSUP, 11194 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11195 "tunnel offload requires DV support"); 11196 } 11197 11198 void 11199 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11200 __rte_unused uint16_t port_id) 11201 { 11202 } 11203 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11204 11205 /* Flex flow item API */ 11206 static struct rte_flow_item_flex_handle * 11207 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11208 const struct rte_flow_item_flex_conf *conf, 11209 struct rte_flow_error *error) 11210 { 11211 static const char err_msg[] = "flex item creation unsupported"; 11212 struct mlx5_priv *priv = dev->data->dev_private; 11213 struct rte_flow_attr attr = { .transfer = 0 }; 11214 const struct mlx5_flow_driver_ops *fops = 11215 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11216 11217 if (!priv->pci_dev) { 11218 rte_flow_error_set(error, ENOTSUP, 11219 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11220 "create flex item on PF only"); 11221 return NULL; 11222 } 11223 switch (priv->pci_dev->id.device_id) { 11224 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11225 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11226 break; 11227 default: 11228 rte_flow_error_set(error, ENOTSUP, 11229 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11230 "flex item available on BlueField ports only"); 11231 return NULL; 11232 } 11233 if (!fops->item_create) { 11234 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11235 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11236 NULL, err_msg); 11237 return NULL; 11238 } 11239 return fops->item_create(dev, conf, error); 11240 } 11241 11242 static int 11243 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11244 const struct rte_flow_item_flex_handle *handle, 11245 struct rte_flow_error *error) 11246 { 11247 static const char err_msg[] = "flex item release unsupported"; 11248 struct rte_flow_attr attr = { .transfer = 0 }; 11249 const struct mlx5_flow_driver_ops *fops = 11250 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11251 11252 if (!fops->item_release) { 11253 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11254 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11255 NULL, err_msg); 11256 return -rte_errno; 11257 } 11258 return fops->item_release(dev, handle, error); 11259 } 11260 11261 static void 11262 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11263 { 11264 int ret; 11265 struct rte_flow_error error; 11266 11267 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11268 char *item_name; 11269 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11270 sizeof(item_name), 11271 (void *)(uintptr_t)item->type, &error); 11272 if (ret > 0) 11273 printf("%s ", item_name); 11274 else 11275 printf("%d\n", (int)item->type); 11276 } 11277 printf("END\n"); 11278 } 11279 11280 static int 11281 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11282 { 11283 const struct rte_flow_item_udp *spec = udp_item->spec; 11284 const struct rte_flow_item_udp *mask = udp_item->mask; 11285 uint16_t udp_dport = 0; 11286 11287 if (spec != NULL) { 11288 if (!mask) 11289 mask = &rte_flow_item_udp_mask; 11290 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11291 mask->hdr.dst_port); 11292 } 11293 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11294 } 11295 11296 static const struct mlx5_flow_expand_node * 11297 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11298 unsigned int item_idx, 11299 const struct mlx5_flow_expand_node graph[], 11300 const struct mlx5_flow_expand_node *node) 11301 { 11302 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11303 11304 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11305 node != NULL && 11306 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11307 /* 11308 * The expansion node is VXLAN and it is also the last 11309 * expandable item in the pattern, so need to continue 11310 * expansion of the inner tunnel. 11311 */ 11312 MLX5_ASSERT(item_idx > 0); 11313 prev_item = pattern + item_idx - 1; 11314 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11315 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11316 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11317 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11318 } 11319 return node; 11320 } 11321 11322 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11323 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11324 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11325 }; 11326 11327 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11328 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11329 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11330 { 9, 10, 11 }, { 12, 13, 14 }, 11331 }; 11332 11333 /** 11334 * Discover the number of available flow priorities. 11335 * 11336 * @param dev 11337 * Ethernet device. 11338 * 11339 * @return 11340 * On success, number of available flow priorities. 11341 * On failure, a negative errno-style code and rte_errno is set. 11342 */ 11343 int 11344 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11345 { 11346 static const uint16_t vprio[] = {8, 16}; 11347 const struct mlx5_priv *priv = dev->data->dev_private; 11348 const struct mlx5_flow_driver_ops *fops; 11349 enum mlx5_flow_drv_type type; 11350 int ret; 11351 11352 type = mlx5_flow_os_get_type(); 11353 if (type == MLX5_FLOW_TYPE_MAX) { 11354 type = MLX5_FLOW_TYPE_VERBS; 11355 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11356 type = MLX5_FLOW_TYPE_DV; 11357 } 11358 fops = flow_get_drv_ops(type); 11359 if (fops->discover_priorities == NULL) { 11360 DRV_LOG(ERR, "Priority discovery not supported"); 11361 rte_errno = ENOTSUP; 11362 return -rte_errno; 11363 } 11364 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11365 if (ret < 0) 11366 return ret; 11367 switch (ret) { 11368 case 8: 11369 ret = RTE_DIM(priority_map_3); 11370 break; 11371 case 16: 11372 ret = RTE_DIM(priority_map_5); 11373 break; 11374 default: 11375 rte_errno = ENOTSUP; 11376 DRV_LOG(ERR, 11377 "port %u maximum priority: %d expected 8/16", 11378 dev->data->port_id, ret); 11379 return -rte_errno; 11380 } 11381 DRV_LOG(INFO, "port %u supported flow priorities:" 11382 " 0-%d for ingress or egress root table," 11383 " 0-%d for non-root table or transfer root table.", 11384 dev->data->port_id, ret - 2, 11385 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11386 return ret; 11387 } 11388 11389 /** 11390 * Adjust flow priority based on the highest layer and the request priority. 11391 * 11392 * @param[in] dev 11393 * Pointer to the Ethernet device structure. 11394 * @param[in] priority 11395 * The rule base priority. 11396 * @param[in] subpriority 11397 * The priority based on the items. 11398 * 11399 * @return 11400 * The new priority. 11401 */ 11402 uint32_t 11403 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11404 uint32_t subpriority) 11405 { 11406 uint32_t res = 0; 11407 struct mlx5_priv *priv = dev->data->dev_private; 11408 11409 switch (priv->sh->flow_max_priority) { 11410 case RTE_DIM(priority_map_3): 11411 res = priority_map_3[priority][subpriority]; 11412 break; 11413 case RTE_DIM(priority_map_5): 11414 res = priority_map_5[priority][subpriority]; 11415 break; 11416 } 11417 return res; 11418 } 11419 11420 /** 11421 * Get the priority for sending traffic to kernel table. 11422 * 11423 * @param[in] dev 11424 * Pointer to the Ethernet device structure. 11425 * 11426 * @return 11427 * On success: the value of priority for sending traffic to kernel table 11428 * On failure: -1 11429 */ 11430 uint32_t 11431 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11432 { 11433 struct mlx5_priv *priv = dev->data->dev_private; 11434 uint32_t res; 11435 11436 switch (priv->sh->flow_max_priority) { 11437 case RTE_DIM(priority_map_5): 11438 res = 15; 11439 break; 11440 case RTE_DIM(priority_map_3): 11441 res = 7; 11442 break; 11443 default: 11444 DRV_LOG(ERR, 11445 "port %u maximum priority: %d expected 8/16", 11446 dev->data->port_id, priv->sh->flow_max_priority); 11447 res = (uint32_t)-1; 11448 } 11449 return res; 11450 } 11451 11452 /** 11453 * Get the E-Switch Manager vport id. 11454 * 11455 * @param[in] dev 11456 * Pointer to the Ethernet device structure. 11457 * 11458 * @return 11459 * The vport id. 11460 */ 11461 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11462 { 11463 struct mlx5_priv *priv = dev->data->dev_private; 11464 struct mlx5_common_device *cdev = priv->sh->cdev; 11465 11466 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11467 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11468 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11469 11470 if (priv->pci_dev == NULL) 11471 return 0; 11472 switch (priv->pci_dev->id.device_id) { 11473 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 11474 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11475 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11476 /* 11477 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11478 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11479 * ECPF vport ID is selected and not the host port (0) in any BF case. 11480 */ 11481 return (int16_t)MLX5_ECPF_VPORT_ID; 11482 default: 11483 return MLX5_PF_VPORT_ID; 11484 } 11485 } 11486 11487 /** 11488 * Parse item to get the vport id. 11489 * 11490 * @param[in] dev 11491 * Pointer to the Ethernet device structure. 11492 * @param[in] item 11493 * The src port id match item. 11494 * @param[out] vport_id 11495 * Pointer to put the vport id. 11496 * @param[out] all_ports 11497 * Indicate if the item matches all ports. 11498 * @param[out] error 11499 * Pointer to error structure. 11500 * 11501 * @return 11502 * 0 on success, a negative errno value otherwise and rte_errno is set. 11503 */ 11504 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11505 const struct rte_flow_item *item, 11506 uint16_t *vport_id, 11507 bool *all_ports, 11508 struct rte_flow_error *error) 11509 { 11510 struct mlx5_priv *port_priv; 11511 const struct rte_flow_item_port_id *pid_v = NULL; 11512 const struct rte_flow_item_ethdev *dev_v = NULL; 11513 uint32_t esw_mgr_port; 11514 uint32_t src_port; 11515 11516 if (all_ports) 11517 *all_ports = false; 11518 switch (item->type) { 11519 case RTE_FLOW_ITEM_TYPE_PORT_ID: 11520 pid_v = item->spec; 11521 if (!pid_v) 11522 return 0; 11523 src_port = pid_v->id; 11524 esw_mgr_port = MLX5_PORT_ESW_MGR; 11525 break; 11526 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 11527 dev_v = item->spec; 11528 if (!dev_v) { 11529 if (all_ports) 11530 *all_ports = true; 11531 return 0; 11532 } 11533 src_port = dev_v->port_id; 11534 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11535 break; 11536 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 11537 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11538 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11539 break; 11540 default: 11541 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11542 NULL, "Incorrect item type."); 11543 } 11544 if (src_port == esw_mgr_port) { 11545 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11546 } else { 11547 port_priv = mlx5_port_to_eswitch_info(src_port, false); 11548 if (!port_priv) 11549 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11550 NULL, "Failed to get port info."); 11551 *vport_id = port_priv->representor_id; 11552 } 11553 11554 return 0; 11555 } 11556 11557 int 11558 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11559 uint16_t *proxy_port_id, 11560 struct rte_flow_error *error) 11561 { 11562 const struct mlx5_priv *priv = dev->data->dev_private; 11563 uint16_t port_id; 11564 11565 if (!priv->sh->config.dv_esw_en) 11566 return rte_flow_error_set(error, EINVAL, 11567 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11568 NULL, 11569 "unable to provide a proxy port" 11570 " without E-Switch configured"); 11571 if (!priv->master && !priv->representor) 11572 return rte_flow_error_set(error, EINVAL, 11573 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11574 NULL, 11575 "unable to provide a proxy port" 11576 " for port which is not a master" 11577 " or a representor port"); 11578 if (priv->master) { 11579 *proxy_port_id = dev->data->port_id; 11580 return 0; 11581 } 11582 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11583 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11584 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11585 11586 if (port_priv->master && 11587 port_priv->domain_id == priv->domain_id) { 11588 *proxy_port_id = port_id; 11589 return 0; 11590 } 11591 } 11592 return rte_flow_error_set(error, ENODEV, 11593 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11594 NULL, "unable to find a proxy port"); 11595 } 11596