1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static __rte_always_inline int 168 mlx5_need_cache_flow(const struct mlx5_priv *priv, 169 const struct rte_flow_attr *attr) 170 { 171 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 172 (attr ? !attr->group : true) && 173 priv->mode_info.mode == MLX5_FLOW_ENGINE_MODE_STANDBY && 174 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 175 } 176 177 static bool 178 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 179 { 180 switch (item->type) { 181 case RTE_FLOW_ITEM_TYPE_ETH: 182 case RTE_FLOW_ITEM_TYPE_VLAN: 183 case RTE_FLOW_ITEM_TYPE_IPV4: 184 case RTE_FLOW_ITEM_TYPE_IPV6: 185 case RTE_FLOW_ITEM_TYPE_UDP: 186 case RTE_FLOW_ITEM_TYPE_TCP: 187 case RTE_FLOW_ITEM_TYPE_ESP: 188 case RTE_FLOW_ITEM_TYPE_ICMP: 189 case RTE_FLOW_ITEM_TYPE_ICMP6: 190 case RTE_FLOW_ITEM_TYPE_VXLAN: 191 case RTE_FLOW_ITEM_TYPE_NVGRE: 192 case RTE_FLOW_ITEM_TYPE_GRE: 193 case RTE_FLOW_ITEM_TYPE_GENEVE: 194 case RTE_FLOW_ITEM_TYPE_MPLS: 195 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 196 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 197 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 198 case RTE_FLOW_ITEM_TYPE_GTP: 199 return true; 200 default: 201 break; 202 } 203 return false; 204 } 205 206 /** 207 * Network Service Header (NSH) and its next protocol values 208 * are described in RFC-8393. 209 */ 210 static enum rte_flow_item_type 211 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 212 { 213 enum rte_flow_item_type type; 214 215 switch (proto_mask & proto_spec) { 216 case 0: 217 type = RTE_FLOW_ITEM_TYPE_VOID; 218 break; 219 case RTE_VXLAN_GPE_TYPE_IPV4: 220 type = RTE_FLOW_ITEM_TYPE_IPV4; 221 break; 222 case RTE_VXLAN_GPE_TYPE_IPV6: 223 type = RTE_VXLAN_GPE_TYPE_IPV6; 224 break; 225 case RTE_VXLAN_GPE_TYPE_ETH: 226 type = RTE_FLOW_ITEM_TYPE_ETH; 227 break; 228 default: 229 type = RTE_FLOW_ITEM_TYPE_END; 230 } 231 return type; 232 } 233 234 static enum rte_flow_item_type 235 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 236 { 237 enum rte_flow_item_type type; 238 239 switch (proto_mask & proto_spec) { 240 case 0: 241 type = RTE_FLOW_ITEM_TYPE_VOID; 242 break; 243 case IPPROTO_UDP: 244 type = RTE_FLOW_ITEM_TYPE_UDP; 245 break; 246 case IPPROTO_TCP: 247 type = RTE_FLOW_ITEM_TYPE_TCP; 248 break; 249 case IPPROTO_IPIP: 250 type = RTE_FLOW_ITEM_TYPE_IPV4; 251 break; 252 case IPPROTO_IPV6: 253 type = RTE_FLOW_ITEM_TYPE_IPV6; 254 break; 255 case IPPROTO_ESP: 256 type = RTE_FLOW_ITEM_TYPE_ESP; 257 break; 258 default: 259 type = RTE_FLOW_ITEM_TYPE_END; 260 } 261 return type; 262 } 263 264 static enum rte_flow_item_type 265 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 266 rte_be16_t type_mask, bool is_tunnel) 267 { 268 enum rte_flow_item_type type; 269 270 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 271 case 0: 272 type = RTE_FLOW_ITEM_TYPE_VOID; 273 break; 274 case RTE_ETHER_TYPE_TEB: 275 type = is_tunnel ? 276 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 277 break; 278 case RTE_ETHER_TYPE_VLAN: 279 type = !is_tunnel ? 280 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 281 break; 282 case RTE_ETHER_TYPE_IPV4: 283 type = RTE_FLOW_ITEM_TYPE_IPV4; 284 break; 285 case RTE_ETHER_TYPE_IPV6: 286 type = RTE_FLOW_ITEM_TYPE_IPV6; 287 break; 288 default: 289 type = RTE_FLOW_ITEM_TYPE_END; 290 } 291 return type; 292 } 293 294 static enum rte_flow_item_type 295 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 296 { 297 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 298 do { \ 299 const void *m = item->mask; \ 300 const void *s = item->spec; \ 301 mask = m ? \ 302 ((const struct rte_flow_item_##type *)m)->fld : \ 303 rte_flow_item_##type##_mask.fld; \ 304 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 305 } while (0) 306 307 enum rte_flow_item_type ret; 308 uint16_t spec, mask; 309 310 if (item == NULL || item->spec == NULL) 311 return RTE_FLOW_ITEM_TYPE_VOID; 312 switch (item->type) { 313 case RTE_FLOW_ITEM_TYPE_ETH: 314 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 315 if (!mask) 316 return RTE_FLOW_ITEM_TYPE_VOID; 317 ret = mlx5_ethertype_to_item_type(spec, mask, false); 318 break; 319 case RTE_FLOW_ITEM_TYPE_VLAN: 320 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 321 if (!mask) 322 return RTE_FLOW_ITEM_TYPE_VOID; 323 ret = mlx5_ethertype_to_item_type(spec, mask, false); 324 break; 325 case RTE_FLOW_ITEM_TYPE_IPV4: 326 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 327 if (!mask) 328 return RTE_FLOW_ITEM_TYPE_VOID; 329 ret = mlx5_inet_proto_to_item_type(spec, mask); 330 break; 331 case RTE_FLOW_ITEM_TYPE_IPV6: 332 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 333 if (!mask) 334 return RTE_FLOW_ITEM_TYPE_VOID; 335 ret = mlx5_inet_proto_to_item_type(spec, mask); 336 break; 337 case RTE_FLOW_ITEM_TYPE_GENEVE: 338 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 339 ret = mlx5_ethertype_to_item_type(spec, mask, true); 340 break; 341 case RTE_FLOW_ITEM_TYPE_GRE: 342 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 343 ret = mlx5_ethertype_to_item_type(spec, mask, true); 344 break; 345 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 346 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 347 ret = mlx5_nsh_proto_to_item_type(spec, mask); 348 break; 349 default: 350 ret = RTE_FLOW_ITEM_TYPE_VOID; 351 break; 352 } 353 return ret; 354 #undef MLX5_XSET_ITEM_MASK_SPEC 355 } 356 357 static const int * 358 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 359 const int *next_node) 360 { 361 const struct mlx5_flow_expand_node *node = NULL; 362 const int *next = next_node; 363 364 while (next && *next) { 365 /* 366 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 367 * flag set, because they were not found in the flow pattern. 368 */ 369 node = &graph[*next]; 370 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 371 break; 372 next = node->next; 373 } 374 return next; 375 } 376 377 #define MLX5_RSS_EXP_ELT_N 16 378 379 /** 380 * Expand RSS flows into several possible flows according to the RSS hash 381 * fields requested and the driver capabilities. 382 * 383 * @param[out] buf 384 * Buffer to store the result expansion. 385 * @param[in] size 386 * Buffer size in bytes. If 0, @p buf can be NULL. 387 * @param[in] pattern 388 * User flow pattern. 389 * @param[in] types 390 * RSS types to expand (see RTE_ETH_RSS_* definitions). 391 * @param[in] graph 392 * Input graph to expand @p pattern according to @p types. 393 * @param[in] graph_root_index 394 * Index of root node in @p graph, typically 0. 395 * 396 * @return 397 * A positive value representing the size of @p buf in bytes regardless of 398 * @p size on success, a negative errno value otherwise and rte_errno is 399 * set, the following errors are defined: 400 * 401 * -E2BIG: graph-depth @p graph is too deep. 402 * -EINVAL: @p size has not enough space for expanded pattern. 403 */ 404 static int 405 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 406 const struct rte_flow_item *pattern, uint64_t types, 407 const struct mlx5_flow_expand_node graph[], 408 int graph_root_index) 409 { 410 const struct rte_flow_item *item; 411 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 412 const int *next_node; 413 const int *stack[MLX5_RSS_EXP_ELT_N]; 414 int stack_pos = 0; 415 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 416 unsigned int i, item_idx, last_expand_item_idx = 0; 417 size_t lsize; 418 size_t user_pattern_size = 0; 419 void *addr = NULL; 420 const struct mlx5_flow_expand_node *next = NULL; 421 struct rte_flow_item missed_item; 422 int missed = 0; 423 int elt = 0; 424 const struct rte_flow_item *last_expand_item = NULL; 425 426 memset(&missed_item, 0, sizeof(missed_item)); 427 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 428 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 429 if (lsize > size) 430 return -EINVAL; 431 buf->entry[0].priority = 0; 432 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 433 buf->entries = 0; 434 addr = buf->entry[0].pattern; 435 for (item = pattern, item_idx = 0; 436 item->type != RTE_FLOW_ITEM_TYPE_END; 437 item++, item_idx++) { 438 if (!mlx5_flow_is_rss_expandable_item(item)) { 439 user_pattern_size += sizeof(*item); 440 continue; 441 } 442 last_expand_item = item; 443 last_expand_item_idx = item_idx; 444 i = 0; 445 while (node->next && node->next[i]) { 446 next = &graph[node->next[i]]; 447 if (next->type == item->type) 448 break; 449 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 450 node = next; 451 i = 0; 452 } else { 453 ++i; 454 } 455 } 456 if (next) 457 node = next; 458 user_pattern_size += sizeof(*item); 459 } 460 user_pattern_size += sizeof(*item); /* Handle END item. */ 461 lsize += user_pattern_size; 462 if (lsize > size) 463 return -EINVAL; 464 /* Copy the user pattern in the first entry of the buffer. */ 465 rte_memcpy(addr, pattern, user_pattern_size); 466 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 467 buf->entries = 1; 468 /* Start expanding. */ 469 memset(flow_items, 0, sizeof(flow_items)); 470 user_pattern_size -= sizeof(*item); 471 /* 472 * Check if the last valid item has spec set, need complete pattern, 473 * and the pattern can be used for expansion. 474 */ 475 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 476 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 477 /* Item type END indicates expansion is not required. */ 478 return lsize; 479 } 480 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 481 next = NULL; 482 missed = 1; 483 i = 0; 484 while (node->next && node->next[i]) { 485 next = &graph[node->next[i]]; 486 if (next->type == missed_item.type) { 487 flow_items[0].type = missed_item.type; 488 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 489 break; 490 } 491 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 492 node = next; 493 i = 0; 494 } else { 495 ++i; 496 } 497 next = NULL; 498 } 499 } 500 if (next && missed) { 501 elt = 2; /* missed item + item end. */ 502 node = next; 503 lsize += elt * sizeof(*item) + user_pattern_size; 504 if (lsize > size) 505 return -EINVAL; 506 if (node->rss_types & types) { 507 buf->entry[buf->entries].priority = 1; 508 buf->entry[buf->entries].pattern = addr; 509 buf->entries++; 510 rte_memcpy(addr, buf->entry[0].pattern, 511 user_pattern_size); 512 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 513 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 514 addr = (void *)(((uintptr_t)addr) + 515 elt * sizeof(*item)); 516 } 517 } else if (last_expand_item != NULL) { 518 node = mlx5_flow_expand_rss_adjust_node(pattern, 519 last_expand_item_idx, graph, node); 520 } 521 memset(flow_items, 0, sizeof(flow_items)); 522 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 523 node->next); 524 stack[stack_pos] = next_node; 525 node = next_node ? &graph[*next_node] : NULL; 526 while (node) { 527 flow_items[stack_pos].type = node->type; 528 if (node->rss_types & types) { 529 size_t n; 530 /* 531 * compute the number of items to copy from the 532 * expansion and copy it. 533 * When the stack_pos is 0, there are 1 element in it, 534 * plus the addition END item. 535 */ 536 elt = stack_pos + 2; 537 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 538 lsize += elt * sizeof(*item) + user_pattern_size; 539 if (lsize > size) 540 return -EINVAL; 541 n = elt * sizeof(*item); 542 buf->entry[buf->entries].priority = 543 stack_pos + 1 + missed; 544 buf->entry[buf->entries].pattern = addr; 545 buf->entries++; 546 rte_memcpy(addr, buf->entry[0].pattern, 547 user_pattern_size); 548 addr = (void *)(((uintptr_t)addr) + 549 user_pattern_size); 550 rte_memcpy(addr, &missed_item, 551 missed * sizeof(*item)); 552 addr = (void *)(((uintptr_t)addr) + 553 missed * sizeof(*item)); 554 rte_memcpy(addr, flow_items, n); 555 addr = (void *)(((uintptr_t)addr) + n); 556 } 557 /* Go deeper. */ 558 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 559 node->next) { 560 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 561 node->next); 562 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 563 rte_errno = E2BIG; 564 return -rte_errno; 565 } 566 stack[stack_pos] = next_node; 567 } else if (*(next_node + 1)) { 568 /* Follow up with the next possibility. */ 569 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 570 ++next_node); 571 } else if (!stack_pos) { 572 /* 573 * Completing the traverse over the different paths. 574 * The next_node is advanced to the terminator. 575 */ 576 ++next_node; 577 } else { 578 /* Move to the next path. */ 579 while (stack_pos) { 580 next_node = stack[--stack_pos]; 581 next_node++; 582 if (*next_node) 583 break; 584 } 585 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 586 next_node); 587 stack[stack_pos] = next_node; 588 } 589 node = next_node && *next_node ? &graph[*next_node] : NULL; 590 }; 591 return lsize; 592 } 593 594 /** 595 * Expand SQN flows into several possible flows according to the Tx queue 596 * number 597 * 598 * @param[in] buf 599 * Buffer to store the result expansion. 600 * @param[in] size 601 * Buffer size in bytes. If 0, @p buf can be NULL. 602 * @param[in] pattern 603 * User flow pattern. 604 * @param[in] sq_specs 605 * Buffer to store sq spec. 606 * 607 * @return 608 * 0 for success and negative value for failure 609 * 610 */ 611 static int 612 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 613 const struct rte_flow_item *pattern, 614 struct mlx5_rte_flow_item_sq *sq_specs) 615 { 616 const struct rte_flow_item *item; 617 bool port_representor = false; 618 size_t user_pattern_size = 0; 619 struct rte_eth_dev *dev; 620 struct mlx5_priv *priv; 621 void *addr = NULL; 622 uint16_t port_id; 623 size_t lsize; 624 int elt = 2; 625 uint16_t i; 626 627 buf->entries = 0; 628 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 629 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 630 const struct rte_flow_item_ethdev *pid_v = item->spec; 631 632 if (!pid_v) 633 return 0; 634 port_id = pid_v->port_id; 635 port_representor = true; 636 } 637 user_pattern_size += sizeof(*item); 638 } 639 if (!port_representor) 640 return 0; 641 dev = &rte_eth_devices[port_id]; 642 priv = dev->data->dev_private; 643 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 644 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 645 sizeof(buf->entry[0]) * priv->txqs_n; 646 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 647 return -EINVAL; 648 addr = buf->entry[0].pattern; 649 for (i = 0; i != priv->txqs_n; ++i) { 650 struct rte_flow_item pattern_add[] = { 651 { 652 .type = (enum rte_flow_item_type) 653 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 654 .spec = &sq_specs[i], 655 }, 656 { 657 .type = RTE_FLOW_ITEM_TYPE_END, 658 }, 659 }; 660 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 661 662 if (txq == NULL) 663 return -EINVAL; 664 buf->entry[i].pattern = addr; 665 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 666 mlx5_txq_release(dev, i); 667 rte_memcpy(addr, pattern, user_pattern_size); 668 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 669 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 670 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 671 buf->entries++; 672 } 673 return 0; 674 } 675 676 enum mlx5_expansion { 677 MLX5_EXPANSION_ROOT, 678 MLX5_EXPANSION_ROOT_OUTER, 679 MLX5_EXPANSION_OUTER_ETH, 680 MLX5_EXPANSION_OUTER_VLAN, 681 MLX5_EXPANSION_OUTER_IPV4, 682 MLX5_EXPANSION_OUTER_IPV4_UDP, 683 MLX5_EXPANSION_OUTER_IPV4_TCP, 684 MLX5_EXPANSION_OUTER_IPV4_ESP, 685 MLX5_EXPANSION_OUTER_IPV4_ICMP, 686 MLX5_EXPANSION_OUTER_IPV6, 687 MLX5_EXPANSION_OUTER_IPV6_UDP, 688 MLX5_EXPANSION_OUTER_IPV6_TCP, 689 MLX5_EXPANSION_OUTER_IPV6_ESP, 690 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 691 MLX5_EXPANSION_VXLAN, 692 MLX5_EXPANSION_STD_VXLAN, 693 MLX5_EXPANSION_L3_VXLAN, 694 MLX5_EXPANSION_VXLAN_GPE, 695 MLX5_EXPANSION_GRE, 696 MLX5_EXPANSION_NVGRE, 697 MLX5_EXPANSION_GRE_KEY, 698 MLX5_EXPANSION_MPLS, 699 MLX5_EXPANSION_ETH, 700 MLX5_EXPANSION_VLAN, 701 MLX5_EXPANSION_IPV4, 702 MLX5_EXPANSION_IPV4_UDP, 703 MLX5_EXPANSION_IPV4_TCP, 704 MLX5_EXPANSION_IPV4_ESP, 705 MLX5_EXPANSION_IPV4_ICMP, 706 MLX5_EXPANSION_IPV6, 707 MLX5_EXPANSION_IPV6_UDP, 708 MLX5_EXPANSION_IPV6_TCP, 709 MLX5_EXPANSION_IPV6_ESP, 710 MLX5_EXPANSION_IPV6_ICMP6, 711 MLX5_EXPANSION_IPV6_FRAG_EXT, 712 MLX5_EXPANSION_GTP, 713 MLX5_EXPANSION_GENEVE, 714 }; 715 716 /** Supported expansion of items. */ 717 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 718 [MLX5_EXPANSION_ROOT] = { 719 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 720 MLX5_EXPANSION_IPV4, 721 MLX5_EXPANSION_IPV6), 722 .type = RTE_FLOW_ITEM_TYPE_END, 723 }, 724 [MLX5_EXPANSION_ROOT_OUTER] = { 725 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 726 MLX5_EXPANSION_OUTER_IPV4, 727 MLX5_EXPANSION_OUTER_IPV6), 728 .type = RTE_FLOW_ITEM_TYPE_END, 729 }, 730 [MLX5_EXPANSION_OUTER_ETH] = { 731 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 732 .type = RTE_FLOW_ITEM_TYPE_ETH, 733 .rss_types = 0, 734 }, 735 [MLX5_EXPANSION_OUTER_VLAN] = { 736 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 737 MLX5_EXPANSION_OUTER_IPV6), 738 .type = RTE_FLOW_ITEM_TYPE_VLAN, 739 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 740 }, 741 [MLX5_EXPANSION_OUTER_IPV4] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT 743 (MLX5_EXPANSION_OUTER_IPV4_UDP, 744 MLX5_EXPANSION_OUTER_IPV4_TCP, 745 MLX5_EXPANSION_OUTER_IPV4_ESP, 746 MLX5_EXPANSION_OUTER_IPV4_ICMP, 747 MLX5_EXPANSION_GRE, 748 MLX5_EXPANSION_NVGRE, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_IPV4, 752 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 753 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 754 }, 755 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 756 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 757 MLX5_EXPANSION_VXLAN_GPE, 758 MLX5_EXPANSION_MPLS, 759 MLX5_EXPANSION_GENEVE, 760 MLX5_EXPANSION_GTP), 761 .type = RTE_FLOW_ITEM_TYPE_UDP, 762 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 763 }, 764 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 765 .type = RTE_FLOW_ITEM_TYPE_TCP, 766 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 767 }, 768 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 769 .type = RTE_FLOW_ITEM_TYPE_ESP, 770 .rss_types = RTE_ETH_RSS_ESP, 771 }, 772 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 773 .type = RTE_FLOW_ITEM_TYPE_ICMP, 774 }, 775 [MLX5_EXPANSION_OUTER_IPV6] = { 776 .next = MLX5_FLOW_EXPAND_RSS_NEXT 777 (MLX5_EXPANSION_OUTER_IPV6_UDP, 778 MLX5_EXPANSION_OUTER_IPV6_TCP, 779 MLX5_EXPANSION_OUTER_IPV6_ESP, 780 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 781 MLX5_EXPANSION_IPV4, 782 MLX5_EXPANSION_IPV6, 783 MLX5_EXPANSION_GRE, 784 MLX5_EXPANSION_NVGRE), 785 .type = RTE_FLOW_ITEM_TYPE_IPV6, 786 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 787 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 788 }, 789 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 790 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 791 MLX5_EXPANSION_VXLAN_GPE, 792 MLX5_EXPANSION_MPLS, 793 MLX5_EXPANSION_GENEVE, 794 MLX5_EXPANSION_GTP), 795 .type = RTE_FLOW_ITEM_TYPE_UDP, 796 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 797 }, 798 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 799 .type = RTE_FLOW_ITEM_TYPE_TCP, 800 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 801 }, 802 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 803 .type = RTE_FLOW_ITEM_TYPE_ESP, 804 .rss_types = RTE_ETH_RSS_ESP, 805 }, 806 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 807 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 808 }, 809 [MLX5_EXPANSION_VXLAN] = { 810 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 811 MLX5_EXPANSION_IPV4, 812 MLX5_EXPANSION_IPV6), 813 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 814 }, 815 [MLX5_EXPANSION_STD_VXLAN] = { 816 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 817 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 818 }, 819 [MLX5_EXPANSION_L3_VXLAN] = { 820 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 821 MLX5_EXPANSION_IPV6), 822 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 823 }, 824 [MLX5_EXPANSION_VXLAN_GPE] = { 825 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 826 MLX5_EXPANSION_IPV4, 827 MLX5_EXPANSION_IPV6), 828 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 829 }, 830 [MLX5_EXPANSION_GRE] = { 831 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 832 MLX5_EXPANSION_IPV4, 833 MLX5_EXPANSION_IPV6, 834 MLX5_EXPANSION_GRE_KEY, 835 MLX5_EXPANSION_MPLS), 836 .type = RTE_FLOW_ITEM_TYPE_GRE, 837 }, 838 [MLX5_EXPANSION_GRE_KEY] = { 839 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 840 MLX5_EXPANSION_IPV6, 841 MLX5_EXPANSION_MPLS), 842 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 843 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 844 }, 845 [MLX5_EXPANSION_NVGRE] = { 846 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 847 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 848 }, 849 [MLX5_EXPANSION_MPLS] = { 850 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 851 MLX5_EXPANSION_IPV6, 852 MLX5_EXPANSION_ETH), 853 .type = RTE_FLOW_ITEM_TYPE_MPLS, 854 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 855 }, 856 [MLX5_EXPANSION_ETH] = { 857 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 858 .type = RTE_FLOW_ITEM_TYPE_ETH, 859 }, 860 [MLX5_EXPANSION_VLAN] = { 861 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 862 MLX5_EXPANSION_IPV6), 863 .type = RTE_FLOW_ITEM_TYPE_VLAN, 864 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 865 }, 866 [MLX5_EXPANSION_IPV4] = { 867 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 868 MLX5_EXPANSION_IPV4_TCP, 869 MLX5_EXPANSION_IPV4_ESP, 870 MLX5_EXPANSION_IPV4_ICMP), 871 .type = RTE_FLOW_ITEM_TYPE_IPV4, 872 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 873 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 874 }, 875 [MLX5_EXPANSION_IPV4_UDP] = { 876 .type = RTE_FLOW_ITEM_TYPE_UDP, 877 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 878 }, 879 [MLX5_EXPANSION_IPV4_TCP] = { 880 .type = RTE_FLOW_ITEM_TYPE_TCP, 881 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 882 }, 883 [MLX5_EXPANSION_IPV4_ESP] = { 884 .type = RTE_FLOW_ITEM_TYPE_ESP, 885 .rss_types = RTE_ETH_RSS_ESP, 886 }, 887 [MLX5_EXPANSION_IPV4_ICMP] = { 888 .type = RTE_FLOW_ITEM_TYPE_ICMP, 889 }, 890 [MLX5_EXPANSION_IPV6] = { 891 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 892 MLX5_EXPANSION_IPV6_TCP, 893 MLX5_EXPANSION_IPV6_ESP, 894 MLX5_EXPANSION_IPV6_ICMP6, 895 MLX5_EXPANSION_IPV6_FRAG_EXT), 896 .type = RTE_FLOW_ITEM_TYPE_IPV6, 897 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 898 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 899 }, 900 [MLX5_EXPANSION_IPV6_UDP] = { 901 .type = RTE_FLOW_ITEM_TYPE_UDP, 902 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 903 }, 904 [MLX5_EXPANSION_IPV6_TCP] = { 905 .type = RTE_FLOW_ITEM_TYPE_TCP, 906 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 907 }, 908 [MLX5_EXPANSION_IPV6_ESP] = { 909 .type = RTE_FLOW_ITEM_TYPE_ESP, 910 .rss_types = RTE_ETH_RSS_ESP, 911 }, 912 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 913 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 914 }, 915 [MLX5_EXPANSION_IPV6_ICMP6] = { 916 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 917 }, 918 [MLX5_EXPANSION_GTP] = { 919 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 920 MLX5_EXPANSION_IPV6), 921 .type = RTE_FLOW_ITEM_TYPE_GTP, 922 }, 923 [MLX5_EXPANSION_GENEVE] = { 924 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 925 MLX5_EXPANSION_IPV4, 926 MLX5_EXPANSION_IPV6), 927 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 928 }, 929 }; 930 931 static struct rte_flow_action_handle * 932 mlx5_action_handle_create(struct rte_eth_dev *dev, 933 const struct rte_flow_indir_action_conf *conf, 934 const struct rte_flow_action *action, 935 struct rte_flow_error *error); 936 static int mlx5_action_handle_destroy 937 (struct rte_eth_dev *dev, 938 struct rte_flow_action_handle *handle, 939 struct rte_flow_error *error); 940 static int mlx5_action_handle_update 941 (struct rte_eth_dev *dev, 942 struct rte_flow_action_handle *handle, 943 const void *update, 944 struct rte_flow_error *error); 945 static int mlx5_action_handle_query 946 (struct rte_eth_dev *dev, 947 const struct rte_flow_action_handle *handle, 948 void *data, 949 struct rte_flow_error *error); 950 static int 951 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 952 struct rte_flow_tunnel *app_tunnel, 953 struct rte_flow_action **actions, 954 uint32_t *num_of_actions, 955 struct rte_flow_error *error); 956 static int 957 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 958 struct rte_flow_tunnel *app_tunnel, 959 struct rte_flow_item **items, 960 uint32_t *num_of_items, 961 struct rte_flow_error *error); 962 static int 963 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 964 struct rte_flow_item *pmd_items, 965 uint32_t num_items, struct rte_flow_error *err); 966 static int 967 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 968 struct rte_flow_action *pmd_actions, 969 uint32_t num_actions, 970 struct rte_flow_error *err); 971 static int 972 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 973 struct rte_mbuf *m, 974 struct rte_flow_restore_info *info, 975 struct rte_flow_error *err); 976 static struct rte_flow_item_flex_handle * 977 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 978 const struct rte_flow_item_flex_conf *conf, 979 struct rte_flow_error *error); 980 static int 981 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 982 const struct rte_flow_item_flex_handle *handle, 983 struct rte_flow_error *error); 984 static int 985 mlx5_flow_info_get(struct rte_eth_dev *dev, 986 struct rte_flow_port_info *port_info, 987 struct rte_flow_queue_info *queue_info, 988 struct rte_flow_error *error); 989 static int 990 mlx5_flow_port_configure(struct rte_eth_dev *dev, 991 const struct rte_flow_port_attr *port_attr, 992 uint16_t nb_queue, 993 const struct rte_flow_queue_attr *queue_attr[], 994 struct rte_flow_error *err); 995 996 static struct rte_flow_pattern_template * 997 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 998 const struct rte_flow_pattern_template_attr *attr, 999 const struct rte_flow_item items[], 1000 struct rte_flow_error *error); 1001 1002 static int 1003 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1004 struct rte_flow_pattern_template *template, 1005 struct rte_flow_error *error); 1006 static struct rte_flow_actions_template * 1007 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1008 const struct rte_flow_actions_template_attr *attr, 1009 const struct rte_flow_action actions[], 1010 const struct rte_flow_action masks[], 1011 struct rte_flow_error *error); 1012 static int 1013 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1014 struct rte_flow_actions_template *template, 1015 struct rte_flow_error *error); 1016 1017 static struct rte_flow_template_table * 1018 mlx5_flow_table_create(struct rte_eth_dev *dev, 1019 const struct rte_flow_template_table_attr *attr, 1020 struct rte_flow_pattern_template *item_templates[], 1021 uint8_t nb_item_templates, 1022 struct rte_flow_actions_template *action_templates[], 1023 uint8_t nb_action_templates, 1024 struct rte_flow_error *error); 1025 static int 1026 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1027 struct rte_flow_template_table *table, 1028 struct rte_flow_error *error); 1029 static struct rte_flow * 1030 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1031 uint32_t queue, 1032 const struct rte_flow_op_attr *attr, 1033 struct rte_flow_template_table *table, 1034 const struct rte_flow_item items[], 1035 uint8_t pattern_template_index, 1036 const struct rte_flow_action actions[], 1037 uint8_t action_template_index, 1038 void *user_data, 1039 struct rte_flow_error *error); 1040 static struct rte_flow * 1041 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 1042 uint32_t queue, 1043 const struct rte_flow_op_attr *attr, 1044 struct rte_flow_template_table *table, 1045 uint32_t rule_index, 1046 const struct rte_flow_action actions[], 1047 uint8_t action_template_index, 1048 void *user_data, 1049 struct rte_flow_error *error); 1050 static int 1051 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1052 uint32_t queue, 1053 const struct rte_flow_op_attr *attr, 1054 struct rte_flow *flow, 1055 void *user_data, 1056 struct rte_flow_error *error); 1057 static int 1058 mlx5_flow_pull(struct rte_eth_dev *dev, 1059 uint32_t queue, 1060 struct rte_flow_op_result res[], 1061 uint16_t n_res, 1062 struct rte_flow_error *error); 1063 static int 1064 mlx5_flow_push(struct rte_eth_dev *dev, 1065 uint32_t queue, 1066 struct rte_flow_error *error); 1067 1068 static struct rte_flow_action_handle * 1069 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1070 const struct rte_flow_op_attr *attr, 1071 const struct rte_flow_indir_action_conf *conf, 1072 const struct rte_flow_action *action, 1073 void *user_data, 1074 struct rte_flow_error *error); 1075 1076 static int 1077 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1078 const struct rte_flow_op_attr *attr, 1079 struct rte_flow_action_handle *handle, 1080 const void *update, 1081 void *user_data, 1082 struct rte_flow_error *error); 1083 1084 static int 1085 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1086 const struct rte_flow_op_attr *attr, 1087 struct rte_flow_action_handle *handle, 1088 void *user_data, 1089 struct rte_flow_error *error); 1090 1091 static int 1092 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1093 const struct rte_flow_op_attr *attr, 1094 const struct rte_flow_action_handle *handle, 1095 void *data, 1096 void *user_data, 1097 struct rte_flow_error *error); 1098 static int 1099 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1100 struct rte_flow_action_handle *handle, 1101 const void *update, void *query, 1102 enum rte_flow_query_update_mode qu_mode, 1103 struct rte_flow_error *error); 1104 static int 1105 mlx5_flow_async_action_handle_query_update 1106 (struct rte_eth_dev *dev, uint32_t queue_id, 1107 const struct rte_flow_op_attr *op_attr, 1108 struct rte_flow_action_handle *action_handle, 1109 const void *update, void *query, 1110 enum rte_flow_query_update_mode qu_mode, 1111 void *user_data, struct rte_flow_error *error); 1112 1113 static const struct rte_flow_ops mlx5_flow_ops = { 1114 .validate = mlx5_flow_validate, 1115 .create = mlx5_flow_create, 1116 .destroy = mlx5_flow_destroy, 1117 .flush = mlx5_flow_flush, 1118 .isolate = mlx5_flow_isolate, 1119 .query = mlx5_flow_query, 1120 .dev_dump = mlx5_flow_dev_dump, 1121 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1122 .get_aged_flows = mlx5_flow_get_aged_flows, 1123 .action_handle_create = mlx5_action_handle_create, 1124 .action_handle_destroy = mlx5_action_handle_destroy, 1125 .action_handle_update = mlx5_action_handle_update, 1126 .action_handle_query = mlx5_action_handle_query, 1127 .action_handle_query_update = mlx5_action_handle_query_update, 1128 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1129 .tunnel_match = mlx5_flow_tunnel_match, 1130 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1131 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1132 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1133 .flex_item_create = mlx5_flow_flex_item_create, 1134 .flex_item_release = mlx5_flow_flex_item_release, 1135 .info_get = mlx5_flow_info_get, 1136 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1137 .configure = mlx5_flow_port_configure, 1138 .pattern_template_create = mlx5_flow_pattern_template_create, 1139 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1140 .actions_template_create = mlx5_flow_actions_template_create, 1141 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1142 .template_table_create = mlx5_flow_table_create, 1143 .template_table_destroy = mlx5_flow_table_destroy, 1144 .async_create = mlx5_flow_async_flow_create, 1145 .async_create_by_index = mlx5_flow_async_flow_create_by_index, 1146 .async_destroy = mlx5_flow_async_flow_destroy, 1147 .pull = mlx5_flow_pull, 1148 .push = mlx5_flow_push, 1149 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1150 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1151 .async_action_handle_query_update = 1152 mlx5_flow_async_action_handle_query_update, 1153 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1154 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1155 }; 1156 1157 /* Tunnel information. */ 1158 struct mlx5_flow_tunnel_info { 1159 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1160 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1161 }; 1162 1163 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1164 { 1165 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1166 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1167 }, 1168 { 1169 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1170 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1171 }, 1172 { 1173 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1174 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1175 }, 1176 { 1177 .tunnel = MLX5_FLOW_LAYER_GRE, 1178 .ptype = RTE_PTYPE_TUNNEL_GRE, 1179 }, 1180 { 1181 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1182 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1183 }, 1184 { 1185 .tunnel = MLX5_FLOW_LAYER_MPLS, 1186 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1187 }, 1188 { 1189 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1190 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1191 }, 1192 { 1193 .tunnel = MLX5_FLOW_LAYER_IPIP, 1194 .ptype = RTE_PTYPE_TUNNEL_IP, 1195 }, 1196 { 1197 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1198 .ptype = RTE_PTYPE_TUNNEL_IP, 1199 }, 1200 { 1201 .tunnel = MLX5_FLOW_LAYER_GTP, 1202 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1203 }, 1204 }; 1205 1206 1207 1208 /** 1209 * Translate tag ID to register. 1210 * 1211 * @param[in] dev 1212 * Pointer to the Ethernet device structure. 1213 * @param[in] feature 1214 * The feature that request the register. 1215 * @param[in] id 1216 * The request register ID. 1217 * @param[out] error 1218 * Error description in case of any. 1219 * 1220 * @return 1221 * The request register on success, a negative errno 1222 * value otherwise and rte_errno is set. 1223 */ 1224 int 1225 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1226 enum mlx5_feature_name feature, 1227 uint32_t id, 1228 struct rte_flow_error *error) 1229 { 1230 struct mlx5_priv *priv = dev->data->dev_private; 1231 struct mlx5_sh_config *config = &priv->sh->config; 1232 enum modify_reg start_reg; 1233 bool skip_mtr_reg = false; 1234 1235 switch (feature) { 1236 case MLX5_HAIRPIN_RX: 1237 return REG_B; 1238 case MLX5_HAIRPIN_TX: 1239 return REG_A; 1240 case MLX5_METADATA_RX: 1241 switch (config->dv_xmeta_en) { 1242 case MLX5_XMETA_MODE_LEGACY: 1243 return REG_B; 1244 case MLX5_XMETA_MODE_META16: 1245 return REG_C_0; 1246 case MLX5_XMETA_MODE_META32: 1247 return REG_C_1; 1248 case MLX5_XMETA_MODE_META32_HWS: 1249 return REG_C_1; 1250 } 1251 break; 1252 case MLX5_METADATA_TX: 1253 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1254 return REG_C_1; 1255 } else { 1256 return REG_A; 1257 } 1258 case MLX5_METADATA_FDB: 1259 switch (config->dv_xmeta_en) { 1260 case MLX5_XMETA_MODE_LEGACY: 1261 return REG_NON; 1262 case MLX5_XMETA_MODE_META16: 1263 return REG_C_0; 1264 case MLX5_XMETA_MODE_META32: 1265 return REG_C_1; 1266 case MLX5_XMETA_MODE_META32_HWS: 1267 return REG_C_1; 1268 } 1269 break; 1270 case MLX5_FLOW_MARK: 1271 switch (config->dv_xmeta_en) { 1272 case MLX5_XMETA_MODE_LEGACY: 1273 case MLX5_XMETA_MODE_META32_HWS: 1274 return REG_NON; 1275 case MLX5_XMETA_MODE_META16: 1276 return REG_C_1; 1277 case MLX5_XMETA_MODE_META32: 1278 return REG_C_0; 1279 } 1280 break; 1281 case MLX5_MTR_ID: 1282 /* 1283 * If meter color and meter id share one register, flow match 1284 * should use the meter color register for match. 1285 */ 1286 if (priv->mtr_reg_share) 1287 return priv->mtr_color_reg; 1288 else 1289 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1290 REG_C_3; 1291 case MLX5_MTR_COLOR: 1292 case MLX5_ASO_FLOW_HIT: 1293 case MLX5_ASO_CONNTRACK: 1294 case MLX5_SAMPLE_ID: 1295 /* All features use the same REG_C. */ 1296 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1297 return priv->mtr_color_reg; 1298 case MLX5_COPY_MARK: 1299 /* 1300 * Metadata COPY_MARK register using is in meter suffix sub 1301 * flow while with meter. It's safe to share the same register. 1302 */ 1303 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1304 case MLX5_APP_TAG: 1305 /* 1306 * If meter is enable, it will engage the register for color 1307 * match and flow match. If meter color match is not using the 1308 * REG_C_2, need to skip the REG_C_x be used by meter color 1309 * match. 1310 * If meter is disable, free to use all available registers. 1311 */ 1312 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1313 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1314 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1315 if (id > (uint32_t)(REG_C_7 - start_reg)) 1316 return rte_flow_error_set(error, EINVAL, 1317 RTE_FLOW_ERROR_TYPE_ITEM, 1318 NULL, "invalid tag id"); 1319 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1320 return rte_flow_error_set(error, ENOTSUP, 1321 RTE_FLOW_ERROR_TYPE_ITEM, 1322 NULL, "unsupported tag id"); 1323 /* 1324 * This case means meter is using the REG_C_x great than 2. 1325 * Take care not to conflict with meter color REG_C_x. 1326 * If the available index REG_C_y >= REG_C_x, skip the 1327 * color register. 1328 */ 1329 if (skip_mtr_reg && priv->sh->flow_mreg_c 1330 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1331 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1332 return rte_flow_error_set(error, EINVAL, 1333 RTE_FLOW_ERROR_TYPE_ITEM, 1334 NULL, "invalid tag id"); 1335 if (priv->sh->flow_mreg_c 1336 [id + 1 + start_reg - REG_C_0] != REG_NON) 1337 return priv->sh->flow_mreg_c 1338 [id + 1 + start_reg - REG_C_0]; 1339 return rte_flow_error_set(error, ENOTSUP, 1340 RTE_FLOW_ERROR_TYPE_ITEM, 1341 NULL, "unsupported tag id"); 1342 } 1343 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1344 } 1345 MLX5_ASSERT(false); 1346 return rte_flow_error_set(error, EINVAL, 1347 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1348 NULL, "invalid feature name"); 1349 } 1350 1351 /** 1352 * Check extensive flow metadata register support. 1353 * 1354 * @param dev 1355 * Pointer to rte_eth_dev structure. 1356 * 1357 * @return 1358 * True if device supports extensive flow metadata register, otherwise false. 1359 */ 1360 bool 1361 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1362 { 1363 struct mlx5_priv *priv = dev->data->dev_private; 1364 1365 /* 1366 * Having available reg_c can be regarded inclusively as supporting 1367 * extensive flow metadata register, which could mean, 1368 * - metadata register copy action by modify header. 1369 * - 16 modify header actions is supported. 1370 * - reg_c's are preserved across different domain (FDB and NIC) on 1371 * packet loopback by flow lookup miss. 1372 */ 1373 return priv->sh->flow_mreg_c[2] != REG_NON; 1374 } 1375 1376 /** 1377 * Get the lowest priority. 1378 * 1379 * @param[in] dev 1380 * Pointer to the Ethernet device structure. 1381 * @param[in] attributes 1382 * Pointer to device flow rule attributes. 1383 * 1384 * @return 1385 * The value of lowest priority of flow. 1386 */ 1387 uint32_t 1388 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1389 const struct rte_flow_attr *attr) 1390 { 1391 struct mlx5_priv *priv = dev->data->dev_private; 1392 1393 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1394 return priv->sh->flow_max_priority - 2; 1395 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1396 } 1397 1398 /** 1399 * Calculate matcher priority of the flow. 1400 * 1401 * @param[in] dev 1402 * Pointer to the Ethernet device structure. 1403 * @param[in] attr 1404 * Pointer to device flow rule attributes. 1405 * @param[in] subpriority 1406 * The priority based on the items. 1407 * @param[in] external 1408 * Flow is user flow. 1409 * @return 1410 * The matcher priority of the flow. 1411 */ 1412 uint16_t 1413 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1414 const struct rte_flow_attr *attr, 1415 uint32_t subpriority, bool external) 1416 { 1417 uint16_t priority = (uint16_t)attr->priority; 1418 struct mlx5_priv *priv = dev->data->dev_private; 1419 1420 /* NIC root rules */ 1421 if (!attr->group && !attr->transfer) { 1422 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1423 priority = priv->sh->flow_max_priority - 1; 1424 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1425 /* FDB root rules */ 1426 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1427 attr->group == 0 && 1428 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1429 return (priv->sh->flow_max_priority - 1) * 3; 1430 } 1431 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1432 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1433 return priority * 3 + subpriority; 1434 } 1435 1436 /** 1437 * Verify the @p item specifications (spec, last, mask) are compatible with the 1438 * NIC capabilities. 1439 * 1440 * @param[in] item 1441 * Item specification. 1442 * @param[in] mask 1443 * @p item->mask or flow default bit-masks. 1444 * @param[in] nic_mask 1445 * Bit-masks covering supported fields by the NIC to compare with user mask. 1446 * @param[in] size 1447 * Bit-masks size in bytes. 1448 * @param[in] range_accepted 1449 * True if range of values is accepted for specific fields, false otherwise. 1450 * @param[out] error 1451 * Pointer to error structure. 1452 * 1453 * @return 1454 * 0 on success, a negative errno value otherwise and rte_errno is set. 1455 */ 1456 int 1457 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1458 const uint8_t *mask, 1459 const uint8_t *nic_mask, 1460 unsigned int size, 1461 bool range_accepted, 1462 struct rte_flow_error *error) 1463 { 1464 unsigned int i; 1465 1466 MLX5_ASSERT(nic_mask); 1467 for (i = 0; i < size; ++i) 1468 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1469 return rte_flow_error_set(error, ENOTSUP, 1470 RTE_FLOW_ERROR_TYPE_ITEM, 1471 item, 1472 "mask enables non supported" 1473 " bits"); 1474 if (!item->spec && (item->mask || item->last)) 1475 return rte_flow_error_set(error, EINVAL, 1476 RTE_FLOW_ERROR_TYPE_ITEM, item, 1477 "mask/last without a spec is not" 1478 " supported"); 1479 if (item->spec && item->last && !range_accepted) { 1480 uint8_t spec[size]; 1481 uint8_t last[size]; 1482 unsigned int i; 1483 int ret; 1484 1485 for (i = 0; i < size; ++i) { 1486 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1487 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1488 } 1489 ret = memcmp(spec, last, size); 1490 if (ret != 0) 1491 return rte_flow_error_set(error, EINVAL, 1492 RTE_FLOW_ERROR_TYPE_ITEM, 1493 item, 1494 "range is not valid"); 1495 } 1496 return 0; 1497 } 1498 1499 /** 1500 * Adjust the hash fields according to the @p flow information. 1501 * 1502 * @param[in] dev_flow. 1503 * Pointer to the mlx5_flow. 1504 * @param[in] tunnel 1505 * 1 when the hash field is for a tunnel item. 1506 * @param[in] layer_types 1507 * RTE_ETH_RSS_* types. 1508 * @param[in] hash_fields 1509 * Item hash fields. 1510 * 1511 * @return 1512 * The hash fields that should be used. 1513 */ 1514 uint64_t 1515 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1516 int tunnel __rte_unused, uint64_t layer_types, 1517 uint64_t hash_fields) 1518 { 1519 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1520 int rss_request_inner = rss_desc->level >= 2; 1521 1522 /* Check RSS hash level for tunnel. */ 1523 if (tunnel && rss_request_inner) 1524 hash_fields |= IBV_RX_HASH_INNER; 1525 else if (tunnel || rss_request_inner) 1526 return 0; 1527 #endif 1528 /* Check if requested layer matches RSS hash fields. */ 1529 if (!(rss_desc->types & layer_types)) 1530 return 0; 1531 return hash_fields; 1532 } 1533 1534 /** 1535 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1536 * if several tunnel rules are used on this queue, the tunnel ptype will be 1537 * cleared. 1538 * 1539 * @param rxq_ctrl 1540 * Rx queue to update. 1541 */ 1542 static void 1543 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1544 { 1545 unsigned int i; 1546 uint32_t tunnel_ptype = 0; 1547 1548 /* Look up for the ptype to use. */ 1549 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1550 if (!rxq_ctrl->flow_tunnels_n[i]) 1551 continue; 1552 if (!tunnel_ptype) { 1553 tunnel_ptype = tunnels_info[i].ptype; 1554 } else { 1555 tunnel_ptype = 0; 1556 break; 1557 } 1558 } 1559 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1560 } 1561 1562 /** 1563 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1564 * flow. 1565 * 1566 * @param[in] dev 1567 * Pointer to the Ethernet device structure. 1568 * @param[in] dev_handle 1569 * Pointer to device flow handle structure. 1570 */ 1571 void 1572 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1573 struct mlx5_flow_handle *dev_handle) 1574 { 1575 struct mlx5_priv *priv = dev->data->dev_private; 1576 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1577 struct mlx5_ind_table_obj *ind_tbl = NULL; 1578 unsigned int i; 1579 1580 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1581 struct mlx5_hrxq *hrxq; 1582 1583 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1584 dev_handle->rix_hrxq); 1585 if (hrxq) 1586 ind_tbl = hrxq->ind_table; 1587 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1588 struct mlx5_shared_action_rss *shared_rss; 1589 1590 shared_rss = mlx5_ipool_get 1591 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1592 dev_handle->rix_srss); 1593 if (shared_rss) 1594 ind_tbl = shared_rss->ind_tbl; 1595 } 1596 if (!ind_tbl) 1597 return; 1598 for (i = 0; i != ind_tbl->queues_n; ++i) { 1599 int idx = ind_tbl->queues[i]; 1600 struct mlx5_rxq_ctrl *rxq_ctrl; 1601 1602 if (mlx5_is_external_rxq(dev, idx)) 1603 continue; 1604 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1605 MLX5_ASSERT(rxq_ctrl != NULL); 1606 if (rxq_ctrl == NULL) 1607 continue; 1608 /* 1609 * To support metadata register copy on Tx loopback, 1610 * this must be always enabled (metadata may arive 1611 * from other port - not from local flows only. 1612 */ 1613 if (tunnel) { 1614 unsigned int j; 1615 1616 /* Increase the counter matching the flow. */ 1617 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1618 if ((tunnels_info[j].tunnel & 1619 dev_handle->layers) == 1620 tunnels_info[j].tunnel) { 1621 rxq_ctrl->flow_tunnels_n[j]++; 1622 break; 1623 } 1624 } 1625 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1626 } 1627 } 1628 } 1629 1630 static void 1631 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1632 { 1633 struct mlx5_priv *priv = dev->data->dev_private; 1634 struct mlx5_rxq_ctrl *rxq_ctrl; 1635 uint16_t port_id; 1636 1637 if (priv->sh->shared_mark_enabled) 1638 return; 1639 if (priv->master || priv->representor) { 1640 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1641 struct mlx5_priv *opriv = 1642 rte_eth_devices[port_id].data->dev_private; 1643 1644 if (!opriv || 1645 opriv->sh != priv->sh || 1646 opriv->domain_id != priv->domain_id || 1647 opriv->mark_enabled) 1648 continue; 1649 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1650 rxq_ctrl->rxq.mark = 1; 1651 } 1652 opriv->mark_enabled = 1; 1653 } 1654 } else { 1655 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1656 rxq_ctrl->rxq.mark = 1; 1657 } 1658 priv->mark_enabled = 1; 1659 } 1660 priv->sh->shared_mark_enabled = 1; 1661 } 1662 1663 /** 1664 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1665 * 1666 * @param[in] dev 1667 * Pointer to the Ethernet device structure. 1668 * @param[in] flow 1669 * Pointer to flow structure. 1670 */ 1671 static void 1672 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1673 { 1674 struct mlx5_priv *priv = dev->data->dev_private; 1675 uint32_t handle_idx; 1676 struct mlx5_flow_handle *dev_handle; 1677 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1678 1679 MLX5_ASSERT(wks); 1680 if (wks->mark) 1681 flow_rxq_mark_flag_set(dev); 1682 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1683 handle_idx, dev_handle, next) 1684 flow_drv_rxq_flags_set(dev, dev_handle); 1685 } 1686 1687 /** 1688 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1689 * device flow if no other flow uses it with the same kind of request. 1690 * 1691 * @param dev 1692 * Pointer to Ethernet device. 1693 * @param[in] dev_handle 1694 * Pointer to the device flow handle structure. 1695 */ 1696 static void 1697 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1698 struct mlx5_flow_handle *dev_handle) 1699 { 1700 struct mlx5_priv *priv = dev->data->dev_private; 1701 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1702 struct mlx5_ind_table_obj *ind_tbl = NULL; 1703 unsigned int i; 1704 1705 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1706 struct mlx5_hrxq *hrxq; 1707 1708 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1709 dev_handle->rix_hrxq); 1710 if (hrxq) 1711 ind_tbl = hrxq->ind_table; 1712 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1713 struct mlx5_shared_action_rss *shared_rss; 1714 1715 shared_rss = mlx5_ipool_get 1716 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1717 dev_handle->rix_srss); 1718 if (shared_rss) 1719 ind_tbl = shared_rss->ind_tbl; 1720 } 1721 if (!ind_tbl) 1722 return; 1723 MLX5_ASSERT(dev->data->dev_started); 1724 for (i = 0; i != ind_tbl->queues_n; ++i) { 1725 int idx = ind_tbl->queues[i]; 1726 struct mlx5_rxq_ctrl *rxq_ctrl; 1727 1728 if (mlx5_is_external_rxq(dev, idx)) 1729 continue; 1730 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1731 MLX5_ASSERT(rxq_ctrl != NULL); 1732 if (rxq_ctrl == NULL) 1733 continue; 1734 if (tunnel) { 1735 unsigned int j; 1736 1737 /* Decrease the counter matching the flow. */ 1738 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1739 if ((tunnels_info[j].tunnel & 1740 dev_handle->layers) == 1741 tunnels_info[j].tunnel) { 1742 rxq_ctrl->flow_tunnels_n[j]--; 1743 break; 1744 } 1745 } 1746 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1747 } 1748 } 1749 } 1750 1751 /** 1752 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1753 * @p flow if no other flow uses it with the same kind of request. 1754 * 1755 * @param dev 1756 * Pointer to Ethernet device. 1757 * @param[in] flow 1758 * Pointer to the flow. 1759 */ 1760 static void 1761 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1762 { 1763 struct mlx5_priv *priv = dev->data->dev_private; 1764 uint32_t handle_idx; 1765 struct mlx5_flow_handle *dev_handle; 1766 1767 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1768 handle_idx, dev_handle, next) 1769 flow_drv_rxq_flags_trim(dev, dev_handle); 1770 } 1771 1772 /** 1773 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1774 * 1775 * @param dev 1776 * Pointer to Ethernet device. 1777 */ 1778 static void 1779 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1780 { 1781 struct mlx5_priv *priv = dev->data->dev_private; 1782 unsigned int i; 1783 1784 for (i = 0; i != priv->rxqs_n; ++i) { 1785 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1786 unsigned int j; 1787 1788 if (rxq == NULL || rxq->ctrl == NULL) 1789 continue; 1790 rxq->ctrl->rxq.mark = 0; 1791 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1792 rxq->ctrl->flow_tunnels_n[j] = 0; 1793 rxq->ctrl->rxq.tunnel = 0; 1794 } 1795 priv->mark_enabled = 0; 1796 priv->sh->shared_mark_enabled = 0; 1797 } 1798 1799 /** 1800 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1801 * 1802 * @param[in] dev 1803 * Pointer to the Ethernet device structure. 1804 */ 1805 void 1806 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1807 { 1808 struct mlx5_priv *priv = dev->data->dev_private; 1809 unsigned int i; 1810 1811 for (i = 0; i != priv->rxqs_n; ++i) { 1812 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1813 struct mlx5_rxq_data *data; 1814 1815 if (rxq == NULL || rxq->ctrl == NULL) 1816 continue; 1817 data = &rxq->ctrl->rxq; 1818 if (!rte_flow_dynf_metadata_avail()) { 1819 data->dynf_meta = 0; 1820 data->flow_meta_mask = 0; 1821 data->flow_meta_offset = -1; 1822 data->flow_meta_port_mask = 0; 1823 } else { 1824 data->dynf_meta = 1; 1825 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1826 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1827 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1828 } 1829 } 1830 } 1831 1832 /* 1833 * return a pointer to the desired action in the list of actions. 1834 * 1835 * @param[in] actions 1836 * The list of actions to search the action in. 1837 * @param[in] action 1838 * The action to find. 1839 * 1840 * @return 1841 * Pointer to the action in the list, if found. NULL otherwise. 1842 */ 1843 const struct rte_flow_action * 1844 mlx5_flow_find_action(const struct rte_flow_action *actions, 1845 enum rte_flow_action_type action) 1846 { 1847 if (actions == NULL) 1848 return NULL; 1849 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1850 if (actions->type == action) 1851 return actions; 1852 return NULL; 1853 } 1854 1855 /* 1856 * Validate the flag action. 1857 * 1858 * @param[in] action_flags 1859 * Bit-fields that holds the actions detected until now. 1860 * @param[in] attr 1861 * Attributes of flow that includes this action. 1862 * @param[out] error 1863 * Pointer to error structure. 1864 * 1865 * @return 1866 * 0 on success, a negative errno value otherwise and rte_errno is set. 1867 */ 1868 int 1869 mlx5_flow_validate_action_flag(uint64_t action_flags, 1870 const struct rte_flow_attr *attr, 1871 struct rte_flow_error *error) 1872 { 1873 if (action_flags & MLX5_FLOW_ACTION_MARK) 1874 return rte_flow_error_set(error, EINVAL, 1875 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1876 "can't mark and flag in same flow"); 1877 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1878 return rte_flow_error_set(error, EINVAL, 1879 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1880 "can't have 2 flag" 1881 " actions in same flow"); 1882 if (attr->egress) 1883 return rte_flow_error_set(error, ENOTSUP, 1884 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1885 "flag action not supported for " 1886 "egress"); 1887 return 0; 1888 } 1889 1890 /* 1891 * Validate the mark action. 1892 * 1893 * @param[in] action 1894 * Pointer to the queue action. 1895 * @param[in] action_flags 1896 * Bit-fields that holds the actions detected until now. 1897 * @param[in] attr 1898 * Attributes of flow that includes this action. 1899 * @param[out] error 1900 * Pointer to error structure. 1901 * 1902 * @return 1903 * 0 on success, a negative errno value otherwise and rte_errno is set. 1904 */ 1905 int 1906 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1907 uint64_t action_flags, 1908 const struct rte_flow_attr *attr, 1909 struct rte_flow_error *error) 1910 { 1911 const struct rte_flow_action_mark *mark = action->conf; 1912 1913 if (!mark) 1914 return rte_flow_error_set(error, EINVAL, 1915 RTE_FLOW_ERROR_TYPE_ACTION, 1916 action, 1917 "configuration cannot be null"); 1918 if (mark->id >= MLX5_FLOW_MARK_MAX) 1919 return rte_flow_error_set(error, EINVAL, 1920 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1921 &mark->id, 1922 "mark id must in 0 <= id < " 1923 RTE_STR(MLX5_FLOW_MARK_MAX)); 1924 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1925 return rte_flow_error_set(error, EINVAL, 1926 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1927 "can't flag and mark in same flow"); 1928 if (action_flags & MLX5_FLOW_ACTION_MARK) 1929 return rte_flow_error_set(error, EINVAL, 1930 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1931 "can't have 2 mark actions in same" 1932 " flow"); 1933 if (attr->egress) 1934 return rte_flow_error_set(error, ENOTSUP, 1935 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1936 "mark action not supported for " 1937 "egress"); 1938 return 0; 1939 } 1940 1941 /* 1942 * Validate the drop action. 1943 * 1944 * @param[in] dev 1945 * Pointer to the Ethernet device structure. 1946 * @param[in] is_root 1947 * True if flow is validated for root table. False otherwise. 1948 * @param[in] attr 1949 * Attributes of flow that includes this action. 1950 * @param[out] error 1951 * Pointer to error structure. 1952 * 1953 * @return 1954 * 0 on success, a negative errno value otherwise and rte_errno is set. 1955 */ 1956 int 1957 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 1958 bool is_root, 1959 const struct rte_flow_attr *attr, 1960 struct rte_flow_error *error) 1961 { 1962 struct mlx5_priv *priv = dev->data->dev_private; 1963 1964 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 1965 return rte_flow_error_set(error, ENOTSUP, 1966 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1967 "drop action not supported for " 1968 "egress"); 1969 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 1970 !priv->sh->dr_root_drop_action_en) { 1971 return rte_flow_error_set(error, ENOTSUP, 1972 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 1973 "drop action not supported for " 1974 "egress and transfer on group 0"); 1975 } 1976 return 0; 1977 } 1978 1979 /* 1980 * Validate the queue action. 1981 * 1982 * @param[in] action 1983 * Pointer to the queue action. 1984 * @param[in] action_flags 1985 * Bit-fields that holds the actions detected until now. 1986 * @param[in] dev 1987 * Pointer to the Ethernet device structure. 1988 * @param[in] attr 1989 * Attributes of flow that includes this action. 1990 * @param[out] error 1991 * Pointer to error structure. 1992 * 1993 * @return 1994 * 0 on success, a negative errno value otherwise and rte_errno is set. 1995 */ 1996 int 1997 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1998 uint64_t action_flags, 1999 struct rte_eth_dev *dev, 2000 const struct rte_flow_attr *attr, 2001 struct rte_flow_error *error) 2002 { 2003 struct mlx5_priv *priv = dev->data->dev_private; 2004 const struct rte_flow_action_queue *queue = action->conf; 2005 2006 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2007 return rte_flow_error_set(error, EINVAL, 2008 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2009 "can't have 2 fate actions in" 2010 " same flow"); 2011 if (attr->egress) 2012 return rte_flow_error_set(error, ENOTSUP, 2013 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2014 "queue action not supported for egress."); 2015 if (mlx5_is_external_rxq(dev, queue->index)) 2016 return 0; 2017 if (!priv->rxqs_n) 2018 return rte_flow_error_set(error, EINVAL, 2019 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2020 NULL, "No Rx queues configured"); 2021 if (queue->index >= priv->rxqs_n) 2022 return rte_flow_error_set(error, EINVAL, 2023 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2024 &queue->index, 2025 "queue index out of range"); 2026 if (mlx5_rxq_get(dev, queue->index) == NULL) 2027 return rte_flow_error_set(error, EINVAL, 2028 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2029 &queue->index, 2030 "queue is not configured"); 2031 return 0; 2032 } 2033 2034 /** 2035 * Validate queue numbers for device RSS. 2036 * 2037 * @param[in] dev 2038 * Configured device. 2039 * @param[in] queues 2040 * Array of queue numbers. 2041 * @param[in] queues_n 2042 * Size of the @p queues array. 2043 * @param[out] error 2044 * On error, filled with a textual error description. 2045 * @param[out] queue_idx 2046 * On error, filled with an offending queue index in @p queues array. 2047 * 2048 * @return 2049 * 0 on success, a negative errno code on error. 2050 */ 2051 static int 2052 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2053 const uint16_t *queues, uint32_t queues_n, 2054 const char **error, uint32_t *queue_idx) 2055 { 2056 const struct mlx5_priv *priv = dev->data->dev_private; 2057 bool is_hairpin = false; 2058 bool is_ext_rss = false; 2059 uint32_t i; 2060 2061 for (i = 0; i != queues_n; ++i) { 2062 struct mlx5_rxq_ctrl *rxq_ctrl; 2063 2064 if (mlx5_is_external_rxq(dev, queues[0])) { 2065 is_ext_rss = true; 2066 continue; 2067 } 2068 if (is_ext_rss) { 2069 *error = "Combining external and regular RSS queues is not supported"; 2070 *queue_idx = i; 2071 return -ENOTSUP; 2072 } 2073 if (queues[i] >= priv->rxqs_n) { 2074 *error = "queue index out of range"; 2075 *queue_idx = i; 2076 return -EINVAL; 2077 } 2078 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2079 if (rxq_ctrl == NULL) { 2080 *error = "queue is not configured"; 2081 *queue_idx = i; 2082 return -EINVAL; 2083 } 2084 if (i == 0 && rxq_ctrl->is_hairpin) 2085 is_hairpin = true; 2086 if (is_hairpin != rxq_ctrl->is_hairpin) { 2087 *error = "combining hairpin and regular RSS queues is not supported"; 2088 *queue_idx = i; 2089 return -ENOTSUP; 2090 } 2091 } 2092 return 0; 2093 } 2094 2095 /* 2096 * Validate the rss action. 2097 * 2098 * @param[in] dev 2099 * Pointer to the Ethernet device structure. 2100 * @param[in] action 2101 * Pointer to the queue action. 2102 * @param[out] error 2103 * Pointer to error structure. 2104 * 2105 * @return 2106 * 0 on success, a negative errno value otherwise and rte_errno is set. 2107 */ 2108 int 2109 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2110 const struct rte_flow_action *action, 2111 struct rte_flow_error *error) 2112 { 2113 struct mlx5_priv *priv = dev->data->dev_private; 2114 const struct rte_flow_action_rss *rss = action->conf; 2115 int ret; 2116 const char *message; 2117 uint32_t queue_idx; 2118 2119 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2120 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2121 return rte_flow_error_set(error, ENOTSUP, 2122 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2123 &rss->func, 2124 "RSS hash function not supported"); 2125 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2126 if (rss->level > 2) 2127 #else 2128 if (rss->level > 1) 2129 #endif 2130 return rte_flow_error_set(error, ENOTSUP, 2131 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2132 &rss->level, 2133 "tunnel RSS is not supported"); 2134 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2135 if (rss->key_len == 0 && rss->key != NULL) 2136 return rte_flow_error_set(error, ENOTSUP, 2137 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2138 &rss->key_len, 2139 "RSS hash key length 0"); 2140 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2141 return rte_flow_error_set(error, ENOTSUP, 2142 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2143 &rss->key_len, 2144 "RSS hash key too small"); 2145 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2146 return rte_flow_error_set(error, ENOTSUP, 2147 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2148 &rss->key_len, 2149 "RSS hash key too large"); 2150 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2151 return rte_flow_error_set(error, ENOTSUP, 2152 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2153 &rss->queue_num, 2154 "number of queues too large"); 2155 if (rss->types & MLX5_RSS_HF_MASK) 2156 return rte_flow_error_set(error, ENOTSUP, 2157 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2158 &rss->types, 2159 "some RSS protocols are not" 2160 " supported"); 2161 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2162 !(rss->types & RTE_ETH_RSS_IP)) 2163 return rte_flow_error_set(error, EINVAL, 2164 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2165 "L3 partial RSS requested but L3 RSS" 2166 " type not specified"); 2167 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2168 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2169 return rte_flow_error_set(error, EINVAL, 2170 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2171 "L4 partial RSS requested but L4 RSS" 2172 " type not specified"); 2173 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2174 return rte_flow_error_set(error, EINVAL, 2175 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2176 NULL, "No Rx queues configured"); 2177 if (!rss->queue_num) 2178 return rte_flow_error_set(error, EINVAL, 2179 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2180 NULL, "No queues configured"); 2181 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2182 &message, &queue_idx); 2183 if (ret != 0) { 2184 return rte_flow_error_set(error, -ret, 2185 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2186 &rss->queue[queue_idx], message); 2187 } 2188 return 0; 2189 } 2190 2191 /* 2192 * Validate the rss action. 2193 * 2194 * @param[in] action 2195 * Pointer to the queue action. 2196 * @param[in] action_flags 2197 * Bit-fields that holds the actions detected until now. 2198 * @param[in] dev 2199 * Pointer to the Ethernet device structure. 2200 * @param[in] attr 2201 * Attributes of flow that includes this action. 2202 * @param[in] item_flags 2203 * Items that were detected. 2204 * @param[out] error 2205 * Pointer to error structure. 2206 * 2207 * @return 2208 * 0 on success, a negative errno value otherwise and rte_errno is set. 2209 */ 2210 int 2211 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2212 uint64_t action_flags, 2213 struct rte_eth_dev *dev, 2214 const struct rte_flow_attr *attr, 2215 uint64_t item_flags, 2216 struct rte_flow_error *error) 2217 { 2218 const struct rte_flow_action_rss *rss = action->conf; 2219 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2220 int ret; 2221 2222 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2223 return rte_flow_error_set(error, EINVAL, 2224 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2225 "can't have 2 fate actions" 2226 " in same flow"); 2227 ret = mlx5_validate_action_rss(dev, action, error); 2228 if (ret) 2229 return ret; 2230 if (attr->egress) 2231 return rte_flow_error_set(error, ENOTSUP, 2232 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2233 "rss action not supported for " 2234 "egress"); 2235 if (rss->level > 1 && !tunnel) 2236 return rte_flow_error_set(error, EINVAL, 2237 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2238 "inner RSS is not supported for " 2239 "non-tunnel flows"); 2240 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2241 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2242 return rte_flow_error_set(error, EINVAL, 2243 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2244 "RSS on eCPRI is not supported now"); 2245 } 2246 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2247 !(item_flags & 2248 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2249 rss->level > 1) 2250 return rte_flow_error_set(error, EINVAL, 2251 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2252 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2253 return 0; 2254 } 2255 2256 /* 2257 * Validate the default miss action. 2258 * 2259 * @param[in] action_flags 2260 * Bit-fields that holds the actions detected until now. 2261 * @param[out] error 2262 * Pointer to error structure. 2263 * 2264 * @return 2265 * 0 on success, a negative errno value otherwise and rte_errno is set. 2266 */ 2267 int 2268 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2269 const struct rte_flow_attr *attr, 2270 struct rte_flow_error *error) 2271 { 2272 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2273 return rte_flow_error_set(error, EINVAL, 2274 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2275 "can't have 2 fate actions in" 2276 " same flow"); 2277 if (attr->egress) 2278 return rte_flow_error_set(error, ENOTSUP, 2279 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2280 "default miss action not supported " 2281 "for egress"); 2282 if (attr->group) 2283 return rte_flow_error_set(error, ENOTSUP, 2284 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2285 "only group 0 is supported"); 2286 if (attr->transfer) 2287 return rte_flow_error_set(error, ENOTSUP, 2288 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2289 NULL, "transfer is not supported"); 2290 return 0; 2291 } 2292 2293 /* 2294 * Validate the count action. 2295 * 2296 * @param[in] dev 2297 * Pointer to the Ethernet device structure. 2298 * @param[in] attr 2299 * Attributes of flow that includes this action. 2300 * @param[out] error 2301 * Pointer to error structure. 2302 * 2303 * @return 2304 * 0 on success, a negative errno value otherwise and rte_errno is set. 2305 */ 2306 int 2307 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2308 const struct rte_flow_attr *attr, 2309 struct rte_flow_error *error) 2310 { 2311 if (attr->egress) 2312 return rte_flow_error_set(error, ENOTSUP, 2313 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2314 "count action not supported for " 2315 "egress"); 2316 return 0; 2317 } 2318 2319 /* 2320 * Validate the ASO CT action. 2321 * 2322 * @param[in] dev 2323 * Pointer to the Ethernet device structure. 2324 * @param[in] conntrack 2325 * Pointer to the CT action profile. 2326 * @param[out] error 2327 * Pointer to error structure. 2328 * 2329 * @return 2330 * 0 on success, a negative errno value otherwise and rte_errno is set. 2331 */ 2332 int 2333 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2334 const struct rte_flow_action_conntrack *conntrack, 2335 struct rte_flow_error *error) 2336 { 2337 RTE_SET_USED(dev); 2338 2339 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2340 return rte_flow_error_set(error, EINVAL, 2341 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2342 "Invalid CT state"); 2343 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2344 return rte_flow_error_set(error, EINVAL, 2345 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2346 "Invalid last TCP packet flag"); 2347 return 0; 2348 } 2349 2350 /** 2351 * Validate the level value for modify field action. 2352 * 2353 * @param[in] data 2354 * Pointer to the rte_flow_action_modify_data structure either src or dst. 2355 * @param[out] error 2356 * Pointer to error structure. 2357 * 2358 * @return 2359 * 0 on success, a negative errno value otherwise and rte_errno is set. 2360 */ 2361 int 2362 flow_validate_modify_field_level(const struct rte_flow_action_modify_data *data, 2363 struct rte_flow_error *error) 2364 { 2365 if (data->level == 0) 2366 return 0; 2367 if (data->field != RTE_FLOW_FIELD_TAG) 2368 return rte_flow_error_set(error, ENOTSUP, 2369 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2370 "inner header fields modification is not supported"); 2371 if (data->tag_index != 0) 2372 return rte_flow_error_set(error, EINVAL, 2373 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2374 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2375 /* 2376 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2377 * 'tag_index' field. In old API, it was provided using 'level' field 2378 * and it is still supported for backwards compatibility. 2379 */ 2380 DRV_LOG(WARNING, "tag array provided in 'level' field instead of 'tag_index' field."); 2381 return 0; 2382 } 2383 2384 /** 2385 * Validate ICMP6 item. 2386 * 2387 * @param[in] item 2388 * Item specification. 2389 * @param[in] item_flags 2390 * Bit-fields that holds the items detected until now. 2391 * @param[in] ext_vlan_sup 2392 * Whether extended VLAN features are supported or not. 2393 * @param[out] error 2394 * Pointer to error structure. 2395 * 2396 * @return 2397 * 0 on success, a negative errno value otherwise and rte_errno is set. 2398 */ 2399 int 2400 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2401 uint64_t item_flags, 2402 uint8_t target_protocol, 2403 struct rte_flow_error *error) 2404 { 2405 const struct rte_flow_item_icmp6 *mask = item->mask; 2406 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2407 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2408 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2409 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2410 MLX5_FLOW_LAYER_OUTER_L4; 2411 int ret; 2412 2413 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2414 return rte_flow_error_set(error, EINVAL, 2415 RTE_FLOW_ERROR_TYPE_ITEM, item, 2416 "protocol filtering not compatible" 2417 " with ICMP6 layer"); 2418 if (!(item_flags & l3m)) 2419 return rte_flow_error_set(error, EINVAL, 2420 RTE_FLOW_ERROR_TYPE_ITEM, item, 2421 "IPv6 is mandatory to filter on" 2422 " ICMP6"); 2423 if (item_flags & l4m) 2424 return rte_flow_error_set(error, EINVAL, 2425 RTE_FLOW_ERROR_TYPE_ITEM, item, 2426 "multiple L4 layers not supported"); 2427 if (!mask) 2428 mask = &rte_flow_item_icmp6_mask; 2429 ret = mlx5_flow_item_acceptable 2430 (item, (const uint8_t *)mask, 2431 (const uint8_t *)&rte_flow_item_icmp6_mask, 2432 sizeof(struct rte_flow_item_icmp6), 2433 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2434 if (ret < 0) 2435 return ret; 2436 return 0; 2437 } 2438 2439 /** 2440 * Validate ICMP6 echo request/reply item. 2441 * 2442 * @param[in] item 2443 * Item specification. 2444 * @param[in] item_flags 2445 * Bit-fields that holds the items detected until now. 2446 * @param[in] ext_vlan_sup 2447 * Whether extended VLAN features are supported or not. 2448 * @param[out] error 2449 * Pointer to error structure. 2450 * 2451 * @return 2452 * 0 on success, a negative errno value otherwise and rte_errno is set. 2453 */ 2454 int 2455 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2456 uint64_t item_flags, 2457 uint8_t target_protocol, 2458 struct rte_flow_error *error) 2459 { 2460 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2461 const struct rte_flow_item_icmp6_echo nic_mask = { 2462 .hdr.base.type = 0xff, 2463 .hdr.base.code = 0xff, 2464 .hdr.identifier = RTE_BE16(0xffff), 2465 .hdr.sequence = RTE_BE16(0xffff), 2466 }; 2467 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2468 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2469 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2470 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2471 MLX5_FLOW_LAYER_OUTER_L4; 2472 int ret; 2473 2474 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2475 return rte_flow_error_set(error, EINVAL, 2476 RTE_FLOW_ERROR_TYPE_ITEM, item, 2477 "protocol filtering not compatible" 2478 " with ICMP6 layer"); 2479 if (!(item_flags & l3m)) 2480 return rte_flow_error_set(error, EINVAL, 2481 RTE_FLOW_ERROR_TYPE_ITEM, item, 2482 "IPv6 is mandatory to filter on" 2483 " ICMP6"); 2484 if (item_flags & l4m) 2485 return rte_flow_error_set(error, EINVAL, 2486 RTE_FLOW_ERROR_TYPE_ITEM, item, 2487 "multiple L4 layers not supported"); 2488 if (!mask) 2489 mask = &nic_mask; 2490 ret = mlx5_flow_item_acceptable 2491 (item, (const uint8_t *)mask, 2492 (const uint8_t *)&nic_mask, 2493 sizeof(struct rte_flow_item_icmp6_echo), 2494 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2495 if (ret < 0) 2496 return ret; 2497 return 0; 2498 } 2499 2500 /** 2501 * Validate ICMP item. 2502 * 2503 * @param[in] item 2504 * Item specification. 2505 * @param[in] item_flags 2506 * Bit-fields that holds the items detected until now. 2507 * @param[out] error 2508 * Pointer to error structure. 2509 * 2510 * @return 2511 * 0 on success, a negative errno value otherwise and rte_errno is set. 2512 */ 2513 int 2514 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2515 uint64_t item_flags, 2516 uint8_t target_protocol, 2517 struct rte_flow_error *error) 2518 { 2519 const struct rte_flow_item_icmp *mask = item->mask; 2520 const struct rte_flow_item_icmp nic_mask = { 2521 .hdr.icmp_type = 0xff, 2522 .hdr.icmp_code = 0xff, 2523 .hdr.icmp_ident = RTE_BE16(0xffff), 2524 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2525 }; 2526 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2527 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2528 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2529 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2530 MLX5_FLOW_LAYER_OUTER_L4; 2531 int ret; 2532 2533 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2534 return rte_flow_error_set(error, EINVAL, 2535 RTE_FLOW_ERROR_TYPE_ITEM, item, 2536 "protocol filtering not compatible" 2537 " with ICMP layer"); 2538 if (!(item_flags & l3m)) 2539 return rte_flow_error_set(error, EINVAL, 2540 RTE_FLOW_ERROR_TYPE_ITEM, item, 2541 "IPv4 is mandatory to filter" 2542 " on ICMP"); 2543 if (item_flags & l4m) 2544 return rte_flow_error_set(error, EINVAL, 2545 RTE_FLOW_ERROR_TYPE_ITEM, item, 2546 "multiple L4 layers not supported"); 2547 if (!mask) 2548 mask = &nic_mask; 2549 ret = mlx5_flow_item_acceptable 2550 (item, (const uint8_t *)mask, 2551 (const uint8_t *)&nic_mask, 2552 sizeof(struct rte_flow_item_icmp), 2553 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2554 if (ret < 0) 2555 return ret; 2556 return 0; 2557 } 2558 2559 /** 2560 * Validate Ethernet item. 2561 * 2562 * @param[in] item 2563 * Item specification. 2564 * @param[in] item_flags 2565 * Bit-fields that holds the items detected until now. 2566 * @param[out] error 2567 * Pointer to error structure. 2568 * 2569 * @return 2570 * 0 on success, a negative errno value otherwise and rte_errno is set. 2571 */ 2572 int 2573 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2574 uint64_t item_flags, bool ext_vlan_sup, 2575 struct rte_flow_error *error) 2576 { 2577 const struct rte_flow_item_eth *mask = item->mask; 2578 const struct rte_flow_item_eth nic_mask = { 2579 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2580 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2581 .hdr.ether_type = RTE_BE16(0xffff), 2582 .has_vlan = ext_vlan_sup ? 1 : 0, 2583 }; 2584 int ret; 2585 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2586 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2587 MLX5_FLOW_LAYER_OUTER_L2; 2588 2589 if (item_flags & ethm) 2590 return rte_flow_error_set(error, ENOTSUP, 2591 RTE_FLOW_ERROR_TYPE_ITEM, item, 2592 "multiple L2 layers not supported"); 2593 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2594 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2595 return rte_flow_error_set(error, EINVAL, 2596 RTE_FLOW_ERROR_TYPE_ITEM, item, 2597 "L2 layer should not follow " 2598 "L3 layers"); 2599 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2600 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2601 return rte_flow_error_set(error, EINVAL, 2602 RTE_FLOW_ERROR_TYPE_ITEM, item, 2603 "L2 layer should not follow VLAN"); 2604 if (item_flags & MLX5_FLOW_LAYER_GTP) 2605 return rte_flow_error_set(error, EINVAL, 2606 RTE_FLOW_ERROR_TYPE_ITEM, item, 2607 "L2 layer should not follow GTP"); 2608 if (!mask) 2609 mask = &rte_flow_item_eth_mask; 2610 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2611 (const uint8_t *)&nic_mask, 2612 sizeof(struct rte_flow_item_eth), 2613 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2614 return ret; 2615 } 2616 2617 /** 2618 * Validate VLAN item. 2619 * 2620 * @param[in] item 2621 * Item specification. 2622 * @param[in] item_flags 2623 * Bit-fields that holds the items detected until now. 2624 * @param[in] dev 2625 * Ethernet device flow is being created on. 2626 * @param[out] error 2627 * Pointer to error structure. 2628 * 2629 * @return 2630 * 0 on success, a negative errno value otherwise and rte_errno is set. 2631 */ 2632 int 2633 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2634 uint64_t item_flags, 2635 struct rte_eth_dev *dev, 2636 struct rte_flow_error *error) 2637 { 2638 const struct rte_flow_item_vlan *spec = item->spec; 2639 const struct rte_flow_item_vlan *mask = item->mask; 2640 const struct rte_flow_item_vlan nic_mask = { 2641 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2642 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2643 }; 2644 uint16_t vlan_tag = 0; 2645 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2646 int ret; 2647 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2648 MLX5_FLOW_LAYER_INNER_L4) : 2649 (MLX5_FLOW_LAYER_OUTER_L3 | 2650 MLX5_FLOW_LAYER_OUTER_L4); 2651 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2652 MLX5_FLOW_LAYER_OUTER_VLAN; 2653 2654 if (item_flags & vlanm) 2655 return rte_flow_error_set(error, EINVAL, 2656 RTE_FLOW_ERROR_TYPE_ITEM, item, 2657 "multiple VLAN layers not supported"); 2658 else if ((item_flags & l34m) != 0) 2659 return rte_flow_error_set(error, EINVAL, 2660 RTE_FLOW_ERROR_TYPE_ITEM, item, 2661 "VLAN cannot follow L3/L4 layer"); 2662 if (!mask) 2663 mask = &rte_flow_item_vlan_mask; 2664 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2665 (const uint8_t *)&nic_mask, 2666 sizeof(struct rte_flow_item_vlan), 2667 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2668 if (ret) 2669 return ret; 2670 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2671 struct mlx5_priv *priv = dev->data->dev_private; 2672 2673 if (priv->vmwa_context) { 2674 /* 2675 * Non-NULL context means we have a virtual machine 2676 * and SR-IOV enabled, we have to create VLAN interface 2677 * to make hypervisor to setup E-Switch vport 2678 * context correctly. We avoid creating the multiple 2679 * VLAN interfaces, so we cannot support VLAN tag mask. 2680 */ 2681 return rte_flow_error_set(error, EINVAL, 2682 RTE_FLOW_ERROR_TYPE_ITEM, 2683 item, 2684 "VLAN tag mask is not" 2685 " supported in virtual" 2686 " environment"); 2687 } 2688 } 2689 if (spec) { 2690 vlan_tag = spec->hdr.vlan_tci; 2691 vlan_tag &= mask->hdr.vlan_tci; 2692 } 2693 /* 2694 * From verbs perspective an empty VLAN is equivalent 2695 * to a packet without VLAN layer. 2696 */ 2697 if (!vlan_tag) 2698 return rte_flow_error_set(error, EINVAL, 2699 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2700 item->spec, 2701 "VLAN cannot be empty"); 2702 return 0; 2703 } 2704 2705 /** 2706 * Validate IPV4 item. 2707 * 2708 * @param[in] item 2709 * Item specification. 2710 * @param[in] item_flags 2711 * Bit-fields that holds the items detected until now. 2712 * @param[in] last_item 2713 * Previous validated item in the pattern items. 2714 * @param[in] ether_type 2715 * Type in the ethernet layer header (including dot1q). 2716 * @param[in] acc_mask 2717 * Acceptable mask, if NULL default internal default mask 2718 * will be used to check whether item fields are supported. 2719 * @param[in] range_accepted 2720 * True if range of values is accepted for specific fields, false otherwise. 2721 * @param[out] error 2722 * Pointer to error structure. 2723 * 2724 * @return 2725 * 0 on success, a negative errno value otherwise and rte_errno is set. 2726 */ 2727 int 2728 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2729 uint64_t item_flags, 2730 uint64_t last_item, 2731 uint16_t ether_type, 2732 const struct rte_flow_item_ipv4 *acc_mask, 2733 bool range_accepted, 2734 struct rte_flow_error *error) 2735 { 2736 const struct rte_flow_item_ipv4 *mask = item->mask; 2737 const struct rte_flow_item_ipv4 *spec = item->spec; 2738 const struct rte_flow_item_ipv4 nic_mask = { 2739 .hdr = { 2740 .src_addr = RTE_BE32(0xffffffff), 2741 .dst_addr = RTE_BE32(0xffffffff), 2742 .type_of_service = 0xff, 2743 .next_proto_id = 0xff, 2744 }, 2745 }; 2746 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2747 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2748 MLX5_FLOW_LAYER_OUTER_L3; 2749 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2750 MLX5_FLOW_LAYER_OUTER_L4; 2751 int ret; 2752 uint8_t next_proto = 0xFF; 2753 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2754 MLX5_FLOW_LAYER_OUTER_VLAN | 2755 MLX5_FLOW_LAYER_INNER_VLAN); 2756 2757 if ((last_item & l2_vlan) && ether_type && 2758 ether_type != RTE_ETHER_TYPE_IPV4) 2759 return rte_flow_error_set(error, EINVAL, 2760 RTE_FLOW_ERROR_TYPE_ITEM, item, 2761 "IPv4 cannot follow L2/VLAN layer " 2762 "which ether type is not IPv4"); 2763 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2764 if (mask && spec) 2765 next_proto = mask->hdr.next_proto_id & 2766 spec->hdr.next_proto_id; 2767 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2768 return rte_flow_error_set(error, EINVAL, 2769 RTE_FLOW_ERROR_TYPE_ITEM, 2770 item, 2771 "multiple tunnel " 2772 "not supported"); 2773 } 2774 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2775 return rte_flow_error_set(error, EINVAL, 2776 RTE_FLOW_ERROR_TYPE_ITEM, item, 2777 "wrong tunnel type - IPv6 specified " 2778 "but IPv4 item provided"); 2779 if (item_flags & l3m) 2780 return rte_flow_error_set(error, ENOTSUP, 2781 RTE_FLOW_ERROR_TYPE_ITEM, item, 2782 "multiple L3 layers not supported"); 2783 else if (item_flags & l4m) 2784 return rte_flow_error_set(error, EINVAL, 2785 RTE_FLOW_ERROR_TYPE_ITEM, item, 2786 "L3 cannot follow an L4 layer."); 2787 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2788 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2789 return rte_flow_error_set(error, EINVAL, 2790 RTE_FLOW_ERROR_TYPE_ITEM, item, 2791 "L3 cannot follow an NVGRE layer."); 2792 if (!mask) 2793 mask = &rte_flow_item_ipv4_mask; 2794 else if (mask->hdr.next_proto_id != 0 && 2795 mask->hdr.next_proto_id != 0xff) 2796 return rte_flow_error_set(error, EINVAL, 2797 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2798 "partial mask is not supported" 2799 " for protocol"); 2800 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2801 acc_mask ? (const uint8_t *)acc_mask 2802 : (const uint8_t *)&nic_mask, 2803 sizeof(struct rte_flow_item_ipv4), 2804 range_accepted, error); 2805 if (ret < 0) 2806 return ret; 2807 return 0; 2808 } 2809 2810 /** 2811 * Validate IPV6 item. 2812 * 2813 * @param[in] item 2814 * Item specification. 2815 * @param[in] item_flags 2816 * Bit-fields that holds the items detected until now. 2817 * @param[in] last_item 2818 * Previous validated item in the pattern items. 2819 * @param[in] ether_type 2820 * Type in the ethernet layer header (including dot1q). 2821 * @param[in] acc_mask 2822 * Acceptable mask, if NULL default internal default mask 2823 * will be used to check whether item fields are supported. 2824 * @param[out] error 2825 * Pointer to error structure. 2826 * 2827 * @return 2828 * 0 on success, a negative errno value otherwise and rte_errno is set. 2829 */ 2830 int 2831 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2832 uint64_t item_flags, 2833 uint64_t last_item, 2834 uint16_t ether_type, 2835 const struct rte_flow_item_ipv6 *acc_mask, 2836 struct rte_flow_error *error) 2837 { 2838 const struct rte_flow_item_ipv6 *mask = item->mask; 2839 const struct rte_flow_item_ipv6 *spec = item->spec; 2840 const struct rte_flow_item_ipv6 nic_mask = { 2841 .hdr = { 2842 .src_addr = 2843 "\xff\xff\xff\xff\xff\xff\xff\xff" 2844 "\xff\xff\xff\xff\xff\xff\xff\xff", 2845 .dst_addr = 2846 "\xff\xff\xff\xff\xff\xff\xff\xff" 2847 "\xff\xff\xff\xff\xff\xff\xff\xff", 2848 .vtc_flow = RTE_BE32(0xffffffff), 2849 .proto = 0xff, 2850 }, 2851 }; 2852 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2853 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2854 MLX5_FLOW_LAYER_OUTER_L3; 2855 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2856 MLX5_FLOW_LAYER_OUTER_L4; 2857 int ret; 2858 uint8_t next_proto = 0xFF; 2859 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2860 MLX5_FLOW_LAYER_OUTER_VLAN | 2861 MLX5_FLOW_LAYER_INNER_VLAN); 2862 2863 if ((last_item & l2_vlan) && ether_type && 2864 ether_type != RTE_ETHER_TYPE_IPV6) 2865 return rte_flow_error_set(error, EINVAL, 2866 RTE_FLOW_ERROR_TYPE_ITEM, item, 2867 "IPv6 cannot follow L2/VLAN layer " 2868 "which ether type is not IPv6"); 2869 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2870 next_proto = spec->hdr.proto; 2871 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2872 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2873 return rte_flow_error_set(error, EINVAL, 2874 RTE_FLOW_ERROR_TYPE_ITEM, 2875 item, 2876 "multiple tunnel " 2877 "not supported"); 2878 } 2879 if (next_proto == IPPROTO_HOPOPTS || 2880 next_proto == IPPROTO_ROUTING || 2881 next_proto == IPPROTO_FRAGMENT || 2882 next_proto == IPPROTO_ESP || 2883 next_proto == IPPROTO_AH || 2884 next_proto == IPPROTO_DSTOPTS) 2885 return rte_flow_error_set(error, EINVAL, 2886 RTE_FLOW_ERROR_TYPE_ITEM, item, 2887 "IPv6 proto (next header) should " 2888 "not be set as extension header"); 2889 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2890 return rte_flow_error_set(error, EINVAL, 2891 RTE_FLOW_ERROR_TYPE_ITEM, item, 2892 "wrong tunnel type - IPv4 specified " 2893 "but IPv6 item provided"); 2894 if (item_flags & l3m) 2895 return rte_flow_error_set(error, ENOTSUP, 2896 RTE_FLOW_ERROR_TYPE_ITEM, item, 2897 "multiple L3 layers not supported"); 2898 else if (item_flags & l4m) 2899 return rte_flow_error_set(error, EINVAL, 2900 RTE_FLOW_ERROR_TYPE_ITEM, item, 2901 "L3 cannot follow an L4 layer."); 2902 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2903 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2904 return rte_flow_error_set(error, EINVAL, 2905 RTE_FLOW_ERROR_TYPE_ITEM, item, 2906 "L3 cannot follow an NVGRE layer."); 2907 if (!mask) 2908 mask = &rte_flow_item_ipv6_mask; 2909 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2910 acc_mask ? (const uint8_t *)acc_mask 2911 : (const uint8_t *)&nic_mask, 2912 sizeof(struct rte_flow_item_ipv6), 2913 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2914 if (ret < 0) 2915 return ret; 2916 return 0; 2917 } 2918 2919 /** 2920 * Validate UDP item. 2921 * 2922 * @param[in] item 2923 * Item specification. 2924 * @param[in] item_flags 2925 * Bit-fields that holds the items detected until now. 2926 * @param[in] target_protocol 2927 * The next protocol in the previous item. 2928 * @param[in] flow_mask 2929 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2930 * @param[out] error 2931 * Pointer to error structure. 2932 * 2933 * @return 2934 * 0 on success, a negative errno value otherwise and rte_errno is set. 2935 */ 2936 int 2937 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2938 uint64_t item_flags, 2939 uint8_t target_protocol, 2940 struct rte_flow_error *error) 2941 { 2942 const struct rte_flow_item_udp *mask = item->mask; 2943 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2944 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2945 MLX5_FLOW_LAYER_OUTER_L3; 2946 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2947 MLX5_FLOW_LAYER_OUTER_L4; 2948 int ret; 2949 2950 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2951 return rte_flow_error_set(error, EINVAL, 2952 RTE_FLOW_ERROR_TYPE_ITEM, item, 2953 "protocol filtering not compatible" 2954 " with UDP layer"); 2955 if (!(item_flags & l3m)) 2956 return rte_flow_error_set(error, EINVAL, 2957 RTE_FLOW_ERROR_TYPE_ITEM, item, 2958 "L3 is mandatory to filter on L4"); 2959 if (item_flags & l4m) 2960 return rte_flow_error_set(error, EINVAL, 2961 RTE_FLOW_ERROR_TYPE_ITEM, item, 2962 "multiple L4 layers not supported"); 2963 if (!mask) 2964 mask = &rte_flow_item_udp_mask; 2965 ret = mlx5_flow_item_acceptable 2966 (item, (const uint8_t *)mask, 2967 (const uint8_t *)&rte_flow_item_udp_mask, 2968 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2969 error); 2970 if (ret < 0) 2971 return ret; 2972 return 0; 2973 } 2974 2975 /** 2976 * Validate TCP item. 2977 * 2978 * @param[in] item 2979 * Item specification. 2980 * @param[in] item_flags 2981 * Bit-fields that holds the items detected until now. 2982 * @param[in] target_protocol 2983 * The next protocol in the previous item. 2984 * @param[out] error 2985 * Pointer to error structure. 2986 * 2987 * @return 2988 * 0 on success, a negative errno value otherwise and rte_errno is set. 2989 */ 2990 int 2991 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2992 uint64_t item_flags, 2993 uint8_t target_protocol, 2994 const struct rte_flow_item_tcp *flow_mask, 2995 struct rte_flow_error *error) 2996 { 2997 const struct rte_flow_item_tcp *mask = item->mask; 2998 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2999 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3000 MLX5_FLOW_LAYER_OUTER_L3; 3001 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3002 MLX5_FLOW_LAYER_OUTER_L4; 3003 int ret; 3004 3005 MLX5_ASSERT(flow_mask); 3006 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3007 return rte_flow_error_set(error, EINVAL, 3008 RTE_FLOW_ERROR_TYPE_ITEM, item, 3009 "protocol filtering not compatible" 3010 " with TCP layer"); 3011 if (!(item_flags & l3m)) 3012 return rte_flow_error_set(error, EINVAL, 3013 RTE_FLOW_ERROR_TYPE_ITEM, item, 3014 "L3 is mandatory to filter on L4"); 3015 if (item_flags & l4m) 3016 return rte_flow_error_set(error, EINVAL, 3017 RTE_FLOW_ERROR_TYPE_ITEM, item, 3018 "multiple L4 layers not supported"); 3019 if (!mask) 3020 mask = &rte_flow_item_tcp_mask; 3021 ret = mlx5_flow_item_acceptable 3022 (item, (const uint8_t *)mask, 3023 (const uint8_t *)flow_mask, 3024 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3025 error); 3026 if (ret < 0) 3027 return ret; 3028 return 0; 3029 } 3030 3031 /** 3032 * Validate VXLAN item. 3033 * 3034 * @param[in] dev 3035 * Pointer to the Ethernet device structure. 3036 * @param[in] udp_dport 3037 * UDP destination port 3038 * @param[in] item 3039 * Item specification. 3040 * @param[in] item_flags 3041 * Bit-fields that holds the items detected until now. 3042 * @param root 3043 * Whether action is on root table. 3044 * @param[out] error 3045 * Pointer to error structure. 3046 * 3047 * @return 3048 * 0 on success, a negative errno value otherwise and rte_errno is set. 3049 */ 3050 int 3051 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3052 uint16_t udp_dport, 3053 const struct rte_flow_item *item, 3054 uint64_t item_flags, 3055 bool root, 3056 struct rte_flow_error *error) 3057 { 3058 const struct rte_flow_item_vxlan *spec = item->spec; 3059 const struct rte_flow_item_vxlan *mask = item->mask; 3060 int ret; 3061 struct mlx5_priv *priv = dev->data->dev_private; 3062 union vni { 3063 uint32_t vlan_id; 3064 uint8_t vni[4]; 3065 } id = { .vlan_id = 0, }; 3066 const struct rte_flow_item_vxlan nic_mask = { 3067 .hdr.vni = "\xff\xff\xff", 3068 .hdr.rsvd1 = 0xff, 3069 }; 3070 const struct rte_flow_item_vxlan *valid_mask; 3071 3072 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3073 return rte_flow_error_set(error, ENOTSUP, 3074 RTE_FLOW_ERROR_TYPE_ITEM, item, 3075 "multiple tunnel layers not" 3076 " supported"); 3077 valid_mask = &rte_flow_item_vxlan_mask; 3078 /* 3079 * Verify only UDPv4 is present as defined in 3080 * https://tools.ietf.org/html/rfc7348 3081 */ 3082 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3083 return rte_flow_error_set(error, EINVAL, 3084 RTE_FLOW_ERROR_TYPE_ITEM, item, 3085 "no outer UDP layer found"); 3086 if (!mask) 3087 mask = &rte_flow_item_vxlan_mask; 3088 3089 if (priv->sh->steering_format_version != 3090 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3091 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3092 /* non-root table */ 3093 if (!root && priv->sh->misc5_cap) 3094 valid_mask = &nic_mask; 3095 /* Group zero in NIC domain */ 3096 if (!root && priv->sh->tunnel_header_0_1) 3097 valid_mask = &nic_mask; 3098 } 3099 ret = mlx5_flow_item_acceptable 3100 (item, (const uint8_t *)mask, 3101 (const uint8_t *)valid_mask, 3102 sizeof(struct rte_flow_item_vxlan), 3103 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3104 if (ret < 0) 3105 return ret; 3106 if (spec) { 3107 memcpy(&id.vni[1], spec->hdr.vni, 3); 3108 memcpy(&id.vni[1], mask->hdr.vni, 3); 3109 } 3110 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3111 return rte_flow_error_set(error, ENOTSUP, 3112 RTE_FLOW_ERROR_TYPE_ITEM, item, 3113 "VXLAN tunnel must be fully defined"); 3114 return 0; 3115 } 3116 3117 /** 3118 * Validate VXLAN_GPE item. 3119 * 3120 * @param[in] item 3121 * Item specification. 3122 * @param[in] item_flags 3123 * Bit-fields that holds the items detected until now. 3124 * @param[in] priv 3125 * Pointer to the private data structure. 3126 * @param[in] target_protocol 3127 * The next protocol in the previous item. 3128 * @param[out] error 3129 * Pointer to error structure. 3130 * 3131 * @return 3132 * 0 on success, a negative errno value otherwise and rte_errno is set. 3133 */ 3134 int 3135 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3136 uint64_t item_flags, 3137 struct rte_eth_dev *dev, 3138 struct rte_flow_error *error) 3139 { 3140 struct mlx5_priv *priv = dev->data->dev_private; 3141 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3142 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3143 int ret; 3144 union vni { 3145 uint32_t vlan_id; 3146 uint8_t vni[4]; 3147 } id = { .vlan_id = 0, }; 3148 3149 if (!priv->sh->config.l3_vxlan_en) 3150 return rte_flow_error_set(error, ENOTSUP, 3151 RTE_FLOW_ERROR_TYPE_ITEM, item, 3152 "L3 VXLAN is not enabled by device" 3153 " parameter and/or not configured in" 3154 " firmware"); 3155 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3156 return rte_flow_error_set(error, ENOTSUP, 3157 RTE_FLOW_ERROR_TYPE_ITEM, item, 3158 "multiple tunnel layers not" 3159 " supported"); 3160 /* 3161 * Verify only UDPv4 is present as defined in 3162 * https://tools.ietf.org/html/rfc7348 3163 */ 3164 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3165 return rte_flow_error_set(error, EINVAL, 3166 RTE_FLOW_ERROR_TYPE_ITEM, item, 3167 "no outer UDP layer found"); 3168 if (!mask) 3169 mask = &rte_flow_item_vxlan_gpe_mask; 3170 ret = mlx5_flow_item_acceptable 3171 (item, (const uint8_t *)mask, 3172 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 3173 sizeof(struct rte_flow_item_vxlan_gpe), 3174 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3175 if (ret < 0) 3176 return ret; 3177 if (spec) { 3178 if (spec->hdr.proto) 3179 return rte_flow_error_set(error, ENOTSUP, 3180 RTE_FLOW_ERROR_TYPE_ITEM, 3181 item, 3182 "VxLAN-GPE protocol" 3183 " not supported"); 3184 memcpy(&id.vni[1], spec->hdr.vni, 3); 3185 memcpy(&id.vni[1], mask->hdr.vni, 3); 3186 } 3187 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3188 return rte_flow_error_set(error, ENOTSUP, 3189 RTE_FLOW_ERROR_TYPE_ITEM, item, 3190 "VXLAN-GPE tunnel must be fully" 3191 " defined"); 3192 return 0; 3193 } 3194 /** 3195 * Validate GRE Key item. 3196 * 3197 * @param[in] item 3198 * Item specification. 3199 * @param[in] item_flags 3200 * Bit flags to mark detected items. 3201 * @param[in] gre_item 3202 * Pointer to gre_item 3203 * @param[out] error 3204 * Pointer to error structure. 3205 * 3206 * @return 3207 * 0 on success, a negative errno value otherwise and rte_errno is set. 3208 */ 3209 int 3210 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3211 uint64_t item_flags, 3212 const struct rte_flow_item *gre_item, 3213 struct rte_flow_error *error) 3214 { 3215 const rte_be32_t *mask = item->mask; 3216 int ret = 0; 3217 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3218 const struct rte_flow_item_gre *gre_spec; 3219 const struct rte_flow_item_gre *gre_mask; 3220 3221 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3222 return rte_flow_error_set(error, ENOTSUP, 3223 RTE_FLOW_ERROR_TYPE_ITEM, item, 3224 "Multiple GRE key not support"); 3225 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3226 return rte_flow_error_set(error, ENOTSUP, 3227 RTE_FLOW_ERROR_TYPE_ITEM, item, 3228 "No preceding GRE header"); 3229 if (item_flags & MLX5_FLOW_LAYER_INNER) 3230 return rte_flow_error_set(error, ENOTSUP, 3231 RTE_FLOW_ERROR_TYPE_ITEM, item, 3232 "GRE key following a wrong item"); 3233 gre_mask = gre_item->mask; 3234 if (!gre_mask) 3235 gre_mask = &rte_flow_item_gre_mask; 3236 gre_spec = gre_item->spec; 3237 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3238 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3239 return rte_flow_error_set(error, EINVAL, 3240 RTE_FLOW_ERROR_TYPE_ITEM, item, 3241 "Key bit must be on"); 3242 3243 if (!mask) 3244 mask = &gre_key_default_mask; 3245 ret = mlx5_flow_item_acceptable 3246 (item, (const uint8_t *)mask, 3247 (const uint8_t *)&gre_key_default_mask, 3248 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3249 return ret; 3250 } 3251 3252 /** 3253 * Validate GRE optional item. 3254 * 3255 * @param[in] dev 3256 * Pointer to the Ethernet device structure. 3257 * @param[in] item 3258 * Item specification. 3259 * @param[in] item_flags 3260 * Bit flags to mark detected items. 3261 * @param[in] attr 3262 * Flow rule attributes. 3263 * @param[in] gre_item 3264 * Pointer to gre_item 3265 * @param[out] error 3266 * Pointer to error structure. 3267 * 3268 * @return 3269 * 0 on success, a negative errno value otherwise and rte_errno is set. 3270 */ 3271 int 3272 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3273 const struct rte_flow_item *item, 3274 uint64_t item_flags, 3275 const struct rte_flow_attr *attr, 3276 const struct rte_flow_item *gre_item, 3277 struct rte_flow_error *error) 3278 { 3279 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3280 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3281 const struct rte_flow_item_gre_opt *spec = item->spec; 3282 const struct rte_flow_item_gre_opt *mask = item->mask; 3283 struct mlx5_priv *priv = dev->data->dev_private; 3284 int ret = 0; 3285 struct rte_flow_item_gre_opt nic_mask = { 3286 .checksum_rsvd = { 3287 .checksum = RTE_BE16(UINT16_MAX), 3288 .reserved1 = 0x0, 3289 }, 3290 .key = { 3291 .key = RTE_BE32(UINT32_MAX), 3292 }, 3293 .sequence = { 3294 .sequence = RTE_BE32(UINT32_MAX), 3295 }, 3296 }; 3297 3298 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3299 return rte_flow_error_set(error, ENOTSUP, 3300 RTE_FLOW_ERROR_TYPE_ITEM, item, 3301 "No preceding GRE header"); 3302 if (item_flags & MLX5_FLOW_LAYER_INNER) 3303 return rte_flow_error_set(error, ENOTSUP, 3304 RTE_FLOW_ERROR_TYPE_ITEM, item, 3305 "GRE option following a wrong item"); 3306 if (!spec || !mask) 3307 return rte_flow_error_set(error, EINVAL, 3308 RTE_FLOW_ERROR_TYPE_ITEM, item, 3309 "At least one field gre_option(checksum/key/sequence) must be specified"); 3310 if (!gre_mask) 3311 gre_mask = &rte_flow_item_gre_mask; 3312 if (mask->checksum_rsvd.checksum) 3313 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3314 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3315 return rte_flow_error_set(error, EINVAL, 3316 RTE_FLOW_ERROR_TYPE_ITEM, 3317 item, 3318 "Checksum bit must be on"); 3319 if (mask->key.key) 3320 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3321 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3322 return rte_flow_error_set(error, EINVAL, 3323 RTE_FLOW_ERROR_TYPE_ITEM, 3324 item, "Key bit must be on"); 3325 if (mask->sequence.sequence) 3326 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3327 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3328 return rte_flow_error_set(error, EINVAL, 3329 RTE_FLOW_ERROR_TYPE_ITEM, 3330 item, 3331 "Sequence bit must be on"); 3332 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3333 if (priv->sh->steering_format_version == 3334 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3335 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3336 !priv->sh->misc5_cap) || 3337 (!(priv->sh->tunnel_header_0_1 && 3338 priv->sh->tunnel_header_2_3) && 3339 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3340 return rte_flow_error_set(error, EINVAL, 3341 RTE_FLOW_ERROR_TYPE_ITEM, 3342 item, 3343 "Checksum/Sequence not supported"); 3344 } 3345 ret = mlx5_flow_item_acceptable 3346 (item, (const uint8_t *)mask, 3347 (const uint8_t *)&nic_mask, 3348 sizeof(struct rte_flow_item_gre_opt), 3349 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3350 return ret; 3351 } 3352 3353 /** 3354 * Validate GRE item. 3355 * 3356 * @param[in] item 3357 * Item specification. 3358 * @param[in] item_flags 3359 * Bit flags to mark detected items. 3360 * @param[in] target_protocol 3361 * The next protocol in the previous item. 3362 * @param[out] error 3363 * Pointer to error structure. 3364 * 3365 * @return 3366 * 0 on success, a negative errno value otherwise and rte_errno is set. 3367 */ 3368 int 3369 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3370 uint64_t item_flags, 3371 uint8_t target_protocol, 3372 struct rte_flow_error *error) 3373 { 3374 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3375 const struct rte_flow_item_gre *mask = item->mask; 3376 int ret; 3377 const struct rte_flow_item_gre nic_mask = { 3378 .c_rsvd0_ver = RTE_BE16(0xB000), 3379 .protocol = RTE_BE16(UINT16_MAX), 3380 }; 3381 3382 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3383 return rte_flow_error_set(error, EINVAL, 3384 RTE_FLOW_ERROR_TYPE_ITEM, item, 3385 "protocol filtering not compatible" 3386 " with this GRE layer"); 3387 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3388 return rte_flow_error_set(error, ENOTSUP, 3389 RTE_FLOW_ERROR_TYPE_ITEM, item, 3390 "multiple tunnel layers not" 3391 " supported"); 3392 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3393 return rte_flow_error_set(error, ENOTSUP, 3394 RTE_FLOW_ERROR_TYPE_ITEM, item, 3395 "L3 Layer is missing"); 3396 if (!mask) 3397 mask = &rte_flow_item_gre_mask; 3398 ret = mlx5_flow_item_acceptable 3399 (item, (const uint8_t *)mask, 3400 (const uint8_t *)&nic_mask, 3401 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3402 error); 3403 if (ret < 0) 3404 return ret; 3405 #ifndef HAVE_MLX5DV_DR 3406 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3407 if (spec && (spec->protocol & mask->protocol)) 3408 return rte_flow_error_set(error, ENOTSUP, 3409 RTE_FLOW_ERROR_TYPE_ITEM, item, 3410 "without MPLS support the" 3411 " specification cannot be used for" 3412 " filtering"); 3413 #endif 3414 #endif 3415 return 0; 3416 } 3417 3418 /** 3419 * Validate Geneve item. 3420 * 3421 * @param[in] item 3422 * Item specification. 3423 * @param[in] itemFlags 3424 * Bit-fields that holds the items detected until now. 3425 * @param[in] enPriv 3426 * Pointer to the private data structure. 3427 * @param[out] error 3428 * Pointer to error structure. 3429 * 3430 * @return 3431 * 0 on success, a negative errno value otherwise and rte_errno is set. 3432 */ 3433 3434 int 3435 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3436 uint64_t item_flags, 3437 struct rte_eth_dev *dev, 3438 struct rte_flow_error *error) 3439 { 3440 struct mlx5_priv *priv = dev->data->dev_private; 3441 const struct rte_flow_item_geneve *spec = item->spec; 3442 const struct rte_flow_item_geneve *mask = item->mask; 3443 int ret; 3444 uint16_t gbhdr; 3445 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3446 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3447 const struct rte_flow_item_geneve nic_mask = { 3448 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3449 .vni = "\xff\xff\xff", 3450 .protocol = RTE_BE16(UINT16_MAX), 3451 }; 3452 3453 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3454 return rte_flow_error_set(error, ENOTSUP, 3455 RTE_FLOW_ERROR_TYPE_ITEM, item, 3456 "L3 Geneve is not enabled by device" 3457 " parameter and/or not configured in" 3458 " firmware"); 3459 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3460 return rte_flow_error_set(error, ENOTSUP, 3461 RTE_FLOW_ERROR_TYPE_ITEM, item, 3462 "multiple tunnel layers not" 3463 " supported"); 3464 /* 3465 * Verify only UDPv4 is present as defined in 3466 * https://tools.ietf.org/html/rfc7348 3467 */ 3468 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3469 return rte_flow_error_set(error, EINVAL, 3470 RTE_FLOW_ERROR_TYPE_ITEM, item, 3471 "no outer UDP layer found"); 3472 if (!mask) 3473 mask = &rte_flow_item_geneve_mask; 3474 ret = mlx5_flow_item_acceptable 3475 (item, (const uint8_t *)mask, 3476 (const uint8_t *)&nic_mask, 3477 sizeof(struct rte_flow_item_geneve), 3478 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3479 if (ret) 3480 return ret; 3481 if (spec) { 3482 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3483 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3484 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3485 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3486 return rte_flow_error_set(error, ENOTSUP, 3487 RTE_FLOW_ERROR_TYPE_ITEM, 3488 item, 3489 "Geneve protocol unsupported" 3490 " fields are being used"); 3491 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3492 return rte_flow_error_set 3493 (error, ENOTSUP, 3494 RTE_FLOW_ERROR_TYPE_ITEM, 3495 item, 3496 "Unsupported Geneve options length"); 3497 } 3498 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3499 return rte_flow_error_set 3500 (error, ENOTSUP, 3501 RTE_FLOW_ERROR_TYPE_ITEM, item, 3502 "Geneve tunnel must be fully defined"); 3503 return 0; 3504 } 3505 3506 /** 3507 * Validate Geneve TLV option item. 3508 * 3509 * @param[in] item 3510 * Item specification. 3511 * @param[in] last_item 3512 * Previous validated item in the pattern items. 3513 * @param[in] geneve_item 3514 * Previous GENEVE item specification. 3515 * @param[in] dev 3516 * Pointer to the rte_eth_dev structure. 3517 * @param[out] error 3518 * Pointer to error structure. 3519 * 3520 * @return 3521 * 0 on success, a negative errno value otherwise and rte_errno is set. 3522 */ 3523 int 3524 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3525 uint64_t last_item, 3526 const struct rte_flow_item *geneve_item, 3527 struct rte_eth_dev *dev, 3528 struct rte_flow_error *error) 3529 { 3530 struct mlx5_priv *priv = dev->data->dev_private; 3531 struct mlx5_dev_ctx_shared *sh = priv->sh; 3532 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3533 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3534 uint8_t data_max_supported = 3535 hca_attr->max_geneve_tlv_option_data_len * 4; 3536 const struct rte_flow_item_geneve *geneve_spec; 3537 const struct rte_flow_item_geneve *geneve_mask; 3538 const struct rte_flow_item_geneve_opt *spec = item->spec; 3539 const struct rte_flow_item_geneve_opt *mask = item->mask; 3540 unsigned int i; 3541 unsigned int data_len; 3542 uint8_t tlv_option_len; 3543 uint16_t optlen_m, optlen_v; 3544 const struct rte_flow_item_geneve_opt full_mask = { 3545 .option_class = RTE_BE16(0xffff), 3546 .option_type = 0xff, 3547 .option_len = 0x1f, 3548 }; 3549 3550 if (!mask) 3551 mask = &rte_flow_item_geneve_opt_mask; 3552 if (!spec) 3553 return rte_flow_error_set 3554 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3555 "Geneve TLV opt class/type/length must be specified"); 3556 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3557 return rte_flow_error_set 3558 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3559 "Geneve TLV opt length exceeds the limit (31)"); 3560 /* Check if class type and length masks are full. */ 3561 if (full_mask.option_class != mask->option_class || 3562 full_mask.option_type != mask->option_type || 3563 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3564 return rte_flow_error_set 3565 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3566 "Geneve TLV opt class/type/length masks must be full"); 3567 /* Check if length is supported */ 3568 if ((uint32_t)spec->option_len > 3569 hca_attr->max_geneve_tlv_option_data_len) 3570 return rte_flow_error_set 3571 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3572 "Geneve TLV opt length not supported"); 3573 if (hca_attr->max_geneve_tlv_options > 1) 3574 DRV_LOG(DEBUG, 3575 "max_geneve_tlv_options supports more than 1 option"); 3576 /* Check GENEVE item preceding. */ 3577 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3578 return rte_flow_error_set 3579 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3580 "Geneve opt item must be preceded with Geneve item"); 3581 geneve_spec = geneve_item->spec; 3582 geneve_mask = geneve_item->mask ? geneve_item->mask : 3583 &rte_flow_item_geneve_mask; 3584 /* Check if GENEVE TLV option size doesn't exceed option length */ 3585 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3586 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3587 tlv_option_len = spec->option_len & mask->option_len; 3588 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3589 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3590 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3591 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3592 if ((optlen_v & optlen_m) <= tlv_option_len) 3593 return rte_flow_error_set 3594 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3595 "GENEVE TLV option length exceeds optlen"); 3596 } 3597 /* Check if length is 0 or data is 0. */ 3598 if (spec->data == NULL || spec->option_len == 0) 3599 return rte_flow_error_set 3600 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3601 "Geneve TLV opt with zero data/length not supported"); 3602 /* Check not all data & mask are 0. */ 3603 data_len = spec->option_len * 4; 3604 if (mask->data == NULL) { 3605 for (i = 0; i < data_len; i++) 3606 if (spec->data[i]) 3607 break; 3608 if (i == data_len) 3609 return rte_flow_error_set(error, ENOTSUP, 3610 RTE_FLOW_ERROR_TYPE_ITEM, item, 3611 "Can't match on Geneve option data 0"); 3612 } else { 3613 for (i = 0; i < data_len; i++) 3614 if (spec->data[i] & mask->data[i]) 3615 break; 3616 if (i == data_len) 3617 return rte_flow_error_set(error, ENOTSUP, 3618 RTE_FLOW_ERROR_TYPE_ITEM, item, 3619 "Can't match on Geneve option data and mask 0"); 3620 /* Check data mask supported. */ 3621 for (i = data_max_supported; i < data_len ; i++) 3622 if (mask->data[i]) 3623 return rte_flow_error_set(error, ENOTSUP, 3624 RTE_FLOW_ERROR_TYPE_ITEM, item, 3625 "Data mask is of unsupported size"); 3626 } 3627 /* Check GENEVE option is supported in NIC. */ 3628 if (!hca_attr->geneve_tlv_opt) 3629 return rte_flow_error_set 3630 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3631 "Geneve TLV opt not supported"); 3632 /* Check if we already have geneve option with different type/class. */ 3633 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3634 geneve_opt_resource = sh->geneve_tlv_option_resource; 3635 if (geneve_opt_resource != NULL) 3636 if (geneve_opt_resource->option_class != spec->option_class || 3637 geneve_opt_resource->option_type != spec->option_type || 3638 geneve_opt_resource->length != spec->option_len) { 3639 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3640 return rte_flow_error_set(error, ENOTSUP, 3641 RTE_FLOW_ERROR_TYPE_ITEM, item, 3642 "Only one Geneve TLV option supported"); 3643 } 3644 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3645 return 0; 3646 } 3647 3648 /** 3649 * Validate MPLS item. 3650 * 3651 * @param[in] dev 3652 * Pointer to the rte_eth_dev structure. 3653 * @param[in] item 3654 * Item specification. 3655 * @param[in] item_flags 3656 * Bit-fields that holds the items detected until now. 3657 * @param[in] prev_layer 3658 * The protocol layer indicated in previous item. 3659 * @param[out] error 3660 * Pointer to error structure. 3661 * 3662 * @return 3663 * 0 on success, a negative errno value otherwise and rte_errno is set. 3664 */ 3665 int 3666 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3667 const struct rte_flow_item *item __rte_unused, 3668 uint64_t item_flags __rte_unused, 3669 uint64_t prev_layer __rte_unused, 3670 struct rte_flow_error *error) 3671 { 3672 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3673 const struct rte_flow_item_mpls *mask = item->mask; 3674 struct mlx5_priv *priv = dev->data->dev_private; 3675 int ret; 3676 3677 if (!priv->sh->dev_cap.mpls_en) 3678 return rte_flow_error_set(error, ENOTSUP, 3679 RTE_FLOW_ERROR_TYPE_ITEM, item, 3680 "MPLS not supported or" 3681 " disabled in firmware" 3682 " configuration."); 3683 /* MPLS over UDP, GRE is allowed */ 3684 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3685 MLX5_FLOW_LAYER_GRE | 3686 MLX5_FLOW_LAYER_GRE_KEY))) 3687 return rte_flow_error_set(error, EINVAL, 3688 RTE_FLOW_ERROR_TYPE_ITEM, item, 3689 "protocol filtering not compatible" 3690 " with MPLS layer"); 3691 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3692 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3693 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3694 return rte_flow_error_set(error, ENOTSUP, 3695 RTE_FLOW_ERROR_TYPE_ITEM, item, 3696 "multiple tunnel layers not" 3697 " supported"); 3698 if (!mask) 3699 mask = &rte_flow_item_mpls_mask; 3700 ret = mlx5_flow_item_acceptable 3701 (item, (const uint8_t *)mask, 3702 (const uint8_t *)&rte_flow_item_mpls_mask, 3703 sizeof(struct rte_flow_item_mpls), 3704 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3705 if (ret < 0) 3706 return ret; 3707 return 0; 3708 #else 3709 return rte_flow_error_set(error, ENOTSUP, 3710 RTE_FLOW_ERROR_TYPE_ITEM, item, 3711 "MPLS is not supported by Verbs, please" 3712 " update."); 3713 #endif 3714 } 3715 3716 /** 3717 * Validate NVGRE item. 3718 * 3719 * @param[in] item 3720 * Item specification. 3721 * @param[in] item_flags 3722 * Bit flags to mark detected items. 3723 * @param[in] target_protocol 3724 * The next protocol in the previous item. 3725 * @param[out] error 3726 * Pointer to error structure. 3727 * 3728 * @return 3729 * 0 on success, a negative errno value otherwise and rte_errno is set. 3730 */ 3731 int 3732 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3733 uint64_t item_flags, 3734 uint8_t target_protocol, 3735 struct rte_flow_error *error) 3736 { 3737 const struct rte_flow_item_nvgre *mask = item->mask; 3738 int ret; 3739 3740 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3741 return rte_flow_error_set(error, EINVAL, 3742 RTE_FLOW_ERROR_TYPE_ITEM, item, 3743 "protocol filtering not compatible" 3744 " with this GRE layer"); 3745 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3746 return rte_flow_error_set(error, ENOTSUP, 3747 RTE_FLOW_ERROR_TYPE_ITEM, item, 3748 "multiple tunnel layers not" 3749 " supported"); 3750 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3751 return rte_flow_error_set(error, ENOTSUP, 3752 RTE_FLOW_ERROR_TYPE_ITEM, item, 3753 "L3 Layer is missing"); 3754 if (!mask) 3755 mask = &rte_flow_item_nvgre_mask; 3756 ret = mlx5_flow_item_acceptable 3757 (item, (const uint8_t *)mask, 3758 (const uint8_t *)&rte_flow_item_nvgre_mask, 3759 sizeof(struct rte_flow_item_nvgre), 3760 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3761 if (ret < 0) 3762 return ret; 3763 return 0; 3764 } 3765 3766 /** 3767 * Validate eCPRI item. 3768 * 3769 * @param[in] item 3770 * Item specification. 3771 * @param[in] item_flags 3772 * Bit-fields that holds the items detected until now. 3773 * @param[in] last_item 3774 * Previous validated item in the pattern items. 3775 * @param[in] ether_type 3776 * Type in the ethernet layer header (including dot1q). 3777 * @param[in] acc_mask 3778 * Acceptable mask, if NULL default internal default mask 3779 * will be used to check whether item fields are supported. 3780 * @param[out] error 3781 * Pointer to error structure. 3782 * 3783 * @return 3784 * 0 on success, a negative errno value otherwise and rte_errno is set. 3785 */ 3786 int 3787 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3788 uint64_t item_flags, 3789 uint64_t last_item, 3790 uint16_t ether_type, 3791 const struct rte_flow_item_ecpri *acc_mask, 3792 struct rte_flow_error *error) 3793 { 3794 const struct rte_flow_item_ecpri *mask = item->mask; 3795 const struct rte_flow_item_ecpri nic_mask = { 3796 .hdr = { 3797 .common = { 3798 .u32 = 3799 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3800 .type = 0xFF, 3801 }).u32), 3802 }, 3803 .dummy[0] = 0xFFFFFFFF, 3804 }, 3805 }; 3806 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3807 MLX5_FLOW_LAYER_OUTER_VLAN); 3808 struct rte_flow_item_ecpri mask_lo; 3809 3810 if (!(last_item & outer_l2_vlan) && 3811 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3812 return rte_flow_error_set(error, EINVAL, 3813 RTE_FLOW_ERROR_TYPE_ITEM, item, 3814 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3815 if ((last_item & outer_l2_vlan) && ether_type && 3816 ether_type != RTE_ETHER_TYPE_ECPRI) 3817 return rte_flow_error_set(error, EINVAL, 3818 RTE_FLOW_ERROR_TYPE_ITEM, item, 3819 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3820 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3821 return rte_flow_error_set(error, EINVAL, 3822 RTE_FLOW_ERROR_TYPE_ITEM, item, 3823 "eCPRI with tunnel is not supported right now"); 3824 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3825 return rte_flow_error_set(error, ENOTSUP, 3826 RTE_FLOW_ERROR_TYPE_ITEM, item, 3827 "multiple L3 layers not supported"); 3828 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3829 return rte_flow_error_set(error, EINVAL, 3830 RTE_FLOW_ERROR_TYPE_ITEM, item, 3831 "eCPRI cannot coexist with a TCP layer"); 3832 /* In specification, eCPRI could be over UDP layer. */ 3833 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3834 return rte_flow_error_set(error, EINVAL, 3835 RTE_FLOW_ERROR_TYPE_ITEM, item, 3836 "eCPRI over UDP layer is not yet supported right now"); 3837 /* Mask for type field in common header could be zero. */ 3838 if (!mask) 3839 mask = &rte_flow_item_ecpri_mask; 3840 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3841 /* Input mask is in big-endian format. */ 3842 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3843 return rte_flow_error_set(error, EINVAL, 3844 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3845 "partial mask is not supported for protocol"); 3846 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3847 return rte_flow_error_set(error, EINVAL, 3848 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3849 "message header mask must be after a type mask"); 3850 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3851 acc_mask ? (const uint8_t *)acc_mask 3852 : (const uint8_t *)&nic_mask, 3853 sizeof(struct rte_flow_item_ecpri), 3854 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3855 } 3856 3857 static int 3858 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3859 const struct rte_flow_attr *attr __rte_unused, 3860 const struct rte_flow_item items[] __rte_unused, 3861 const struct rte_flow_action actions[] __rte_unused, 3862 bool external __rte_unused, 3863 int hairpin __rte_unused, 3864 struct rte_flow_error *error) 3865 { 3866 return rte_flow_error_set(error, ENOTSUP, 3867 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3868 } 3869 3870 static struct mlx5_flow * 3871 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3872 const struct rte_flow_attr *attr __rte_unused, 3873 const struct rte_flow_item items[] __rte_unused, 3874 const struct rte_flow_action actions[] __rte_unused, 3875 struct rte_flow_error *error) 3876 { 3877 rte_flow_error_set(error, ENOTSUP, 3878 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3879 return NULL; 3880 } 3881 3882 static int 3883 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3884 struct mlx5_flow *dev_flow __rte_unused, 3885 const struct rte_flow_attr *attr __rte_unused, 3886 const struct rte_flow_item items[] __rte_unused, 3887 const struct rte_flow_action actions[] __rte_unused, 3888 struct rte_flow_error *error) 3889 { 3890 return rte_flow_error_set(error, ENOTSUP, 3891 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3892 } 3893 3894 static int 3895 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3896 struct rte_flow *flow __rte_unused, 3897 struct rte_flow_error *error) 3898 { 3899 return rte_flow_error_set(error, ENOTSUP, 3900 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3901 } 3902 3903 static void 3904 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3905 struct rte_flow *flow __rte_unused) 3906 { 3907 } 3908 3909 static void 3910 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3911 struct rte_flow *flow __rte_unused) 3912 { 3913 } 3914 3915 static int 3916 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3917 struct rte_flow *flow __rte_unused, 3918 const struct rte_flow_action *actions __rte_unused, 3919 void *data __rte_unused, 3920 struct rte_flow_error *error) 3921 { 3922 return rte_flow_error_set(error, ENOTSUP, 3923 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3924 } 3925 3926 static int 3927 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3928 uint32_t domains __rte_unused, 3929 uint32_t flags __rte_unused) 3930 { 3931 return 0; 3932 } 3933 3934 int 3935 flow_null_get_aged_flows(struct rte_eth_dev *dev, 3936 void **context __rte_unused, 3937 uint32_t nb_contexts __rte_unused, 3938 struct rte_flow_error *error __rte_unused) 3939 { 3940 DRV_LOG(ERR, "port %u get aged flows is not supported.", 3941 dev->data->port_id); 3942 return -ENOTSUP; 3943 } 3944 3945 uint32_t 3946 flow_null_counter_allocate(struct rte_eth_dev *dev) 3947 { 3948 DRV_LOG(ERR, "port %u counter allocate is not supported.", 3949 dev->data->port_id); 3950 return 0; 3951 } 3952 3953 void 3954 flow_null_counter_free(struct rte_eth_dev *dev, 3955 uint32_t counter __rte_unused) 3956 { 3957 DRV_LOG(ERR, "port %u counter free is not supported.", 3958 dev->data->port_id); 3959 } 3960 3961 int 3962 flow_null_counter_query(struct rte_eth_dev *dev, 3963 uint32_t counter __rte_unused, 3964 bool clear __rte_unused, 3965 uint64_t *pkts __rte_unused, 3966 uint64_t *bytes __rte_unused, 3967 void **action __rte_unused) 3968 { 3969 DRV_LOG(ERR, "port %u counter query is not supported.", 3970 dev->data->port_id); 3971 return -ENOTSUP; 3972 } 3973 3974 /* Void driver to protect from null pointer reference. */ 3975 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3976 .validate = flow_null_validate, 3977 .prepare = flow_null_prepare, 3978 .translate = flow_null_translate, 3979 .apply = flow_null_apply, 3980 .remove = flow_null_remove, 3981 .destroy = flow_null_destroy, 3982 .query = flow_null_query, 3983 .sync_domain = flow_null_sync_domain, 3984 .get_aged_flows = flow_null_get_aged_flows, 3985 .counter_alloc = flow_null_counter_allocate, 3986 .counter_free = flow_null_counter_free, 3987 .counter_query = flow_null_counter_query 3988 }; 3989 3990 /** 3991 * Select flow driver type according to flow attributes and device 3992 * configuration. 3993 * 3994 * @param[in] dev 3995 * Pointer to the dev structure. 3996 * @param[in] attr 3997 * Pointer to the flow attributes. 3998 * 3999 * @return 4000 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4001 */ 4002 static enum mlx5_flow_drv_type 4003 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4004 { 4005 struct mlx5_priv *priv = dev->data->dev_private; 4006 /* The OS can determine first a specific flow type (DV, VERBS) */ 4007 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4008 4009 if (type != MLX5_FLOW_TYPE_MAX) 4010 return type; 4011 /* 4012 * Currently when dv_flow_en == 2, only HW steering engine is 4013 * supported. New engines can also be chosen here if ready. 4014 */ 4015 if (priv->sh->config.dv_flow_en == 2) 4016 return MLX5_FLOW_TYPE_HW; 4017 if (!attr) 4018 return MLX5_FLOW_TYPE_MIN; 4019 /* If no OS specific type - continue with DV/VERBS selection */ 4020 if (attr->transfer && priv->sh->config.dv_esw_en) 4021 type = MLX5_FLOW_TYPE_DV; 4022 if (!attr->transfer) 4023 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4024 MLX5_FLOW_TYPE_VERBS; 4025 return type; 4026 } 4027 4028 #define flow_get_drv_ops(type) flow_drv_ops[type] 4029 4030 /** 4031 * Flow driver validation API. This abstracts calling driver specific functions. 4032 * The type of flow driver is determined according to flow attributes. 4033 * 4034 * @param[in] dev 4035 * Pointer to the dev structure. 4036 * @param[in] attr 4037 * Pointer to the flow attributes. 4038 * @param[in] items 4039 * Pointer to the list of items. 4040 * @param[in] actions 4041 * Pointer to the list of actions. 4042 * @param[in] external 4043 * This flow rule is created by request external to PMD. 4044 * @param[in] hairpin 4045 * Number of hairpin TX actions, 0 means classic flow. 4046 * @param[out] error 4047 * Pointer to the error structure. 4048 * 4049 * @return 4050 * 0 on success, a negative errno value otherwise and rte_errno is set. 4051 */ 4052 static inline int 4053 flow_drv_validate(struct rte_eth_dev *dev, 4054 const struct rte_flow_attr *attr, 4055 const struct rte_flow_item items[], 4056 const struct rte_flow_action actions[], 4057 bool external, int hairpin, struct rte_flow_error *error) 4058 { 4059 const struct mlx5_flow_driver_ops *fops; 4060 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4061 4062 fops = flow_get_drv_ops(type); 4063 return fops->validate(dev, attr, items, actions, external, 4064 hairpin, error); 4065 } 4066 4067 /** 4068 * Flow driver preparation API. This abstracts calling driver specific 4069 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4070 * calculates the size of memory required for device flow, allocates the memory, 4071 * initializes the device flow and returns the pointer. 4072 * 4073 * @note 4074 * This function initializes device flow structure such as dv or verbs in 4075 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4076 * rest. For example, adding returning device flow to flow->dev_flow list and 4077 * setting backward reference to the flow should be done out of this function. 4078 * layers field is not filled either. 4079 * 4080 * @param[in] dev 4081 * Pointer to the dev structure. 4082 * @param[in] attr 4083 * Pointer to the flow attributes. 4084 * @param[in] items 4085 * Pointer to the list of items. 4086 * @param[in] actions 4087 * Pointer to the list of actions. 4088 * @param[in] flow_idx 4089 * This memory pool index to the flow. 4090 * @param[out] error 4091 * Pointer to the error structure. 4092 * 4093 * @return 4094 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4095 */ 4096 static inline struct mlx5_flow * 4097 flow_drv_prepare(struct rte_eth_dev *dev, 4098 const struct rte_flow *flow, 4099 const struct rte_flow_attr *attr, 4100 const struct rte_flow_item items[], 4101 const struct rte_flow_action actions[], 4102 uint32_t flow_idx, 4103 struct rte_flow_error *error) 4104 { 4105 const struct mlx5_flow_driver_ops *fops; 4106 enum mlx5_flow_drv_type type = flow->drv_type; 4107 struct mlx5_flow *mlx5_flow = NULL; 4108 4109 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4110 fops = flow_get_drv_ops(type); 4111 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4112 if (mlx5_flow) 4113 mlx5_flow->flow_idx = flow_idx; 4114 return mlx5_flow; 4115 } 4116 4117 /** 4118 * Flow driver translation API. This abstracts calling driver specific 4119 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4120 * translates a generic flow into a driver flow. flow_drv_prepare() must 4121 * precede. 4122 * 4123 * @note 4124 * dev_flow->layers could be filled as a result of parsing during translation 4125 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4126 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4127 * flow->actions could be overwritten even though all the expanded dev_flows 4128 * have the same actions. 4129 * 4130 * @param[in] dev 4131 * Pointer to the rte dev structure. 4132 * @param[in, out] dev_flow 4133 * Pointer to the mlx5 flow. 4134 * @param[in] attr 4135 * Pointer to the flow attributes. 4136 * @param[in] items 4137 * Pointer to the list of items. 4138 * @param[in] actions 4139 * Pointer to the list of actions. 4140 * @param[out] error 4141 * Pointer to the error structure. 4142 * 4143 * @return 4144 * 0 on success, a negative errno value otherwise and rte_errno is set. 4145 */ 4146 static inline int 4147 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4148 const struct rte_flow_attr *attr, 4149 const struct rte_flow_item items[], 4150 const struct rte_flow_action actions[], 4151 struct rte_flow_error *error) 4152 { 4153 const struct mlx5_flow_driver_ops *fops; 4154 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4155 4156 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4157 fops = flow_get_drv_ops(type); 4158 return fops->translate(dev, dev_flow, attr, items, actions, error); 4159 } 4160 4161 /** 4162 * Flow driver apply API. This abstracts calling driver specific functions. 4163 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4164 * translated driver flows on to device. flow_drv_translate() must precede. 4165 * 4166 * @param[in] dev 4167 * Pointer to Ethernet device structure. 4168 * @param[in, out] flow 4169 * Pointer to flow structure. 4170 * @param[out] error 4171 * Pointer to error structure. 4172 * 4173 * @return 4174 * 0 on success, a negative errno value otherwise and rte_errno is set. 4175 */ 4176 static inline int 4177 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4178 struct rte_flow_error *error) 4179 { 4180 const struct mlx5_flow_driver_ops *fops; 4181 enum mlx5_flow_drv_type type = flow->drv_type; 4182 4183 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4184 fops = flow_get_drv_ops(type); 4185 return fops->apply(dev, flow, error); 4186 } 4187 4188 /** 4189 * Flow driver destroy API. This abstracts calling driver specific functions. 4190 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4191 * on device and releases resources of the flow. 4192 * 4193 * @param[in] dev 4194 * Pointer to Ethernet device. 4195 * @param[in, out] flow 4196 * Pointer to flow structure. 4197 */ 4198 static inline void 4199 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4200 { 4201 const struct mlx5_flow_driver_ops *fops; 4202 enum mlx5_flow_drv_type type = flow->drv_type; 4203 4204 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4205 fops = flow_get_drv_ops(type); 4206 fops->destroy(dev, flow); 4207 } 4208 4209 /** 4210 * Flow driver find RSS policy tbl API. This abstracts calling driver 4211 * specific functions. Parent flow (rte_flow) should have driver 4212 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4213 * 4214 * @param[in] dev 4215 * Pointer to Ethernet device. 4216 * @param[in, out] flow 4217 * Pointer to flow structure. 4218 * @param[in] policy 4219 * Pointer to meter policy table. 4220 * @param[in] rss_desc 4221 * Pointer to rss_desc 4222 */ 4223 static struct mlx5_flow_meter_sub_policy * 4224 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4225 struct rte_flow *flow, 4226 struct mlx5_flow_meter_policy *policy, 4227 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4228 { 4229 const struct mlx5_flow_driver_ops *fops; 4230 enum mlx5_flow_drv_type type = flow->drv_type; 4231 4232 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4233 fops = flow_get_drv_ops(type); 4234 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4235 } 4236 4237 /** 4238 * Flow driver color tag rule API. This abstracts calling driver 4239 * specific functions. Parent flow (rte_flow) should have driver 4240 * type (drv_type). It will create the color tag rules in hierarchy meter. 4241 * 4242 * @param[in] dev 4243 * Pointer to Ethernet device. 4244 * @param[in, out] flow 4245 * Pointer to flow structure. 4246 * @param[in] fm 4247 * Pointer to flow meter structure. 4248 * @param[in] src_port 4249 * The src port this extra rule should use. 4250 * @param[in] item 4251 * The src port id match item. 4252 * @param[out] error 4253 * Pointer to error structure. 4254 */ 4255 static int 4256 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4257 struct rte_flow *flow, 4258 struct mlx5_flow_meter_info *fm, 4259 int32_t src_port, 4260 const struct rte_flow_item *item, 4261 struct rte_flow_error *error) 4262 { 4263 const struct mlx5_flow_driver_ops *fops; 4264 enum mlx5_flow_drv_type type = flow->drv_type; 4265 4266 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4267 fops = flow_get_drv_ops(type); 4268 return fops->meter_hierarchy_rule_create(dev, fm, 4269 src_port, item, error); 4270 } 4271 4272 /** 4273 * Get RSS action from the action list. 4274 * 4275 * @param[in] dev 4276 * Pointer to Ethernet device. 4277 * @param[in] actions 4278 * Pointer to the list of actions. 4279 * @param[in] flow 4280 * Parent flow structure pointer. 4281 * 4282 * @return 4283 * Pointer to the RSS action if exist, else return NULL. 4284 */ 4285 static const struct rte_flow_action_rss* 4286 flow_get_rss_action(struct rte_eth_dev *dev, 4287 const struct rte_flow_action actions[]) 4288 { 4289 struct mlx5_priv *priv = dev->data->dev_private; 4290 const struct rte_flow_action_rss *rss = NULL; 4291 struct mlx5_meter_policy_action_container *acg; 4292 struct mlx5_meter_policy_action_container *acy; 4293 4294 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4295 switch (actions->type) { 4296 case RTE_FLOW_ACTION_TYPE_RSS: 4297 rss = actions->conf; 4298 break; 4299 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4300 { 4301 const struct rte_flow_action_sample *sample = 4302 actions->conf; 4303 const struct rte_flow_action *act = sample->actions; 4304 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4305 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4306 rss = act->conf; 4307 break; 4308 } 4309 case RTE_FLOW_ACTION_TYPE_METER: 4310 { 4311 uint32_t mtr_idx; 4312 struct mlx5_flow_meter_info *fm; 4313 struct mlx5_flow_meter_policy *policy; 4314 const struct rte_flow_action_meter *mtr = actions->conf; 4315 4316 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4317 if (fm && !fm->def_policy) { 4318 policy = mlx5_flow_meter_policy_find(dev, 4319 fm->policy_id, NULL); 4320 MLX5_ASSERT(policy); 4321 if (policy->is_hierarchy) { 4322 policy = 4323 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4324 policy); 4325 if (!policy) 4326 return NULL; 4327 } 4328 if (policy->is_rss) { 4329 acg = 4330 &policy->act_cnt[RTE_COLOR_GREEN]; 4331 acy = 4332 &policy->act_cnt[RTE_COLOR_YELLOW]; 4333 if (acg->fate_action == 4334 MLX5_FLOW_FATE_SHARED_RSS) 4335 rss = acg->rss->conf; 4336 else if (acy->fate_action == 4337 MLX5_FLOW_FATE_SHARED_RSS) 4338 rss = acy->rss->conf; 4339 } 4340 } 4341 break; 4342 } 4343 default: 4344 break; 4345 } 4346 } 4347 return rss; 4348 } 4349 4350 /** 4351 * Get ASO age action by index. 4352 * 4353 * @param[in] dev 4354 * Pointer to the Ethernet device structure. 4355 * @param[in] age_idx 4356 * Index to the ASO age action. 4357 * 4358 * @return 4359 * The specified ASO age action. 4360 */ 4361 struct mlx5_aso_age_action* 4362 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4363 { 4364 uint16_t pool_idx = age_idx & UINT16_MAX; 4365 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4366 struct mlx5_priv *priv = dev->data->dev_private; 4367 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4368 struct mlx5_aso_age_pool *pool; 4369 4370 rte_rwlock_read_lock(&mng->resize_rwl); 4371 pool = mng->pools[pool_idx]; 4372 rte_rwlock_read_unlock(&mng->resize_rwl); 4373 return &pool->actions[offset - 1]; 4374 } 4375 4376 /* maps indirect action to translated direct in some actions array */ 4377 struct mlx5_translated_action_handle { 4378 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4379 int index; /**< Index in related array of rte_flow_action. */ 4380 }; 4381 4382 /** 4383 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4384 * direct action if translation possible. 4385 * This functionality used to run same execution path for both direct and 4386 * indirect actions on flow create. All necessary preparations for indirect 4387 * action handling should be performed on *handle* actions list returned 4388 * from this call. 4389 * 4390 * @param[in] dev 4391 * Pointer to Ethernet device. 4392 * @param[in] actions 4393 * List of actions to translate. 4394 * @param[out] handle 4395 * List to store translated indirect action object handles. 4396 * @param[in, out] indir_n 4397 * Size of *handle* array. On return should be updated with number of 4398 * indirect actions retrieved from the *actions* list. 4399 * @param[out] translated_actions 4400 * List of actions where all indirect actions were translated to direct 4401 * if possible. NULL if no translation took place. 4402 * @param[out] error 4403 * Pointer to the error structure. 4404 * 4405 * @return 4406 * 0 on success, a negative errno value otherwise and rte_errno is set. 4407 */ 4408 static int 4409 flow_action_handles_translate(struct rte_eth_dev *dev, 4410 const struct rte_flow_action actions[], 4411 struct mlx5_translated_action_handle *handle, 4412 int *indir_n, 4413 struct rte_flow_action **translated_actions, 4414 struct rte_flow_error *error) 4415 { 4416 struct mlx5_priv *priv = dev->data->dev_private; 4417 struct rte_flow_action *translated = NULL; 4418 size_t actions_size; 4419 int n; 4420 int copied_n = 0; 4421 struct mlx5_translated_action_handle *handle_end = NULL; 4422 4423 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4424 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4425 continue; 4426 if (copied_n == *indir_n) { 4427 return rte_flow_error_set 4428 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4429 NULL, "too many shared actions"); 4430 } 4431 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4432 sizeof(actions[n].conf)); 4433 handle[copied_n].index = n; 4434 copied_n++; 4435 } 4436 n++; 4437 *indir_n = copied_n; 4438 if (!copied_n) 4439 return 0; 4440 actions_size = sizeof(struct rte_flow_action) * n; 4441 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4442 if (!translated) { 4443 rte_errno = ENOMEM; 4444 return -ENOMEM; 4445 } 4446 memcpy(translated, actions, actions_size); 4447 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4448 struct mlx5_shared_action_rss *shared_rss; 4449 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4450 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4451 uint32_t idx = act_idx & 4452 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4453 4454 switch (type) { 4455 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4456 shared_rss = mlx5_ipool_get 4457 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4458 translated[handle->index].type = 4459 RTE_FLOW_ACTION_TYPE_RSS; 4460 translated[handle->index].conf = 4461 &shared_rss->origin; 4462 break; 4463 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4464 translated[handle->index].type = 4465 (enum rte_flow_action_type) 4466 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4467 translated[handle->index].conf = (void *)(uintptr_t)idx; 4468 break; 4469 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4470 translated[handle->index].type = 4471 (enum rte_flow_action_type) 4472 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4473 translated[handle->index].conf = (void *)(uintptr_t)idx; 4474 break; 4475 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4476 if (priv->sh->flow_hit_aso_en) { 4477 translated[handle->index].type = 4478 (enum rte_flow_action_type) 4479 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4480 translated[handle->index].conf = 4481 (void *)(uintptr_t)idx; 4482 break; 4483 } 4484 /* Fall-through */ 4485 case MLX5_INDIRECT_ACTION_TYPE_CT: 4486 if (priv->sh->ct_aso_en) { 4487 translated[handle->index].type = 4488 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4489 translated[handle->index].conf = 4490 (void *)(uintptr_t)idx; 4491 break; 4492 } 4493 /* Fall-through */ 4494 default: 4495 mlx5_free(translated); 4496 return rte_flow_error_set 4497 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4498 NULL, "invalid indirect action type"); 4499 } 4500 } 4501 *translated_actions = translated; 4502 return 0; 4503 } 4504 4505 /** 4506 * Get Shared RSS action from the action list. 4507 * 4508 * @param[in] dev 4509 * Pointer to Ethernet device. 4510 * @param[in] shared 4511 * Pointer to the list of actions. 4512 * @param[in] shared_n 4513 * Actions list length. 4514 * 4515 * @return 4516 * The MLX5 RSS action ID if exists, otherwise return 0. 4517 */ 4518 static uint32_t 4519 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4520 struct mlx5_translated_action_handle *handle, 4521 int shared_n) 4522 { 4523 struct mlx5_translated_action_handle *handle_end; 4524 struct mlx5_priv *priv = dev->data->dev_private; 4525 struct mlx5_shared_action_rss *shared_rss; 4526 4527 4528 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4529 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4530 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4531 uint32_t idx = act_idx & 4532 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4533 switch (type) { 4534 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4535 shared_rss = mlx5_ipool_get 4536 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4537 idx); 4538 __atomic_fetch_add(&shared_rss->refcnt, 1, 4539 __ATOMIC_RELAXED); 4540 return idx; 4541 default: 4542 break; 4543 } 4544 } 4545 return 0; 4546 } 4547 4548 static unsigned int 4549 find_graph_root(uint32_t rss_level) 4550 { 4551 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4552 MLX5_EXPANSION_ROOT_OUTER; 4553 } 4554 4555 /** 4556 * Get layer flags from the prefix flow. 4557 * 4558 * Some flows may be split to several subflows, the prefix subflow gets the 4559 * match items and the suffix sub flow gets the actions. 4560 * Some actions need the user defined match item flags to get the detail for 4561 * the action. 4562 * This function helps the suffix flow to get the item layer flags from prefix 4563 * subflow. 4564 * 4565 * @param[in] dev_flow 4566 * Pointer the created prefix subflow. 4567 * 4568 * @return 4569 * The layers get from prefix subflow. 4570 */ 4571 static inline uint64_t 4572 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4573 { 4574 uint64_t layers = 0; 4575 4576 /* 4577 * Layers bits could be localization, but usually the compiler will 4578 * help to do the optimization work for source code. 4579 * If no decap actions, use the layers directly. 4580 */ 4581 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4582 return dev_flow->handle->layers; 4583 /* Convert L3 layers with decap action. */ 4584 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4585 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4586 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4587 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4588 /* Convert L4 layers with decap action. */ 4589 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4590 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4591 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4592 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4593 return layers; 4594 } 4595 4596 /** 4597 * Get metadata split action information. 4598 * 4599 * @param[in] actions 4600 * Pointer to the list of actions. 4601 * @param[out] qrss 4602 * Pointer to the return pointer. 4603 * @param[out] qrss_type 4604 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4605 * if no QUEUE/RSS is found. 4606 * @param[out] encap_idx 4607 * Pointer to the index of the encap action if exists, otherwise the last 4608 * action index. 4609 * 4610 * @return 4611 * Total number of actions. 4612 */ 4613 static int 4614 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4615 const struct rte_flow_action **qrss, 4616 int *encap_idx) 4617 { 4618 const struct rte_flow_action_raw_encap *raw_encap; 4619 int actions_n = 0; 4620 int raw_decap_idx = -1; 4621 4622 *encap_idx = -1; 4623 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4624 switch (actions->type) { 4625 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4626 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4627 *encap_idx = actions_n; 4628 break; 4629 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4630 raw_decap_idx = actions_n; 4631 break; 4632 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4633 raw_encap = actions->conf; 4634 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4635 *encap_idx = raw_decap_idx != -1 ? 4636 raw_decap_idx : actions_n; 4637 break; 4638 case RTE_FLOW_ACTION_TYPE_QUEUE: 4639 case RTE_FLOW_ACTION_TYPE_RSS: 4640 *qrss = actions; 4641 break; 4642 default: 4643 break; 4644 } 4645 actions_n++; 4646 } 4647 if (*encap_idx == -1) 4648 *encap_idx = actions_n; 4649 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4650 return actions_n + 1; 4651 } 4652 4653 /** 4654 * Check if the action will change packet. 4655 * 4656 * @param dev 4657 * Pointer to Ethernet device. 4658 * @param[in] type 4659 * action type. 4660 * 4661 * @return 4662 * true if action will change packet, false otherwise. 4663 */ 4664 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4665 enum rte_flow_action_type type) 4666 { 4667 struct mlx5_priv *priv = dev->data->dev_private; 4668 4669 switch (type) { 4670 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4671 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4672 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4673 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4674 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4675 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4676 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4677 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4678 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4679 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4680 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4681 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4682 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4683 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4684 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4685 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4686 case RTE_FLOW_ACTION_TYPE_SET_META: 4687 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4688 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4689 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4690 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4691 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4692 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4693 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4694 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4695 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4696 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4697 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4698 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4699 return true; 4700 case RTE_FLOW_ACTION_TYPE_FLAG: 4701 case RTE_FLOW_ACTION_TYPE_MARK: 4702 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4703 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4704 return true; 4705 else 4706 return false; 4707 default: 4708 return false; 4709 } 4710 } 4711 4712 /** 4713 * Check meter action from the action list. 4714 * 4715 * @param dev 4716 * Pointer to Ethernet device. 4717 * @param[in] actions 4718 * Pointer to the list of actions. 4719 * @param[out] has_mtr 4720 * Pointer to the meter exist flag. 4721 * @param[out] has_modify 4722 * Pointer to the flag showing there's packet change action. 4723 * @param[out] meter_id 4724 * Pointer to the meter id. 4725 * 4726 * @return 4727 * Total number of actions. 4728 */ 4729 static int 4730 flow_check_meter_action(struct rte_eth_dev *dev, 4731 const struct rte_flow_action actions[], 4732 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4733 { 4734 const struct rte_flow_action_meter *mtr = NULL; 4735 int actions_n = 0; 4736 4737 MLX5_ASSERT(has_mtr); 4738 *has_mtr = false; 4739 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4740 switch (actions->type) { 4741 case RTE_FLOW_ACTION_TYPE_METER: 4742 mtr = actions->conf; 4743 *meter_id = mtr->mtr_id; 4744 *has_mtr = true; 4745 break; 4746 default: 4747 break; 4748 } 4749 if (!*has_mtr) 4750 *has_modify |= flow_check_modify_action_type(dev, 4751 actions->type); 4752 actions_n++; 4753 } 4754 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4755 return actions_n + 1; 4756 } 4757 4758 /** 4759 * Check if the flow should be split due to hairpin. 4760 * The reason for the split is that in current HW we can't 4761 * support encap and push-vlan on Rx, so if a flow contains 4762 * these actions we move it to Tx. 4763 * 4764 * @param dev 4765 * Pointer to Ethernet device. 4766 * @param[in] attr 4767 * Flow rule attributes. 4768 * @param[in] actions 4769 * Associated actions (list terminated by the END action). 4770 * 4771 * @return 4772 * > 0 the number of actions and the flow should be split, 4773 * 0 when no split required. 4774 */ 4775 static int 4776 flow_check_hairpin_split(struct rte_eth_dev *dev, 4777 const struct rte_flow_attr *attr, 4778 const struct rte_flow_action actions[]) 4779 { 4780 int queue_action = 0; 4781 int action_n = 0; 4782 int split = 0; 4783 int push_vlan = 0; 4784 const struct rte_flow_action_queue *queue; 4785 const struct rte_flow_action_rss *rss; 4786 const struct rte_flow_action_raw_encap *raw_encap; 4787 const struct rte_eth_hairpin_conf *conf; 4788 4789 if (!attr->ingress) 4790 return 0; 4791 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4792 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4793 push_vlan = 1; 4794 switch (actions->type) { 4795 case RTE_FLOW_ACTION_TYPE_QUEUE: 4796 queue = actions->conf; 4797 if (queue == NULL) 4798 return 0; 4799 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4800 if (conf == NULL || conf->tx_explicit != 0) 4801 return 0; 4802 queue_action = 1; 4803 action_n++; 4804 break; 4805 case RTE_FLOW_ACTION_TYPE_RSS: 4806 rss = actions->conf; 4807 if (rss == NULL || rss->queue_num == 0) 4808 return 0; 4809 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4810 if (conf == NULL || conf->tx_explicit != 0) 4811 return 0; 4812 queue_action = 1; 4813 action_n++; 4814 break; 4815 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4816 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4817 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4818 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4819 split++; 4820 action_n++; 4821 break; 4822 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4823 if (push_vlan) 4824 split++; 4825 action_n++; 4826 break; 4827 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4828 raw_encap = actions->conf; 4829 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4830 split++; 4831 action_n++; 4832 break; 4833 default: 4834 action_n++; 4835 break; 4836 } 4837 } 4838 if (split && queue_action) 4839 return action_n; 4840 return 0; 4841 } 4842 4843 /* Declare flow create/destroy prototype in advance. */ 4844 static uint32_t 4845 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4846 const struct rte_flow_attr *attr, 4847 const struct rte_flow_item items[], 4848 const struct rte_flow_action actions[], 4849 bool external, struct rte_flow_error *error); 4850 4851 static void 4852 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4853 uint32_t flow_idx); 4854 4855 int 4856 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4857 struct mlx5_list_entry *entry, void *cb_ctx) 4858 { 4859 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4860 struct mlx5_flow_mreg_copy_resource *mcp_res = 4861 container_of(entry, typeof(*mcp_res), hlist_ent); 4862 4863 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4864 } 4865 4866 struct mlx5_list_entry * 4867 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4868 { 4869 struct rte_eth_dev *dev = tool_ctx; 4870 struct mlx5_priv *priv = dev->data->dev_private; 4871 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4872 struct mlx5_flow_mreg_copy_resource *mcp_res; 4873 struct rte_flow_error *error = ctx->error; 4874 uint32_t idx = 0; 4875 int ret; 4876 uint32_t mark_id = *(uint32_t *)(ctx->data); 4877 struct rte_flow_attr attr = { 4878 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4879 .ingress = 1, 4880 }; 4881 struct mlx5_rte_flow_item_tag tag_spec = { 4882 .data = mark_id, 4883 }; 4884 struct rte_flow_item items[] = { 4885 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4886 }; 4887 struct rte_flow_action_mark ftag = { 4888 .id = mark_id, 4889 }; 4890 struct mlx5_flow_action_copy_mreg cp_mreg = { 4891 .dst = REG_B, 4892 .src = REG_NON, 4893 }; 4894 struct rte_flow_action_jump jump = { 4895 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4896 }; 4897 struct rte_flow_action actions[] = { 4898 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4899 }; 4900 4901 /* Fill the register fields in the flow. */ 4902 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4903 if (ret < 0) 4904 return NULL; 4905 tag_spec.id = ret; 4906 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4907 if (ret < 0) 4908 return NULL; 4909 cp_mreg.src = ret; 4910 /* Provide the full width of FLAG specific value. */ 4911 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4912 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4913 /* Build a new flow. */ 4914 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4915 items[0] = (struct rte_flow_item){ 4916 .type = (enum rte_flow_item_type) 4917 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4918 .spec = &tag_spec, 4919 }; 4920 items[1] = (struct rte_flow_item){ 4921 .type = RTE_FLOW_ITEM_TYPE_END, 4922 }; 4923 actions[0] = (struct rte_flow_action){ 4924 .type = (enum rte_flow_action_type) 4925 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4926 .conf = &ftag, 4927 }; 4928 actions[1] = (struct rte_flow_action){ 4929 .type = (enum rte_flow_action_type) 4930 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4931 .conf = &cp_mreg, 4932 }; 4933 actions[2] = (struct rte_flow_action){ 4934 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4935 .conf = &jump, 4936 }; 4937 actions[3] = (struct rte_flow_action){ 4938 .type = RTE_FLOW_ACTION_TYPE_END, 4939 }; 4940 } else { 4941 /* Default rule, wildcard match. */ 4942 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4943 items[0] = (struct rte_flow_item){ 4944 .type = RTE_FLOW_ITEM_TYPE_END, 4945 }; 4946 actions[0] = (struct rte_flow_action){ 4947 .type = (enum rte_flow_action_type) 4948 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4949 .conf = &cp_mreg, 4950 }; 4951 actions[1] = (struct rte_flow_action){ 4952 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4953 .conf = &jump, 4954 }; 4955 actions[2] = (struct rte_flow_action){ 4956 .type = RTE_FLOW_ACTION_TYPE_END, 4957 }; 4958 } 4959 /* Build a new entry. */ 4960 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4961 if (!mcp_res) { 4962 rte_errno = ENOMEM; 4963 return NULL; 4964 } 4965 mcp_res->idx = idx; 4966 mcp_res->mark_id = mark_id; 4967 /* 4968 * The copy Flows are not included in any list. There 4969 * ones are referenced from other Flows and can not 4970 * be applied, removed, deleted in arbitrary order 4971 * by list traversing. 4972 */ 4973 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4974 &attr, items, actions, false, error); 4975 if (!mcp_res->rix_flow) { 4976 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4977 return NULL; 4978 } 4979 return &mcp_res->hlist_ent; 4980 } 4981 4982 struct mlx5_list_entry * 4983 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4984 void *cb_ctx __rte_unused) 4985 { 4986 struct rte_eth_dev *dev = tool_ctx; 4987 struct mlx5_priv *priv = dev->data->dev_private; 4988 struct mlx5_flow_mreg_copy_resource *mcp_res; 4989 uint32_t idx = 0; 4990 4991 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4992 if (!mcp_res) { 4993 rte_errno = ENOMEM; 4994 return NULL; 4995 } 4996 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4997 mcp_res->idx = idx; 4998 return &mcp_res->hlist_ent; 4999 } 5000 5001 void 5002 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5003 { 5004 struct mlx5_flow_mreg_copy_resource *mcp_res = 5005 container_of(entry, typeof(*mcp_res), hlist_ent); 5006 struct rte_eth_dev *dev = tool_ctx; 5007 struct mlx5_priv *priv = dev->data->dev_private; 5008 5009 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5010 } 5011 5012 /** 5013 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5014 * 5015 * As mark_id is unique, if there's already a registered flow for the mark_id, 5016 * return by increasing the reference counter of the resource. Otherwise, create 5017 * the resource (mcp_res) and flow. 5018 * 5019 * Flow looks like, 5020 * - If ingress port is ANY and reg_c[1] is mark_id, 5021 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5022 * 5023 * For default flow (zero mark_id), flow is like, 5024 * - If ingress port is ANY, 5025 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5026 * 5027 * @param dev 5028 * Pointer to Ethernet device. 5029 * @param mark_id 5030 * ID of MARK action, zero means default flow for META. 5031 * @param[out] error 5032 * Perform verbose error reporting if not NULL. 5033 * 5034 * @return 5035 * Associated resource on success, NULL otherwise and rte_errno is set. 5036 */ 5037 static struct mlx5_flow_mreg_copy_resource * 5038 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5039 struct rte_flow_error *error) 5040 { 5041 struct mlx5_priv *priv = dev->data->dev_private; 5042 struct mlx5_list_entry *entry; 5043 struct mlx5_flow_cb_ctx ctx = { 5044 .dev = dev, 5045 .error = error, 5046 .data = &mark_id, 5047 }; 5048 5049 /* Check if already registered. */ 5050 MLX5_ASSERT(priv->mreg_cp_tbl); 5051 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5052 if (!entry) 5053 return NULL; 5054 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5055 hlist_ent); 5056 } 5057 5058 void 5059 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5060 { 5061 struct mlx5_flow_mreg_copy_resource *mcp_res = 5062 container_of(entry, typeof(*mcp_res), hlist_ent); 5063 struct rte_eth_dev *dev = tool_ctx; 5064 struct mlx5_priv *priv = dev->data->dev_private; 5065 5066 MLX5_ASSERT(mcp_res->rix_flow); 5067 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5068 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5069 } 5070 5071 /** 5072 * Release flow in RX_CP_TBL. 5073 * 5074 * @param dev 5075 * Pointer to Ethernet device. 5076 * @flow 5077 * Parent flow for wich copying is provided. 5078 */ 5079 static void 5080 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5081 struct rte_flow *flow) 5082 { 5083 struct mlx5_flow_mreg_copy_resource *mcp_res; 5084 struct mlx5_priv *priv = dev->data->dev_private; 5085 5086 if (!flow->rix_mreg_copy) 5087 return; 5088 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5089 flow->rix_mreg_copy); 5090 if (!mcp_res || !priv->mreg_cp_tbl) 5091 return; 5092 MLX5_ASSERT(mcp_res->rix_flow); 5093 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5094 flow->rix_mreg_copy = 0; 5095 } 5096 5097 /** 5098 * Remove the default copy action from RX_CP_TBL. 5099 * 5100 * This functions is called in the mlx5_dev_start(). No thread safe 5101 * is guaranteed. 5102 * 5103 * @param dev 5104 * Pointer to Ethernet device. 5105 */ 5106 static void 5107 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5108 { 5109 struct mlx5_list_entry *entry; 5110 struct mlx5_priv *priv = dev->data->dev_private; 5111 struct mlx5_flow_cb_ctx ctx; 5112 uint32_t mark_id; 5113 5114 /* Check if default flow is registered. */ 5115 if (!priv->mreg_cp_tbl) 5116 return; 5117 mark_id = MLX5_DEFAULT_COPY_ID; 5118 ctx.data = &mark_id; 5119 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5120 if (!entry) 5121 return; 5122 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5123 } 5124 5125 /** 5126 * Add the default copy action in RX_CP_TBL. 5127 * 5128 * This functions is called in the mlx5_dev_start(). No thread safe 5129 * is guaranteed. 5130 * 5131 * @param dev 5132 * Pointer to Ethernet device. 5133 * @param[out] error 5134 * Perform verbose error reporting if not NULL. 5135 * 5136 * @return 5137 * 0 for success, negative value otherwise and rte_errno is set. 5138 */ 5139 static int 5140 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5141 struct rte_flow_error *error) 5142 { 5143 struct mlx5_priv *priv = dev->data->dev_private; 5144 struct mlx5_flow_mreg_copy_resource *mcp_res; 5145 struct mlx5_flow_cb_ctx ctx; 5146 uint32_t mark_id; 5147 5148 /* Check whether extensive metadata feature is engaged. */ 5149 if (!priv->sh->config.dv_flow_en || 5150 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5151 !mlx5_flow_ext_mreg_supported(dev) || 5152 !priv->sh->dv_regc0_mask) 5153 return 0; 5154 /* 5155 * Add default mreg copy flow may be called multiple time, but 5156 * only be called once in stop. Avoid register it twice. 5157 */ 5158 mark_id = MLX5_DEFAULT_COPY_ID; 5159 ctx.data = &mark_id; 5160 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5161 return 0; 5162 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5163 if (!mcp_res) 5164 return -rte_errno; 5165 return 0; 5166 } 5167 5168 /** 5169 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5170 * 5171 * All the flow having Q/RSS action should be split by 5172 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5173 * performs the following, 5174 * - CQE->flow_tag := reg_c[1] (MARK) 5175 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5176 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5177 * but there should be a flow per each MARK ID set by MARK action. 5178 * 5179 * For the aforementioned reason, if there's a MARK action in flow's action 5180 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5181 * the MARK ID to CQE's flow_tag like, 5182 * - If reg_c[1] is mark_id, 5183 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5184 * 5185 * For SET_META action which stores value in reg_c[0], as the destination is 5186 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5187 * MARK ID means the default flow. The default flow looks like, 5188 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5189 * 5190 * @param dev 5191 * Pointer to Ethernet device. 5192 * @param flow 5193 * Pointer to flow structure. 5194 * @param[in] actions 5195 * Pointer to the list of actions. 5196 * @param[out] error 5197 * Perform verbose error reporting if not NULL. 5198 * 5199 * @return 5200 * 0 on success, negative value otherwise and rte_errno is set. 5201 */ 5202 static int 5203 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5204 struct rte_flow *flow, 5205 const struct rte_flow_action *actions, 5206 struct rte_flow_error *error) 5207 { 5208 struct mlx5_priv *priv = dev->data->dev_private; 5209 struct mlx5_sh_config *config = &priv->sh->config; 5210 struct mlx5_flow_mreg_copy_resource *mcp_res; 5211 const struct rte_flow_action_mark *mark; 5212 5213 /* Check whether extensive metadata feature is engaged. */ 5214 if (!config->dv_flow_en || 5215 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5216 !mlx5_flow_ext_mreg_supported(dev) || 5217 !priv->sh->dv_regc0_mask) 5218 return 0; 5219 /* Find MARK action. */ 5220 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5221 switch (actions->type) { 5222 case RTE_FLOW_ACTION_TYPE_FLAG: 5223 mcp_res = flow_mreg_add_copy_action 5224 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5225 if (!mcp_res) 5226 return -rte_errno; 5227 flow->rix_mreg_copy = mcp_res->idx; 5228 return 0; 5229 case RTE_FLOW_ACTION_TYPE_MARK: 5230 mark = (const struct rte_flow_action_mark *) 5231 actions->conf; 5232 mcp_res = 5233 flow_mreg_add_copy_action(dev, mark->id, error); 5234 if (!mcp_res) 5235 return -rte_errno; 5236 flow->rix_mreg_copy = mcp_res->idx; 5237 return 0; 5238 default: 5239 break; 5240 } 5241 } 5242 return 0; 5243 } 5244 5245 #define MLX5_MAX_SPLIT_ACTIONS 24 5246 #define MLX5_MAX_SPLIT_ITEMS 24 5247 5248 /** 5249 * Split the hairpin flow. 5250 * Since HW can't support encap and push-vlan on Rx, we move these 5251 * actions to Tx. 5252 * If the count action is after the encap then we also 5253 * move the count action. in this case the count will also measure 5254 * the outer bytes. 5255 * 5256 * @param dev 5257 * Pointer to Ethernet device. 5258 * @param[in] actions 5259 * Associated actions (list terminated by the END action). 5260 * @param[out] actions_rx 5261 * Rx flow actions. 5262 * @param[out] actions_tx 5263 * Tx flow actions.. 5264 * @param[out] pattern_tx 5265 * The pattern items for the Tx flow. 5266 * @param[out] flow_id 5267 * The flow ID connected to this flow. 5268 * 5269 * @return 5270 * 0 on success. 5271 */ 5272 static int 5273 flow_hairpin_split(struct rte_eth_dev *dev, 5274 const struct rte_flow_action actions[], 5275 struct rte_flow_action actions_rx[], 5276 struct rte_flow_action actions_tx[], 5277 struct rte_flow_item pattern_tx[], 5278 uint32_t flow_id) 5279 { 5280 const struct rte_flow_action_raw_encap *raw_encap; 5281 const struct rte_flow_action_raw_decap *raw_decap; 5282 struct mlx5_rte_flow_action_set_tag *set_tag; 5283 struct rte_flow_action *tag_action; 5284 struct mlx5_rte_flow_item_tag *tag_item; 5285 struct rte_flow_item *item; 5286 char *addr; 5287 int push_vlan = 0; 5288 int encap = 0; 5289 5290 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5291 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5292 push_vlan = 1; 5293 switch (actions->type) { 5294 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5295 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5296 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5297 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5298 rte_memcpy(actions_tx, actions, 5299 sizeof(struct rte_flow_action)); 5300 actions_tx++; 5301 break; 5302 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5303 if (push_vlan) { 5304 rte_memcpy(actions_tx, actions, 5305 sizeof(struct rte_flow_action)); 5306 actions_tx++; 5307 } else { 5308 rte_memcpy(actions_rx, actions, 5309 sizeof(struct rte_flow_action)); 5310 actions_rx++; 5311 } 5312 break; 5313 case RTE_FLOW_ACTION_TYPE_COUNT: 5314 if (encap) { 5315 rte_memcpy(actions_tx, actions, 5316 sizeof(struct rte_flow_action)); 5317 actions_tx++; 5318 } else { 5319 rte_memcpy(actions_rx, actions, 5320 sizeof(struct rte_flow_action)); 5321 actions_rx++; 5322 } 5323 break; 5324 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5325 raw_encap = actions->conf; 5326 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5327 memcpy(actions_tx, actions, 5328 sizeof(struct rte_flow_action)); 5329 actions_tx++; 5330 encap = 1; 5331 } else { 5332 rte_memcpy(actions_rx, actions, 5333 sizeof(struct rte_flow_action)); 5334 actions_rx++; 5335 } 5336 break; 5337 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5338 raw_decap = actions->conf; 5339 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5340 memcpy(actions_tx, actions, 5341 sizeof(struct rte_flow_action)); 5342 actions_tx++; 5343 } else { 5344 rte_memcpy(actions_rx, actions, 5345 sizeof(struct rte_flow_action)); 5346 actions_rx++; 5347 } 5348 break; 5349 default: 5350 rte_memcpy(actions_rx, actions, 5351 sizeof(struct rte_flow_action)); 5352 actions_rx++; 5353 break; 5354 } 5355 } 5356 /* Add set meta action and end action for the Rx flow. */ 5357 tag_action = actions_rx; 5358 tag_action->type = (enum rte_flow_action_type) 5359 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5360 actions_rx++; 5361 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5362 actions_rx++; 5363 set_tag = (void *)actions_rx; 5364 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5365 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5366 .data = flow_id, 5367 }; 5368 MLX5_ASSERT(set_tag->id > REG_NON); 5369 tag_action->conf = set_tag; 5370 /* Create Tx item list. */ 5371 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5372 addr = (void *)&pattern_tx[2]; 5373 item = pattern_tx; 5374 item->type = (enum rte_flow_item_type) 5375 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5376 tag_item = (void *)addr; 5377 tag_item->data = flow_id; 5378 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5379 MLX5_ASSERT(set_tag->id > REG_NON); 5380 item->spec = tag_item; 5381 addr += sizeof(struct mlx5_rte_flow_item_tag); 5382 tag_item = (void *)addr; 5383 tag_item->data = UINT32_MAX; 5384 tag_item->id = UINT16_MAX; 5385 item->mask = tag_item; 5386 item->last = NULL; 5387 item++; 5388 item->type = RTE_FLOW_ITEM_TYPE_END; 5389 return 0; 5390 } 5391 5392 /** 5393 * The last stage of splitting chain, just creates the subflow 5394 * without any modification. 5395 * 5396 * @param[in] dev 5397 * Pointer to Ethernet device. 5398 * @param[in] flow 5399 * Parent flow structure pointer. 5400 * @param[in, out] sub_flow 5401 * Pointer to return the created subflow, may be NULL. 5402 * @param[in] attr 5403 * Flow rule attributes. 5404 * @param[in] items 5405 * Pattern specification (list terminated by the END pattern item). 5406 * @param[in] actions 5407 * Associated actions (list terminated by the END action). 5408 * @param[in] flow_split_info 5409 * Pointer to flow split info structure. 5410 * @param[out] error 5411 * Perform verbose error reporting if not NULL. 5412 * @return 5413 * 0 on success, negative value otherwise 5414 */ 5415 static int 5416 flow_create_split_inner(struct rte_eth_dev *dev, 5417 struct rte_flow *flow, 5418 struct mlx5_flow **sub_flow, 5419 const struct rte_flow_attr *attr, 5420 const struct rte_flow_item items[], 5421 const struct rte_flow_action actions[], 5422 struct mlx5_flow_split_info *flow_split_info, 5423 struct rte_flow_error *error) 5424 { 5425 struct mlx5_flow *dev_flow; 5426 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5427 5428 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5429 flow_split_info->flow_idx, error); 5430 if (!dev_flow) 5431 return -rte_errno; 5432 dev_flow->flow = flow; 5433 dev_flow->external = flow_split_info->external; 5434 dev_flow->skip_scale = flow_split_info->skip_scale; 5435 /* Subflow object was created, we must include one in the list. */ 5436 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5437 dev_flow->handle, next); 5438 /* 5439 * If dev_flow is as one of the suffix flow, some actions in suffix 5440 * flow may need some user defined item layer flags, and pass the 5441 * Metadata rxq mark flag to suffix flow as well. 5442 */ 5443 if (flow_split_info->prefix_layers) 5444 dev_flow->handle->layers = flow_split_info->prefix_layers; 5445 if (flow_split_info->prefix_mark) { 5446 MLX5_ASSERT(wks); 5447 wks->mark = 1; 5448 } 5449 if (sub_flow) 5450 *sub_flow = dev_flow; 5451 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5452 dev_flow->dv.table_id = flow_split_info->table_id; 5453 #endif 5454 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5455 } 5456 5457 /** 5458 * Get the sub policy of a meter. 5459 * 5460 * @param[in] dev 5461 * Pointer to Ethernet device. 5462 * @param[in] flow 5463 * Parent flow structure pointer. 5464 * @param wks 5465 * Pointer to thread flow work space. 5466 * @param[in] attr 5467 * Flow rule attributes. 5468 * @param[in] items 5469 * Pattern specification (list terminated by the END pattern item). 5470 * @param[out] error 5471 * Perform verbose error reporting if not NULL. 5472 * 5473 * @return 5474 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5475 */ 5476 static struct mlx5_flow_meter_sub_policy * 5477 get_meter_sub_policy(struct rte_eth_dev *dev, 5478 struct rte_flow *flow, 5479 struct mlx5_flow_workspace *wks, 5480 const struct rte_flow_attr *attr, 5481 const struct rte_flow_item items[], 5482 struct rte_flow_error *error) 5483 { 5484 struct mlx5_flow_meter_policy *policy; 5485 struct mlx5_flow_meter_policy *final_policy; 5486 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5487 5488 policy = wks->policy; 5489 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5490 if (final_policy->is_rss || final_policy->is_queue) { 5491 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5492 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5493 uint32_t i; 5494 5495 /* 5496 * This is a tmp dev_flow, 5497 * no need to register any matcher for it in translate. 5498 */ 5499 wks->skip_matcher_reg = 1; 5500 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5501 struct mlx5_flow dev_flow = {0}; 5502 struct mlx5_flow_handle dev_handle = { {0} }; 5503 uint8_t fate = final_policy->act_cnt[i].fate_action; 5504 5505 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5506 const struct rte_flow_action_rss *rss_act = 5507 final_policy->act_cnt[i].rss->conf; 5508 struct rte_flow_action rss_actions[2] = { 5509 [0] = { 5510 .type = RTE_FLOW_ACTION_TYPE_RSS, 5511 .conf = rss_act, 5512 }, 5513 [1] = { 5514 .type = RTE_FLOW_ACTION_TYPE_END, 5515 .conf = NULL, 5516 } 5517 }; 5518 5519 dev_flow.handle = &dev_handle; 5520 dev_flow.ingress = attr->ingress; 5521 dev_flow.flow = flow; 5522 dev_flow.external = 0; 5523 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5524 dev_flow.dv.transfer = attr->transfer; 5525 #endif 5526 /** 5527 * Translate RSS action to get rss hash fields. 5528 */ 5529 if (flow_drv_translate(dev, &dev_flow, attr, 5530 items, rss_actions, error)) 5531 goto exit; 5532 rss_desc_v[i] = wks->rss_desc; 5533 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5534 rss_desc_v[i].hash_fields = 5535 dev_flow.hash_fields; 5536 rss_desc_v[i].queue_num = 5537 rss_desc_v[i].hash_fields ? 5538 rss_desc_v[i].queue_num : 1; 5539 rss_desc_v[i].tunnel = 5540 !!(dev_flow.handle->layers & 5541 MLX5_FLOW_LAYER_TUNNEL); 5542 /* Use the RSS queues in the containers. */ 5543 rss_desc_v[i].queue = 5544 (uint16_t *)(uintptr_t)rss_act->queue; 5545 rss_desc[i] = &rss_desc_v[i]; 5546 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5547 /* This is queue action. */ 5548 rss_desc_v[i] = wks->rss_desc; 5549 rss_desc_v[i].key_len = 0; 5550 rss_desc_v[i].hash_fields = 0; 5551 rss_desc_v[i].queue = 5552 &final_policy->act_cnt[i].queue; 5553 rss_desc_v[i].queue_num = 1; 5554 rss_desc[i] = &rss_desc_v[i]; 5555 } else { 5556 rss_desc[i] = NULL; 5557 } 5558 } 5559 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5560 flow, policy, rss_desc); 5561 } else { 5562 enum mlx5_meter_domain mtr_domain = 5563 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5564 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5565 MLX5_MTR_DOMAIN_INGRESS); 5566 sub_policy = policy->sub_policys[mtr_domain][0]; 5567 } 5568 if (!sub_policy) 5569 rte_flow_error_set(error, EINVAL, 5570 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5571 "Failed to get meter sub-policy."); 5572 exit: 5573 return sub_policy; 5574 } 5575 5576 /** 5577 * Split the meter flow. 5578 * 5579 * As meter flow will split to three sub flow, other than meter 5580 * action, the other actions make sense to only meter accepts 5581 * the packet. If it need to be dropped, no other additional 5582 * actions should be take. 5583 * 5584 * One kind of special action which decapsulates the L3 tunnel 5585 * header will be in the prefix sub flow, as not to take the 5586 * L3 tunnel header into account. 5587 * 5588 * @param[in] dev 5589 * Pointer to Ethernet device. 5590 * @param[in] flow 5591 * Parent flow structure pointer. 5592 * @param wks 5593 * Pointer to thread flow work space. 5594 * @param[in] attr 5595 * Flow rule attributes. 5596 * @param[in] items 5597 * Pattern specification (list terminated by the END pattern item). 5598 * @param[out] sfx_items 5599 * Suffix flow match items (list terminated by the END pattern item). 5600 * @param[in] actions 5601 * Associated actions (list terminated by the END action). 5602 * @param[out] actions_sfx 5603 * Suffix flow actions. 5604 * @param[out] actions_pre 5605 * Prefix flow actions. 5606 * @param[out] mtr_flow_id 5607 * Pointer to meter flow id. 5608 * @param[out] error 5609 * Perform verbose error reporting if not NULL. 5610 * 5611 * @return 5612 * 0 on success, a negative errno value otherwise and rte_errno is set. 5613 */ 5614 static int 5615 flow_meter_split_prep(struct rte_eth_dev *dev, 5616 struct rte_flow *flow, 5617 struct mlx5_flow_workspace *wks, 5618 const struct rte_flow_attr *attr, 5619 const struct rte_flow_item items[], 5620 struct rte_flow_item sfx_items[], 5621 const struct rte_flow_action actions[], 5622 struct rte_flow_action actions_sfx[], 5623 struct rte_flow_action actions_pre[], 5624 uint32_t *mtr_flow_id, 5625 struct rte_flow_error *error) 5626 { 5627 struct mlx5_priv *priv = dev->data->dev_private; 5628 struct mlx5_flow_meter_info *fm = wks->fm; 5629 struct rte_flow_action *tag_action = NULL; 5630 struct rte_flow_item *tag_item; 5631 struct mlx5_rte_flow_action_set_tag *set_tag; 5632 const struct rte_flow_action_raw_encap *raw_encap; 5633 const struct rte_flow_action_raw_decap *raw_decap; 5634 struct mlx5_rte_flow_item_tag *tag_item_spec; 5635 struct mlx5_rte_flow_item_tag *tag_item_mask; 5636 uint32_t tag_id = 0; 5637 struct rte_flow_item *vlan_item_dst = NULL; 5638 const struct rte_flow_item *vlan_item_src = NULL; 5639 const struct rte_flow_item *orig_items = items; 5640 struct rte_flow_action *hw_mtr_action; 5641 struct rte_flow_action *action_pre_head = NULL; 5642 uint16_t flow_src_port = priv->representor_id; 5643 bool mtr_first; 5644 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5645 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5646 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5647 uint32_t flow_id = 0; 5648 uint32_t flow_id_reversed = 0; 5649 uint8_t flow_id_bits = 0; 5650 bool after_meter = false; 5651 int shift; 5652 5653 /* Prepare the suffix subflow items. */ 5654 tag_item = sfx_items++; 5655 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5656 int item_type = items->type; 5657 5658 switch (item_type) { 5659 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5660 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5661 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5662 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5663 return -rte_errno; 5664 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5665 flow_src_port != priv->representor_id) { 5666 if (flow_drv_mtr_hierarchy_rule_create(dev, 5667 flow, fm, 5668 flow_src_port, 5669 items, 5670 error)) 5671 return -rte_errno; 5672 } 5673 memcpy(sfx_items, items, sizeof(*sfx_items)); 5674 sfx_items++; 5675 break; 5676 case RTE_FLOW_ITEM_TYPE_VLAN: 5677 /* Determine if copy vlan item below. */ 5678 vlan_item_src = items; 5679 vlan_item_dst = sfx_items++; 5680 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5681 break; 5682 default: 5683 break; 5684 } 5685 } 5686 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5687 sfx_items++; 5688 mtr_first = priv->sh->meter_aso_en && 5689 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5690 /* For ASO meter, meter must be before tag in TX direction. */ 5691 if (mtr_first) { 5692 action_pre_head = actions_pre++; 5693 /* Leave space for tag action. */ 5694 tag_action = actions_pre++; 5695 } 5696 /* Prepare the actions for prefix and suffix flow. */ 5697 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5698 struct rte_flow_action *action_cur = NULL; 5699 5700 switch (actions->type) { 5701 case RTE_FLOW_ACTION_TYPE_METER: 5702 if (mtr_first) { 5703 action_cur = action_pre_head; 5704 } else { 5705 /* Leave space for tag action. */ 5706 tag_action = actions_pre++; 5707 action_cur = actions_pre++; 5708 } 5709 after_meter = true; 5710 break; 5711 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5712 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5713 action_cur = actions_pre++; 5714 break; 5715 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5716 raw_encap = actions->conf; 5717 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5718 action_cur = actions_pre++; 5719 break; 5720 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5721 raw_decap = actions->conf; 5722 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5723 action_cur = actions_pre++; 5724 break; 5725 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5726 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5727 if (vlan_item_dst && vlan_item_src) { 5728 memcpy(vlan_item_dst, vlan_item_src, 5729 sizeof(*vlan_item_dst)); 5730 /* 5731 * Convert to internal match item, it is used 5732 * for vlan push and set vid. 5733 */ 5734 vlan_item_dst->type = (enum rte_flow_item_type) 5735 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5736 } 5737 break; 5738 case RTE_FLOW_ACTION_TYPE_COUNT: 5739 if (fm->def_policy) 5740 action_cur = after_meter ? 5741 actions_sfx++ : actions_pre++; 5742 break; 5743 default: 5744 break; 5745 } 5746 if (!action_cur) 5747 action_cur = (fm->def_policy) ? 5748 actions_sfx++ : actions_pre++; 5749 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5750 } 5751 /* Add end action to the actions. */ 5752 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5753 if (priv->sh->meter_aso_en) { 5754 /** 5755 * For ASO meter, need to add an extra jump action explicitly, 5756 * to jump from meter to policer table. 5757 */ 5758 struct mlx5_flow_meter_sub_policy *sub_policy; 5759 struct mlx5_flow_tbl_data_entry *tbl_data; 5760 5761 if (!fm->def_policy) { 5762 sub_policy = get_meter_sub_policy(dev, flow, wks, 5763 attr, orig_items, 5764 error); 5765 if (!sub_policy) 5766 return -rte_errno; 5767 } else { 5768 enum mlx5_meter_domain mtr_domain = 5769 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5770 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5771 MLX5_MTR_DOMAIN_INGRESS); 5772 5773 sub_policy = 5774 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5775 } 5776 tbl_data = container_of(sub_policy->tbl_rsc, 5777 struct mlx5_flow_tbl_data_entry, tbl); 5778 hw_mtr_action = actions_pre++; 5779 hw_mtr_action->type = (enum rte_flow_action_type) 5780 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5781 hw_mtr_action->conf = tbl_data->jump.action; 5782 } 5783 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5784 actions_pre++; 5785 if (!tag_action) 5786 return rte_flow_error_set(error, ENOMEM, 5787 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5788 NULL, "No tag action space."); 5789 if (!mtr_flow_id) { 5790 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5791 goto exit; 5792 } 5793 /* Only default-policy Meter creates mtr flow id. */ 5794 if (fm->def_policy) { 5795 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5796 if (!tag_id) 5797 return rte_flow_error_set(error, ENOMEM, 5798 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5799 "Failed to allocate meter flow id."); 5800 flow_id = tag_id - 1; 5801 flow_id_bits = (!flow_id) ? 1 : 5802 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5803 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5804 mtr_reg_bits) { 5805 mlx5_ipool_free(fm->flow_ipool, tag_id); 5806 return rte_flow_error_set(error, EINVAL, 5807 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5808 "Meter flow id exceeds max limit."); 5809 } 5810 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5811 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5812 } 5813 /* Build tag actions and items for meter_id/meter flow_id. */ 5814 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5815 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5816 tag_item_mask = tag_item_spec + 1; 5817 /* Both flow_id and meter_id share the same register. */ 5818 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5819 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5820 0, error), 5821 .offset = mtr_id_offset, 5822 .length = mtr_reg_bits, 5823 .data = flow->meter, 5824 }; 5825 /* 5826 * The color Reg bits used by flow_id are growing from 5827 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5828 */ 5829 for (shift = 0; shift < flow_id_bits; shift++) 5830 flow_id_reversed = (flow_id_reversed << 1) | 5831 ((flow_id >> shift) & 0x1); 5832 set_tag->data |= 5833 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5834 tag_item_spec->id = set_tag->id; 5835 tag_item_spec->data = set_tag->data << mtr_id_offset; 5836 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5837 tag_action->type = (enum rte_flow_action_type) 5838 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5839 tag_action->conf = set_tag; 5840 tag_item->type = (enum rte_flow_item_type) 5841 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5842 tag_item->spec = tag_item_spec; 5843 tag_item->last = NULL; 5844 tag_item->mask = tag_item_mask; 5845 exit: 5846 if (mtr_flow_id) 5847 *mtr_flow_id = tag_id; 5848 return 0; 5849 } 5850 5851 /** 5852 * Split action list having QUEUE/RSS for metadata register copy. 5853 * 5854 * Once Q/RSS action is detected in user's action list, the flow action 5855 * should be split in order to copy metadata registers, which will happen in 5856 * RX_CP_TBL like, 5857 * - CQE->flow_tag := reg_c[1] (MARK) 5858 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5859 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5860 * This is because the last action of each flow must be a terminal action 5861 * (QUEUE, RSS or DROP). 5862 * 5863 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5864 * stored and kept in the mlx5_flow structure per each sub_flow. 5865 * 5866 * The Q/RSS action is replaced with, 5867 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5868 * And the following JUMP action is added at the end, 5869 * - JUMP, to RX_CP_TBL. 5870 * 5871 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5872 * flow_create_split_metadata() routine. The flow will look like, 5873 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5874 * 5875 * @param dev 5876 * Pointer to Ethernet device. 5877 * @param[out] split_actions 5878 * Pointer to store split actions to jump to CP_TBL. 5879 * @param[in] actions 5880 * Pointer to the list of original flow actions. 5881 * @param[in] qrss 5882 * Pointer to the Q/RSS action. 5883 * @param[in] actions_n 5884 * Number of original actions. 5885 * @param[in] mtr_sfx 5886 * Check if it is in meter suffix table. 5887 * @param[out] error 5888 * Perform verbose error reporting if not NULL. 5889 * 5890 * @return 5891 * non-zero unique flow_id on success, otherwise 0 and 5892 * error/rte_error are set. 5893 */ 5894 static uint32_t 5895 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5896 struct rte_flow_action *split_actions, 5897 const struct rte_flow_action *actions, 5898 const struct rte_flow_action *qrss, 5899 int actions_n, int mtr_sfx, 5900 struct rte_flow_error *error) 5901 { 5902 struct mlx5_priv *priv = dev->data->dev_private; 5903 struct mlx5_rte_flow_action_set_tag *set_tag; 5904 struct rte_flow_action_jump *jump; 5905 const int qrss_idx = qrss - actions; 5906 uint32_t flow_id = 0; 5907 int ret = 0; 5908 5909 /* 5910 * Given actions will be split 5911 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5912 * - Add jump to mreg CP_TBL. 5913 * As a result, there will be one more action. 5914 */ 5915 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5916 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5917 ++actions_n; 5918 set_tag = (void *)(split_actions + actions_n); 5919 /* 5920 * If we are not the meter suffix flow, add the tag action. 5921 * Since meter suffix flow already has the tag added. 5922 */ 5923 if (!mtr_sfx) { 5924 /* 5925 * Allocate the new subflow ID. This one is unique within 5926 * device and not shared with representors. Otherwise, 5927 * we would have to resolve multi-thread access synch 5928 * issue. Each flow on the shared device is appended 5929 * with source vport identifier, so the resulting 5930 * flows will be unique in the shared (by master and 5931 * representors) domain even if they have coinciding 5932 * IDs. 5933 */ 5934 mlx5_ipool_malloc(priv->sh->ipool 5935 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5936 if (!flow_id) 5937 return rte_flow_error_set(error, ENOMEM, 5938 RTE_FLOW_ERROR_TYPE_ACTION, 5939 NULL, "can't allocate id " 5940 "for split Q/RSS subflow"); 5941 /* Internal SET_TAG action to set flow ID. */ 5942 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5943 .data = flow_id, 5944 }; 5945 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5946 if (ret < 0) 5947 return ret; 5948 set_tag->id = ret; 5949 /* Construct new actions array. */ 5950 /* Replace QUEUE/RSS action. */ 5951 split_actions[qrss_idx] = (struct rte_flow_action){ 5952 .type = (enum rte_flow_action_type) 5953 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5954 .conf = set_tag, 5955 }; 5956 } else { 5957 /* 5958 * If we are the suffix flow of meter, tag already exist. 5959 * Set the QUEUE/RSS action to void. 5960 */ 5961 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5962 } 5963 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5964 jump = (void *)(set_tag + 1); 5965 *jump = (struct rte_flow_action_jump){ 5966 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5967 }; 5968 split_actions[actions_n - 2] = (struct rte_flow_action){ 5969 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5970 .conf = jump, 5971 }; 5972 split_actions[actions_n - 1] = (struct rte_flow_action){ 5973 .type = RTE_FLOW_ACTION_TYPE_END, 5974 }; 5975 return flow_id; 5976 } 5977 5978 /** 5979 * Extend the given action list for Tx metadata copy. 5980 * 5981 * Copy the given action list to the ext_actions and add flow metadata register 5982 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5983 * 5984 * @param[out] ext_actions 5985 * Pointer to the extended action list. 5986 * @param[in] actions 5987 * Pointer to the list of actions. 5988 * @param[in] actions_n 5989 * Number of actions in the list. 5990 * @param[out] error 5991 * Perform verbose error reporting if not NULL. 5992 * @param[in] encap_idx 5993 * The encap action index. 5994 * 5995 * @return 5996 * 0 on success, negative value otherwise 5997 */ 5998 static int 5999 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6000 struct rte_flow_action *ext_actions, 6001 const struct rte_flow_action *actions, 6002 int actions_n, struct rte_flow_error *error, 6003 int encap_idx) 6004 { 6005 struct mlx5_flow_action_copy_mreg *cp_mreg = 6006 (struct mlx5_flow_action_copy_mreg *) 6007 (ext_actions + actions_n + 1); 6008 int ret; 6009 6010 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6011 if (ret < 0) 6012 return ret; 6013 cp_mreg->dst = ret; 6014 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6015 if (ret < 0) 6016 return ret; 6017 cp_mreg->src = ret; 6018 if (encap_idx != 0) 6019 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6020 if (encap_idx == actions_n - 1) { 6021 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6022 .type = (enum rte_flow_action_type) 6023 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6024 .conf = cp_mreg, 6025 }; 6026 ext_actions[actions_n] = (struct rte_flow_action){ 6027 .type = RTE_FLOW_ACTION_TYPE_END, 6028 }; 6029 } else { 6030 ext_actions[encap_idx] = (struct rte_flow_action){ 6031 .type = (enum rte_flow_action_type) 6032 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6033 .conf = cp_mreg, 6034 }; 6035 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6036 sizeof(*ext_actions) * (actions_n - encap_idx)); 6037 } 6038 return 0; 6039 } 6040 6041 /** 6042 * Check the match action from the action list. 6043 * 6044 * @param[in] actions 6045 * Pointer to the list of actions. 6046 * @param[in] attr 6047 * Flow rule attributes. 6048 * @param[in] action 6049 * The action to be check if exist. 6050 * @param[out] match_action_pos 6051 * Pointer to the position of the matched action if exists, otherwise is -1. 6052 * @param[out] qrss_action_pos 6053 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6054 * @param[out] modify_after_mirror 6055 * Pointer to the flag of modify action after FDB mirroring. 6056 * 6057 * @return 6058 * > 0 the total number of actions. 6059 * 0 if not found match action in action list. 6060 */ 6061 static int 6062 flow_check_match_action(const struct rte_flow_action actions[], 6063 const struct rte_flow_attr *attr, 6064 enum rte_flow_action_type action, 6065 int *match_action_pos, int *qrss_action_pos, 6066 int *modify_after_mirror) 6067 { 6068 const struct rte_flow_action_sample *sample; 6069 const struct rte_flow_action_raw_decap *decap; 6070 int actions_n = 0; 6071 uint32_t ratio = 0; 6072 int sub_type = 0; 6073 int flag = 0; 6074 int fdb_mirror = 0; 6075 6076 *match_action_pos = -1; 6077 *qrss_action_pos = -1; 6078 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6079 if (actions->type == action) { 6080 flag = 1; 6081 *match_action_pos = actions_n; 6082 } 6083 switch (actions->type) { 6084 case RTE_FLOW_ACTION_TYPE_QUEUE: 6085 case RTE_FLOW_ACTION_TYPE_RSS: 6086 *qrss_action_pos = actions_n; 6087 break; 6088 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6089 sample = actions->conf; 6090 ratio = sample->ratio; 6091 sub_type = ((const struct rte_flow_action *) 6092 (sample->actions))->type; 6093 if (ratio == 1 && attr->transfer && 6094 sub_type != RTE_FLOW_ACTION_TYPE_END) 6095 fdb_mirror = 1; 6096 break; 6097 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6098 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6099 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6100 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6101 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6102 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6103 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6104 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6105 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6106 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6107 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6108 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6109 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6110 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6111 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6112 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6113 case RTE_FLOW_ACTION_TYPE_FLAG: 6114 case RTE_FLOW_ACTION_TYPE_MARK: 6115 case RTE_FLOW_ACTION_TYPE_SET_META: 6116 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6117 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6118 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6119 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6120 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6121 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6122 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6123 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6124 case RTE_FLOW_ACTION_TYPE_METER: 6125 if (fdb_mirror) 6126 *modify_after_mirror = 1; 6127 break; 6128 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6129 decap = actions->conf; 6130 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 6131 ; 6132 actions_n++; 6133 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6134 const struct rte_flow_action_raw_encap *encap = 6135 actions->conf; 6136 if (decap->size <= 6137 MLX5_ENCAPSULATION_DECISION_SIZE && 6138 encap->size > 6139 MLX5_ENCAPSULATION_DECISION_SIZE) 6140 /* L3 encap. */ 6141 break; 6142 } 6143 if (fdb_mirror) 6144 *modify_after_mirror = 1; 6145 break; 6146 default: 6147 break; 6148 } 6149 actions_n++; 6150 } 6151 if (flag && fdb_mirror && !*modify_after_mirror) { 6152 /* FDB mirroring uses the destination array to implement 6153 * instead of FLOW_SAMPLER object. 6154 */ 6155 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6156 flag = 0; 6157 } 6158 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6159 return flag ? actions_n + 1 : 0; 6160 } 6161 6162 #define SAMPLE_SUFFIX_ITEM 3 6163 6164 /** 6165 * Split the sample flow. 6166 * 6167 * As sample flow will split to two sub flow, sample flow with 6168 * sample action, the other actions will move to new suffix flow. 6169 * 6170 * Also add unique tag id with tag action in the sample flow, 6171 * the same tag id will be as match in the suffix flow. 6172 * 6173 * @param dev 6174 * Pointer to Ethernet device. 6175 * @param[in] add_tag 6176 * Add extra tag action flag. 6177 * @param[out] sfx_items 6178 * Suffix flow match items (list terminated by the END pattern item). 6179 * @param[in] actions 6180 * Associated actions (list terminated by the END action). 6181 * @param[out] actions_sfx 6182 * Suffix flow actions. 6183 * @param[out] actions_pre 6184 * Prefix flow actions. 6185 * @param[in] actions_n 6186 * The total number of actions. 6187 * @param[in] sample_action_pos 6188 * The sample action position. 6189 * @param[in] qrss_action_pos 6190 * The Queue/RSS action position. 6191 * @param[in] jump_table 6192 * Add extra jump action flag. 6193 * @param[out] error 6194 * Perform verbose error reporting if not NULL. 6195 * 6196 * @return 6197 * 0 on success, or unique flow_id, a negative errno value 6198 * otherwise and rte_errno is set. 6199 */ 6200 static int 6201 flow_sample_split_prep(struct rte_eth_dev *dev, 6202 int add_tag, 6203 const struct rte_flow_item items[], 6204 struct rte_flow_item sfx_items[], 6205 const struct rte_flow_action actions[], 6206 struct rte_flow_action actions_sfx[], 6207 struct rte_flow_action actions_pre[], 6208 int actions_n, 6209 int sample_action_pos, 6210 int qrss_action_pos, 6211 int jump_table, 6212 struct rte_flow_error *error) 6213 { 6214 struct mlx5_priv *priv = dev->data->dev_private; 6215 struct mlx5_rte_flow_action_set_tag *set_tag; 6216 struct mlx5_rte_flow_item_tag *tag_spec; 6217 struct mlx5_rte_flow_item_tag *tag_mask; 6218 struct rte_flow_action_jump *jump_action; 6219 uint32_t tag_id = 0; 6220 int append_index = 0; 6221 int set_tag_idx = -1; 6222 int index; 6223 int ret; 6224 6225 if (sample_action_pos < 0) 6226 return rte_flow_error_set(error, EINVAL, 6227 RTE_FLOW_ERROR_TYPE_ACTION, 6228 NULL, "invalid position of sample " 6229 "action in list"); 6230 /* Prepare the actions for prefix and suffix flow. */ 6231 if (add_tag) { 6232 /* Update the new added tag action index preceding 6233 * the PUSH_VLAN or ENCAP action. 6234 */ 6235 const struct rte_flow_action_raw_encap *raw_encap; 6236 const struct rte_flow_action *action = actions; 6237 int encap_idx; 6238 int action_idx = 0; 6239 int raw_decap_idx = -1; 6240 int push_vlan_idx = -1; 6241 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6242 switch (action->type) { 6243 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6244 raw_decap_idx = action_idx; 6245 break; 6246 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6247 raw_encap = action->conf; 6248 if (raw_encap->size > 6249 MLX5_ENCAPSULATION_DECISION_SIZE) { 6250 encap_idx = raw_decap_idx != -1 ? 6251 raw_decap_idx : action_idx; 6252 if (encap_idx < sample_action_pos && 6253 push_vlan_idx == -1) 6254 set_tag_idx = encap_idx; 6255 } 6256 break; 6257 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6258 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6259 encap_idx = action_idx; 6260 if (encap_idx < sample_action_pos && 6261 push_vlan_idx == -1) 6262 set_tag_idx = encap_idx; 6263 break; 6264 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6265 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6266 if (action_idx < sample_action_pos && 6267 push_vlan_idx == -1) { 6268 set_tag_idx = action_idx; 6269 push_vlan_idx = action_idx; 6270 } 6271 break; 6272 default: 6273 break; 6274 } 6275 action_idx++; 6276 } 6277 } 6278 /* Prepare the actions for prefix and suffix flow. */ 6279 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6280 index = qrss_action_pos; 6281 /* Put the preceding the Queue/RSS action into prefix flow. */ 6282 if (index != 0) 6283 memcpy(actions_pre, actions, 6284 sizeof(struct rte_flow_action) * index); 6285 /* Put others preceding the sample action into prefix flow. */ 6286 if (sample_action_pos > index + 1) 6287 memcpy(actions_pre + index, actions + index + 1, 6288 sizeof(struct rte_flow_action) * 6289 (sample_action_pos - index - 1)); 6290 index = sample_action_pos - 1; 6291 /* Put Queue/RSS action into Suffix flow. */ 6292 memcpy(actions_sfx, actions + qrss_action_pos, 6293 sizeof(struct rte_flow_action)); 6294 actions_sfx++; 6295 } else if (add_tag && set_tag_idx >= 0) { 6296 if (set_tag_idx > 0) 6297 memcpy(actions_pre, actions, 6298 sizeof(struct rte_flow_action) * set_tag_idx); 6299 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6300 sizeof(struct rte_flow_action) * 6301 (sample_action_pos - set_tag_idx)); 6302 index = sample_action_pos; 6303 } else { 6304 index = sample_action_pos; 6305 if (index != 0) 6306 memcpy(actions_pre, actions, 6307 sizeof(struct rte_flow_action) * index); 6308 } 6309 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6310 * For CX6DX and above, metadata registers Cx preserve their value, 6311 * add an extra tag action for NIC-RX and E-Switch Domain. 6312 */ 6313 if (add_tag) { 6314 /* Prepare the prefix tag action. */ 6315 append_index++; 6316 set_tag = (void *)(actions_pre + actions_n + append_index); 6317 /* Trust VF/SF on CX5 not supported meter so that the reserved 6318 * metadata regC is REG_NON, back to use application tag 6319 * index 0. 6320 */ 6321 if (unlikely(priv->mtr_color_reg == REG_NON)) 6322 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6323 else 6324 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6325 if (ret < 0) 6326 return ret; 6327 mlx5_ipool_malloc(priv->sh->ipool 6328 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6329 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6330 .id = ret, 6331 .data = tag_id, 6332 }; 6333 /* Prepare the suffix subflow items. */ 6334 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6335 tag_spec->data = tag_id; 6336 tag_spec->id = set_tag->id; 6337 tag_mask = tag_spec + 1; 6338 tag_mask->data = UINT32_MAX; 6339 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6340 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6341 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6342 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6343 memcpy(sfx_items, items, sizeof(*sfx_items)); 6344 sfx_items++; 6345 break; 6346 } 6347 } 6348 sfx_items[0] = (struct rte_flow_item){ 6349 .type = (enum rte_flow_item_type) 6350 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6351 .spec = tag_spec, 6352 .last = NULL, 6353 .mask = tag_mask, 6354 }; 6355 sfx_items[1] = (struct rte_flow_item){ 6356 .type = (enum rte_flow_item_type) 6357 RTE_FLOW_ITEM_TYPE_END, 6358 }; 6359 /* Prepare the tag action in prefix subflow. */ 6360 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6361 actions_pre[set_tag_idx] = 6362 (struct rte_flow_action){ 6363 .type = (enum rte_flow_action_type) 6364 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6365 .conf = set_tag, 6366 }; 6367 /* Update next sample position due to add one tag action */ 6368 index += 1; 6369 } 6370 /* Copy the sample action into prefix flow. */ 6371 memcpy(actions_pre + index, actions + sample_action_pos, 6372 sizeof(struct rte_flow_action)); 6373 index += 1; 6374 /* For the modify action after the sample action in E-Switch mirroring, 6375 * Add the extra jump action in prefix subflow and jump into the next 6376 * table, then do the modify action in the new table. 6377 */ 6378 if (jump_table) { 6379 /* Prepare the prefix jump action. */ 6380 append_index++; 6381 jump_action = (void *)(actions_pre + actions_n + append_index); 6382 jump_action->group = jump_table; 6383 actions_pre[index++] = 6384 (struct rte_flow_action){ 6385 .type = (enum rte_flow_action_type) 6386 RTE_FLOW_ACTION_TYPE_JUMP, 6387 .conf = jump_action, 6388 }; 6389 } 6390 actions_pre[index] = (struct rte_flow_action){ 6391 .type = (enum rte_flow_action_type) 6392 RTE_FLOW_ACTION_TYPE_END, 6393 }; 6394 /* Put the actions after sample into Suffix flow. */ 6395 memcpy(actions_sfx, actions + sample_action_pos + 1, 6396 sizeof(struct rte_flow_action) * 6397 (actions_n - sample_action_pos - 1)); 6398 return tag_id; 6399 } 6400 6401 /** 6402 * The splitting for metadata feature. 6403 * 6404 * - Q/RSS action on NIC Rx should be split in order to pass by 6405 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6406 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6407 * 6408 * - All the actions on NIC Tx should have a mreg copy action to 6409 * copy reg_a from WQE to reg_c[0]. 6410 * 6411 * @param dev 6412 * Pointer to Ethernet device. 6413 * @param[in] flow 6414 * Parent flow structure pointer. 6415 * @param[in] attr 6416 * Flow rule attributes. 6417 * @param[in] items 6418 * Pattern specification (list terminated by the END pattern item). 6419 * @param[in] actions 6420 * Associated actions (list terminated by the END action). 6421 * @param[in] flow_split_info 6422 * Pointer to flow split info structure. 6423 * @param[out] error 6424 * Perform verbose error reporting if not NULL. 6425 * @return 6426 * 0 on success, negative value otherwise 6427 */ 6428 static int 6429 flow_create_split_metadata(struct rte_eth_dev *dev, 6430 struct rte_flow *flow, 6431 const struct rte_flow_attr *attr, 6432 const struct rte_flow_item items[], 6433 const struct rte_flow_action actions[], 6434 struct mlx5_flow_split_info *flow_split_info, 6435 struct rte_flow_error *error) 6436 { 6437 struct mlx5_priv *priv = dev->data->dev_private; 6438 struct mlx5_sh_config *config = &priv->sh->config; 6439 const struct rte_flow_action *qrss = NULL; 6440 struct rte_flow_action *ext_actions = NULL; 6441 struct mlx5_flow *dev_flow = NULL; 6442 uint32_t qrss_id = 0; 6443 int mtr_sfx = 0; 6444 size_t act_size; 6445 int actions_n; 6446 int encap_idx; 6447 int ret; 6448 6449 /* Check whether extensive metadata feature is engaged. */ 6450 if (!config->dv_flow_en || 6451 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6452 !mlx5_flow_ext_mreg_supported(dev)) 6453 return flow_create_split_inner(dev, flow, NULL, attr, items, 6454 actions, flow_split_info, error); 6455 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6456 &encap_idx); 6457 if (qrss) { 6458 /* Exclude hairpin flows from splitting. */ 6459 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6460 const struct rte_flow_action_queue *queue; 6461 6462 queue = qrss->conf; 6463 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6464 qrss = NULL; 6465 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6466 const struct rte_flow_action_rss *rss; 6467 6468 rss = qrss->conf; 6469 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6470 qrss = NULL; 6471 } 6472 } 6473 if (qrss) { 6474 /* Check if it is in meter suffix table. */ 6475 mtr_sfx = attr->group == 6476 ((attr->transfer && priv->fdb_def_rule) ? 6477 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6478 MLX5_FLOW_TABLE_LEVEL_METER); 6479 /* 6480 * Q/RSS action on NIC Rx should be split in order to pass by 6481 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6482 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6483 */ 6484 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6485 sizeof(struct rte_flow_action_set_tag) + 6486 sizeof(struct rte_flow_action_jump); 6487 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6488 SOCKET_ID_ANY); 6489 if (!ext_actions) 6490 return rte_flow_error_set(error, ENOMEM, 6491 RTE_FLOW_ERROR_TYPE_ACTION, 6492 NULL, "no memory to split " 6493 "metadata flow"); 6494 /* 6495 * Create the new actions list with removed Q/RSS action 6496 * and appended set tag and jump to register copy table 6497 * (RX_CP_TBL). We should preallocate unique tag ID here 6498 * in advance, because it is needed for set tag action. 6499 */ 6500 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6501 qrss, actions_n, 6502 mtr_sfx, error); 6503 if (!mtr_sfx && !qrss_id) { 6504 ret = -rte_errno; 6505 goto exit; 6506 } 6507 } else if (attr->egress) { 6508 /* 6509 * All the actions on NIC Tx should have a metadata register 6510 * copy action to copy reg_a from WQE to reg_c[meta] 6511 */ 6512 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6513 sizeof(struct mlx5_flow_action_copy_mreg); 6514 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6515 SOCKET_ID_ANY); 6516 if (!ext_actions) 6517 return rte_flow_error_set(error, ENOMEM, 6518 RTE_FLOW_ERROR_TYPE_ACTION, 6519 NULL, "no memory to split " 6520 "metadata flow"); 6521 /* Create the action list appended with copy register. */ 6522 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6523 actions_n, error, encap_idx); 6524 if (ret < 0) 6525 goto exit; 6526 } 6527 /* Add the unmodified original or prefix subflow. */ 6528 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6529 items, ext_actions ? ext_actions : 6530 actions, flow_split_info, error); 6531 if (ret < 0) 6532 goto exit; 6533 MLX5_ASSERT(dev_flow); 6534 if (qrss) { 6535 const struct rte_flow_attr q_attr = { 6536 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6537 .ingress = 1, 6538 }; 6539 /* Internal PMD action to set register. */ 6540 struct mlx5_rte_flow_item_tag q_tag_spec = { 6541 .data = qrss_id, 6542 .id = REG_NON, 6543 }; 6544 struct rte_flow_item q_items[] = { 6545 { 6546 .type = (enum rte_flow_item_type) 6547 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6548 .spec = &q_tag_spec, 6549 .last = NULL, 6550 .mask = NULL, 6551 }, 6552 { 6553 .type = RTE_FLOW_ITEM_TYPE_END, 6554 }, 6555 }; 6556 struct rte_flow_action q_actions[] = { 6557 { 6558 .type = qrss->type, 6559 .conf = qrss->conf, 6560 }, 6561 { 6562 .type = RTE_FLOW_ACTION_TYPE_END, 6563 }, 6564 }; 6565 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6566 6567 /* 6568 * Configure the tag item only if there is no meter subflow. 6569 * Since tag is already marked in the meter suffix subflow 6570 * we can just use the meter suffix items as is. 6571 */ 6572 if (qrss_id) { 6573 /* Not meter subflow. */ 6574 MLX5_ASSERT(!mtr_sfx); 6575 /* 6576 * Put unique id in prefix flow due to it is destroyed 6577 * after suffix flow and id will be freed after there 6578 * is no actual flows with this id and identifier 6579 * reallocation becomes possible (for example, for 6580 * other flows in other threads). 6581 */ 6582 dev_flow->handle->split_flow_id = qrss_id; 6583 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6584 error); 6585 if (ret < 0) 6586 goto exit; 6587 q_tag_spec.id = ret; 6588 } 6589 dev_flow = NULL; 6590 /* Add suffix subflow to execute Q/RSS. */ 6591 flow_split_info->prefix_layers = layers; 6592 flow_split_info->prefix_mark = 0; 6593 flow_split_info->table_id = 0; 6594 ret = flow_create_split_inner(dev, flow, &dev_flow, 6595 &q_attr, mtr_sfx ? items : 6596 q_items, q_actions, 6597 flow_split_info, error); 6598 if (ret < 0) 6599 goto exit; 6600 /* qrss ID should be freed if failed. */ 6601 qrss_id = 0; 6602 MLX5_ASSERT(dev_flow); 6603 } 6604 6605 exit: 6606 /* 6607 * We do not destroy the partially created sub_flows in case of error. 6608 * These ones are included into parent flow list and will be destroyed 6609 * by flow_drv_destroy. 6610 */ 6611 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6612 qrss_id); 6613 mlx5_free(ext_actions); 6614 return ret; 6615 } 6616 6617 /** 6618 * Create meter internal drop flow with the original pattern. 6619 * 6620 * @param dev 6621 * Pointer to Ethernet device. 6622 * @param[in] flow 6623 * Parent flow structure pointer. 6624 * @param[in] attr 6625 * Flow rule attributes. 6626 * @param[in] items 6627 * Pattern specification (list terminated by the END pattern item). 6628 * @param[in] flow_split_info 6629 * Pointer to flow split info structure. 6630 * @param[in] fm 6631 * Pointer to flow meter structure. 6632 * @param[out] error 6633 * Perform verbose error reporting if not NULL. 6634 * @return 6635 * 0 on success, negative value otherwise 6636 */ 6637 static uint32_t 6638 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6639 struct rte_flow *flow, 6640 const struct rte_flow_attr *attr, 6641 const struct rte_flow_item items[], 6642 struct mlx5_flow_split_info *flow_split_info, 6643 struct mlx5_flow_meter_info *fm, 6644 struct rte_flow_error *error) 6645 { 6646 struct mlx5_flow *dev_flow = NULL; 6647 struct rte_flow_attr drop_attr = *attr; 6648 struct rte_flow_action drop_actions[3]; 6649 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6650 6651 MLX5_ASSERT(fm->drop_cnt); 6652 drop_actions[0].type = 6653 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6654 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6655 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6656 drop_actions[1].conf = NULL; 6657 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6658 drop_actions[2].conf = NULL; 6659 drop_split_info.external = false; 6660 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6661 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6662 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6663 return flow_create_split_inner(dev, flow, &dev_flow, 6664 &drop_attr, items, drop_actions, 6665 &drop_split_info, error); 6666 } 6667 6668 /** 6669 * The splitting for meter feature. 6670 * 6671 * - The meter flow will be split to two flows as prefix and 6672 * suffix flow. The packets make sense only it pass the prefix 6673 * meter action. 6674 * 6675 * - Reg_C_5 is used for the packet to match betweend prefix and 6676 * suffix flow. 6677 * 6678 * @param dev 6679 * Pointer to Ethernet device. 6680 * @param[in] flow 6681 * Parent flow structure pointer. 6682 * @param[in] attr 6683 * Flow rule attributes. 6684 * @param[in] items 6685 * Pattern specification (list terminated by the END pattern item). 6686 * @param[in] actions 6687 * Associated actions (list terminated by the END action). 6688 * @param[in] flow_split_info 6689 * Pointer to flow split info structure. 6690 * @param[out] error 6691 * Perform verbose error reporting if not NULL. 6692 * @return 6693 * 0 on success, negative value otherwise 6694 */ 6695 static int 6696 flow_create_split_meter(struct rte_eth_dev *dev, 6697 struct rte_flow *flow, 6698 const struct rte_flow_attr *attr, 6699 const struct rte_flow_item items[], 6700 const struct rte_flow_action actions[], 6701 struct mlx5_flow_split_info *flow_split_info, 6702 struct rte_flow_error *error) 6703 { 6704 struct mlx5_priv *priv = dev->data->dev_private; 6705 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6706 struct rte_flow_action *sfx_actions = NULL; 6707 struct rte_flow_action *pre_actions = NULL; 6708 struct rte_flow_item *sfx_items = NULL; 6709 struct mlx5_flow *dev_flow = NULL; 6710 struct rte_flow_attr sfx_attr = *attr; 6711 struct mlx5_flow_meter_info *fm = NULL; 6712 uint8_t skip_scale_restore; 6713 bool has_mtr = false; 6714 bool has_modify = false; 6715 bool set_mtr_reg = true; 6716 bool is_mtr_hierarchy = false; 6717 uint32_t meter_id = 0; 6718 uint32_t mtr_idx = 0; 6719 uint32_t mtr_flow_id = 0; 6720 size_t act_size; 6721 size_t item_size; 6722 int actions_n = 0; 6723 int ret = 0; 6724 6725 if (priv->mtr_en) 6726 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6727 &has_modify, &meter_id); 6728 if (has_mtr) { 6729 if (flow->meter) { 6730 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6731 if (!fm) 6732 return rte_flow_error_set(error, EINVAL, 6733 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6734 NULL, "Meter not found."); 6735 } else { 6736 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6737 if (!fm) 6738 return rte_flow_error_set(error, EINVAL, 6739 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6740 NULL, "Meter not found."); 6741 ret = mlx5_flow_meter_attach(priv, fm, 6742 &sfx_attr, error); 6743 if (ret) 6744 return -rte_errno; 6745 flow->meter = mtr_idx; 6746 } 6747 MLX5_ASSERT(wks); 6748 wks->fm = fm; 6749 if (!fm->def_policy) { 6750 wks->policy = mlx5_flow_meter_policy_find(dev, 6751 fm->policy_id, 6752 NULL); 6753 MLX5_ASSERT(wks->policy); 6754 if (wks->policy->mark) 6755 wks->mark = 1; 6756 if (wks->policy->is_hierarchy) { 6757 wks->final_policy = 6758 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6759 wks->policy); 6760 if (!wks->final_policy) 6761 return rte_flow_error_set(error, 6762 EINVAL, 6763 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6764 "Failed to find terminal policy of hierarchy."); 6765 is_mtr_hierarchy = true; 6766 } 6767 } 6768 /* 6769 * If it isn't default-policy Meter, and 6770 * 1. Not meter hierarchy and there's no action in flow to change 6771 * packet (modify/encap/decap etc.), OR 6772 * 2. No drop count needed for this meter. 6773 * Then no need to use regC to save meter id anymore. 6774 */ 6775 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6776 set_mtr_reg = false; 6777 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6778 #define METER_PREFIX_ACTION 7 6779 act_size = (sizeof(struct rte_flow_action) * 6780 (actions_n + METER_PREFIX_ACTION)) + 6781 sizeof(struct mlx5_rte_flow_action_set_tag); 6782 /* Suffix items: tag, vlan, port id, end. */ 6783 #define METER_SUFFIX_ITEM 4 6784 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6785 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6786 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6787 0, SOCKET_ID_ANY); 6788 if (!sfx_actions) 6789 return rte_flow_error_set(error, ENOMEM, 6790 RTE_FLOW_ERROR_TYPE_ACTION, 6791 NULL, "no memory to split " 6792 "meter flow"); 6793 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6794 act_size); 6795 /* There's no suffix flow for meter of non-default policy. */ 6796 if (!fm->def_policy) 6797 pre_actions = sfx_actions + 1; 6798 else 6799 pre_actions = sfx_actions + actions_n; 6800 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6801 items, sfx_items, actions, 6802 sfx_actions, pre_actions, 6803 (set_mtr_reg ? &mtr_flow_id : NULL), 6804 error); 6805 if (ret) { 6806 ret = -rte_errno; 6807 goto exit; 6808 } 6809 /* Add the prefix subflow. */ 6810 skip_scale_restore = flow_split_info->skip_scale; 6811 flow_split_info->skip_scale |= 6812 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6813 ret = flow_create_split_inner(dev, flow, &dev_flow, 6814 attr, items, pre_actions, 6815 flow_split_info, error); 6816 flow_split_info->skip_scale = skip_scale_restore; 6817 if (ret) { 6818 if (mtr_flow_id) 6819 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6820 ret = -rte_errno; 6821 goto exit; 6822 } 6823 if (mtr_flow_id) { 6824 dev_flow->handle->split_flow_id = mtr_flow_id; 6825 dev_flow->handle->is_meter_flow_id = 1; 6826 } 6827 if (!fm->def_policy) { 6828 if (!set_mtr_reg && fm->drop_cnt) 6829 ret = 6830 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6831 &sfx_attr, items, 6832 flow_split_info, 6833 fm, error); 6834 goto exit; 6835 } 6836 /* Setting the sfx group atrr. */ 6837 sfx_attr.group = sfx_attr.transfer ? 6838 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6839 MLX5_FLOW_TABLE_LEVEL_METER; 6840 flow_split_info->prefix_layers = 6841 flow_get_prefix_layer_flags(dev_flow); 6842 flow_split_info->prefix_mark |= wks->mark; 6843 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6844 } 6845 /* Add the prefix subflow. */ 6846 ret = flow_create_split_metadata(dev, flow, 6847 &sfx_attr, sfx_items ? 6848 sfx_items : items, 6849 sfx_actions ? sfx_actions : actions, 6850 flow_split_info, error); 6851 exit: 6852 if (sfx_actions) 6853 mlx5_free(sfx_actions); 6854 return ret; 6855 } 6856 6857 /** 6858 * The splitting for sample feature. 6859 * 6860 * Once Sample action is detected in the action list, the flow actions should 6861 * be split into prefix sub flow and suffix sub flow. 6862 * 6863 * The original items remain in the prefix sub flow, all actions preceding the 6864 * sample action and the sample action itself will be copied to the prefix 6865 * sub flow, the actions following the sample action will be copied to the 6866 * suffix sub flow, Queue action always be located in the suffix sub flow. 6867 * 6868 * In order to make the packet from prefix sub flow matches with suffix sub 6869 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6870 * flow uses tag item with the unique flow id. 6871 * 6872 * @param dev 6873 * Pointer to Ethernet device. 6874 * @param[in] flow 6875 * Parent flow structure pointer. 6876 * @param[in] attr 6877 * Flow rule attributes. 6878 * @param[in] items 6879 * Pattern specification (list terminated by the END pattern item). 6880 * @param[in] actions 6881 * Associated actions (list terminated by the END action). 6882 * @param[in] flow_split_info 6883 * Pointer to flow split info structure. 6884 * @param[out] error 6885 * Perform verbose error reporting if not NULL. 6886 * @return 6887 * 0 on success, negative value otherwise 6888 */ 6889 static int 6890 flow_create_split_sample(struct rte_eth_dev *dev, 6891 struct rte_flow *flow, 6892 const struct rte_flow_attr *attr, 6893 const struct rte_flow_item items[], 6894 const struct rte_flow_action actions[], 6895 struct mlx5_flow_split_info *flow_split_info, 6896 struct rte_flow_error *error) 6897 { 6898 struct mlx5_priv *priv = dev->data->dev_private; 6899 struct rte_flow_action *sfx_actions = NULL; 6900 struct rte_flow_action *pre_actions = NULL; 6901 struct rte_flow_item *sfx_items = NULL; 6902 struct mlx5_flow *dev_flow = NULL; 6903 struct rte_flow_attr sfx_attr = *attr; 6904 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6905 struct mlx5_flow_dv_sample_resource *sample_res; 6906 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6907 struct mlx5_flow_tbl_resource *sfx_tbl; 6908 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6909 #endif 6910 size_t act_size; 6911 size_t item_size; 6912 uint32_t fdb_tx = 0; 6913 int32_t tag_id = 0; 6914 int actions_n = 0; 6915 int sample_action_pos; 6916 int qrss_action_pos; 6917 int add_tag = 0; 6918 int modify_after_mirror = 0; 6919 uint16_t jump_table = 0; 6920 const uint32_t next_ft_step = 1; 6921 int ret = 0; 6922 struct mlx5_priv *item_port_priv = NULL; 6923 const struct rte_flow_item *item; 6924 6925 if (priv->sampler_en) 6926 actions_n = flow_check_match_action(actions, attr, 6927 RTE_FLOW_ACTION_TYPE_SAMPLE, 6928 &sample_action_pos, &qrss_action_pos, 6929 &modify_after_mirror); 6930 if (actions_n) { 6931 /* The prefix actions must includes sample, tag, end. */ 6932 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6933 + sizeof(struct mlx5_rte_flow_action_set_tag); 6934 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6935 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6936 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6937 item_size), 0, SOCKET_ID_ANY); 6938 if (!sfx_actions) 6939 return rte_flow_error_set(error, ENOMEM, 6940 RTE_FLOW_ERROR_TYPE_ACTION, 6941 NULL, "no memory to split " 6942 "sample flow"); 6943 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 6944 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6945 const struct rte_flow_item_port_id *spec; 6946 6947 spec = (const struct rte_flow_item_port_id *)item->spec; 6948 if (spec) 6949 item_port_priv = 6950 mlx5_port_to_eswitch_info(spec->id, true); 6951 break; 6952 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6953 const struct rte_flow_item_ethdev *spec; 6954 6955 spec = (const struct rte_flow_item_ethdev *)item->spec; 6956 if (spec) 6957 item_port_priv = 6958 mlx5_port_to_eswitch_info(spec->port_id, true); 6959 break; 6960 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 6961 const struct rte_flow_item_ethdev *spec; 6962 6963 spec = (const struct rte_flow_item_ethdev *)item->spec; 6964 if (spec) 6965 item_port_priv = 6966 mlx5_port_to_eswitch_info(spec->port_id, true); 6967 break; 6968 } 6969 } 6970 /* The representor_id is UINT16_MAX for uplink. */ 6971 fdb_tx = (attr->transfer && 6972 flow_source_vport_representor(priv, item_port_priv)); 6973 /* 6974 * When reg_c_preserve is set, metadata registers Cx preserve 6975 * their value even through packet duplication. 6976 */ 6977 add_tag = (!fdb_tx || 6978 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6979 if (add_tag) 6980 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6981 + act_size); 6982 if (modify_after_mirror) 6983 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6984 next_ft_step; 6985 pre_actions = sfx_actions + actions_n; 6986 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6987 actions, sfx_actions, 6988 pre_actions, actions_n, 6989 sample_action_pos, 6990 qrss_action_pos, jump_table, 6991 error); 6992 if (tag_id < 0 || (add_tag && !tag_id)) { 6993 ret = -rte_errno; 6994 goto exit; 6995 } 6996 if (modify_after_mirror) 6997 flow_split_info->skip_scale = 6998 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6999 /* Add the prefix subflow. */ 7000 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7001 items, pre_actions, 7002 flow_split_info, error); 7003 if (ret) { 7004 ret = -rte_errno; 7005 goto exit; 7006 } 7007 dev_flow->handle->split_flow_id = tag_id; 7008 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7009 if (!modify_after_mirror) { 7010 /* Set the sfx group attr. */ 7011 sample_res = (struct mlx5_flow_dv_sample_resource *) 7012 dev_flow->dv.sample_res; 7013 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7014 sample_res->normal_path_tbl; 7015 sfx_tbl_data = container_of(sfx_tbl, 7016 struct mlx5_flow_tbl_data_entry, 7017 tbl); 7018 sfx_attr.group = sfx_attr.transfer ? 7019 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7020 } else { 7021 MLX5_ASSERT(attr->transfer); 7022 sfx_attr.group = jump_table; 7023 } 7024 flow_split_info->prefix_layers = 7025 flow_get_prefix_layer_flags(dev_flow); 7026 MLX5_ASSERT(wks); 7027 flow_split_info->prefix_mark |= wks->mark; 7028 /* Suffix group level already be scaled with factor, set 7029 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7030 * again in translation. 7031 */ 7032 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7033 #endif 7034 } 7035 /* Add the suffix subflow. */ 7036 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7037 sfx_items ? sfx_items : items, 7038 sfx_actions ? sfx_actions : actions, 7039 flow_split_info, error); 7040 exit: 7041 if (sfx_actions) 7042 mlx5_free(sfx_actions); 7043 return ret; 7044 } 7045 7046 /** 7047 * Split the flow to subflow set. The splitters might be linked 7048 * in the chain, like this: 7049 * flow_create_split_outer() calls: 7050 * flow_create_split_meter() calls: 7051 * flow_create_split_metadata(meter_subflow_0) calls: 7052 * flow_create_split_inner(metadata_subflow_0) 7053 * flow_create_split_inner(metadata_subflow_1) 7054 * flow_create_split_inner(metadata_subflow_2) 7055 * flow_create_split_metadata(meter_subflow_1) calls: 7056 * flow_create_split_inner(metadata_subflow_0) 7057 * flow_create_split_inner(metadata_subflow_1) 7058 * flow_create_split_inner(metadata_subflow_2) 7059 * 7060 * This provide flexible way to add new levels of flow splitting. 7061 * The all of successfully created subflows are included to the 7062 * parent flow dev_flow list. 7063 * 7064 * @param dev 7065 * Pointer to Ethernet device. 7066 * @param[in] flow 7067 * Parent flow structure pointer. 7068 * @param[in] attr 7069 * Flow rule attributes. 7070 * @param[in] items 7071 * Pattern specification (list terminated by the END pattern item). 7072 * @param[in] actions 7073 * Associated actions (list terminated by the END action). 7074 * @param[in] flow_split_info 7075 * Pointer to flow split info structure. 7076 * @param[out] error 7077 * Perform verbose error reporting if not NULL. 7078 * @return 7079 * 0 on success, negative value otherwise 7080 */ 7081 static int 7082 flow_create_split_outer(struct rte_eth_dev *dev, 7083 struct rte_flow *flow, 7084 const struct rte_flow_attr *attr, 7085 const struct rte_flow_item items[], 7086 const struct rte_flow_action actions[], 7087 struct mlx5_flow_split_info *flow_split_info, 7088 struct rte_flow_error *error) 7089 { 7090 int ret; 7091 7092 ret = flow_create_split_sample(dev, flow, attr, items, 7093 actions, flow_split_info, error); 7094 MLX5_ASSERT(ret <= 0); 7095 return ret; 7096 } 7097 7098 static inline struct mlx5_flow_tunnel * 7099 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7100 { 7101 struct mlx5_flow_tunnel *tunnel; 7102 7103 #pragma GCC diagnostic push 7104 #pragma GCC diagnostic ignored "-Wcast-qual" 7105 tunnel = (typeof(tunnel))flow->tunnel; 7106 #pragma GCC diagnostic pop 7107 7108 return tunnel; 7109 } 7110 7111 /** 7112 * Adjust flow RSS workspace if needed. 7113 * 7114 * @param wks 7115 * Pointer to thread flow work space. 7116 * @param rss_desc 7117 * Pointer to RSS descriptor. 7118 * @param[in] nrssq_num 7119 * New RSS queue number. 7120 * 7121 * @return 7122 * 0 on success, -1 otherwise and rte_errno is set. 7123 */ 7124 static int 7125 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 7126 struct mlx5_flow_rss_desc *rss_desc, 7127 uint32_t nrssq_num) 7128 { 7129 if (likely(nrssq_num <= wks->rssq_num)) 7130 return 0; 7131 rss_desc->queue = realloc(rss_desc->queue, 7132 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 7133 if (!rss_desc->queue) { 7134 rte_errno = ENOMEM; 7135 return -1; 7136 } 7137 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 7138 return 0; 7139 } 7140 7141 /** 7142 * Create a flow and add it to @p list. 7143 * 7144 * @param dev 7145 * Pointer to Ethernet device. 7146 * @param list 7147 * Pointer to a TAILQ flow list. If this parameter NULL, 7148 * no list insertion occurred, flow is just created, 7149 * this is caller's responsibility to track the 7150 * created flow. 7151 * @param[in] attr 7152 * Flow rule attributes. 7153 * @param[in] items 7154 * Pattern specification (list terminated by the END pattern item). 7155 * @param[in] actions 7156 * Associated actions (list terminated by the END action). 7157 * @param[in] external 7158 * This flow rule is created by request external to PMD. 7159 * @param[out] error 7160 * Perform verbose error reporting if not NULL. 7161 * 7162 * @return 7163 * A flow index on success, 0 otherwise and rte_errno is set. 7164 */ 7165 static uint32_t 7166 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7167 const struct rte_flow_attr *attr, 7168 const struct rte_flow_item items[], 7169 const struct rte_flow_action original_actions[], 7170 bool external, struct rte_flow_error *error) 7171 { 7172 struct mlx5_priv *priv = dev->data->dev_private; 7173 struct rte_flow *flow = NULL; 7174 struct mlx5_flow *dev_flow; 7175 const struct rte_flow_action_rss *rss = NULL; 7176 struct mlx5_translated_action_handle 7177 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7178 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7179 union { 7180 struct mlx5_flow_expand_rss buf; 7181 uint8_t buffer[8192]; 7182 } expand_buffer; 7183 union { 7184 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7185 uint8_t buffer[2048]; 7186 } actions_rx; 7187 union { 7188 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7189 uint8_t buffer[2048]; 7190 } actions_hairpin_tx; 7191 union { 7192 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7193 uint8_t buffer[2048]; 7194 } items_tx; 7195 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7196 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7197 struct mlx5_flow_rss_desc *rss_desc; 7198 const struct rte_flow_action *p_actions_rx; 7199 uint32_t i; 7200 uint32_t idx = 0; 7201 int hairpin_flow; 7202 struct rte_flow_attr attr_tx = { .priority = 0 }; 7203 const struct rte_flow_action *actions; 7204 struct rte_flow_action *translated_actions = NULL; 7205 struct mlx5_flow_tunnel *tunnel; 7206 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7207 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7208 struct mlx5_flow_split_info flow_split_info = { 7209 .external = !!external, 7210 .skip_scale = 0, 7211 .flow_idx = 0, 7212 .prefix_mark = 0, 7213 .prefix_layers = 0, 7214 .table_id = 0 7215 }; 7216 int ret; 7217 7218 MLX5_ASSERT(wks); 7219 rss_desc = &wks->rss_desc; 7220 ret = flow_action_handles_translate(dev, original_actions, 7221 indir_actions, 7222 &indir_actions_n, 7223 &translated_actions, error); 7224 if (ret < 0) { 7225 MLX5_ASSERT(translated_actions == NULL); 7226 return 0; 7227 } 7228 actions = translated_actions ? translated_actions : original_actions; 7229 p_actions_rx = actions; 7230 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7231 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7232 external, hairpin_flow, error); 7233 if (ret < 0) 7234 goto error_before_hairpin_split; 7235 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7236 if (!flow) { 7237 rte_errno = ENOMEM; 7238 goto error_before_hairpin_split; 7239 } 7240 if (hairpin_flow > 0) { 7241 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7242 rte_errno = EINVAL; 7243 goto error_before_hairpin_split; 7244 } 7245 flow_hairpin_split(dev, actions, actions_rx.actions, 7246 actions_hairpin_tx.actions, items_tx.items, 7247 idx); 7248 p_actions_rx = actions_rx.actions; 7249 } 7250 flow_split_info.flow_idx = idx; 7251 flow->drv_type = flow_get_drv_type(dev, attr); 7252 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7253 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7254 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7255 /* RSS Action only works on NIC RX domain */ 7256 if (attr->ingress) 7257 rss = flow_get_rss_action(dev, p_actions_rx); 7258 if (rss) { 7259 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 7260 return 0; 7261 /* 7262 * The following information is required by 7263 * mlx5_flow_hashfields_adjust() in advance. 7264 */ 7265 rss_desc->level = rss->level; 7266 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7267 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7268 } 7269 flow->dev_handles = 0; 7270 if (rss && rss->types) { 7271 unsigned int graph_root; 7272 7273 graph_root = find_graph_root(rss->level); 7274 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7275 items, rss->types, 7276 mlx5_support_expansion, graph_root); 7277 MLX5_ASSERT(ret > 0 && 7278 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7279 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7280 for (i = 0; i < buf->entries; ++i) 7281 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7282 } 7283 } else { 7284 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7285 sizeof(expand_buffer.buffer), 7286 items, sq_specs); 7287 if (ret) { 7288 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7289 NULL, "not enough memory for rte_flow"); 7290 goto error; 7291 } 7292 if (buf->entries == 0) { 7293 buf->entries = 1; 7294 buf->entry[0].pattern = (void *)(uintptr_t)items; 7295 } 7296 } 7297 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7298 indir_actions_n); 7299 for (i = 0; i < buf->entries; ++i) { 7300 /* Initialize flow split data. */ 7301 flow_split_info.prefix_layers = 0; 7302 flow_split_info.prefix_mark = 0; 7303 flow_split_info.skip_scale = 0; 7304 /* 7305 * The splitter may create multiple dev_flows, 7306 * depending on configuration. In the simplest 7307 * case it just creates unmodified original flow. 7308 */ 7309 ret = flow_create_split_outer(dev, flow, attr, 7310 buf->entry[i].pattern, 7311 p_actions_rx, &flow_split_info, 7312 error); 7313 if (ret < 0) 7314 goto error; 7315 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7316 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7317 p_actions_rx, 7318 idx, 7319 wks->flows[0].tunnel, 7320 &default_miss_ctx, 7321 error); 7322 if (ret < 0) { 7323 mlx5_free(default_miss_ctx.queue); 7324 goto error; 7325 } 7326 } 7327 } 7328 /* Create the tx flow. */ 7329 if (hairpin_flow) { 7330 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7331 attr_tx.ingress = 0; 7332 attr_tx.egress = 1; 7333 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7334 actions_hairpin_tx.actions, 7335 idx, error); 7336 if (!dev_flow) 7337 goto error; 7338 dev_flow->flow = flow; 7339 dev_flow->external = 0; 7340 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7341 dev_flow->handle, next); 7342 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7343 items_tx.items, 7344 actions_hairpin_tx.actions, error); 7345 if (ret < 0) 7346 goto error; 7347 } 7348 /* 7349 * Update the metadata register copy table. If extensive 7350 * metadata feature is enabled and registers are supported 7351 * we might create the extra rte_flow for each unique 7352 * MARK/FLAG action ID. 7353 * 7354 * The table is updated for ingress and transfer flows only, because 7355 * the egress Flows belong to the different device and 7356 * copy table should be updated in peer NIC Rx domain. 7357 */ 7358 if ((attr->ingress || attr->transfer) && 7359 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7360 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7361 if (ret) 7362 goto error; 7363 } 7364 /* 7365 * If the flow is external (from application) OR device is started, 7366 * OR mreg discover, then apply immediately. 7367 */ 7368 if (external || dev->data->dev_started || 7369 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7370 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7371 ret = flow_drv_apply(dev, flow, error); 7372 if (ret < 0) 7373 goto error; 7374 } 7375 flow->type = type; 7376 flow_rxq_flags_set(dev, flow); 7377 rte_free(translated_actions); 7378 tunnel = flow_tunnel_from_rule(wks->flows); 7379 if (tunnel) { 7380 flow->tunnel = 1; 7381 flow->tunnel_id = tunnel->tunnel_id; 7382 __atomic_fetch_add(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7383 mlx5_free(default_miss_ctx.queue); 7384 } 7385 mlx5_flow_pop_thread_workspace(); 7386 return idx; 7387 error: 7388 MLX5_ASSERT(flow); 7389 ret = rte_errno; /* Save rte_errno before cleanup. */ 7390 flow_mreg_del_copy_action(dev, flow); 7391 flow_drv_destroy(dev, flow); 7392 if (rss_desc->shared_rss) 7393 __atomic_fetch_sub(&((struct mlx5_shared_action_rss *) 7394 mlx5_ipool_get 7395 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7396 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7397 mlx5_ipool_free(priv->flows[type], idx); 7398 rte_errno = ret; /* Restore rte_errno. */ 7399 ret = rte_errno; 7400 rte_errno = ret; 7401 error_before_hairpin_split: 7402 mlx5_flow_pop_thread_workspace(); 7403 rte_free(translated_actions); 7404 return 0; 7405 } 7406 7407 /** 7408 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7409 * incoming packets to table 1. 7410 * 7411 * Other flow rules, requested for group n, will be created in 7412 * e-switch table n+1. 7413 * Jump action to e-switch group n will be created to group n+1. 7414 * 7415 * Used when working in switchdev mode, to utilise advantages of table 1 7416 * and above. 7417 * 7418 * @param dev 7419 * Pointer to Ethernet device. 7420 * 7421 * @return 7422 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7423 */ 7424 struct rte_flow * 7425 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7426 { 7427 const struct rte_flow_attr attr = { 7428 .group = 0, 7429 .priority = 0, 7430 .ingress = 0, 7431 .egress = 0, 7432 .transfer = 1, 7433 }; 7434 const struct rte_flow_item pattern = { 7435 .type = RTE_FLOW_ITEM_TYPE_END, 7436 }; 7437 struct rte_flow_action_jump jump = { 7438 .group = 1, 7439 }; 7440 const struct rte_flow_action actions[] = { 7441 { 7442 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7443 .conf = &jump, 7444 }, 7445 { 7446 .type = RTE_FLOW_ACTION_TYPE_END, 7447 }, 7448 }; 7449 struct rte_flow_error error; 7450 7451 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7452 &attr, &pattern, 7453 actions, false, &error); 7454 } 7455 7456 /** 7457 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7458 * and sq number, directs all packets to peer vport. 7459 * 7460 * @param dev 7461 * Pointer to Ethernet device. 7462 * @param sq_num 7463 * SQ number. 7464 * 7465 * @return 7466 * Flow ID on success, 0 otherwise and rte_errno is set. 7467 */ 7468 uint32_t 7469 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7470 { 7471 struct rte_flow_attr attr = { 7472 .group = 0, 7473 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7474 .ingress = 0, 7475 .egress = 0, 7476 .transfer = 1, 7477 }; 7478 struct rte_flow_item_port_id port_spec = { 7479 .id = MLX5_PORT_ESW_MGR, 7480 }; 7481 struct mlx5_rte_flow_item_sq sq_spec = { 7482 .queue = sq_num, 7483 }; 7484 struct rte_flow_item pattern[] = { 7485 { 7486 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7487 .spec = &port_spec, 7488 }, 7489 { 7490 .type = (enum rte_flow_item_type) 7491 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7492 .spec = &sq_spec, 7493 }, 7494 { 7495 .type = RTE_FLOW_ITEM_TYPE_END, 7496 }, 7497 }; 7498 struct rte_flow_action_jump jump = { 7499 .group = 1, 7500 }; 7501 struct rte_flow_action_port_id port = { 7502 .id = dev->data->port_id, 7503 }; 7504 struct rte_flow_action actions[] = { 7505 { 7506 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7507 .conf = &jump, 7508 }, 7509 { 7510 .type = RTE_FLOW_ACTION_TYPE_END, 7511 }, 7512 }; 7513 struct rte_flow_error error; 7514 7515 /* 7516 * Creates group 0, highest priority jump flow. 7517 * Matches txq to bypass kernel packets. 7518 */ 7519 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7520 false, &error) == 0) 7521 return 0; 7522 /* Create group 1, lowest priority redirect flow for txq. */ 7523 attr.group = 1; 7524 actions[0].conf = &port; 7525 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7526 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7527 actions, false, &error); 7528 } 7529 7530 /** 7531 * Validate a flow supported by the NIC. 7532 * 7533 * @see rte_flow_validate() 7534 * @see rte_flow_ops 7535 */ 7536 int 7537 mlx5_flow_validate(struct rte_eth_dev *dev, 7538 const struct rte_flow_attr *attr, 7539 const struct rte_flow_item items[], 7540 const struct rte_flow_action original_actions[], 7541 struct rte_flow_error *error) 7542 { 7543 int hairpin_flow; 7544 struct mlx5_translated_action_handle 7545 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7546 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7547 const struct rte_flow_action *actions; 7548 struct rte_flow_action *translated_actions = NULL; 7549 int ret = flow_action_handles_translate(dev, original_actions, 7550 indir_actions, 7551 &indir_actions_n, 7552 &translated_actions, error); 7553 7554 if (ret) 7555 return ret; 7556 actions = translated_actions ? translated_actions : original_actions; 7557 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7558 ret = flow_drv_validate(dev, attr, items, actions, 7559 true, hairpin_flow, error); 7560 rte_free(translated_actions); 7561 return ret; 7562 } 7563 7564 static int 7565 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7566 const struct rte_flow_attr *attr, 7567 const uint32_t orig_prio, 7568 const struct rte_flow_item *items, 7569 const struct rte_flow_action *actions, 7570 uint32_t flow_idx) 7571 { 7572 struct mlx5_priv *priv = dev->data->dev_private; 7573 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7574 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7575 struct rte_flow_error error; 7576 int len, ret; 7577 7578 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7579 if (!flow_info) { 7580 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7581 return -1; 7582 } 7583 flow_info->orig_prio = orig_prio; 7584 flow_info->attr = *attr; 7585 /* Standby mode rule awlays saves it in low priority entry. */ 7586 flow_info->flow_idx_low_prio = flow_idx; 7587 7588 /* Store matching items. */ 7589 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7590 if (ret <= 0) { 7591 DRV_LOG(ERR, "Can't get items length."); 7592 goto end; 7593 } 7594 len = RTE_ALIGN(ret, 16); 7595 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7596 if (!flow_info->items) { 7597 DRV_LOG(ERR, "No enough memory for items caching."); 7598 goto end; 7599 } 7600 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7601 if (ret <= 0) { 7602 DRV_LOG(ERR, "Can't duplicate items."); 7603 goto end; 7604 } 7605 7606 /* Store flow actions. */ 7607 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7608 if (ret <= 0) { 7609 DRV_LOG(ERR, "Can't get actions length."); 7610 goto end; 7611 } 7612 len = RTE_ALIGN(ret, 16); 7613 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7614 if (!flow_info->actions) { 7615 DRV_LOG(ERR, "No enough memory for actions caching."); 7616 goto end; 7617 } 7618 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7619 if (ret <= 0) { 7620 DRV_LOG(ERR, "Can't duplicate actions."); 7621 goto end; 7622 } 7623 7624 /* Insert to the list end. */ 7625 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7626 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7627 } else { 7628 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7629 while (LIST_NEXT(tmp_info, next)) 7630 tmp_info = LIST_NEXT(tmp_info, next); 7631 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7632 } 7633 return 0; 7634 end: 7635 if (flow_info->items) 7636 mlx5_free(flow_info->items); 7637 if (flow_info->actions) 7638 mlx5_free(flow_info->actions); 7639 mlx5_free(flow_info); 7640 return -1; 7641 } 7642 7643 static int 7644 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7645 { 7646 struct mlx5_priv *priv = dev->data->dev_private; 7647 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7648 struct mlx5_dv_flow_info *flow_info; 7649 struct rte_flow_attr attr; 7650 struct rte_flow_error error; 7651 struct rte_flow *high, *low; 7652 7653 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7654 while (flow_info) { 7655 /* DUP flow may have the same priority. */ 7656 if (flow_info->orig_prio != flow_info->attr.priority) { 7657 attr = flow_info->attr; 7658 if (orig_prio) 7659 attr.priority = flow_info->orig_prio; 7660 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7661 &attr, flow_info->items, flow_info->actions, 7662 true, &error); 7663 if (!flow_info->flow_idx_high_prio) { 7664 DRV_LOG(ERR, "Priority toggle failed internally."); 7665 goto err; 7666 } 7667 } 7668 flow_info = LIST_NEXT(flow_info, next); 7669 } 7670 /* Delete the low priority rules and swap the flow handle. */ 7671 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7672 while (flow_info) { 7673 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7674 if (flow_info->orig_prio != flow_info->attr.priority) { 7675 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7676 flow_info->flow_idx_high_prio); 7677 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7678 flow_info->flow_idx_low_prio); 7679 if (high && low) { 7680 RTE_SWAP(*low, *high); 7681 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7682 flow_info->flow_idx_low_prio); 7683 flow_info->flow_idx_high_prio = 0; 7684 } 7685 } 7686 flow_info = LIST_NEXT(flow_info, next); 7687 } 7688 return 0; 7689 err: 7690 /* Destroy preceding successful high priority rules. */ 7691 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7692 while (flow_info) { 7693 if (flow_info->orig_prio != flow_info->attr.priority) { 7694 if (flow_info->flow_idx_high_prio) 7695 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7696 flow_info->flow_idx_high_prio); 7697 else 7698 break; 7699 flow_info->flow_idx_high_prio = 0; 7700 } 7701 flow_info = LIST_NEXT(flow_info, next); 7702 } 7703 return -1; 7704 } 7705 7706 /** 7707 * Set the mode of the flow engine of a process to active or standby during live migration. 7708 * 7709 * @param[in] mode 7710 * MLX5 flow engine mode, @see `enum mlx5_flow_engine_mode`. 7711 * @param[in] flags 7712 * Flow engine mode specific flags. 7713 * 7714 * @return 7715 * Negative value on error, positive on success. 7716 */ 7717 int 7718 rte_pmd_mlx5_flow_engine_set_mode(enum mlx5_flow_engine_mode mode, uint32_t flags) 7719 { 7720 struct mlx5_priv *priv; 7721 struct mlx5_flow_engine_mode_info *mode_info; 7722 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7723 uint16_t port, port_id; 7724 uint16_t toggle_num = 0; 7725 struct rte_eth_dev *dev; 7726 enum mlx5_flow_engine_mode orig_mode; 7727 uint32_t orig_flags; 7728 bool need_toggle = false; 7729 7730 /* Check if flags combinations are supported. */ 7731 if (flags && flags != MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7732 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7733 return -1; 7734 } 7735 MLX5_ETH_FOREACH_DEV(port, NULL) { 7736 dev = &rte_eth_devices[port]; 7737 priv = dev->data->dev_private; 7738 mode_info = &priv->mode_info; 7739 /* No mode change. Assume all devices hold the same mode. */ 7740 if (mode_info->mode == mode) { 7741 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7742 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7743 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7744 port, mode_info->mode_flag); 7745 orig_mode = mode_info->mode; 7746 orig_flags = mode_info->mode_flag; 7747 goto err; 7748 } 7749 mode_info->mode_flag = flags; 7750 toggle_num++; 7751 continue; 7752 } 7753 /* Active -> standby. */ 7754 if (mode == MLX5_FLOW_ENGINE_MODE_STANDBY) { 7755 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7756 DRV_LOG(ERR, "Cached rule existed"); 7757 orig_mode = mode_info->mode; 7758 orig_flags = mode_info->mode_flag; 7759 goto err; 7760 } 7761 mode_info->mode_flag = flags; 7762 mode_info->mode = mode; 7763 toggle_num++; 7764 /* Standby -> active. */ 7765 } else if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7766 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7767 DRV_LOG(INFO, "No cached rule existed"); 7768 } else { 7769 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7770 orig_mode = mode_info->mode; 7771 orig_flags = mode_info->mode_flag; 7772 need_toggle = true; 7773 goto err; 7774 } 7775 } 7776 toggle_num++; 7777 } 7778 } 7779 if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7780 /* Clear cache flow rules. */ 7781 MLX5_ETH_FOREACH_DEV(port, NULL) { 7782 priv = rte_eth_devices[port].data->dev_private; 7783 mode_info = &priv->mode_info; 7784 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7785 while (flow_info) { 7786 tmp_info = LIST_NEXT(flow_info, next); 7787 LIST_REMOVE(flow_info, next); 7788 mlx5_free(flow_info->actions); 7789 mlx5_free(flow_info->items); 7790 mlx5_free(flow_info); 7791 flow_info = tmp_info; 7792 } 7793 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7794 } 7795 } 7796 return toggle_num; 7797 err: 7798 /* Rollback all preceding successful ports. */ 7799 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7800 if (port_id == port) 7801 break; 7802 priv = rte_eth_devices[port_id].data->dev_private; 7803 mode_info = &priv->mode_info; 7804 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7805 mlx5_flow_cache_flow_toggle(dev, false)) 7806 return -EPERM; 7807 mode_info->mode = orig_mode; 7808 mode_info->mode_flag = orig_flags; 7809 } 7810 return -EINVAL; 7811 } 7812 /** 7813 * Create a flow. 7814 * 7815 * @see rte_flow_create() 7816 * @see rte_flow_ops 7817 */ 7818 struct rte_flow * 7819 mlx5_flow_create(struct rte_eth_dev *dev, 7820 const struct rte_flow_attr *attr, 7821 const struct rte_flow_item items[], 7822 const struct rte_flow_action actions[], 7823 struct rte_flow_error *error) 7824 { 7825 struct mlx5_priv *priv = dev->data->dev_private; 7826 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7827 uint32_t prio = attr->priority; 7828 uint32_t flow_idx; 7829 7830 if (priv->sh->config.dv_flow_en == 2) { 7831 rte_flow_error_set(error, ENOTSUP, 7832 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7833 NULL, 7834 "Flow non-Q creation not supported"); 7835 return NULL; 7836 } 7837 /* 7838 * If the device is not started yet, it is not allowed to created a 7839 * flow from application. PMD default flows and traffic control flows 7840 * are not affected. 7841 */ 7842 if (unlikely(!dev->data->dev_started)) { 7843 DRV_LOG(DEBUG, "port %u is not started when " 7844 "inserting a flow", dev->data->port_id); 7845 rte_flow_error_set(error, ENODEV, 7846 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7847 NULL, 7848 "port not started"); 7849 return NULL; 7850 } 7851 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7852 if (attr->transfer || 7853 (attr->ingress && 7854 !(priv->mode_info.mode_flag & MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 7855 new_attr->priority += 1; 7856 } 7857 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 7858 if (!flow_idx) 7859 return NULL; 7860 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7861 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 7862 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7863 flow_idx = 0; 7864 } 7865 } 7866 return (void *)(uintptr_t)flow_idx; 7867 } 7868 7869 /** 7870 * Destroy a flow in a list. 7871 * 7872 * @param dev 7873 * Pointer to Ethernet device. 7874 * @param[in] flow_idx 7875 * Index of flow to destroy. 7876 */ 7877 static void 7878 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7879 uint32_t flow_idx) 7880 { 7881 struct mlx5_priv *priv = dev->data->dev_private; 7882 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7883 7884 if (!flow) 7885 return; 7886 MLX5_ASSERT(flow->type == type); 7887 /* 7888 * Update RX queue flags only if port is started, otherwise it is 7889 * already clean. 7890 */ 7891 if (dev->data->dev_started) 7892 flow_rxq_flags_trim(dev, flow); 7893 flow_drv_destroy(dev, flow); 7894 if (flow->tunnel) { 7895 struct mlx5_flow_tunnel *tunnel; 7896 7897 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7898 RTE_VERIFY(tunnel); 7899 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 7900 mlx5_flow_tunnel_free(dev, tunnel); 7901 } 7902 flow_mreg_del_copy_action(dev, flow); 7903 mlx5_ipool_free(priv->flows[type], flow_idx); 7904 } 7905 7906 /** 7907 * Destroy all flows. 7908 * 7909 * @param dev 7910 * Pointer to Ethernet device. 7911 * @param type 7912 * Flow type to be flushed. 7913 * @param active 7914 * If flushing is called actively. 7915 */ 7916 void 7917 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7918 bool active) 7919 { 7920 struct mlx5_priv *priv = dev->data->dev_private; 7921 uint32_t num_flushed = 0, fidx = 1; 7922 struct rte_flow *flow; 7923 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7924 struct mlx5_dv_flow_info *flow_info; 7925 7926 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7927 if (priv->sh->config.dv_flow_en == 2 && 7928 type == MLX5_FLOW_TYPE_GEN) { 7929 flow_hw_q_flow_flush(dev, NULL); 7930 return; 7931 } 7932 #endif 7933 7934 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7935 flow_list_destroy(dev, type, fidx); 7936 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 7937 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7938 while (flow_info) { 7939 /* Romove the cache flow info. */ 7940 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 7941 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 7942 LIST_REMOVE(flow_info, next); 7943 mlx5_free(flow_info->items); 7944 mlx5_free(flow_info->actions); 7945 mlx5_free(flow_info); 7946 break; 7947 } 7948 flow_info = LIST_NEXT(flow_info, next); 7949 } 7950 } 7951 num_flushed++; 7952 } 7953 if (active) { 7954 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7955 dev->data->port_id, num_flushed); 7956 } 7957 } 7958 7959 /** 7960 * Stop all default actions for flows. 7961 * 7962 * @param dev 7963 * Pointer to Ethernet device. 7964 */ 7965 void 7966 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7967 { 7968 flow_mreg_del_default_copy_action(dev); 7969 flow_rxq_flags_clear(dev); 7970 } 7971 7972 /** 7973 * Set rxq flag. 7974 * 7975 * @param[in] dev 7976 * Pointer to the rte_eth_dev structure. 7977 * @param[in] enable 7978 * Flag to enable or not. 7979 */ 7980 void 7981 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 7982 { 7983 struct mlx5_priv *priv = dev->data->dev_private; 7984 unsigned int i; 7985 7986 if ((!priv->mark_enabled && !enable) || 7987 (priv->mark_enabled && enable)) 7988 return; 7989 for (i = 0; i < priv->rxqs_n; ++i) { 7990 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 7991 7992 /* With RXQ start/stop feature, RXQ might be stopped. */ 7993 if (!rxq_ctrl) 7994 continue; 7995 rxq_ctrl->rxq.mark = enable; 7996 } 7997 priv->mark_enabled = enable; 7998 } 7999 8000 /** 8001 * Start all default actions for flows. 8002 * 8003 * @param dev 8004 * Pointer to Ethernet device. 8005 * @return 8006 * 0 on success, a negative errno value otherwise and rte_errno is set. 8007 */ 8008 int 8009 mlx5_flow_start_default(struct rte_eth_dev *dev) 8010 { 8011 struct rte_flow_error error; 8012 8013 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8014 return flow_mreg_add_default_copy_action(dev, &error); 8015 } 8016 8017 /** 8018 * Release key of thread specific flow workspace data. 8019 */ 8020 void 8021 flow_release_workspace(void *data) 8022 { 8023 struct mlx5_flow_workspace *wks = data; 8024 struct mlx5_flow_workspace *next; 8025 8026 while (wks) { 8027 next = wks->next; 8028 free(wks->rss_desc.queue); 8029 free(wks); 8030 wks = next; 8031 } 8032 } 8033 8034 /** 8035 * Get thread specific current flow workspace. 8036 * 8037 * @return pointer to thread specific flow workspace data, NULL on error. 8038 */ 8039 struct mlx5_flow_workspace* 8040 mlx5_flow_get_thread_workspace(void) 8041 { 8042 struct mlx5_flow_workspace *data; 8043 8044 data = mlx5_flow_os_get_specific_workspace(); 8045 MLX5_ASSERT(data && data->inuse); 8046 if (!data || !data->inuse) 8047 DRV_LOG(ERR, "flow workspace not initialized."); 8048 return data; 8049 } 8050 8051 /** 8052 * Allocate and init new flow workspace. 8053 * 8054 * @return pointer to flow workspace data, NULL on error. 8055 */ 8056 static struct mlx5_flow_workspace* 8057 flow_alloc_thread_workspace(void) 8058 { 8059 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 8060 8061 if (!data) { 8062 DRV_LOG(ERR, "Failed to allocate flow workspace " 8063 "memory."); 8064 return NULL; 8065 } 8066 data->rss_desc.queue = calloc(1, 8067 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 8068 if (!data->rss_desc.queue) 8069 goto err; 8070 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 8071 return data; 8072 err: 8073 free(data->rss_desc.queue); 8074 free(data); 8075 return NULL; 8076 } 8077 8078 /** 8079 * Get new thread specific flow workspace. 8080 * 8081 * If current workspace inuse, create new one and set as current. 8082 * 8083 * @return pointer to thread specific flow workspace data, NULL on error. 8084 */ 8085 struct mlx5_flow_workspace* 8086 mlx5_flow_push_thread_workspace(void) 8087 { 8088 struct mlx5_flow_workspace *curr; 8089 struct mlx5_flow_workspace *data; 8090 8091 curr = mlx5_flow_os_get_specific_workspace(); 8092 if (!curr) { 8093 data = flow_alloc_thread_workspace(); 8094 if (!data) 8095 return NULL; 8096 } else if (!curr->inuse) { 8097 data = curr; 8098 } else if (curr->next) { 8099 data = curr->next; 8100 } else { 8101 data = flow_alloc_thread_workspace(); 8102 if (!data) 8103 return NULL; 8104 curr->next = data; 8105 data->prev = curr; 8106 } 8107 data->inuse = 1; 8108 data->flow_idx = 0; 8109 /* Set as current workspace */ 8110 if (mlx5_flow_os_set_specific_workspace(data)) 8111 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8112 return data; 8113 } 8114 8115 /** 8116 * Close current thread specific flow workspace. 8117 * 8118 * If previous workspace available, set it as current. 8119 * 8120 * @return pointer to thread specific flow workspace data, NULL on error. 8121 */ 8122 void 8123 mlx5_flow_pop_thread_workspace(void) 8124 { 8125 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8126 8127 if (!data) 8128 return; 8129 if (!data->inuse) { 8130 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8131 return; 8132 } 8133 data->inuse = 0; 8134 if (!data->prev) 8135 return; 8136 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8137 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8138 } 8139 8140 /** 8141 * Verify the flow list is empty 8142 * 8143 * @param dev 8144 * Pointer to Ethernet device. 8145 * 8146 * @return the number of flows not released. 8147 */ 8148 int 8149 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8150 { 8151 struct mlx5_priv *priv = dev->data->dev_private; 8152 struct rte_flow *flow; 8153 uint32_t idx = 0; 8154 int ret = 0, i; 8155 8156 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8157 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8158 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8159 dev->data->port_id, (void *)flow); 8160 ret++; 8161 } 8162 } 8163 return ret; 8164 } 8165 8166 /** 8167 * Enable default hairpin egress flow. 8168 * 8169 * @param dev 8170 * Pointer to Ethernet device. 8171 * @param sq_num 8172 * The SQ hw number. 8173 * 8174 * @return 8175 * 0 on success, a negative errno value otherwise and rte_errno is set. 8176 */ 8177 int 8178 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8179 uint32_t sq_num) 8180 { 8181 const struct rte_flow_attr attr = { 8182 .egress = 1, 8183 .priority = 0, 8184 }; 8185 struct mlx5_rte_flow_item_sq queue_spec = { 8186 .queue = sq_num, 8187 }; 8188 struct mlx5_rte_flow_item_sq queue_mask = { 8189 .queue = UINT32_MAX, 8190 }; 8191 struct rte_flow_item items[] = { 8192 { 8193 .type = (enum rte_flow_item_type) 8194 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8195 .spec = &queue_spec, 8196 .last = NULL, 8197 .mask = &queue_mask, 8198 }, 8199 { 8200 .type = RTE_FLOW_ITEM_TYPE_END, 8201 }, 8202 }; 8203 struct rte_flow_action_jump jump = { 8204 .group = MLX5_HAIRPIN_TX_TABLE, 8205 }; 8206 struct rte_flow_action actions[2]; 8207 uint32_t flow_idx; 8208 struct rte_flow_error error; 8209 8210 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8211 actions[0].conf = &jump; 8212 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8213 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8214 &attr, items, actions, false, &error); 8215 if (!flow_idx) { 8216 DRV_LOG(DEBUG, 8217 "Failed to create ctrl flow: rte_errno(%d)," 8218 " type(%d), message(%s)", 8219 rte_errno, error.type, 8220 error.message ? error.message : " (no stated reason)"); 8221 return -rte_errno; 8222 } 8223 return 0; 8224 } 8225 8226 /** 8227 * Enable a control flow configured from the control plane. 8228 * 8229 * @param dev 8230 * Pointer to Ethernet device. 8231 * @param eth_spec 8232 * An Ethernet flow spec to apply. 8233 * @param eth_mask 8234 * An Ethernet flow mask to apply. 8235 * @param vlan_spec 8236 * A VLAN flow spec to apply. 8237 * @param vlan_mask 8238 * A VLAN flow mask to apply. 8239 * 8240 * @return 8241 * 0 on success, a negative errno value otherwise and rte_errno is set. 8242 */ 8243 int 8244 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8245 struct rte_flow_item_eth *eth_spec, 8246 struct rte_flow_item_eth *eth_mask, 8247 struct rte_flow_item_vlan *vlan_spec, 8248 struct rte_flow_item_vlan *vlan_mask) 8249 { 8250 struct mlx5_priv *priv = dev->data->dev_private; 8251 const struct rte_flow_attr attr = { 8252 .ingress = 1, 8253 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8254 }; 8255 struct rte_flow_item items[] = { 8256 { 8257 .type = RTE_FLOW_ITEM_TYPE_ETH, 8258 .spec = eth_spec, 8259 .last = NULL, 8260 .mask = eth_mask, 8261 }, 8262 { 8263 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8264 RTE_FLOW_ITEM_TYPE_END, 8265 .spec = vlan_spec, 8266 .last = NULL, 8267 .mask = vlan_mask, 8268 }, 8269 { 8270 .type = RTE_FLOW_ITEM_TYPE_END, 8271 }, 8272 }; 8273 uint16_t queue[priv->reta_idx_n]; 8274 struct rte_flow_action_rss action_rss = { 8275 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8276 .level = 0, 8277 .types = priv->rss_conf.rss_hf, 8278 .key_len = priv->rss_conf.rss_key_len, 8279 .queue_num = priv->reta_idx_n, 8280 .key = priv->rss_conf.rss_key, 8281 .queue = queue, 8282 }; 8283 struct rte_flow_action actions[] = { 8284 { 8285 .type = RTE_FLOW_ACTION_TYPE_RSS, 8286 .conf = &action_rss, 8287 }, 8288 { 8289 .type = RTE_FLOW_ACTION_TYPE_END, 8290 }, 8291 }; 8292 uint32_t flow_idx; 8293 struct rte_flow_error error; 8294 unsigned int i; 8295 8296 if (!priv->reta_idx_n || !priv->rxqs_n) { 8297 return 0; 8298 } 8299 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8300 action_rss.types = 0; 8301 for (i = 0; i != priv->reta_idx_n; ++i) 8302 queue[i] = (*priv->reta_idx)[i]; 8303 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8304 &attr, items, actions, false, &error); 8305 if (!flow_idx) 8306 return -rte_errno; 8307 return 0; 8308 } 8309 8310 /** 8311 * Enable a flow control configured from the control plane. 8312 * 8313 * @param dev 8314 * Pointer to Ethernet device. 8315 * @param eth_spec 8316 * An Ethernet flow spec to apply. 8317 * @param eth_mask 8318 * An Ethernet flow mask to apply. 8319 * 8320 * @return 8321 * 0 on success, a negative errno value otherwise and rte_errno is set. 8322 */ 8323 int 8324 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8325 struct rte_flow_item_eth *eth_spec, 8326 struct rte_flow_item_eth *eth_mask) 8327 { 8328 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8329 } 8330 8331 /** 8332 * Create default miss flow rule matching lacp traffic 8333 * 8334 * @param dev 8335 * Pointer to Ethernet device. 8336 * @param eth_spec 8337 * An Ethernet flow spec to apply. 8338 * 8339 * @return 8340 * 0 on success, a negative errno value otherwise and rte_errno is set. 8341 */ 8342 int 8343 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8344 { 8345 /* 8346 * The LACP matching is done by only using ether type since using 8347 * a multicast dst mac causes kernel to give low priority to this flow. 8348 */ 8349 static const struct rte_flow_item_eth lacp_spec = { 8350 .hdr.ether_type = RTE_BE16(0x8809), 8351 }; 8352 static const struct rte_flow_item_eth lacp_mask = { 8353 .hdr.ether_type = 0xffff, 8354 }; 8355 const struct rte_flow_attr attr = { 8356 .ingress = 1, 8357 }; 8358 struct rte_flow_item items[] = { 8359 { 8360 .type = RTE_FLOW_ITEM_TYPE_ETH, 8361 .spec = &lacp_spec, 8362 .mask = &lacp_mask, 8363 }, 8364 { 8365 .type = RTE_FLOW_ITEM_TYPE_END, 8366 }, 8367 }; 8368 struct rte_flow_action actions[] = { 8369 { 8370 .type = (enum rte_flow_action_type) 8371 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8372 }, 8373 { 8374 .type = RTE_FLOW_ACTION_TYPE_END, 8375 }, 8376 }; 8377 struct rte_flow_error error; 8378 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8379 &attr, items, actions, 8380 false, &error); 8381 8382 if (!flow_idx) 8383 return -rte_errno; 8384 return 0; 8385 } 8386 8387 /** 8388 * Destroy a flow. 8389 * 8390 * @see rte_flow_destroy() 8391 * @see rte_flow_ops 8392 */ 8393 int 8394 mlx5_flow_destroy(struct rte_eth_dev *dev, 8395 struct rte_flow *flow, 8396 struct rte_flow_error *error __rte_unused) 8397 { 8398 struct mlx5_priv *priv = dev->data->dev_private; 8399 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8400 struct mlx5_dv_flow_info *flow_info; 8401 8402 if (priv->sh->config.dv_flow_en == 2) 8403 return rte_flow_error_set(error, ENOTSUP, 8404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8405 NULL, 8406 "Flow non-Q destruction not supported"); 8407 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8408 (uintptr_t)(void *)flow); 8409 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8410 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8411 while (flow_info) { 8412 /* Romove the cache flow info. */ 8413 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8414 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8415 LIST_REMOVE(flow_info, next); 8416 mlx5_free(flow_info->items); 8417 mlx5_free(flow_info->actions); 8418 mlx5_free(flow_info); 8419 break; 8420 } 8421 flow_info = LIST_NEXT(flow_info, next); 8422 } 8423 } 8424 return 0; 8425 } 8426 8427 /** 8428 * Destroy all flows. 8429 * 8430 * @see rte_flow_flush() 8431 * @see rte_flow_ops 8432 */ 8433 int 8434 mlx5_flow_flush(struct rte_eth_dev *dev, 8435 struct rte_flow_error *error __rte_unused) 8436 { 8437 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8438 return 0; 8439 } 8440 8441 /** 8442 * Isolated mode. 8443 * 8444 * @see rte_flow_isolate() 8445 * @see rte_flow_ops 8446 */ 8447 int 8448 mlx5_flow_isolate(struct rte_eth_dev *dev, 8449 int enable, 8450 struct rte_flow_error *error) 8451 { 8452 struct mlx5_priv *priv = dev->data->dev_private; 8453 8454 if (dev->data->dev_started) { 8455 rte_flow_error_set(error, EBUSY, 8456 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8457 NULL, 8458 "port must be stopped first"); 8459 return -rte_errno; 8460 } 8461 if (!enable && !priv->sh->config.repr_matching) 8462 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8463 "isolated mode cannot be disabled when " 8464 "representor matching is disabled"); 8465 priv->isolated = !!enable; 8466 if (enable) 8467 dev->dev_ops = &mlx5_dev_ops_isolate; 8468 else 8469 dev->dev_ops = &mlx5_dev_ops; 8470 8471 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8472 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8473 8474 return 0; 8475 } 8476 8477 /** 8478 * Query a flow. 8479 * 8480 * @see rte_flow_query() 8481 * @see rte_flow_ops 8482 */ 8483 static int 8484 flow_drv_query(struct rte_eth_dev *dev, 8485 struct rte_flow *eflow, 8486 const struct rte_flow_action *actions, 8487 void *data, 8488 struct rte_flow_error *error) 8489 { 8490 struct mlx5_priv *priv = dev->data->dev_private; 8491 const struct mlx5_flow_driver_ops *fops; 8492 struct rte_flow *flow = NULL; 8493 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8494 8495 if (priv->sh->config.dv_flow_en == 2) { 8496 #ifdef HAVE_MLX5_HWS_SUPPORT 8497 flow = eflow; 8498 ftype = MLX5_FLOW_TYPE_HW; 8499 #endif 8500 } else { 8501 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8502 (uintptr_t)(void *)eflow); 8503 } 8504 if (!flow) { 8505 return rte_flow_error_set(error, ENOENT, 8506 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8507 NULL, 8508 "invalid flow handle"); 8509 } 8510 if (ftype == MLX5_FLOW_TYPE_MIN) 8511 ftype = flow->drv_type; 8512 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8513 fops = flow_get_drv_ops(ftype); 8514 8515 return fops->query(dev, flow, actions, data, error); 8516 } 8517 8518 /** 8519 * Query a flow. 8520 * 8521 * @see rte_flow_query() 8522 * @see rte_flow_ops 8523 */ 8524 int 8525 mlx5_flow_query(struct rte_eth_dev *dev, 8526 struct rte_flow *flow, 8527 const struct rte_flow_action *actions, 8528 void *data, 8529 struct rte_flow_error *error) 8530 { 8531 int ret; 8532 8533 ret = flow_drv_query(dev, flow, actions, data, 8534 error); 8535 if (ret < 0) 8536 return ret; 8537 return 0; 8538 } 8539 8540 /** 8541 * Get rte_flow callbacks. 8542 * 8543 * @param dev 8544 * Pointer to Ethernet device structure. 8545 * @param ops 8546 * Pointer to operation-specific structure. 8547 * 8548 * @return 0 8549 */ 8550 int 8551 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8552 const struct rte_flow_ops **ops) 8553 { 8554 *ops = &mlx5_flow_ops; 8555 return 0; 8556 } 8557 8558 /** 8559 * Validate meter policy actions. 8560 * Dispatcher for action type specific validation. 8561 * 8562 * @param[in] dev 8563 * Pointer to the Ethernet device structure. 8564 * @param[in] action 8565 * The meter policy action object to validate. 8566 * @param[in] attr 8567 * Attributes of flow to determine steering domain. 8568 * @param[out] is_rss 8569 * Is RSS or not. 8570 * @param[out] domain_bitmap 8571 * Domain bitmap. 8572 * @param[out] is_def_policy 8573 * Is default policy or not. 8574 * @param[out] error 8575 * Perform verbose error reporting if not NULL. Initialized in case of 8576 * error only. 8577 * 8578 * @return 8579 * 0 on success, otherwise negative errno value. 8580 */ 8581 int 8582 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8583 const struct rte_flow_action *actions[RTE_COLORS], 8584 struct rte_flow_attr *attr, 8585 bool *is_rss, 8586 uint8_t *domain_bitmap, 8587 uint8_t *policy_mode, 8588 struct rte_mtr_error *error) 8589 { 8590 const struct mlx5_flow_driver_ops *fops; 8591 8592 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8593 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8594 domain_bitmap, policy_mode, error); 8595 } 8596 8597 /** 8598 * Destroy the meter table set. 8599 * 8600 * @param[in] dev 8601 * Pointer to Ethernet device. 8602 * @param[in] mtr_policy 8603 * Meter policy struct. 8604 */ 8605 void 8606 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8607 struct mlx5_flow_meter_policy *mtr_policy) 8608 { 8609 const struct mlx5_flow_driver_ops *fops; 8610 8611 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8612 fops->destroy_mtr_acts(dev, mtr_policy); 8613 } 8614 8615 /** 8616 * Create policy action, lock free, 8617 * (mutex should be acquired by caller). 8618 * Dispatcher for action type specific call. 8619 * 8620 * @param[in] dev 8621 * Pointer to the Ethernet device structure. 8622 * @param[in] mtr_policy 8623 * Meter policy struct. 8624 * @param[in] action 8625 * Action specification used to create meter actions. 8626 * @param[in] attr 8627 * Flow rule attributes. 8628 * @param[out] error 8629 * Perform verbose error reporting if not NULL. Initialized in case of 8630 * error only. 8631 * 8632 * @return 8633 * 0 on success, otherwise negative errno value. 8634 */ 8635 int 8636 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8637 struct mlx5_flow_meter_policy *mtr_policy, 8638 const struct rte_flow_action *actions[RTE_COLORS], 8639 struct rte_flow_attr *attr, 8640 struct rte_mtr_error *error) 8641 { 8642 const struct mlx5_flow_driver_ops *fops; 8643 8644 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8645 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8646 } 8647 8648 /** 8649 * Create policy rules, lock free, 8650 * (mutex should be acquired by caller). 8651 * Dispatcher for action type specific call. 8652 * 8653 * @param[in] dev 8654 * Pointer to the Ethernet device structure. 8655 * @param[in] mtr_policy 8656 * Meter policy struct. 8657 * 8658 * @return 8659 * 0 on success, -1 otherwise. 8660 */ 8661 int 8662 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8663 struct mlx5_flow_meter_policy *mtr_policy) 8664 { 8665 const struct mlx5_flow_driver_ops *fops; 8666 8667 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8668 return fops->create_policy_rules(dev, mtr_policy); 8669 } 8670 8671 /** 8672 * Destroy policy rules, lock free, 8673 * (mutex should be acquired by caller). 8674 * Dispatcher for action type specific call. 8675 * 8676 * @param[in] dev 8677 * Pointer to the Ethernet device structure. 8678 * @param[in] mtr_policy 8679 * Meter policy struct. 8680 */ 8681 void 8682 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8683 struct mlx5_flow_meter_policy *mtr_policy) 8684 { 8685 const struct mlx5_flow_driver_ops *fops; 8686 8687 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8688 fops->destroy_policy_rules(dev, mtr_policy); 8689 } 8690 8691 /** 8692 * Destroy the default policy table set. 8693 * 8694 * @param[in] dev 8695 * Pointer to Ethernet device. 8696 */ 8697 void 8698 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8699 { 8700 const struct mlx5_flow_driver_ops *fops; 8701 8702 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8703 fops->destroy_def_policy(dev); 8704 } 8705 8706 /** 8707 * Destroy the default policy table set. 8708 * 8709 * @param[in] dev 8710 * Pointer to Ethernet device. 8711 * 8712 * @return 8713 * 0 on success, -1 otherwise. 8714 */ 8715 int 8716 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8717 { 8718 const struct mlx5_flow_driver_ops *fops; 8719 8720 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8721 return fops->create_def_policy(dev); 8722 } 8723 8724 /** 8725 * Create the needed meter and suffix tables. 8726 * 8727 * @param[in] dev 8728 * Pointer to Ethernet device. 8729 * 8730 * @return 8731 * 0 on success, -1 otherwise. 8732 */ 8733 int 8734 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8735 struct mlx5_flow_meter_info *fm, 8736 uint32_t mtr_idx, 8737 uint8_t domain_bitmap) 8738 { 8739 const struct mlx5_flow_driver_ops *fops; 8740 8741 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8742 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8743 } 8744 8745 /** 8746 * Destroy the meter table set. 8747 * 8748 * @param[in] dev 8749 * Pointer to Ethernet device. 8750 * @param[in] tbl 8751 * Pointer to the meter table set. 8752 */ 8753 void 8754 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8755 struct mlx5_flow_meter_info *fm) 8756 { 8757 const struct mlx5_flow_driver_ops *fops; 8758 8759 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8760 fops->destroy_mtr_tbls(dev, fm); 8761 } 8762 8763 /** 8764 * Destroy the global meter drop table. 8765 * 8766 * @param[in] dev 8767 * Pointer to Ethernet device. 8768 */ 8769 void 8770 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8771 { 8772 const struct mlx5_flow_driver_ops *fops; 8773 8774 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8775 fops->destroy_mtr_drop_tbls(dev); 8776 } 8777 8778 /** 8779 * Destroy the sub policy table with RX queue. 8780 * 8781 * @param[in] dev 8782 * Pointer to Ethernet device. 8783 * @param[in] mtr_policy 8784 * Pointer to meter policy table. 8785 */ 8786 void 8787 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8788 struct mlx5_flow_meter_policy *mtr_policy) 8789 { 8790 const struct mlx5_flow_driver_ops *fops; 8791 8792 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8793 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8794 } 8795 8796 /** 8797 * Allocate the needed aso flow meter id. 8798 * 8799 * @param[in] dev 8800 * Pointer to Ethernet device. 8801 * 8802 * @return 8803 * Index to aso flow meter on success, NULL otherwise. 8804 */ 8805 uint32_t 8806 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8807 { 8808 const struct mlx5_flow_driver_ops *fops; 8809 8810 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8811 return fops->create_meter(dev); 8812 } 8813 8814 /** 8815 * Free the aso flow meter id. 8816 * 8817 * @param[in] dev 8818 * Pointer to Ethernet device. 8819 * @param[in] mtr_idx 8820 * Index to aso flow meter to be free. 8821 * 8822 * @return 8823 * 0 on success. 8824 */ 8825 void 8826 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8827 { 8828 const struct mlx5_flow_driver_ops *fops; 8829 8830 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8831 fops->free_meter(dev, mtr_idx); 8832 } 8833 8834 /** 8835 * Allocate a counter. 8836 * 8837 * @param[in] dev 8838 * Pointer to Ethernet device structure. 8839 * 8840 * @return 8841 * Index to allocated counter on success, 0 otherwise. 8842 */ 8843 uint32_t 8844 mlx5_counter_alloc(struct rte_eth_dev *dev) 8845 { 8846 struct rte_flow_attr attr = { .transfer = 0 }; 8847 8848 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8849 (dev); 8850 } 8851 8852 /** 8853 * Free a counter. 8854 * 8855 * @param[in] dev 8856 * Pointer to Ethernet device structure. 8857 * @param[in] cnt 8858 * Index to counter to be free. 8859 */ 8860 void 8861 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8862 { 8863 struct rte_flow_attr attr = { .transfer = 0 }; 8864 8865 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8866 } 8867 8868 /** 8869 * Query counter statistics. 8870 * 8871 * @param[in] dev 8872 * Pointer to Ethernet device structure. 8873 * @param[in] cnt 8874 * Index to counter to query. 8875 * @param[in] clear 8876 * Set to clear counter statistics. 8877 * @param[out] pkts 8878 * The counter hits packets number to save. 8879 * @param[out] bytes 8880 * The counter hits bytes number to save. 8881 * 8882 * @return 8883 * 0 on success, a negative errno value otherwise. 8884 */ 8885 int 8886 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8887 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8888 { 8889 struct rte_flow_attr attr = { .transfer = 0 }; 8890 8891 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8892 (dev, cnt, clear, pkts, bytes, action); 8893 } 8894 8895 /** 8896 * Get information about HWS pre-configurable resources. 8897 * 8898 * @param[in] dev 8899 * Pointer to the rte_eth_dev structure. 8900 * @param[out] port_info 8901 * Pointer to port information. 8902 * @param[out] queue_info 8903 * Pointer to queue information. 8904 * @param[out] error 8905 * Pointer to error structure. 8906 * 8907 * @return 8908 * 0 on success, a negative errno value otherwise and rte_errno is set. 8909 */ 8910 static int 8911 mlx5_flow_info_get(struct rte_eth_dev *dev, 8912 struct rte_flow_port_info *port_info, 8913 struct rte_flow_queue_info *queue_info, 8914 struct rte_flow_error *error) 8915 { 8916 const struct mlx5_flow_driver_ops *fops; 8917 struct rte_flow_attr attr = {0}; 8918 8919 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8920 return rte_flow_error_set(error, ENOTSUP, 8921 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8922 NULL, 8923 "info get with incorrect steering mode"); 8924 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8925 return fops->info_get(dev, port_info, queue_info, error); 8926 } 8927 8928 /** 8929 * Configure port HWS resources. 8930 * 8931 * @param[in] dev 8932 * Pointer to the rte_eth_dev structure. 8933 * @param[in] port_attr 8934 * Port configuration attributes. 8935 * @param[in] nb_queue 8936 * Number of queue. 8937 * @param[in] queue_attr 8938 * Array that holds attributes for each flow queue. 8939 * @param[out] error 8940 * Pointer to error structure. 8941 * 8942 * @return 8943 * 0 on success, a negative errno value otherwise and rte_errno is set. 8944 */ 8945 static int 8946 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8947 const struct rte_flow_port_attr *port_attr, 8948 uint16_t nb_queue, 8949 const struct rte_flow_queue_attr *queue_attr[], 8950 struct rte_flow_error *error) 8951 { 8952 const struct mlx5_flow_driver_ops *fops; 8953 struct rte_flow_attr attr = {0}; 8954 8955 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8956 return rte_flow_error_set(error, ENOTSUP, 8957 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8958 NULL, 8959 "port configure with incorrect steering mode"); 8960 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8961 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8962 } 8963 8964 /** 8965 * Validate item template. 8966 * 8967 * @param[in] dev 8968 * Pointer to the rte_eth_dev structure. 8969 * @param[in] attr 8970 * Pointer to the item template attributes. 8971 * @param[in] items 8972 * The template item pattern. 8973 * @param[out] error 8974 * Pointer to error structure. 8975 * 8976 * @return 8977 * 0 on success, a negative errno value otherwise and rte_errno is set. 8978 */ 8979 int 8980 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8981 const struct rte_flow_pattern_template_attr *attr, 8982 const struct rte_flow_item items[], 8983 struct rte_flow_error *error) 8984 { 8985 const struct mlx5_flow_driver_ops *fops; 8986 struct rte_flow_attr fattr = {0}; 8987 8988 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8989 rte_flow_error_set(error, ENOTSUP, 8990 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8991 "pattern validate with incorrect steering mode"); 8992 return -ENOTSUP; 8993 } 8994 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8995 return fops->pattern_validate(dev, attr, items, error); 8996 } 8997 8998 /** 8999 * Create flow item template. 9000 * 9001 * @param[in] dev 9002 * Pointer to the rte_eth_dev structure. 9003 * @param[in] attr 9004 * Pointer to the item template attributes. 9005 * @param[in] items 9006 * The template item pattern. 9007 * @param[out] error 9008 * Pointer to error structure. 9009 * 9010 * @return 9011 * 0 on success, a negative errno value otherwise and rte_errno is set. 9012 */ 9013 static struct rte_flow_pattern_template * 9014 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9015 const struct rte_flow_pattern_template_attr *attr, 9016 const struct rte_flow_item items[], 9017 struct rte_flow_error *error) 9018 { 9019 const struct mlx5_flow_driver_ops *fops; 9020 struct rte_flow_attr fattr = {0}; 9021 9022 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9023 rte_flow_error_set(error, ENOTSUP, 9024 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9025 NULL, 9026 "pattern create with incorrect steering mode"); 9027 return NULL; 9028 } 9029 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9030 return fops->pattern_template_create(dev, attr, items, error); 9031 } 9032 9033 /** 9034 * Destroy flow item template. 9035 * 9036 * @param[in] dev 9037 * Pointer to the rte_eth_dev structure. 9038 * @param[in] template 9039 * Pointer to the item template to be destroyed. 9040 * @param[out] error 9041 * Pointer to error structure. 9042 * 9043 * @return 9044 * 0 on success, a negative errno value otherwise and rte_errno is set. 9045 */ 9046 static int 9047 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9048 struct rte_flow_pattern_template *template, 9049 struct rte_flow_error *error) 9050 { 9051 const struct mlx5_flow_driver_ops *fops; 9052 struct rte_flow_attr attr = {0}; 9053 9054 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9055 return rte_flow_error_set(error, ENOTSUP, 9056 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9057 NULL, 9058 "pattern destroy with incorrect steering mode"); 9059 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9060 return fops->pattern_template_destroy(dev, template, error); 9061 } 9062 9063 /** 9064 * Validate flow actions template. 9065 * 9066 * @param[in] dev 9067 * Pointer to the rte_eth_dev structure. 9068 * @param[in] attr 9069 * Pointer to the action template attributes. 9070 * @param[in] actions 9071 * Associated actions (list terminated by the END action). 9072 * @param[in] masks 9073 * List of actions that marks which of the action's member is constant. 9074 * @param[out] error 9075 * Pointer to error structure. 9076 * 9077 * @return 9078 * 0 on success, a negative errno value otherwise and rte_errno is set. 9079 */ 9080 int 9081 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9082 const struct rte_flow_actions_template_attr *attr, 9083 const struct rte_flow_action actions[], 9084 const struct rte_flow_action masks[], 9085 struct rte_flow_error *error) 9086 { 9087 const struct mlx5_flow_driver_ops *fops; 9088 struct rte_flow_attr fattr = {0}; 9089 9090 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9091 rte_flow_error_set(error, ENOTSUP, 9092 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9093 "actions validate with incorrect steering mode"); 9094 return -ENOTSUP; 9095 } 9096 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9097 return fops->actions_validate(dev, attr, actions, masks, error); 9098 } 9099 9100 /** 9101 * Create flow item template. 9102 * 9103 * @param[in] dev 9104 * Pointer to the rte_eth_dev structure. 9105 * @param[in] attr 9106 * Pointer to the action template attributes. 9107 * @param[in] actions 9108 * Associated actions (list terminated by the END action). 9109 * @param[in] masks 9110 * List of actions that marks which of the action's member is constant. 9111 * @param[out] error 9112 * Pointer to error structure. 9113 * 9114 * @return 9115 * 0 on success, a negative errno value otherwise and rte_errno is set. 9116 */ 9117 static struct rte_flow_actions_template * 9118 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9119 const struct rte_flow_actions_template_attr *attr, 9120 const struct rte_flow_action actions[], 9121 const struct rte_flow_action masks[], 9122 struct rte_flow_error *error) 9123 { 9124 const struct mlx5_flow_driver_ops *fops; 9125 struct rte_flow_attr fattr = {0}; 9126 9127 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9128 rte_flow_error_set(error, ENOTSUP, 9129 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9130 NULL, 9131 "action create with incorrect steering mode"); 9132 return NULL; 9133 } 9134 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9135 return fops->actions_template_create(dev, attr, actions, masks, error); 9136 } 9137 9138 /** 9139 * Destroy flow action template. 9140 * 9141 * @param[in] dev 9142 * Pointer to the rte_eth_dev structure. 9143 * @param[in] template 9144 * Pointer to the action template to be destroyed. 9145 * @param[out] error 9146 * Pointer to error structure. 9147 * 9148 * @return 9149 * 0 on success, a negative errno value otherwise and rte_errno is set. 9150 */ 9151 static int 9152 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9153 struct rte_flow_actions_template *template, 9154 struct rte_flow_error *error) 9155 { 9156 const struct mlx5_flow_driver_ops *fops; 9157 struct rte_flow_attr attr = {0}; 9158 9159 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9160 return rte_flow_error_set(error, ENOTSUP, 9161 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9162 NULL, 9163 "action destroy with incorrect steering mode"); 9164 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9165 return fops->actions_template_destroy(dev, template, error); 9166 } 9167 9168 /** 9169 * Create flow table. 9170 * 9171 * @param[in] dev 9172 * Pointer to the rte_eth_dev structure. 9173 * @param[in] attr 9174 * Pointer to the table attributes. 9175 * @param[in] item_templates 9176 * Item template array to be binded to the table. 9177 * @param[in] nb_item_templates 9178 * Number of item template. 9179 * @param[in] action_templates 9180 * Action template array to be binded to the table. 9181 * @param[in] nb_action_templates 9182 * Number of action template. 9183 * @param[out] error 9184 * Pointer to error structure. 9185 * 9186 * @return 9187 * Table on success, NULL otherwise and rte_errno is set. 9188 */ 9189 static struct rte_flow_template_table * 9190 mlx5_flow_table_create(struct rte_eth_dev *dev, 9191 const struct rte_flow_template_table_attr *attr, 9192 struct rte_flow_pattern_template *item_templates[], 9193 uint8_t nb_item_templates, 9194 struct rte_flow_actions_template *action_templates[], 9195 uint8_t nb_action_templates, 9196 struct rte_flow_error *error) 9197 { 9198 const struct mlx5_flow_driver_ops *fops; 9199 struct rte_flow_attr fattr = {0}; 9200 9201 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9202 rte_flow_error_set(error, ENOTSUP, 9203 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9204 NULL, 9205 "table create with incorrect steering mode"); 9206 return NULL; 9207 } 9208 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9209 return fops->template_table_create(dev, 9210 attr, 9211 item_templates, 9212 nb_item_templates, 9213 action_templates, 9214 nb_action_templates, 9215 error); 9216 } 9217 9218 /** 9219 * PMD destroy flow table. 9220 * 9221 * @param[in] dev 9222 * Pointer to the rte_eth_dev structure. 9223 * @param[in] table 9224 * Pointer to the table to be destroyed. 9225 * @param[out] error 9226 * Pointer to error structure. 9227 * 9228 * @return 9229 * 0 on success, a negative errno value otherwise and rte_errno is set. 9230 */ 9231 static int 9232 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9233 struct rte_flow_template_table *table, 9234 struct rte_flow_error *error) 9235 { 9236 const struct mlx5_flow_driver_ops *fops; 9237 struct rte_flow_attr attr = {0}; 9238 9239 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9240 return rte_flow_error_set(error, ENOTSUP, 9241 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9242 NULL, 9243 "table destroy with incorrect steering mode"); 9244 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9245 return fops->template_table_destroy(dev, table, error); 9246 } 9247 9248 /** 9249 * Enqueue flow creation. 9250 * 9251 * @param[in] dev 9252 * Pointer to the rte_eth_dev structure. 9253 * @param[in] queue_id 9254 * The queue to create the flow. 9255 * @param[in] attr 9256 * Pointer to the flow operation attributes. 9257 * @param[in] items 9258 * Items with flow spec value. 9259 * @param[in] pattern_template_index 9260 * The item pattern flow follows from the table. 9261 * @param[in] actions 9262 * Action with flow spec value. 9263 * @param[in] action_template_index 9264 * The action pattern flow follows from the table. 9265 * @param[in] user_data 9266 * Pointer to the user_data. 9267 * @param[out] error 9268 * Pointer to error structure. 9269 * 9270 * @return 9271 * Flow pointer on success, NULL otherwise and rte_errno is set. 9272 */ 9273 static struct rte_flow * 9274 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 9275 uint32_t queue_id, 9276 const struct rte_flow_op_attr *attr, 9277 struct rte_flow_template_table *table, 9278 const struct rte_flow_item items[], 9279 uint8_t pattern_template_index, 9280 const struct rte_flow_action actions[], 9281 uint8_t action_template_index, 9282 void *user_data, 9283 struct rte_flow_error *error) 9284 { 9285 const struct mlx5_flow_driver_ops *fops; 9286 struct rte_flow_attr fattr = {0}; 9287 9288 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9289 rte_flow_error_set(error, ENOTSUP, 9290 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9291 NULL, 9292 "flow_q create with incorrect steering mode"); 9293 return NULL; 9294 } 9295 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9296 return fops->async_flow_create(dev, queue_id, attr, table, 9297 items, pattern_template_index, 9298 actions, action_template_index, 9299 user_data, error); 9300 } 9301 9302 /** 9303 * Enqueue flow creation by index. 9304 * 9305 * @param[in] dev 9306 * Pointer to the rte_eth_dev structure. 9307 * @param[in] queue_id 9308 * The queue to create the flow. 9309 * @param[in] attr 9310 * Pointer to the flow operation attributes. 9311 * @param[in] rule_index 9312 * The item pattern flow follows from the table. 9313 * @param[in] actions 9314 * Action with flow spec value. 9315 * @param[in] action_template_index 9316 * The action pattern flow follows from the table. 9317 * @param[in] user_data 9318 * Pointer to the user_data. 9319 * @param[out] error 9320 * Pointer to error structure. 9321 * 9322 * @return 9323 * Flow pointer on success, NULL otherwise and rte_errno is set. 9324 */ 9325 static struct rte_flow * 9326 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 9327 uint32_t queue_id, 9328 const struct rte_flow_op_attr *attr, 9329 struct rte_flow_template_table *table, 9330 uint32_t rule_index, 9331 const struct rte_flow_action actions[], 9332 uint8_t action_template_index, 9333 void *user_data, 9334 struct rte_flow_error *error) 9335 { 9336 const struct mlx5_flow_driver_ops *fops; 9337 struct rte_flow_attr fattr = {0}; 9338 9339 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9340 rte_flow_error_set(error, ENOTSUP, 9341 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9342 NULL, 9343 "flow_q create with incorrect steering mode"); 9344 return NULL; 9345 } 9346 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9347 return fops->async_flow_create_by_index(dev, queue_id, attr, table, 9348 rule_index, actions, action_template_index, 9349 user_data, error); 9350 } 9351 9352 /** 9353 * Enqueue flow destruction. 9354 * 9355 * @param[in] dev 9356 * Pointer to the rte_eth_dev structure. 9357 * @param[in] queue 9358 * The queue to destroy the flow. 9359 * @param[in] attr 9360 * Pointer to the flow operation attributes. 9361 * @param[in] flow 9362 * Pointer to the flow to be destroyed. 9363 * @param[in] user_data 9364 * Pointer to the user_data. 9365 * @param[out] error 9366 * Pointer to error structure. 9367 * 9368 * @return 9369 * 0 on success, negative value otherwise and rte_errno is set. 9370 */ 9371 static int 9372 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 9373 uint32_t queue, 9374 const struct rte_flow_op_attr *attr, 9375 struct rte_flow *flow, 9376 void *user_data, 9377 struct rte_flow_error *error) 9378 { 9379 const struct mlx5_flow_driver_ops *fops; 9380 struct rte_flow_attr fattr = {0}; 9381 9382 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9383 return rte_flow_error_set(error, ENOTSUP, 9384 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9385 NULL, 9386 "flow_q destroy with incorrect steering mode"); 9387 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9388 return fops->async_flow_destroy(dev, queue, attr, flow, 9389 user_data, error); 9390 } 9391 9392 /** 9393 * Pull the enqueued flows. 9394 * 9395 * @param[in] dev 9396 * Pointer to the rte_eth_dev structure. 9397 * @param[in] queue 9398 * The queue to pull the result. 9399 * @param[in/out] res 9400 * Array to save the results. 9401 * @param[in] n_res 9402 * Available result with the array. 9403 * @param[out] error 9404 * Pointer to error structure. 9405 * 9406 * @return 9407 * Result number on success, negative value otherwise and rte_errno is set. 9408 */ 9409 static int 9410 mlx5_flow_pull(struct rte_eth_dev *dev, 9411 uint32_t queue, 9412 struct rte_flow_op_result res[], 9413 uint16_t n_res, 9414 struct rte_flow_error *error) 9415 { 9416 const struct mlx5_flow_driver_ops *fops; 9417 struct rte_flow_attr attr = {0}; 9418 9419 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9420 return rte_flow_error_set(error, ENOTSUP, 9421 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9422 NULL, 9423 "flow_q pull with incorrect steering mode"); 9424 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9425 return fops->pull(dev, queue, res, n_res, error); 9426 } 9427 9428 /** 9429 * Push the enqueued flows. 9430 * 9431 * @param[in] dev 9432 * Pointer to the rte_eth_dev structure. 9433 * @param[in] queue 9434 * The queue to push the flows. 9435 * @param[out] error 9436 * Pointer to error structure. 9437 * 9438 * @return 9439 * 0 on success, negative value otherwise and rte_errno is set. 9440 */ 9441 static int 9442 mlx5_flow_push(struct rte_eth_dev *dev, 9443 uint32_t queue, 9444 struct rte_flow_error *error) 9445 { 9446 const struct mlx5_flow_driver_ops *fops; 9447 struct rte_flow_attr attr = {0}; 9448 9449 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9450 return rte_flow_error_set(error, ENOTSUP, 9451 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9452 NULL, 9453 "flow_q push with incorrect steering mode"); 9454 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9455 return fops->push(dev, queue, error); 9456 } 9457 9458 /** 9459 * Create shared action. 9460 * 9461 * @param[in] dev 9462 * Pointer to the rte_eth_dev structure. 9463 * @param[in] queue 9464 * Which queue to be used.. 9465 * @param[in] attr 9466 * Operation attribute. 9467 * @param[in] conf 9468 * Indirect action configuration. 9469 * @param[in] action 9470 * rte_flow action detail. 9471 * @param[in] user_data 9472 * Pointer to the user_data. 9473 * @param[out] error 9474 * Pointer to error structure. 9475 * 9476 * @return 9477 * Action handle on success, NULL otherwise and rte_errno is set. 9478 */ 9479 static struct rte_flow_action_handle * 9480 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 9481 const struct rte_flow_op_attr *attr, 9482 const struct rte_flow_indir_action_conf *conf, 9483 const struct rte_flow_action *action, 9484 void *user_data, 9485 struct rte_flow_error *error) 9486 { 9487 const struct mlx5_flow_driver_ops *fops = 9488 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9489 9490 return fops->async_action_create(dev, queue, attr, conf, action, 9491 user_data, error); 9492 } 9493 9494 /** 9495 * Update shared action. 9496 * 9497 * @param[in] dev 9498 * Pointer to the rte_eth_dev structure. 9499 * @param[in] queue 9500 * Which queue to be used.. 9501 * @param[in] attr 9502 * Operation attribute. 9503 * @param[in] handle 9504 * Action handle to be updated. 9505 * @param[in] update 9506 * Update value. 9507 * @param[in] user_data 9508 * Pointer to the user_data. 9509 * @param[out] error 9510 * Pointer to error structure. 9511 * 9512 * @return 9513 * 0 on success, negative value otherwise and rte_errno is set. 9514 */ 9515 static int 9516 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 9517 const struct rte_flow_op_attr *attr, 9518 struct rte_flow_action_handle *handle, 9519 const void *update, 9520 void *user_data, 9521 struct rte_flow_error *error) 9522 { 9523 const struct mlx5_flow_driver_ops *fops = 9524 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9525 9526 return fops->async_action_update(dev, queue, attr, handle, 9527 update, user_data, error); 9528 } 9529 9530 static int 9531 mlx5_flow_async_action_handle_query_update 9532 (struct rte_eth_dev *dev, uint32_t queue_id, 9533 const struct rte_flow_op_attr *op_attr, 9534 struct rte_flow_action_handle *action_handle, 9535 const void *update, void *query, 9536 enum rte_flow_query_update_mode qu_mode, 9537 void *user_data, struct rte_flow_error *error) 9538 { 9539 const struct mlx5_flow_driver_ops *fops = 9540 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9541 9542 if (!fops || !fops->async_action_query_update) 9543 return rte_flow_error_set(error, ENOTSUP, 9544 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 9545 "async query_update not supported"); 9546 return fops->async_action_query_update 9547 (dev, queue_id, op_attr, action_handle, 9548 update, query, qu_mode, user_data, error); 9549 } 9550 9551 /** 9552 * Query shared action. 9553 * 9554 * @param[in] dev 9555 * Pointer to the rte_eth_dev structure. 9556 * @param[in] queue 9557 * Which queue to be used.. 9558 * @param[in] attr 9559 * Operation attribute. 9560 * @param[in] handle 9561 * Action handle to be updated. 9562 * @param[in] data 9563 * Pointer query result data. 9564 * @param[in] user_data 9565 * Pointer to the user_data. 9566 * @param[out] error 9567 * Pointer to error structure. 9568 * 9569 * @return 9570 * 0 on success, negative value otherwise and rte_errno is set. 9571 */ 9572 static int 9573 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9574 const struct rte_flow_op_attr *attr, 9575 const struct rte_flow_action_handle *handle, 9576 void *data, 9577 void *user_data, 9578 struct rte_flow_error *error) 9579 { 9580 const struct mlx5_flow_driver_ops *fops = 9581 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9582 9583 return fops->async_action_query(dev, queue, attr, handle, 9584 data, user_data, error); 9585 } 9586 9587 /** 9588 * Destroy shared action. 9589 * 9590 * @param[in] dev 9591 * Pointer to the rte_eth_dev structure. 9592 * @param[in] queue 9593 * Which queue to be used.. 9594 * @param[in] attr 9595 * Operation attribute. 9596 * @param[in] handle 9597 * Action handle to be destroyed. 9598 * @param[in] user_data 9599 * Pointer to the user_data. 9600 * @param[out] error 9601 * Pointer to error structure. 9602 * 9603 * @return 9604 * 0 on success, negative value otherwise and rte_errno is set. 9605 */ 9606 static int 9607 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9608 const struct rte_flow_op_attr *attr, 9609 struct rte_flow_action_handle *handle, 9610 void *user_data, 9611 struct rte_flow_error *error) 9612 { 9613 const struct mlx5_flow_driver_ops *fops = 9614 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9615 9616 return fops->async_action_destroy(dev, queue, attr, handle, 9617 user_data, error); 9618 } 9619 9620 /** 9621 * Allocate a new memory for the counter values wrapped by all the needed 9622 * management. 9623 * 9624 * @param[in] sh 9625 * Pointer to mlx5_dev_ctx_shared object. 9626 * 9627 * @return 9628 * 0 on success, a negative errno value otherwise. 9629 */ 9630 static int 9631 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9632 { 9633 struct mlx5_counter_stats_mem_mng *mem_mng; 9634 volatile struct flow_counter_stats *raw_data; 9635 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9636 int size = (sizeof(struct flow_counter_stats) * 9637 MLX5_COUNTERS_PER_POOL + 9638 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9639 sizeof(struct mlx5_counter_stats_mem_mng); 9640 size_t pgsize = rte_mem_page_size(); 9641 uint8_t *mem; 9642 int ret; 9643 int i; 9644 9645 if (pgsize == (size_t)-1) { 9646 DRV_LOG(ERR, "Failed to get mem page size"); 9647 rte_errno = ENOMEM; 9648 return -ENOMEM; 9649 } 9650 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9651 if (!mem) { 9652 rte_errno = ENOMEM; 9653 return -ENOMEM; 9654 } 9655 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9656 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9657 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9658 sh->cdev->pdn, mem, size, 9659 &mem_mng->wm); 9660 if (ret) { 9661 rte_errno = errno; 9662 mlx5_free(mem); 9663 return -rte_errno; 9664 } 9665 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9666 raw_data = (volatile struct flow_counter_stats *)mem; 9667 for (i = 0; i < raws_n; ++i) { 9668 mem_mng->raws[i].mem_mng = mem_mng; 9669 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9670 } 9671 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9672 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9673 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9674 next); 9675 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9676 sh->sws_cmng.mem_mng = mem_mng; 9677 return 0; 9678 } 9679 9680 /** 9681 * Set the statistic memory to the new counter pool. 9682 * 9683 * @param[in] sh 9684 * Pointer to mlx5_dev_ctx_shared object. 9685 * @param[in] pool 9686 * Pointer to the pool to set the statistic memory. 9687 * 9688 * @return 9689 * 0 on success, a negative errno value otherwise. 9690 */ 9691 static int 9692 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9693 struct mlx5_flow_counter_pool *pool) 9694 { 9695 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9696 /* Resize statistic memory once used out. */ 9697 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9698 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9699 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9700 return -1; 9701 } 9702 rte_spinlock_lock(&pool->sl); 9703 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9704 rte_spinlock_unlock(&pool->sl); 9705 pool->raw_hw = NULL; 9706 return 0; 9707 } 9708 9709 #define MLX5_POOL_QUERY_FREQ_US 1000000 9710 9711 /** 9712 * Set the periodic procedure for triggering asynchronous batch queries for all 9713 * the counter pools. 9714 * 9715 * @param[in] sh 9716 * Pointer to mlx5_dev_ctx_shared object. 9717 */ 9718 void 9719 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9720 { 9721 uint32_t pools_n, us; 9722 9723 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9724 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9725 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9726 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9727 sh->sws_cmng.query_thread_on = 0; 9728 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9729 } else { 9730 sh->sws_cmng.query_thread_on = 1; 9731 } 9732 } 9733 9734 /** 9735 * The periodic procedure for triggering asynchronous batch queries for all the 9736 * counter pools. This function is probably called by the host thread. 9737 * 9738 * @param[in] arg 9739 * The parameter for the alarm process. 9740 */ 9741 void 9742 mlx5_flow_query_alarm(void *arg) 9743 { 9744 struct mlx5_dev_ctx_shared *sh = arg; 9745 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9746 uint16_t pool_index = cmng->pool_index; 9747 struct mlx5_flow_counter_pool *pool; 9748 uint16_t n_valid; 9749 int ret; 9750 9751 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9752 goto set_alarm; 9753 rte_spinlock_lock(&cmng->pool_update_sl); 9754 pool = cmng->pools[pool_index]; 9755 n_valid = cmng->n_valid; 9756 rte_spinlock_unlock(&cmng->pool_update_sl); 9757 /* Set the statistic memory to the new created pool. */ 9758 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9759 goto set_alarm; 9760 if (pool->raw_hw) 9761 /* There is a pool query in progress. */ 9762 goto set_alarm; 9763 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9764 if (!pool->raw_hw) 9765 /* No free counter statistics raw memory. */ 9766 goto set_alarm; 9767 /* 9768 * Identify the counters released between query trigger and query 9769 * handle more efficiently. The counter released in this gap period 9770 * should wait for a new round of query as the new arrived packets 9771 * will not be taken into account. 9772 */ 9773 pool->query_gen++; 9774 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9775 MLX5_COUNTERS_PER_POOL, 9776 NULL, NULL, 9777 pool->raw_hw->mem_mng->wm.lkey, 9778 (void *)(uintptr_t) 9779 pool->raw_hw->data, 9780 sh->devx_comp, 9781 (uint64_t)(uintptr_t)pool); 9782 if (ret) { 9783 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9784 " %d", pool->min_dcs->id); 9785 pool->raw_hw = NULL; 9786 goto set_alarm; 9787 } 9788 LIST_REMOVE(pool->raw_hw, next); 9789 cmng->pending_queries++; 9790 pool_index++; 9791 if (pool_index >= n_valid) 9792 pool_index = 0; 9793 set_alarm: 9794 cmng->pool_index = pool_index; 9795 mlx5_set_query_alarm(sh); 9796 } 9797 9798 /** 9799 * Check and callback event for new aged flow in the counter pool 9800 * 9801 * @param[in] sh 9802 * Pointer to mlx5_dev_ctx_shared object. 9803 * @param[in] pool 9804 * Pointer to Current counter pool. 9805 */ 9806 static void 9807 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9808 struct mlx5_flow_counter_pool *pool) 9809 { 9810 struct mlx5_priv *priv; 9811 struct mlx5_flow_counter *cnt; 9812 struct mlx5_age_info *age_info; 9813 struct mlx5_age_param *age_param; 9814 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9815 struct mlx5_counter_stats_raw *prev = pool->raw; 9816 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9817 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9818 uint16_t expected = AGE_CANDIDATE; 9819 uint32_t i; 9820 9821 pool->time_of_last_age_check = curr_time; 9822 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9823 cnt = MLX5_POOL_GET_CNT(pool, i); 9824 age_param = MLX5_CNT_TO_AGE(cnt); 9825 if (__atomic_load_n(&age_param->state, 9826 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9827 continue; 9828 if (cur->data[i].hits != prev->data[i].hits) { 9829 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9830 __ATOMIC_RELAXED); 9831 continue; 9832 } 9833 if (__atomic_fetch_add(&age_param->sec_since_last_hit, 9834 time_delta, 9835 __ATOMIC_RELAXED) + time_delta <= age_param->timeout) 9836 continue; 9837 /** 9838 * Hold the lock first, or if between the 9839 * state AGE_TMOUT and tailq operation the 9840 * release happened, the release procedure 9841 * may delete a non-existent tailq node. 9842 */ 9843 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9844 age_info = GET_PORT_AGE_INFO(priv); 9845 rte_spinlock_lock(&age_info->aged_sl); 9846 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9847 AGE_TMOUT, false, 9848 __ATOMIC_RELAXED, 9849 __ATOMIC_RELAXED)) { 9850 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9851 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9852 } 9853 rte_spinlock_unlock(&age_info->aged_sl); 9854 } 9855 mlx5_age_event_prepare(sh); 9856 } 9857 9858 /** 9859 * Handler for the HW respond about ready values from an asynchronous batch 9860 * query. This function is probably called by the host thread. 9861 * 9862 * @param[in] sh 9863 * The pointer to the shared device context. 9864 * @param[in] async_id 9865 * The Devx async ID. 9866 * @param[in] status 9867 * The status of the completion. 9868 */ 9869 void 9870 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9871 uint64_t async_id, int status) 9872 { 9873 struct mlx5_flow_counter_pool *pool = 9874 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9875 struct mlx5_counter_stats_raw *raw_to_free; 9876 uint8_t query_gen = pool->query_gen ^ 1; 9877 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9878 enum mlx5_counter_type cnt_type = 9879 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9880 MLX5_COUNTER_TYPE_ORIGIN; 9881 9882 if (unlikely(status)) { 9883 raw_to_free = pool->raw_hw; 9884 } else { 9885 raw_to_free = pool->raw; 9886 if (pool->is_aged) 9887 mlx5_flow_aging_check(sh, pool); 9888 rte_spinlock_lock(&pool->sl); 9889 pool->raw = pool->raw_hw; 9890 rte_spinlock_unlock(&pool->sl); 9891 /* Be sure the new raw counters data is updated in memory. */ 9892 rte_io_wmb(); 9893 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9894 rte_spinlock_lock(&cmng->csl[cnt_type]); 9895 TAILQ_CONCAT(&cmng->counters[cnt_type], 9896 &pool->counters[query_gen], next); 9897 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9898 } 9899 } 9900 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9901 pool->raw_hw = NULL; 9902 sh->sws_cmng.pending_queries--; 9903 } 9904 9905 static int 9906 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9907 const struct flow_grp_info *grp_info, 9908 struct rte_flow_error *error) 9909 { 9910 if (grp_info->transfer && grp_info->external && 9911 grp_info->fdb_def_rule) { 9912 if (group == UINT32_MAX) 9913 return rte_flow_error_set 9914 (error, EINVAL, 9915 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9916 NULL, 9917 "group index not supported"); 9918 *table = group + 1; 9919 } else { 9920 *table = group; 9921 } 9922 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9923 return 0; 9924 } 9925 9926 /** 9927 * Translate the rte_flow group index to HW table value. 9928 * 9929 * If tunnel offload is disabled, all group ids converted to flow table 9930 * id using the standard method. 9931 * If tunnel offload is enabled, group id can be converted using the 9932 * standard or tunnel conversion method. Group conversion method 9933 * selection depends on flags in `grp_info` parameter: 9934 * - Internal (grp_info.external == 0) groups conversion uses the 9935 * standard method. 9936 * - Group ids in JUMP action converted with the tunnel conversion. 9937 * - Group id in rule attribute conversion depends on a rule type and 9938 * group id value: 9939 * ** non zero group attributes converted with the tunnel method 9940 * ** zero group attribute in non-tunnel rule is converted using the 9941 * standard method - there's only one root table 9942 * ** zero group attribute in steer tunnel rule is converted with the 9943 * standard method - single root table 9944 * ** zero group attribute in match tunnel rule is a special OvS 9945 * case: that value is used for portability reasons. That group 9946 * id is converted with the tunnel conversion method. 9947 * 9948 * @param[in] dev 9949 * Port device 9950 * @param[in] tunnel 9951 * PMD tunnel offload object 9952 * @param[in] group 9953 * rte_flow group index value. 9954 * @param[out] table 9955 * HW table value. 9956 * @param[in] grp_info 9957 * flags used for conversion 9958 * @param[out] error 9959 * Pointer to error structure. 9960 * 9961 * @return 9962 * 0 on success, a negative errno value otherwise and rte_errno is set. 9963 */ 9964 int 9965 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9966 const struct mlx5_flow_tunnel *tunnel, 9967 uint32_t group, uint32_t *table, 9968 const struct flow_grp_info *grp_info, 9969 struct rte_flow_error *error) 9970 { 9971 int ret; 9972 bool standard_translation; 9973 9974 if (!grp_info->skip_scale && grp_info->external && 9975 group < MLX5_MAX_TABLES_EXTERNAL) 9976 group *= MLX5_FLOW_TABLE_FACTOR; 9977 if (is_tunnel_offload_active(dev)) { 9978 standard_translation = !grp_info->external || 9979 grp_info->std_tbl_fix; 9980 } else { 9981 standard_translation = true; 9982 } 9983 DRV_LOG(DEBUG, 9984 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9985 dev->data->port_id, group, grp_info->transfer, 9986 grp_info->external, grp_info->fdb_def_rule, 9987 standard_translation ? "STANDARD" : "TUNNEL"); 9988 if (standard_translation) 9989 ret = flow_group_to_table(dev->data->port_id, group, table, 9990 grp_info, error); 9991 else 9992 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9993 table, error); 9994 9995 return ret; 9996 } 9997 9998 /** 9999 * Discover availability of metadata reg_c's. 10000 * 10001 * Iteratively use test flows to check availability. 10002 * 10003 * @param[in] dev 10004 * Pointer to the Ethernet device structure. 10005 * 10006 * @return 10007 * 0 on success, a negative errno value otherwise and rte_errno is set. 10008 */ 10009 int 10010 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 10011 { 10012 struct mlx5_priv *priv = dev->data->dev_private; 10013 enum modify_reg idx; 10014 int n = 0; 10015 10016 /* reg_c[0] and reg_c[1] are reserved. */ 10017 priv->sh->flow_mreg_c[n++] = REG_C_0; 10018 priv->sh->flow_mreg_c[n++] = REG_C_1; 10019 /* Discover availability of other reg_c's. */ 10020 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 10021 struct rte_flow_attr attr = { 10022 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 10023 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 10024 .ingress = 1, 10025 }; 10026 struct rte_flow_item items[] = { 10027 [0] = { 10028 .type = RTE_FLOW_ITEM_TYPE_END, 10029 }, 10030 }; 10031 struct rte_flow_action actions[] = { 10032 [0] = { 10033 .type = (enum rte_flow_action_type) 10034 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 10035 .conf = &(struct mlx5_flow_action_copy_mreg){ 10036 .src = REG_C_1, 10037 .dst = idx, 10038 }, 10039 }, 10040 [1] = { 10041 .type = RTE_FLOW_ACTION_TYPE_JUMP, 10042 .conf = &(struct rte_flow_action_jump){ 10043 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 10044 }, 10045 }, 10046 [2] = { 10047 .type = RTE_FLOW_ACTION_TYPE_END, 10048 }, 10049 }; 10050 uint32_t flow_idx; 10051 struct rte_flow *flow; 10052 struct rte_flow_error error; 10053 10054 if (!priv->sh->config.dv_flow_en) 10055 break; 10056 /* Create internal flow, validation skips copy action. */ 10057 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 10058 items, actions, false, &error); 10059 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10060 flow_idx); 10061 if (!flow) 10062 continue; 10063 priv->sh->flow_mreg_c[n++] = idx; 10064 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 10065 } 10066 for (; n < MLX5_MREG_C_NUM; ++n) 10067 priv->sh->flow_mreg_c[n] = REG_NON; 10068 priv->sh->metadata_regc_check_flag = 1; 10069 return 0; 10070 } 10071 10072 int 10073 save_dump_file(const uint8_t *data, uint32_t size, 10074 uint32_t type, uint64_t id, void *arg, FILE *file) 10075 { 10076 char line[BUF_SIZE]; 10077 uint32_t out = 0; 10078 uint32_t k; 10079 uint32_t actions_num; 10080 struct rte_flow_query_count *count; 10081 10082 memset(line, 0, BUF_SIZE); 10083 switch (type) { 10084 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 10085 actions_num = *(uint32_t *)(arg); 10086 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 10087 type, id, actions_num); 10088 break; 10089 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 10090 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 10091 type, id); 10092 break; 10093 case DR_DUMP_REC_TYPE_PMD_COUNTER: 10094 count = (struct rte_flow_query_count *)arg; 10095 fprintf(file, 10096 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 10097 type, id, count->hits, count->bytes); 10098 return 0; 10099 default: 10100 return -1; 10101 } 10102 10103 for (k = 0; k < size; k++) { 10104 /* Make sure we do not overrun the line buffer length. */ 10105 if (out >= BUF_SIZE - 4) { 10106 line[out] = '\0'; 10107 break; 10108 } 10109 out += snprintf(line + out, BUF_SIZE - out, "%02x", 10110 (data[k]) & 0xff); 10111 } 10112 fprintf(file, "%s\n", line); 10113 return 0; 10114 } 10115 10116 int 10117 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 10118 struct rte_flow_query_count *count, struct rte_flow_error *error) 10119 { 10120 struct rte_flow_action action[2]; 10121 enum mlx5_flow_drv_type ftype; 10122 const struct mlx5_flow_driver_ops *fops; 10123 10124 if (!flow) { 10125 return rte_flow_error_set(error, ENOENT, 10126 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10127 NULL, 10128 "invalid flow handle"); 10129 } 10130 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 10131 action[1].type = RTE_FLOW_ACTION_TYPE_END; 10132 if (flow->counter) { 10133 memset(count, 0, sizeof(struct rte_flow_query_count)); 10134 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 10135 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 10136 ftype < MLX5_FLOW_TYPE_MAX); 10137 fops = flow_get_drv_ops(ftype); 10138 return fops->query(dev, flow, action, count, error); 10139 } 10140 return -1; 10141 } 10142 10143 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10144 /** 10145 * Dump flow ipool data to file 10146 * 10147 * @param[in] dev 10148 * The pointer to Ethernet device. 10149 * @param[in] file 10150 * A pointer to a file for output. 10151 * @param[out] error 10152 * Perform verbose error reporting if not NULL. PMDs initialize this 10153 * structure in case of error only. 10154 * @return 10155 * 0 on success, a negative value otherwise. 10156 */ 10157 int 10158 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 10159 struct rte_flow *flow, FILE *file, 10160 struct rte_flow_error *error) 10161 { 10162 struct mlx5_priv *priv = dev->data->dev_private; 10163 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10164 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10165 uint32_t handle_idx; 10166 struct mlx5_flow_handle *dh; 10167 struct rte_flow_query_count count; 10168 uint32_t actions_num; 10169 const uint8_t *data; 10170 size_t size; 10171 uint64_t id; 10172 uint32_t type; 10173 void *action = NULL; 10174 10175 if (!flow) { 10176 return rte_flow_error_set(error, ENOENT, 10177 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10178 NULL, 10179 "invalid flow handle"); 10180 } 10181 handle_idx = flow->dev_handles; 10182 /* query counter */ 10183 if (flow->counter && 10184 (!mlx5_counter_query(dev, flow->counter, false, 10185 &count.hits, &count.bytes, &action)) && action) { 10186 id = (uint64_t)(uintptr_t)action; 10187 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10188 save_dump_file(NULL, 0, type, 10189 id, (void *)&count, file); 10190 } 10191 10192 while (handle_idx) { 10193 dh = mlx5_ipool_get(priv->sh->ipool 10194 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 10195 if (!dh) 10196 continue; 10197 handle_idx = dh->next.next; 10198 10199 /* Get modify_hdr and encap_decap buf from ipools. */ 10200 encap_decap = NULL; 10201 modify_hdr = dh->dvh.modify_hdr; 10202 10203 if (dh->dvh.rix_encap_decap) { 10204 encap_decap = mlx5_ipool_get(priv->sh->ipool 10205 [MLX5_IPOOL_DECAP_ENCAP], 10206 dh->dvh.rix_encap_decap); 10207 } 10208 if (modify_hdr) { 10209 data = (const uint8_t *)modify_hdr->actions; 10210 size = (size_t)(modify_hdr->actions_num) * 8; 10211 id = (uint64_t)(uintptr_t)modify_hdr->action; 10212 actions_num = modify_hdr->actions_num; 10213 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10214 save_dump_file(data, size, type, id, 10215 (void *)(&actions_num), file); 10216 } 10217 if (encap_decap) { 10218 data = encap_decap->buf; 10219 size = encap_decap->size; 10220 id = (uint64_t)(uintptr_t)encap_decap->action; 10221 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10222 save_dump_file(data, size, type, 10223 id, NULL, file); 10224 } 10225 } 10226 return 0; 10227 } 10228 10229 /** 10230 * Dump all flow's encap_decap/modify_hdr/counter data to file 10231 * 10232 * @param[in] dev 10233 * The pointer to Ethernet device. 10234 * @param[in] file 10235 * A pointer to a file for output. 10236 * @param[out] error 10237 * Perform verbose error reporting if not NULL. PMDs initialize this 10238 * structure in case of error only. 10239 * @return 10240 * 0 on success, a negative value otherwise. 10241 */ 10242 static int 10243 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 10244 FILE *file, struct rte_flow_error *error __rte_unused) 10245 { 10246 struct mlx5_priv *priv = dev->data->dev_private; 10247 struct mlx5_dev_ctx_shared *sh = priv->sh; 10248 struct mlx5_hlist *h; 10249 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10250 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10251 struct rte_flow_query_count count; 10252 uint32_t actions_num; 10253 const uint8_t *data; 10254 size_t size; 10255 uint64_t id; 10256 uint32_t type; 10257 uint32_t i; 10258 uint32_t j; 10259 struct mlx5_list_inconst *l_inconst; 10260 struct mlx5_list_entry *e; 10261 int lcore_index; 10262 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 10263 uint32_t max; 10264 void *action; 10265 10266 /* encap_decap hlist is lcore_share, get global core cache. */ 10267 i = MLX5_LIST_GLOBAL; 10268 h = sh->encaps_decaps; 10269 if (h) { 10270 for (j = 0; j <= h->mask; j++) { 10271 l_inconst = &h->buckets[j].l; 10272 if (!l_inconst || !l_inconst->cache[i]) 10273 continue; 10274 10275 e = LIST_FIRST(&l_inconst->cache[i]->h); 10276 while (e) { 10277 encap_decap = 10278 (struct mlx5_flow_dv_encap_decap_resource *)e; 10279 data = encap_decap->buf; 10280 size = encap_decap->size; 10281 id = (uint64_t)(uintptr_t)encap_decap->action; 10282 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10283 save_dump_file(data, size, type, 10284 id, NULL, file); 10285 e = LIST_NEXT(e, next); 10286 } 10287 } 10288 } 10289 10290 /* get modify_hdr */ 10291 h = sh->modify_cmds; 10292 if (h) { 10293 lcore_index = rte_lcore_index(rte_lcore_id()); 10294 if (unlikely(lcore_index == -1)) { 10295 lcore_index = MLX5_LIST_NLCORE; 10296 rte_spinlock_lock(&h->l_const.lcore_lock); 10297 } 10298 i = lcore_index; 10299 10300 for (j = 0; j <= h->mask; j++) { 10301 l_inconst = &h->buckets[j].l; 10302 if (!l_inconst || !l_inconst->cache[i]) 10303 continue; 10304 10305 e = LIST_FIRST(&l_inconst->cache[i]->h); 10306 while (e) { 10307 modify_hdr = 10308 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10309 data = (const uint8_t *)modify_hdr->actions; 10310 size = (size_t)(modify_hdr->actions_num) * 8; 10311 actions_num = modify_hdr->actions_num; 10312 id = (uint64_t)(uintptr_t)modify_hdr->action; 10313 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10314 save_dump_file(data, size, type, id, 10315 (void *)(&actions_num), file); 10316 e = LIST_NEXT(e, next); 10317 } 10318 } 10319 10320 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10321 rte_spinlock_unlock(&h->l_const.lcore_lock); 10322 } 10323 10324 /* get counter */ 10325 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10326 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10327 for (j = 1; j <= max; j++) { 10328 action = NULL; 10329 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10330 &count.bytes, &action)) && action) { 10331 id = (uint64_t)(uintptr_t)action; 10332 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10333 save_dump_file(NULL, 0, type, 10334 id, (void *)&count, file); 10335 } 10336 } 10337 return 0; 10338 } 10339 #endif 10340 10341 /** 10342 * Dump flow raw hw data to file 10343 * 10344 * @param[in] dev 10345 * The pointer to Ethernet device. 10346 * @param[in] file 10347 * A pointer to a file for output. 10348 * @param[out] error 10349 * Perform verbose error reporting if not NULL. PMDs initialize this 10350 * structure in case of error only. 10351 * @return 10352 * 0 on success, a negative value otherwise. 10353 */ 10354 int 10355 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10356 FILE *file, 10357 struct rte_flow_error *error __rte_unused) 10358 { 10359 struct mlx5_priv *priv = dev->data->dev_private; 10360 struct mlx5_dev_ctx_shared *sh = priv->sh; 10361 uint32_t handle_idx; 10362 int ret; 10363 struct mlx5_flow_handle *dh; 10364 struct rte_flow *flow; 10365 10366 if (!sh->config.dv_flow_en) { 10367 if (fputs("device dv flow disabled\n", file) <= 0) 10368 return -errno; 10369 return -ENOTSUP; 10370 } 10371 10372 /* dump all */ 10373 if (!flow_idx) { 10374 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10375 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10376 return -EINVAL; 10377 10378 if (sh->config.dv_flow_en == 2) 10379 return mlx5dr_debug_dump(priv->dr_ctx, file); 10380 #endif 10381 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10382 sh->rx_domain, 10383 sh->tx_domain, file); 10384 } 10385 /* dump one */ 10386 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10387 (uintptr_t)(void *)flow_idx); 10388 if (!flow) 10389 return -EINVAL; 10390 10391 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10392 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10393 #endif 10394 handle_idx = flow->dev_handles; 10395 while (handle_idx) { 10396 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10397 handle_idx); 10398 if (!dh) 10399 return -ENOENT; 10400 if (dh->drv_flow) { 10401 if (sh->config.dv_flow_en == 2) 10402 return -ENOTSUP; 10403 10404 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10405 file); 10406 if (ret) 10407 return -ENOENT; 10408 } 10409 handle_idx = dh->next.next; 10410 } 10411 return 0; 10412 } 10413 10414 /** 10415 * Get aged-out flows. 10416 * 10417 * @param[in] dev 10418 * Pointer to the Ethernet device structure. 10419 * @param[in] context 10420 * The address of an array of pointers to the aged-out flows contexts. 10421 * @param[in] nb_countexts 10422 * The length of context array pointers. 10423 * @param[out] error 10424 * Perform verbose error reporting if not NULL. Initialized in case of 10425 * error only. 10426 * 10427 * @return 10428 * how many contexts get in success, otherwise negative errno value. 10429 * if nb_contexts is 0, return the amount of all aged contexts. 10430 * if nb_contexts is not 0 , return the amount of aged flows reported 10431 * in the context array. 10432 */ 10433 int 10434 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10435 uint32_t nb_contexts, struct rte_flow_error *error) 10436 { 10437 struct rte_flow_attr attr = { .transfer = 0 }; 10438 10439 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10440 (dev, contexts, nb_contexts, error); 10441 } 10442 10443 /** 10444 * Get aged-out flows per HWS queue. 10445 * 10446 * @param[in] dev 10447 * Pointer to the Ethernet device structure. 10448 * @param[in] queue_id 10449 * Flow queue to query. 10450 * @param[in] context 10451 * The address of an array of pointers to the aged-out flows contexts. 10452 * @param[in] nb_countexts 10453 * The length of context array pointers. 10454 * @param[out] error 10455 * Perform verbose error reporting if not NULL. Initialized in case of 10456 * error only. 10457 * 10458 * @return 10459 * how many contexts get in success, otherwise negative errno value. 10460 * if nb_contexts is 0, return the amount of all aged contexts. 10461 * if nb_contexts is not 0 , return the amount of aged flows reported 10462 * in the context array. 10463 */ 10464 int 10465 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10466 void **contexts, uint32_t nb_contexts, 10467 struct rte_flow_error *error) 10468 { 10469 const struct mlx5_flow_driver_ops *fops; 10470 struct rte_flow_attr attr = { 0 }; 10471 10472 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10473 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10474 return fops->get_q_aged_flows(dev, queue_id, contexts, 10475 nb_contexts, error); 10476 } 10477 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10478 dev->data->port_id, queue_id); 10479 return rte_flow_error_set(error, ENOTSUP, 10480 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10481 "get Q aged flows with incorrect steering mode"); 10482 } 10483 10484 /* Wrapper for driver action_validate op callback */ 10485 static int 10486 flow_drv_action_validate(struct rte_eth_dev *dev, 10487 const struct rte_flow_indir_action_conf *conf, 10488 const struct rte_flow_action *action, 10489 const struct mlx5_flow_driver_ops *fops, 10490 struct rte_flow_error *error) 10491 { 10492 static const char err_msg[] = "indirect action validation unsupported"; 10493 10494 if (!fops->action_validate) { 10495 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10496 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10497 NULL, err_msg); 10498 return -rte_errno; 10499 } 10500 return fops->action_validate(dev, conf, action, error); 10501 } 10502 10503 /** 10504 * Destroys the shared action by handle. 10505 * 10506 * @param dev 10507 * Pointer to Ethernet device structure. 10508 * @param[in] handle 10509 * Handle for the indirect action object to be destroyed. 10510 * @param[out] error 10511 * Perform verbose error reporting if not NULL. PMDs initialize this 10512 * structure in case of error only. 10513 * 10514 * @return 10515 * 0 on success, a negative errno value otherwise and rte_errno is set. 10516 * 10517 * @note: wrapper for driver action_create op callback. 10518 */ 10519 static int 10520 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10521 struct rte_flow_action_handle *handle, 10522 struct rte_flow_error *error) 10523 { 10524 static const char err_msg[] = "indirect action destruction unsupported"; 10525 struct rte_flow_attr attr = { .transfer = 0 }; 10526 const struct mlx5_flow_driver_ops *fops = 10527 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10528 10529 if (!fops->action_destroy) { 10530 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10531 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10532 NULL, err_msg); 10533 return -rte_errno; 10534 } 10535 return fops->action_destroy(dev, handle, error); 10536 } 10537 10538 /* Wrapper for driver action_destroy op callback */ 10539 static int 10540 flow_drv_action_update(struct rte_eth_dev *dev, 10541 struct rte_flow_action_handle *handle, 10542 const void *update, 10543 const struct mlx5_flow_driver_ops *fops, 10544 struct rte_flow_error *error) 10545 { 10546 static const char err_msg[] = "indirect action update unsupported"; 10547 10548 if (!fops->action_update) { 10549 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10550 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10551 NULL, err_msg); 10552 return -rte_errno; 10553 } 10554 return fops->action_update(dev, handle, update, error); 10555 } 10556 10557 /* Wrapper for driver action_destroy op callback */ 10558 static int 10559 flow_drv_action_query(struct rte_eth_dev *dev, 10560 const struct rte_flow_action_handle *handle, 10561 void *data, 10562 const struct mlx5_flow_driver_ops *fops, 10563 struct rte_flow_error *error) 10564 { 10565 static const char err_msg[] = "indirect action query unsupported"; 10566 10567 if (!fops->action_query) { 10568 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10569 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10570 NULL, err_msg); 10571 return -rte_errno; 10572 } 10573 return fops->action_query(dev, handle, data, error); 10574 } 10575 10576 /** 10577 * Create indirect action for reuse in multiple flow rules. 10578 * 10579 * @param dev 10580 * Pointer to Ethernet device structure. 10581 * @param conf 10582 * Pointer to indirect action object configuration. 10583 * @param[in] action 10584 * Action configuration for indirect action object creation. 10585 * @param[out] error 10586 * Perform verbose error reporting if not NULL. PMDs initialize this 10587 * structure in case of error only. 10588 * @return 10589 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10590 */ 10591 static struct rte_flow_action_handle * 10592 mlx5_action_handle_create(struct rte_eth_dev *dev, 10593 const struct rte_flow_indir_action_conf *conf, 10594 const struct rte_flow_action *action, 10595 struct rte_flow_error *error) 10596 { 10597 static const char err_msg[] = "indirect action creation unsupported"; 10598 struct rte_flow_attr attr = { .transfer = 0 }; 10599 const struct mlx5_flow_driver_ops *fops = 10600 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10601 10602 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10603 return NULL; 10604 if (!fops->action_create) { 10605 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10606 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10607 NULL, err_msg); 10608 return NULL; 10609 } 10610 return fops->action_create(dev, conf, action, error); 10611 } 10612 10613 /** 10614 * Updates inplace the indirect action configuration pointed by *handle* 10615 * with the configuration provided as *update* argument. 10616 * The update of the indirect action configuration effects all flow rules 10617 * reusing the action via handle. 10618 * 10619 * @param dev 10620 * Pointer to Ethernet device structure. 10621 * @param[in] handle 10622 * Handle for the indirect action to be updated. 10623 * @param[in] update 10624 * Action specification used to modify the action pointed by handle. 10625 * *update* could be of same type with the action pointed by the *handle* 10626 * handle argument, or some other structures like a wrapper, depending on 10627 * the indirect action type. 10628 * @param[out] error 10629 * Perform verbose error reporting if not NULL. PMDs initialize this 10630 * structure in case of error only. 10631 * 10632 * @return 10633 * 0 on success, a negative errno value otherwise and rte_errno is set. 10634 */ 10635 static int 10636 mlx5_action_handle_update(struct rte_eth_dev *dev, 10637 struct rte_flow_action_handle *handle, 10638 const void *update, 10639 struct rte_flow_error *error) 10640 { 10641 struct rte_flow_attr attr = { .transfer = 0 }; 10642 const struct mlx5_flow_driver_ops *fops = 10643 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10644 int ret; 10645 10646 ret = flow_drv_action_validate(dev, NULL, 10647 (const struct rte_flow_action *)update, fops, error); 10648 if (ret) 10649 return ret; 10650 return flow_drv_action_update(dev, handle, update, fops, 10651 error); 10652 } 10653 10654 /** 10655 * Query the indirect action by handle. 10656 * 10657 * This function allows retrieving action-specific data such as counters. 10658 * Data is gathered by special action which may be present/referenced in 10659 * more than one flow rule definition. 10660 * 10661 * see @RTE_FLOW_ACTION_TYPE_COUNT 10662 * 10663 * @param dev 10664 * Pointer to Ethernet device structure. 10665 * @param[in] handle 10666 * Handle for the indirect action to query. 10667 * @param[in, out] data 10668 * Pointer to storage for the associated query data type. 10669 * @param[out] error 10670 * Perform verbose error reporting if not NULL. PMDs initialize this 10671 * structure in case of error only. 10672 * 10673 * @return 10674 * 0 on success, a negative errno value otherwise and rte_errno is set. 10675 */ 10676 static int 10677 mlx5_action_handle_query(struct rte_eth_dev *dev, 10678 const struct rte_flow_action_handle *handle, 10679 void *data, 10680 struct rte_flow_error *error) 10681 { 10682 struct rte_flow_attr attr = { .transfer = 0 }; 10683 const struct mlx5_flow_driver_ops *fops = 10684 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10685 10686 return flow_drv_action_query(dev, handle, data, fops, error); 10687 } 10688 10689 static int 10690 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10691 struct rte_flow_action_handle *handle, 10692 const void *update, void *query, 10693 enum rte_flow_query_update_mode qu_mode, 10694 struct rte_flow_error *error) 10695 { 10696 struct rte_flow_attr attr = { .transfer = 0 }; 10697 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10698 const struct mlx5_flow_driver_ops *fops; 10699 10700 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10701 return rte_flow_error_set(error, ENOTSUP, 10702 RTE_FLOW_ERROR_TYPE_ACTION, 10703 NULL, "invalid driver type"); 10704 fops = flow_get_drv_ops(drv_type); 10705 if (!fops || !fops->action_query_update) 10706 return rte_flow_error_set(error, ENOTSUP, 10707 RTE_FLOW_ERROR_TYPE_ACTION, 10708 NULL, "no query_update handler"); 10709 return fops->action_query_update(dev, handle, update, 10710 query, qu_mode, error); 10711 } 10712 10713 /** 10714 * Destroy all indirect actions (shared RSS). 10715 * 10716 * @param dev 10717 * Pointer to Ethernet device. 10718 * 10719 * @return 10720 * 0 on success, a negative errno value otherwise and rte_errno is set. 10721 */ 10722 int 10723 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10724 { 10725 struct rte_flow_error error; 10726 struct mlx5_priv *priv = dev->data->dev_private; 10727 struct mlx5_shared_action_rss *shared_rss; 10728 int ret = 0; 10729 uint32_t idx; 10730 10731 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10732 priv->rss_shared_actions, idx, shared_rss, next) { 10733 ret |= mlx5_action_handle_destroy(dev, 10734 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10735 } 10736 return ret; 10737 } 10738 10739 /** 10740 * Validate existing indirect actions against current device configuration 10741 * and attach them to device resources. 10742 * 10743 * @param dev 10744 * Pointer to Ethernet device. 10745 * 10746 * @return 10747 * 0 on success, a negative errno value otherwise and rte_errno is set. 10748 */ 10749 int 10750 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10751 { 10752 struct mlx5_priv *priv = dev->data->dev_private; 10753 int ret = 0; 10754 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10755 10756 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10757 const char *message; 10758 uint32_t queue_idx; 10759 10760 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10761 ind_tbl->queues_n, 10762 &message, &queue_idx); 10763 if (ret != 0) { 10764 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10765 dev->data->port_id, ind_tbl->queues[queue_idx], 10766 message); 10767 break; 10768 } 10769 } 10770 if (ret != 0) 10771 return ret; 10772 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10773 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10774 if (ret != 0) { 10775 DRV_LOG(ERR, "Port %u could not attach " 10776 "indirection table obj %p", 10777 dev->data->port_id, (void *)ind_tbl); 10778 goto error; 10779 } 10780 } 10781 10782 return 0; 10783 error: 10784 ind_tbl_last = ind_tbl; 10785 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10786 if (ind_tbl == ind_tbl_last) 10787 break; 10788 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10789 DRV_LOG(CRIT, "Port %u could not detach " 10790 "indirection table obj %p on rollback", 10791 dev->data->port_id, (void *)ind_tbl); 10792 } 10793 return ret; 10794 } 10795 10796 /** 10797 * Detach indirect actions of the device from its resources. 10798 * 10799 * @param dev 10800 * Pointer to Ethernet device. 10801 * 10802 * @return 10803 * 0 on success, a negative errno value otherwise and rte_errno is set. 10804 */ 10805 int 10806 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10807 { 10808 struct mlx5_priv *priv = dev->data->dev_private; 10809 int ret = 0; 10810 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10811 10812 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10813 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10814 if (ret != 0) { 10815 DRV_LOG(ERR, "Port %u could not detach " 10816 "indirection table obj %p", 10817 dev->data->port_id, (void *)ind_tbl); 10818 goto error; 10819 } 10820 } 10821 return 0; 10822 error: 10823 ind_tbl_last = ind_tbl; 10824 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10825 if (ind_tbl == ind_tbl_last) 10826 break; 10827 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10828 DRV_LOG(CRIT, "Port %u could not attach " 10829 "indirection table obj %p on rollback", 10830 dev->data->port_id, (void *)ind_tbl); 10831 } 10832 return ret; 10833 } 10834 10835 #ifndef HAVE_MLX5DV_DR 10836 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10837 #else 10838 #define MLX5_DOMAIN_SYNC_FLOW \ 10839 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10840 #endif 10841 10842 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10843 { 10844 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10845 const struct mlx5_flow_driver_ops *fops; 10846 int ret; 10847 struct rte_flow_attr attr = { .transfer = 0 }; 10848 10849 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10850 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10851 if (ret > 0) 10852 ret = -ret; 10853 return ret; 10854 } 10855 10856 const struct mlx5_flow_tunnel * 10857 mlx5_get_tof(const struct rte_flow_item *item, 10858 const struct rte_flow_action *action, 10859 enum mlx5_tof_rule_type *rule_type) 10860 { 10861 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10862 if (item->type == (typeof(item->type)) 10863 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10864 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10865 return flow_items_to_tunnel(item); 10866 } 10867 } 10868 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10869 if (action->type == (typeof(action->type)) 10870 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10871 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10872 return flow_actions_to_tunnel(action); 10873 } 10874 } 10875 return NULL; 10876 } 10877 10878 /** 10879 * tunnel offload functionality is defined for DV environment only 10880 */ 10881 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10882 __extension__ 10883 union tunnel_offload_mark { 10884 uint32_t val; 10885 struct { 10886 uint32_t app_reserve:8; 10887 uint32_t table_id:15; 10888 uint32_t transfer:1; 10889 uint32_t _unused_:8; 10890 }; 10891 }; 10892 10893 static bool 10894 mlx5_access_tunnel_offload_db 10895 (struct rte_eth_dev *dev, 10896 bool (*match)(struct rte_eth_dev *, 10897 struct mlx5_flow_tunnel *, const void *), 10898 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10899 void (*miss)(struct rte_eth_dev *, void *), 10900 void *ctx, bool lock_op); 10901 10902 static int 10903 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10904 struct rte_flow *flow, 10905 const struct rte_flow_attr *attr, 10906 const struct rte_flow_action *app_actions, 10907 uint32_t flow_idx, 10908 const struct mlx5_flow_tunnel *tunnel, 10909 struct tunnel_default_miss_ctx *ctx, 10910 struct rte_flow_error *error) 10911 { 10912 struct mlx5_priv *priv = dev->data->dev_private; 10913 struct mlx5_flow *dev_flow; 10914 struct rte_flow_attr miss_attr = *attr; 10915 const struct rte_flow_item miss_items[2] = { 10916 { 10917 .type = RTE_FLOW_ITEM_TYPE_ETH, 10918 .spec = NULL, 10919 .last = NULL, 10920 .mask = NULL 10921 }, 10922 { 10923 .type = RTE_FLOW_ITEM_TYPE_END, 10924 .spec = NULL, 10925 .last = NULL, 10926 .mask = NULL 10927 } 10928 }; 10929 union tunnel_offload_mark mark_id; 10930 struct rte_flow_action_mark miss_mark; 10931 struct rte_flow_action miss_actions[3] = { 10932 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10933 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10934 }; 10935 const struct rte_flow_action_jump *jump_data; 10936 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10937 struct flow_grp_info grp_info = { 10938 .external = 1, 10939 .transfer = attr->transfer, 10940 .fdb_def_rule = !!priv->fdb_def_rule, 10941 .std_tbl_fix = 0, 10942 }; 10943 int ret; 10944 10945 if (!attr->transfer) { 10946 uint32_t q_size; 10947 10948 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10949 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10950 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10951 0, SOCKET_ID_ANY); 10952 if (!ctx->queue) 10953 return rte_flow_error_set 10954 (error, ENOMEM, 10955 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10956 NULL, "invalid default miss RSS"); 10957 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10958 ctx->action_rss.level = 0, 10959 ctx->action_rss.types = priv->rss_conf.rss_hf, 10960 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10961 ctx->action_rss.queue_num = priv->reta_idx_n, 10962 ctx->action_rss.key = priv->rss_conf.rss_key, 10963 ctx->action_rss.queue = ctx->queue; 10964 if (!priv->reta_idx_n || !priv->rxqs_n) 10965 return rte_flow_error_set 10966 (error, EINVAL, 10967 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10968 NULL, "invalid port configuration"); 10969 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10970 ctx->action_rss.types = 0; 10971 for (i = 0; i != priv->reta_idx_n; ++i) 10972 ctx->queue[i] = (*priv->reta_idx)[i]; 10973 } else { 10974 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10975 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10976 } 10977 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10978 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10979 jump_data = app_actions->conf; 10980 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10981 miss_attr.group = jump_data->group; 10982 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10983 &flow_table, &grp_info, error); 10984 if (ret) 10985 return rte_flow_error_set(error, EINVAL, 10986 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10987 NULL, "invalid tunnel id"); 10988 mark_id.app_reserve = 0; 10989 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10990 mark_id.transfer = !!attr->transfer; 10991 mark_id._unused_ = 0; 10992 miss_mark.id = mark_id.val; 10993 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10994 miss_items, miss_actions, flow_idx, error); 10995 if (!dev_flow) 10996 return -rte_errno; 10997 dev_flow->flow = flow; 10998 dev_flow->external = true; 10999 dev_flow->tunnel = tunnel; 11000 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 11001 /* Subflow object was created, we must include one in the list. */ 11002 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 11003 dev_flow->handle, next); 11004 DRV_LOG(DEBUG, 11005 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 11006 dev->data->port_id, tunnel->app_tunnel.type, 11007 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 11008 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 11009 miss_actions, error); 11010 if (!ret) 11011 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 11012 error); 11013 11014 return ret; 11015 } 11016 11017 static const struct mlx5_flow_tbl_data_entry * 11018 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 11019 { 11020 struct mlx5_priv *priv = dev->data->dev_private; 11021 struct mlx5_dev_ctx_shared *sh = priv->sh; 11022 struct mlx5_list_entry *he; 11023 union tunnel_offload_mark mbits = { .val = mark }; 11024 union mlx5_flow_tbl_key table_key = { 11025 { 11026 .level = tunnel_id_to_flow_tbl(mbits.table_id), 11027 .id = 0, 11028 .reserved = 0, 11029 .dummy = 0, 11030 .is_fdb = !!mbits.transfer, 11031 .is_egress = 0, 11032 } 11033 }; 11034 struct mlx5_flow_cb_ctx ctx = { 11035 .data = &table_key.v64, 11036 }; 11037 11038 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 11039 return he ? 11040 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 11041 } 11042 11043 static void 11044 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 11045 struct mlx5_list_entry *entry) 11046 { 11047 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11048 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11049 11050 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11051 tunnel_flow_tbl_to_id(tte->flow_table)); 11052 mlx5_free(tte); 11053 } 11054 11055 static int 11056 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 11057 struct mlx5_list_entry *entry, void *cb_ctx) 11058 { 11059 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11060 union tunnel_tbl_key tbl = { 11061 .val = *(uint64_t *)(ctx->data), 11062 }; 11063 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11064 11065 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 11066 } 11067 11068 static struct mlx5_list_entry * 11069 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 11070 { 11071 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11072 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11073 struct tunnel_tbl_entry *tte; 11074 union tunnel_tbl_key tbl = { 11075 .val = *(uint64_t *)(ctx->data), 11076 }; 11077 11078 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 11079 sizeof(*tte), 0, 11080 SOCKET_ID_ANY); 11081 if (!tte) 11082 goto err; 11083 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11084 &tte->flow_table); 11085 if (tte->flow_table >= MLX5_MAX_TABLES) { 11086 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 11087 tte->flow_table); 11088 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11089 tte->flow_table); 11090 goto err; 11091 } else if (!tte->flow_table) { 11092 goto err; 11093 } 11094 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 11095 tte->tunnel_id = tbl.tunnel_id; 11096 tte->group = tbl.group; 11097 return &tte->hash; 11098 err: 11099 if (tte) 11100 mlx5_free(tte); 11101 return NULL; 11102 } 11103 11104 static struct mlx5_list_entry * 11105 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 11106 struct mlx5_list_entry *oentry, 11107 void *cb_ctx __rte_unused) 11108 { 11109 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 11110 0, SOCKET_ID_ANY); 11111 11112 if (!tte) 11113 return NULL; 11114 memcpy(tte, oentry, sizeof(*tte)); 11115 return &tte->hash; 11116 } 11117 11118 static void 11119 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 11120 struct mlx5_list_entry *entry) 11121 { 11122 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11123 11124 mlx5_free(tte); 11125 } 11126 11127 static uint32_t 11128 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 11129 const struct mlx5_flow_tunnel *tunnel, 11130 uint32_t group, uint32_t *table, 11131 struct rte_flow_error *error) 11132 { 11133 struct mlx5_list_entry *he; 11134 struct tunnel_tbl_entry *tte; 11135 union tunnel_tbl_key key = { 11136 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 11137 .group = group 11138 }; 11139 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11140 struct mlx5_hlist *group_hash; 11141 struct mlx5_flow_cb_ctx ctx = { 11142 .data = &key.val, 11143 }; 11144 11145 group_hash = tunnel ? tunnel->groups : thub->groups; 11146 he = mlx5_hlist_register(group_hash, key.val, &ctx); 11147 if (!he) 11148 return rte_flow_error_set(error, EINVAL, 11149 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 11150 NULL, 11151 "tunnel group index not supported"); 11152 tte = container_of(he, typeof(*tte), hash); 11153 *table = tte->flow_table; 11154 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 11155 dev->data->port_id, key.tunnel_id, group, *table); 11156 return 0; 11157 } 11158 11159 static void 11160 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 11161 struct mlx5_flow_tunnel *tunnel) 11162 { 11163 struct mlx5_priv *priv = dev->data->dev_private; 11164 struct mlx5_indexed_pool *ipool; 11165 11166 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11167 dev->data->port_id, tunnel->tunnel_id); 11168 LIST_REMOVE(tunnel, chain); 11169 mlx5_hlist_destroy(tunnel->groups); 11170 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11171 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11172 } 11173 11174 static bool 11175 mlx5_access_tunnel_offload_db 11176 (struct rte_eth_dev *dev, 11177 bool (*match)(struct rte_eth_dev *, 11178 struct mlx5_flow_tunnel *, const void *), 11179 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11180 void (*miss)(struct rte_eth_dev *, void *), 11181 void *ctx, bool lock_op) 11182 { 11183 bool verdict = false; 11184 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11185 struct mlx5_flow_tunnel *tunnel; 11186 11187 rte_spinlock_lock(&thub->sl); 11188 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11189 verdict = match(dev, tunnel, (const void *)ctx); 11190 if (verdict) 11191 break; 11192 } 11193 if (!lock_op) 11194 rte_spinlock_unlock(&thub->sl); 11195 if (verdict && hit) 11196 hit(dev, tunnel, ctx); 11197 if (!verdict && miss) 11198 miss(dev, ctx); 11199 if (lock_op) 11200 rte_spinlock_unlock(&thub->sl); 11201 11202 return verdict; 11203 } 11204 11205 struct tunnel_db_find_tunnel_id_ctx { 11206 uint32_t tunnel_id; 11207 struct mlx5_flow_tunnel *tunnel; 11208 }; 11209 11210 static bool 11211 find_tunnel_id_match(struct rte_eth_dev *dev, 11212 struct mlx5_flow_tunnel *tunnel, const void *x) 11213 { 11214 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11215 11216 RTE_SET_USED(dev); 11217 return tunnel->tunnel_id == ctx->tunnel_id; 11218 } 11219 11220 static void 11221 find_tunnel_id_hit(struct rte_eth_dev *dev, 11222 struct mlx5_flow_tunnel *tunnel, void *x) 11223 { 11224 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11225 RTE_SET_USED(dev); 11226 ctx->tunnel = tunnel; 11227 } 11228 11229 static struct mlx5_flow_tunnel * 11230 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11231 { 11232 struct tunnel_db_find_tunnel_id_ctx ctx = { 11233 .tunnel_id = id, 11234 }; 11235 11236 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11237 find_tunnel_id_hit, NULL, &ctx, true); 11238 11239 return ctx.tunnel; 11240 } 11241 11242 static struct mlx5_flow_tunnel * 11243 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11244 const struct rte_flow_tunnel *app_tunnel) 11245 { 11246 struct mlx5_priv *priv = dev->data->dev_private; 11247 struct mlx5_indexed_pool *ipool; 11248 struct mlx5_flow_tunnel *tunnel; 11249 uint32_t id; 11250 11251 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11252 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11253 if (!tunnel) 11254 return NULL; 11255 if (id >= MLX5_MAX_TUNNELS) { 11256 mlx5_ipool_free(ipool, id); 11257 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11258 return NULL; 11259 } 11260 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11261 priv->sh, 11262 mlx5_flow_tunnel_grp2tbl_create_cb, 11263 mlx5_flow_tunnel_grp2tbl_match_cb, 11264 mlx5_flow_tunnel_grp2tbl_remove_cb, 11265 mlx5_flow_tunnel_grp2tbl_clone_cb, 11266 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11267 if (!tunnel->groups) { 11268 mlx5_ipool_free(ipool, id); 11269 return NULL; 11270 } 11271 /* initiate new PMD tunnel */ 11272 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11273 tunnel->tunnel_id = id; 11274 tunnel->action.type = (typeof(tunnel->action.type)) 11275 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11276 tunnel->action.conf = tunnel; 11277 tunnel->item.type = (typeof(tunnel->item.type)) 11278 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11279 tunnel->item.spec = tunnel; 11280 tunnel->item.last = NULL; 11281 tunnel->item.mask = NULL; 11282 11283 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11284 dev->data->port_id, tunnel->tunnel_id); 11285 11286 return tunnel; 11287 } 11288 11289 struct tunnel_db_get_tunnel_ctx { 11290 const struct rte_flow_tunnel *app_tunnel; 11291 struct mlx5_flow_tunnel *tunnel; 11292 }; 11293 11294 static bool get_tunnel_match(struct rte_eth_dev *dev, 11295 struct mlx5_flow_tunnel *tunnel, const void *x) 11296 { 11297 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11298 11299 RTE_SET_USED(dev); 11300 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11301 sizeof(*ctx->app_tunnel)); 11302 } 11303 11304 static void get_tunnel_hit(struct rte_eth_dev *dev, 11305 struct mlx5_flow_tunnel *tunnel, void *x) 11306 { 11307 /* called under tunnel spinlock protection */ 11308 struct tunnel_db_get_tunnel_ctx *ctx = x; 11309 11310 RTE_SET_USED(dev); 11311 tunnel->refctn++; 11312 ctx->tunnel = tunnel; 11313 } 11314 11315 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11316 { 11317 /* called under tunnel spinlock protection */ 11318 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11319 struct tunnel_db_get_tunnel_ctx *ctx = x; 11320 11321 rte_spinlock_unlock(&thub->sl); 11322 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11323 rte_spinlock_lock(&thub->sl); 11324 if (ctx->tunnel) { 11325 ctx->tunnel->refctn = 1; 11326 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11327 } 11328 } 11329 11330 11331 static int 11332 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11333 const struct rte_flow_tunnel *app_tunnel, 11334 struct mlx5_flow_tunnel **tunnel) 11335 { 11336 struct tunnel_db_get_tunnel_ctx ctx = { 11337 .app_tunnel = app_tunnel, 11338 }; 11339 11340 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11341 get_tunnel_miss, &ctx, true); 11342 *tunnel = ctx.tunnel; 11343 return ctx.tunnel ? 0 : -ENOMEM; 11344 } 11345 11346 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11347 { 11348 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11349 11350 if (!thub) 11351 return; 11352 if (!LIST_EMPTY(&thub->tunnels)) 11353 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11354 mlx5_hlist_destroy(thub->groups); 11355 mlx5_free(thub); 11356 } 11357 11358 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11359 { 11360 int err; 11361 struct mlx5_flow_tunnel_hub *thub; 11362 11363 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11364 0, SOCKET_ID_ANY); 11365 if (!thub) 11366 return -ENOMEM; 11367 LIST_INIT(&thub->tunnels); 11368 rte_spinlock_init(&thub->sl); 11369 thub->groups = mlx5_hlist_create("flow groups", 64, 11370 false, true, sh, 11371 mlx5_flow_tunnel_grp2tbl_create_cb, 11372 mlx5_flow_tunnel_grp2tbl_match_cb, 11373 mlx5_flow_tunnel_grp2tbl_remove_cb, 11374 mlx5_flow_tunnel_grp2tbl_clone_cb, 11375 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11376 if (!thub->groups) { 11377 err = -rte_errno; 11378 goto err; 11379 } 11380 sh->tunnel_hub = thub; 11381 11382 return 0; 11383 11384 err: 11385 if (thub->groups) 11386 mlx5_hlist_destroy(thub->groups); 11387 if (thub) 11388 mlx5_free(thub); 11389 return err; 11390 } 11391 11392 static inline int 11393 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11394 struct rte_flow_tunnel *tunnel, 11395 struct rte_flow_error *error) 11396 { 11397 struct mlx5_priv *priv = dev->data->dev_private; 11398 11399 if (!priv->sh->config.dv_flow_en) 11400 return rte_flow_error_set(error, ENOTSUP, 11401 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11402 "flow DV interface is off"); 11403 if (!is_tunnel_offload_active(dev)) 11404 return rte_flow_error_set(error, ENOTSUP, 11405 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11406 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11407 if (!tunnel) 11408 return rte_flow_error_set(error, EINVAL, 11409 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11410 "no application tunnel"); 11411 switch (tunnel->type) { 11412 default: 11413 return rte_flow_error_set(error, EINVAL, 11414 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11415 "unsupported tunnel type"); 11416 case RTE_FLOW_ITEM_TYPE_VXLAN: 11417 case RTE_FLOW_ITEM_TYPE_GRE: 11418 case RTE_FLOW_ITEM_TYPE_NVGRE: 11419 case RTE_FLOW_ITEM_TYPE_GENEVE: 11420 break; 11421 } 11422 return 0; 11423 } 11424 11425 static int 11426 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11427 struct rte_flow_tunnel *app_tunnel, 11428 struct rte_flow_action **actions, 11429 uint32_t *num_of_actions, 11430 struct rte_flow_error *error) 11431 { 11432 struct mlx5_flow_tunnel *tunnel; 11433 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11434 11435 if (ret) 11436 return ret; 11437 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11438 if (ret < 0) { 11439 return rte_flow_error_set(error, ret, 11440 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11441 "failed to initialize pmd tunnel"); 11442 } 11443 *actions = &tunnel->action; 11444 *num_of_actions = 1; 11445 return 0; 11446 } 11447 11448 static int 11449 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11450 struct rte_flow_tunnel *app_tunnel, 11451 struct rte_flow_item **items, 11452 uint32_t *num_of_items, 11453 struct rte_flow_error *error) 11454 { 11455 struct mlx5_flow_tunnel *tunnel; 11456 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11457 11458 if (ret) 11459 return ret; 11460 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11461 if (ret < 0) { 11462 return rte_flow_error_set(error, ret, 11463 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11464 "failed to initialize pmd tunnel"); 11465 } 11466 *items = &tunnel->item; 11467 *num_of_items = 1; 11468 return 0; 11469 } 11470 11471 struct tunnel_db_element_release_ctx { 11472 struct rte_flow_item *items; 11473 struct rte_flow_action *actions; 11474 uint32_t num_elements; 11475 struct rte_flow_error *error; 11476 int ret; 11477 }; 11478 11479 static bool 11480 tunnel_element_release_match(struct rte_eth_dev *dev, 11481 struct mlx5_flow_tunnel *tunnel, const void *x) 11482 { 11483 const struct tunnel_db_element_release_ctx *ctx = x; 11484 11485 RTE_SET_USED(dev); 11486 if (ctx->num_elements != 1) 11487 return false; 11488 else if (ctx->items) 11489 return ctx->items == &tunnel->item; 11490 else if (ctx->actions) 11491 return ctx->actions == &tunnel->action; 11492 11493 return false; 11494 } 11495 11496 static void 11497 tunnel_element_release_hit(struct rte_eth_dev *dev, 11498 struct mlx5_flow_tunnel *tunnel, void *x) 11499 { 11500 struct tunnel_db_element_release_ctx *ctx = x; 11501 ctx->ret = 0; 11502 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 11503 mlx5_flow_tunnel_free(dev, tunnel); 11504 } 11505 11506 static void 11507 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11508 { 11509 struct tunnel_db_element_release_ctx *ctx = x; 11510 RTE_SET_USED(dev); 11511 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11512 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11513 "invalid argument"); 11514 } 11515 11516 static int 11517 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11518 struct rte_flow_item *pmd_items, 11519 uint32_t num_items, struct rte_flow_error *err) 11520 { 11521 struct tunnel_db_element_release_ctx ctx = { 11522 .items = pmd_items, 11523 .actions = NULL, 11524 .num_elements = num_items, 11525 .error = err, 11526 }; 11527 11528 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11529 tunnel_element_release_hit, 11530 tunnel_element_release_miss, &ctx, false); 11531 11532 return ctx.ret; 11533 } 11534 11535 static int 11536 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11537 struct rte_flow_action *pmd_actions, 11538 uint32_t num_actions, struct rte_flow_error *err) 11539 { 11540 struct tunnel_db_element_release_ctx ctx = { 11541 .items = NULL, 11542 .actions = pmd_actions, 11543 .num_elements = num_actions, 11544 .error = err, 11545 }; 11546 11547 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11548 tunnel_element_release_hit, 11549 tunnel_element_release_miss, &ctx, false); 11550 11551 return ctx.ret; 11552 } 11553 11554 static int 11555 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11556 struct rte_mbuf *m, 11557 struct rte_flow_restore_info *info, 11558 struct rte_flow_error *err) 11559 { 11560 uint64_t ol_flags = m->ol_flags; 11561 const struct mlx5_flow_tbl_data_entry *tble; 11562 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11563 11564 if (!is_tunnel_offload_active(dev)) { 11565 info->flags = 0; 11566 return 0; 11567 } 11568 11569 if ((ol_flags & mask) != mask) 11570 goto err; 11571 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11572 if (!tble) { 11573 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11574 dev->data->port_id, m->hash.fdir.hi); 11575 goto err; 11576 } 11577 MLX5_ASSERT(tble->tunnel); 11578 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11579 info->group_id = tble->group_id; 11580 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11581 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11582 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11583 11584 return 0; 11585 11586 err: 11587 return rte_flow_error_set(err, EINVAL, 11588 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11589 "failed to get restore info"); 11590 } 11591 11592 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11593 static int 11594 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11595 __rte_unused struct rte_flow_tunnel *app_tunnel, 11596 __rte_unused struct rte_flow_action **actions, 11597 __rte_unused uint32_t *num_of_actions, 11598 __rte_unused struct rte_flow_error *error) 11599 { 11600 return -ENOTSUP; 11601 } 11602 11603 static int 11604 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11605 __rte_unused struct rte_flow_tunnel *app_tunnel, 11606 __rte_unused struct rte_flow_item **items, 11607 __rte_unused uint32_t *num_of_items, 11608 __rte_unused struct rte_flow_error *error) 11609 { 11610 return -ENOTSUP; 11611 } 11612 11613 static int 11614 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11615 __rte_unused struct rte_flow_item *pmd_items, 11616 __rte_unused uint32_t num_items, 11617 __rte_unused struct rte_flow_error *err) 11618 { 11619 return -ENOTSUP; 11620 } 11621 11622 static int 11623 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11624 __rte_unused struct rte_flow_action *pmd_action, 11625 __rte_unused uint32_t num_actions, 11626 __rte_unused struct rte_flow_error *err) 11627 { 11628 return -ENOTSUP; 11629 } 11630 11631 static int 11632 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11633 __rte_unused struct rte_mbuf *m, 11634 __rte_unused struct rte_flow_restore_info *i, 11635 __rte_unused struct rte_flow_error *err) 11636 { 11637 return -ENOTSUP; 11638 } 11639 11640 static int 11641 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11642 __rte_unused struct rte_flow *flow, 11643 __rte_unused const struct rte_flow_attr *attr, 11644 __rte_unused const struct rte_flow_action *actions, 11645 __rte_unused uint32_t flow_idx, 11646 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11647 __rte_unused struct tunnel_default_miss_ctx *ctx, 11648 __rte_unused struct rte_flow_error *error) 11649 { 11650 return -ENOTSUP; 11651 } 11652 11653 static struct mlx5_flow_tunnel * 11654 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11655 __rte_unused uint32_t id) 11656 { 11657 return NULL; 11658 } 11659 11660 static void 11661 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11662 __rte_unused struct mlx5_flow_tunnel *tunnel) 11663 { 11664 } 11665 11666 static uint32_t 11667 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11668 __rte_unused const struct mlx5_flow_tunnel *t, 11669 __rte_unused uint32_t group, 11670 __rte_unused uint32_t *table, 11671 struct rte_flow_error *error) 11672 { 11673 return rte_flow_error_set(error, ENOTSUP, 11674 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11675 "tunnel offload requires DV support"); 11676 } 11677 11678 void 11679 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11680 __rte_unused uint16_t port_id) 11681 { 11682 } 11683 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11684 11685 /* Flex flow item API */ 11686 static struct rte_flow_item_flex_handle * 11687 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11688 const struct rte_flow_item_flex_conf *conf, 11689 struct rte_flow_error *error) 11690 { 11691 static const char err_msg[] = "flex item creation unsupported"; 11692 struct mlx5_priv *priv = dev->data->dev_private; 11693 struct rte_flow_attr attr = { .transfer = 0 }; 11694 const struct mlx5_flow_driver_ops *fops = 11695 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11696 11697 if (!priv->pci_dev) { 11698 rte_flow_error_set(error, ENOTSUP, 11699 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11700 "create flex item on PF only"); 11701 return NULL; 11702 } 11703 switch (priv->pci_dev->id.device_id) { 11704 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11705 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11706 break; 11707 default: 11708 rte_flow_error_set(error, ENOTSUP, 11709 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11710 "flex item available on BlueField ports only"); 11711 return NULL; 11712 } 11713 if (!fops->item_create) { 11714 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11715 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11716 NULL, err_msg); 11717 return NULL; 11718 } 11719 return fops->item_create(dev, conf, error); 11720 } 11721 11722 static int 11723 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11724 const struct rte_flow_item_flex_handle *handle, 11725 struct rte_flow_error *error) 11726 { 11727 static const char err_msg[] = "flex item release unsupported"; 11728 struct rte_flow_attr attr = { .transfer = 0 }; 11729 const struct mlx5_flow_driver_ops *fops = 11730 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11731 11732 if (!fops->item_release) { 11733 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11734 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11735 NULL, err_msg); 11736 return -rte_errno; 11737 } 11738 return fops->item_release(dev, handle, error); 11739 } 11740 11741 static void 11742 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11743 { 11744 int ret; 11745 struct rte_flow_error error; 11746 11747 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11748 char *item_name; 11749 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11750 sizeof(item_name), 11751 (void *)(uintptr_t)item->type, &error); 11752 if (ret > 0) 11753 printf("%s ", item_name); 11754 else 11755 printf("%d\n", (int)item->type); 11756 } 11757 printf("END\n"); 11758 } 11759 11760 static int 11761 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11762 { 11763 const struct rte_flow_item_udp *spec = udp_item->spec; 11764 const struct rte_flow_item_udp *mask = udp_item->mask; 11765 uint16_t udp_dport = 0; 11766 11767 if (spec != NULL) { 11768 if (!mask) 11769 mask = &rte_flow_item_udp_mask; 11770 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11771 mask->hdr.dst_port); 11772 } 11773 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11774 } 11775 11776 static const struct mlx5_flow_expand_node * 11777 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11778 unsigned int item_idx, 11779 const struct mlx5_flow_expand_node graph[], 11780 const struct mlx5_flow_expand_node *node) 11781 { 11782 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11783 11784 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11785 node != NULL && 11786 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11787 /* 11788 * The expansion node is VXLAN and it is also the last 11789 * expandable item in the pattern, so need to continue 11790 * expansion of the inner tunnel. 11791 */ 11792 MLX5_ASSERT(item_idx > 0); 11793 prev_item = pattern + item_idx - 1; 11794 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11795 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11796 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11797 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11798 } 11799 return node; 11800 } 11801 11802 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11803 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11804 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11805 }; 11806 11807 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11808 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11809 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11810 { 9, 10, 11 }, { 12, 13, 14 }, 11811 }; 11812 11813 /** 11814 * Discover the number of available flow priorities. 11815 * 11816 * @param dev 11817 * Ethernet device. 11818 * 11819 * @return 11820 * On success, number of available flow priorities. 11821 * On failure, a negative errno-style code and rte_errno is set. 11822 */ 11823 int 11824 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11825 { 11826 static const uint16_t vprio[] = {8, 16}; 11827 const struct mlx5_priv *priv = dev->data->dev_private; 11828 const struct mlx5_flow_driver_ops *fops; 11829 enum mlx5_flow_drv_type type; 11830 int ret; 11831 11832 type = mlx5_flow_os_get_type(); 11833 if (type == MLX5_FLOW_TYPE_MAX) { 11834 type = MLX5_FLOW_TYPE_VERBS; 11835 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11836 type = MLX5_FLOW_TYPE_DV; 11837 } 11838 fops = flow_get_drv_ops(type); 11839 if (fops->discover_priorities == NULL) { 11840 DRV_LOG(ERR, "Priority discovery not supported"); 11841 rte_errno = ENOTSUP; 11842 return -rte_errno; 11843 } 11844 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11845 if (ret < 0) 11846 return ret; 11847 switch (ret) { 11848 case 8: 11849 ret = RTE_DIM(priority_map_3); 11850 break; 11851 case 16: 11852 ret = RTE_DIM(priority_map_5); 11853 break; 11854 default: 11855 rte_errno = ENOTSUP; 11856 DRV_LOG(ERR, 11857 "port %u maximum priority: %d expected 8/16", 11858 dev->data->port_id, ret); 11859 return -rte_errno; 11860 } 11861 DRV_LOG(INFO, "port %u supported flow priorities:" 11862 " 0-%d for ingress or egress root table," 11863 " 0-%d for non-root table or transfer root table.", 11864 dev->data->port_id, ret - 2, 11865 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11866 return ret; 11867 } 11868 11869 /** 11870 * Adjust flow priority based on the highest layer and the request priority. 11871 * 11872 * @param[in] dev 11873 * Pointer to the Ethernet device structure. 11874 * @param[in] priority 11875 * The rule base priority. 11876 * @param[in] subpriority 11877 * The priority based on the items. 11878 * 11879 * @return 11880 * The new priority. 11881 */ 11882 uint32_t 11883 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11884 uint32_t subpriority) 11885 { 11886 uint32_t res = 0; 11887 struct mlx5_priv *priv = dev->data->dev_private; 11888 11889 switch (priv->sh->flow_max_priority) { 11890 case RTE_DIM(priority_map_3): 11891 res = priority_map_3[priority][subpriority]; 11892 break; 11893 case RTE_DIM(priority_map_5): 11894 res = priority_map_5[priority][subpriority]; 11895 break; 11896 } 11897 return res; 11898 } 11899 11900 /** 11901 * Get the priority for sending traffic to kernel table. 11902 * 11903 * @param[in] dev 11904 * Pointer to the Ethernet device structure. 11905 * 11906 * @return 11907 * On success: the value of priority for sending traffic to kernel table 11908 * On failure: -1 11909 */ 11910 uint32_t 11911 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11912 { 11913 struct mlx5_priv *priv = dev->data->dev_private; 11914 uint32_t res; 11915 11916 switch (priv->sh->flow_max_priority) { 11917 case RTE_DIM(priority_map_5): 11918 res = 15; 11919 break; 11920 case RTE_DIM(priority_map_3): 11921 res = 7; 11922 break; 11923 default: 11924 DRV_LOG(ERR, 11925 "port %u maximum priority: %d expected 8/16", 11926 dev->data->port_id, priv->sh->flow_max_priority); 11927 res = (uint32_t)-1; 11928 } 11929 return res; 11930 } 11931 11932 /** 11933 * Get the E-Switch Manager vport id. 11934 * 11935 * @param[in] dev 11936 * Pointer to the Ethernet device structure. 11937 * 11938 * @return 11939 * The vport id. 11940 */ 11941 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11942 { 11943 struct mlx5_priv *priv = dev->data->dev_private; 11944 struct mlx5_common_device *cdev = priv->sh->cdev; 11945 11946 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11947 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11948 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11949 11950 if (priv->pci_dev == NULL) 11951 return 0; 11952 switch (priv->pci_dev->id.device_id) { 11953 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 11954 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11955 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11956 /* 11957 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11958 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11959 * ECPF vport ID is selected and not the host port (0) in any BF case. 11960 */ 11961 return (int16_t)MLX5_ECPF_VPORT_ID; 11962 default: 11963 return MLX5_PF_VPORT_ID; 11964 } 11965 } 11966 11967 /** 11968 * Parse item to get the vport id. 11969 * 11970 * @param[in] dev 11971 * Pointer to the Ethernet device structure. 11972 * @param[in] item 11973 * The src port id match item. 11974 * @param[out] vport_id 11975 * Pointer to put the vport id. 11976 * @param[out] all_ports 11977 * Indicate if the item matches all ports. 11978 * @param[out] error 11979 * Pointer to error structure. 11980 * 11981 * @return 11982 * 0 on success, a negative errno value otherwise and rte_errno is set. 11983 */ 11984 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11985 const struct rte_flow_item *item, 11986 uint16_t *vport_id, 11987 bool *all_ports, 11988 struct rte_flow_error *error) 11989 { 11990 struct mlx5_priv *port_priv; 11991 const struct rte_flow_item_port_id *pid_v = NULL; 11992 const struct rte_flow_item_ethdev *dev_v = NULL; 11993 uint32_t esw_mgr_port; 11994 uint32_t src_port; 11995 11996 if (all_ports) 11997 *all_ports = false; 11998 switch (item->type) { 11999 case RTE_FLOW_ITEM_TYPE_PORT_ID: 12000 pid_v = item->spec; 12001 if (!pid_v) 12002 return 0; 12003 src_port = pid_v->id; 12004 esw_mgr_port = MLX5_PORT_ESW_MGR; 12005 break; 12006 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 12007 dev_v = item->spec; 12008 if (!dev_v) { 12009 if (all_ports) 12010 *all_ports = true; 12011 return 0; 12012 } 12013 src_port = dev_v->port_id; 12014 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12015 break; 12016 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 12017 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12018 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12019 break; 12020 default: 12021 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12022 NULL, "Incorrect item type."); 12023 } 12024 if (src_port == esw_mgr_port) { 12025 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 12026 } else { 12027 port_priv = mlx5_port_to_eswitch_info(src_port, false); 12028 if (!port_priv) 12029 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12030 NULL, "Failed to get port info."); 12031 *vport_id = port_priv->representor_id; 12032 } 12033 12034 return 0; 12035 } 12036 12037 int 12038 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 12039 uint16_t *proxy_port_id, 12040 struct rte_flow_error *error) 12041 { 12042 const struct mlx5_priv *priv = dev->data->dev_private; 12043 uint16_t port_id; 12044 12045 if (!priv->sh->config.dv_esw_en) 12046 return rte_flow_error_set(error, EINVAL, 12047 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12048 NULL, 12049 "unable to provide a proxy port" 12050 " without E-Switch configured"); 12051 if (!priv->master && !priv->representor) 12052 return rte_flow_error_set(error, EINVAL, 12053 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12054 NULL, 12055 "unable to provide a proxy port" 12056 " for port which is not a master" 12057 " or a representor port"); 12058 if (priv->master) { 12059 *proxy_port_id = dev->data->port_id; 12060 return 0; 12061 } 12062 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 12063 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 12064 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 12065 12066 if (port_priv->master && 12067 port_priv->domain_id == priv->domain_id) { 12068 *proxy_port_id = port_id; 12069 return 0; 12070 } 12071 } 12072 return rte_flow_error_set(error, ENODEV, 12073 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12074 NULL, "unable to find a proxy port"); 12075 } 12076