1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 struct tunnel_default_miss_ctx { 43 uint16_t *queue; 44 __extension__ 45 union { 46 struct rte_flow_action_rss action_rss; 47 struct rte_flow_action_queue miss_queue; 48 struct rte_flow_action_jump miss_jump; 49 uint8_t raw[0]; 50 }; 51 }; 52 53 void 54 mlx5_indirect_list_handles_release(struct rte_eth_dev *dev) 55 { 56 struct mlx5_priv *priv = dev->data->dev_private; 57 #ifdef HAVE_MLX5_HWS_SUPPORT 58 struct rte_flow_error error; 59 #endif 60 61 while (!LIST_EMPTY(&priv->indirect_list_head)) { 62 struct mlx5_indirect_list *e = 63 LIST_FIRST(&priv->indirect_list_head); 64 65 LIST_REMOVE(e, entry); 66 switch (e->type) { 67 #ifdef HAVE_MLX5_HWS_SUPPORT 68 case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR: 69 mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)e); 70 break; 71 case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY: 72 mlx5_destroy_legacy_indirect(dev, e); 73 break; 74 case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT: 75 mlx5_reformat_action_destroy(dev, 76 (struct rte_flow_action_list_handle *)e, &error); 77 break; 78 #endif 79 default: 80 DRV_LOG(ERR, "invalid indirect list type"); 81 MLX5_ASSERT(false); 82 break; 83 } 84 } 85 } 86 87 static int 88 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 89 struct rte_flow *flow, 90 const struct rte_flow_attr *attr, 91 const struct rte_flow_action *app_actions, 92 uint32_t flow_idx, 93 const struct mlx5_flow_tunnel *tunnel, 94 struct tunnel_default_miss_ctx *ctx, 95 struct rte_flow_error *error); 96 static struct mlx5_flow_tunnel * 97 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 98 static void 99 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 100 static uint32_t 101 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 102 const struct mlx5_flow_tunnel *tunnel, 103 uint32_t group, uint32_t *table, 104 struct rte_flow_error *error); 105 106 /** Device flow drivers. */ 107 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 108 109 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 110 111 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 112 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 113 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 114 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 115 #endif 116 #ifdef HAVE_MLX5_HWS_SUPPORT 117 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 118 #endif 119 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 120 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 121 }; 122 123 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 124 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 125 (const int []){ \ 126 __VA_ARGS__, 0, \ 127 } 128 129 /** Node object of input graph for mlx5_flow_expand_rss(). */ 130 struct mlx5_flow_expand_node { 131 const int *const next; 132 /**< 133 * List of next node indexes. Index 0 is interpreted as a terminator. 134 */ 135 const enum rte_flow_item_type type; 136 /**< Pattern item type of current node. */ 137 uint64_t rss_types; 138 /**< 139 * RSS types bit-field associated with this node 140 * (see RTE_ETH_RSS_* definitions). 141 */ 142 uint64_t node_flags; 143 /**< 144 * Bit-fields that define how the node is used in the expansion. 145 * (see MLX5_EXPANSION_NODE_* definitions). 146 */ 147 }; 148 149 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 150 struct mlx5_flow_expand_sqn { 151 uint32_t entries; /** Number of entries */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 /* Optional expand field. The expansion alg will not go deeper. */ 159 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 160 161 /* The node is not added implicitly as expansion to the flow pattern. 162 * If the node type does not match the flow pattern item type, the 163 * expansion alg will go deeper to its next items. 164 * In the current implementation, the list of next nodes indexes can 165 * have up to one node with this flag set and it has to be the last 166 * node index (before the list terminator). 167 */ 168 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 169 170 /** Object returned by mlx5_flow_expand_rss(). */ 171 struct mlx5_flow_expand_rss { 172 uint32_t entries; 173 /**< Number of entries @p patterns and @p priorities. */ 174 struct { 175 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 176 uint32_t priority; /**< Priority offset for each expansion. */ 177 } entry[]; 178 }; 179 180 static void 181 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 182 183 static const struct mlx5_flow_expand_node * 184 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 185 unsigned int item_idx, 186 const struct mlx5_flow_expand_node graph[], 187 const struct mlx5_flow_expand_node *node); 188 189 static __rte_always_inline int 190 mlx5_need_cache_flow(const struct mlx5_priv *priv, 191 const struct rte_flow_attr *attr) 192 { 193 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 194 (attr ? !attr->group : true) && 195 priv->mode_info.mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY && 196 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 197 } 198 199 static bool 200 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 201 { 202 switch (item->type) { 203 case RTE_FLOW_ITEM_TYPE_ETH: 204 case RTE_FLOW_ITEM_TYPE_VLAN: 205 case RTE_FLOW_ITEM_TYPE_IPV4: 206 case RTE_FLOW_ITEM_TYPE_IPV6: 207 case RTE_FLOW_ITEM_TYPE_UDP: 208 case RTE_FLOW_ITEM_TYPE_TCP: 209 case RTE_FLOW_ITEM_TYPE_ESP: 210 case RTE_FLOW_ITEM_TYPE_ICMP: 211 case RTE_FLOW_ITEM_TYPE_ICMP6: 212 case RTE_FLOW_ITEM_TYPE_VXLAN: 213 case RTE_FLOW_ITEM_TYPE_NVGRE: 214 case RTE_FLOW_ITEM_TYPE_GRE: 215 case RTE_FLOW_ITEM_TYPE_GENEVE: 216 case RTE_FLOW_ITEM_TYPE_MPLS: 217 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 218 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 219 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 220 case RTE_FLOW_ITEM_TYPE_GTP: 221 return true; 222 default: 223 break; 224 } 225 return false; 226 } 227 228 /** 229 * Network Service Header (NSH) and its next protocol values 230 * are described in RFC-8393. 231 */ 232 static enum rte_flow_item_type 233 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 234 { 235 enum rte_flow_item_type type; 236 237 switch (proto_mask & proto_spec) { 238 case 0: 239 type = RTE_FLOW_ITEM_TYPE_VOID; 240 break; 241 case RTE_VXLAN_GPE_TYPE_IPV4: 242 type = RTE_FLOW_ITEM_TYPE_IPV4; 243 break; 244 case RTE_VXLAN_GPE_TYPE_IPV6: 245 type = RTE_VXLAN_GPE_TYPE_IPV6; 246 break; 247 case RTE_VXLAN_GPE_TYPE_ETH: 248 type = RTE_FLOW_ITEM_TYPE_ETH; 249 break; 250 default: 251 type = RTE_FLOW_ITEM_TYPE_END; 252 } 253 return type; 254 } 255 256 static enum rte_flow_item_type 257 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 258 { 259 enum rte_flow_item_type type; 260 261 switch (proto_mask & proto_spec) { 262 case 0: 263 type = RTE_FLOW_ITEM_TYPE_VOID; 264 break; 265 case IPPROTO_UDP: 266 type = RTE_FLOW_ITEM_TYPE_UDP; 267 break; 268 case IPPROTO_TCP: 269 type = RTE_FLOW_ITEM_TYPE_TCP; 270 break; 271 case IPPROTO_IPIP: 272 type = RTE_FLOW_ITEM_TYPE_IPV4; 273 break; 274 case IPPROTO_IPV6: 275 type = RTE_FLOW_ITEM_TYPE_IPV6; 276 break; 277 case IPPROTO_ESP: 278 type = RTE_FLOW_ITEM_TYPE_ESP; 279 break; 280 default: 281 type = RTE_FLOW_ITEM_TYPE_END; 282 } 283 return type; 284 } 285 286 static enum rte_flow_item_type 287 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 288 rte_be16_t type_mask, bool is_tunnel) 289 { 290 enum rte_flow_item_type type; 291 292 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 293 case 0: 294 type = RTE_FLOW_ITEM_TYPE_VOID; 295 break; 296 case RTE_ETHER_TYPE_TEB: 297 type = is_tunnel ? 298 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 299 break; 300 case RTE_ETHER_TYPE_VLAN: 301 type = !is_tunnel ? 302 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 303 break; 304 case RTE_ETHER_TYPE_IPV4: 305 type = RTE_FLOW_ITEM_TYPE_IPV4; 306 break; 307 case RTE_ETHER_TYPE_IPV6: 308 type = RTE_FLOW_ITEM_TYPE_IPV6; 309 break; 310 default: 311 type = RTE_FLOW_ITEM_TYPE_END; 312 } 313 return type; 314 } 315 316 static enum rte_flow_item_type 317 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 318 { 319 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 320 do { \ 321 const void *m = item->mask; \ 322 const void *s = item->spec; \ 323 mask = m ? \ 324 ((const struct rte_flow_item_##type *)m)->fld : \ 325 rte_flow_item_##type##_mask.fld; \ 326 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 327 } while (0) 328 329 enum rte_flow_item_type ret; 330 uint16_t spec, mask; 331 332 if (item == NULL || item->spec == NULL) 333 return RTE_FLOW_ITEM_TYPE_VOID; 334 switch (item->type) { 335 case RTE_FLOW_ITEM_TYPE_ETH: 336 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 337 if (!mask) 338 return RTE_FLOW_ITEM_TYPE_VOID; 339 ret = mlx5_ethertype_to_item_type(spec, mask, false); 340 break; 341 case RTE_FLOW_ITEM_TYPE_VLAN: 342 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 343 if (!mask) 344 return RTE_FLOW_ITEM_TYPE_VOID; 345 ret = mlx5_ethertype_to_item_type(spec, mask, false); 346 break; 347 case RTE_FLOW_ITEM_TYPE_IPV4: 348 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 349 if (!mask) 350 return RTE_FLOW_ITEM_TYPE_VOID; 351 ret = mlx5_inet_proto_to_item_type(spec, mask); 352 break; 353 case RTE_FLOW_ITEM_TYPE_IPV6: 354 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 355 if (!mask) 356 return RTE_FLOW_ITEM_TYPE_VOID; 357 ret = mlx5_inet_proto_to_item_type(spec, mask); 358 break; 359 case RTE_FLOW_ITEM_TYPE_GENEVE: 360 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 361 ret = mlx5_ethertype_to_item_type(spec, mask, true); 362 break; 363 case RTE_FLOW_ITEM_TYPE_GRE: 364 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 365 ret = mlx5_ethertype_to_item_type(spec, mask, true); 366 break; 367 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 368 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 369 ret = mlx5_nsh_proto_to_item_type(spec, mask); 370 break; 371 default: 372 ret = RTE_FLOW_ITEM_TYPE_VOID; 373 break; 374 } 375 return ret; 376 #undef MLX5_XSET_ITEM_MASK_SPEC 377 } 378 379 static const int * 380 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 381 const int *next_node) 382 { 383 const struct mlx5_flow_expand_node *node = NULL; 384 const int *next = next_node; 385 386 while (next && *next) { 387 /* 388 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 389 * flag set, because they were not found in the flow pattern. 390 */ 391 node = &graph[*next]; 392 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 393 break; 394 next = node->next; 395 } 396 return next; 397 } 398 399 #define MLX5_RSS_EXP_ELT_N 32 400 401 /** 402 * Expand RSS flows into several possible flows according to the RSS hash 403 * fields requested and the driver capabilities. 404 * 405 * @param[out] buf 406 * Buffer to store the result expansion. 407 * @param[in] size 408 * Buffer size in bytes. If 0, @p buf can be NULL. 409 * @param[in] pattern 410 * User flow pattern. 411 * @param[in] types 412 * RSS types to expand (see RTE_ETH_RSS_* definitions). 413 * @param[in] graph 414 * Input graph to expand @p pattern according to @p types. 415 * @param[in] graph_root_index 416 * Index of root node in @p graph, typically 0. 417 * 418 * @return 419 * A positive value representing the size of @p buf in bytes regardless of 420 * @p size on success, a negative errno value otherwise and rte_errno is 421 * set, the following errors are defined: 422 * 423 * -E2BIG: graph-depth @p graph is too deep. 424 * -EINVAL: @p size has not enough space for expanded pattern. 425 */ 426 static int 427 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 428 const struct rte_flow_item *pattern, uint64_t types, 429 const struct mlx5_flow_expand_node graph[], 430 int graph_root_index) 431 { 432 const struct rte_flow_item *item; 433 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 434 const int *next_node; 435 const int *stack[MLX5_RSS_EXP_ELT_N]; 436 int stack_pos = 0; 437 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 438 unsigned int i, item_idx, last_expand_item_idx = 0; 439 size_t lsize; 440 size_t user_pattern_size = 0; 441 void *addr = NULL; 442 const struct mlx5_flow_expand_node *next = NULL; 443 struct rte_flow_item missed_item; 444 int missed = 0; 445 int elt = 0; 446 const struct rte_flow_item *last_expand_item = NULL; 447 448 memset(&missed_item, 0, sizeof(missed_item)); 449 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 450 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 451 if (lsize > size) 452 return -EINVAL; 453 buf->entry[0].priority = 0; 454 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 455 buf->entries = 0; 456 addr = buf->entry[0].pattern; 457 for (item = pattern, item_idx = 0; 458 item->type != RTE_FLOW_ITEM_TYPE_END; 459 item++, item_idx++) { 460 if (!mlx5_flow_is_rss_expandable_item(item)) { 461 user_pattern_size += sizeof(*item); 462 continue; 463 } 464 last_expand_item = item; 465 last_expand_item_idx = item_idx; 466 i = 0; 467 while (node->next && node->next[i]) { 468 next = &graph[node->next[i]]; 469 if (next->type == item->type) 470 break; 471 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 472 node = next; 473 i = 0; 474 } else { 475 ++i; 476 } 477 } 478 if (next) 479 node = next; 480 user_pattern_size += sizeof(*item); 481 } 482 user_pattern_size += sizeof(*item); /* Handle END item. */ 483 lsize += user_pattern_size; 484 if (lsize > size) 485 return -EINVAL; 486 /* Copy the user pattern in the first entry of the buffer. */ 487 rte_memcpy(addr, pattern, user_pattern_size); 488 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 489 buf->entries = 1; 490 /* Start expanding. */ 491 memset(flow_items, 0, sizeof(flow_items)); 492 user_pattern_size -= sizeof(*item); 493 /* 494 * Check if the last valid item has spec set, need complete pattern, 495 * and the pattern can be used for expansion. 496 */ 497 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 498 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 499 /* Item type END indicates expansion is not required. */ 500 return lsize; 501 } 502 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 503 next = NULL; 504 missed = 1; 505 i = 0; 506 while (node->next && node->next[i]) { 507 next = &graph[node->next[i]]; 508 if (next->type == missed_item.type) { 509 flow_items[0].type = missed_item.type; 510 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 511 break; 512 } 513 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 514 node = next; 515 i = 0; 516 } else { 517 ++i; 518 } 519 next = NULL; 520 } 521 } 522 if (next && missed) { 523 elt = 2; /* missed item + item end. */ 524 node = next; 525 lsize += elt * sizeof(*item) + user_pattern_size; 526 if (lsize > size) 527 return -EINVAL; 528 if (node->rss_types & types) { 529 buf->entry[buf->entries].priority = 1; 530 buf->entry[buf->entries].pattern = addr; 531 buf->entries++; 532 rte_memcpy(addr, buf->entry[0].pattern, 533 user_pattern_size); 534 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 535 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 536 addr = (void *)(((uintptr_t)addr) + 537 elt * sizeof(*item)); 538 } 539 } else if (last_expand_item != NULL) { 540 node = mlx5_flow_expand_rss_adjust_node(pattern, 541 last_expand_item_idx, graph, node); 542 } 543 memset(flow_items, 0, sizeof(flow_items)); 544 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 545 node->next); 546 stack[stack_pos] = next_node; 547 node = next_node ? &graph[*next_node] : NULL; 548 while (node) { 549 flow_items[stack_pos].type = node->type; 550 if (node->rss_types & types) { 551 size_t n; 552 /* 553 * compute the number of items to copy from the 554 * expansion and copy it. 555 * When the stack_pos is 0, there are 1 element in it, 556 * plus the addition END item. 557 */ 558 elt = stack_pos + 2; 559 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 560 lsize += elt * sizeof(*item) + user_pattern_size; 561 if (lsize > size) 562 return -EINVAL; 563 n = elt * sizeof(*item); 564 MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); 565 buf->entry[buf->entries].priority = 566 stack_pos + 1 + missed; 567 buf->entry[buf->entries].pattern = addr; 568 buf->entries++; 569 rte_memcpy(addr, buf->entry[0].pattern, 570 user_pattern_size); 571 addr = (void *)(((uintptr_t)addr) + 572 user_pattern_size); 573 rte_memcpy(addr, &missed_item, 574 missed * sizeof(*item)); 575 addr = (void *)(((uintptr_t)addr) + 576 missed * sizeof(*item)); 577 rte_memcpy(addr, flow_items, n); 578 addr = (void *)(((uintptr_t)addr) + n); 579 } 580 /* Go deeper. */ 581 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 582 node->next) { 583 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 584 node->next); 585 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 586 rte_errno = E2BIG; 587 return -rte_errno; 588 } 589 stack[stack_pos] = next_node; 590 } else if (*(next_node + 1)) { 591 /* Follow up with the next possibility. */ 592 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 593 ++next_node); 594 } else if (!stack_pos) { 595 /* 596 * Completing the traverse over the different paths. 597 * The next_node is advanced to the terminator. 598 */ 599 ++next_node; 600 } else { 601 /* Move to the next path. */ 602 while (stack_pos) { 603 next_node = stack[--stack_pos]; 604 next_node++; 605 if (*next_node) 606 break; 607 } 608 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 609 next_node); 610 stack[stack_pos] = next_node; 611 } 612 node = next_node && *next_node ? &graph[*next_node] : NULL; 613 }; 614 return lsize; 615 } 616 617 /** 618 * Expand SQN flows into several possible flows according to the Tx queue 619 * number 620 * 621 * @param[in] buf 622 * Buffer to store the result expansion. 623 * @param[in] size 624 * Buffer size in bytes. If 0, @p buf can be NULL. 625 * @param[in] pattern 626 * User flow pattern. 627 * @param[in] sq_specs 628 * Buffer to store sq spec. 629 * 630 * @return 631 * 0 for success and negative value for failure 632 * 633 */ 634 static int 635 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 636 const struct rte_flow_item *pattern, 637 struct mlx5_rte_flow_item_sq *sq_specs) 638 { 639 const struct rte_flow_item *item; 640 bool port_representor = false; 641 size_t user_pattern_size = 0; 642 struct rte_eth_dev *dev; 643 struct mlx5_priv *priv; 644 void *addr = NULL; 645 uint16_t port_id; 646 size_t lsize; 647 int elt = 2; 648 uint16_t i; 649 650 buf->entries = 0; 651 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 652 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 653 const struct rte_flow_item_ethdev *pid_v = item->spec; 654 655 if (!pid_v) 656 return 0; 657 port_id = pid_v->port_id; 658 port_representor = true; 659 } 660 user_pattern_size += sizeof(*item); 661 } 662 if (!port_representor) 663 return 0; 664 dev = &rte_eth_devices[port_id]; 665 priv = dev->data->dev_private; 666 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 667 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 668 sizeof(buf->entry[0]) * priv->txqs_n; 669 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 670 return -EINVAL; 671 addr = buf->entry[0].pattern; 672 for (i = 0; i != priv->txqs_n; ++i) { 673 struct rte_flow_item pattern_add[] = { 674 { 675 .type = (enum rte_flow_item_type) 676 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 677 .spec = &sq_specs[i], 678 }, 679 { 680 .type = RTE_FLOW_ITEM_TYPE_END, 681 }, 682 }; 683 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 684 685 if (txq == NULL) 686 return -EINVAL; 687 buf->entry[i].pattern = addr; 688 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 689 mlx5_txq_release(dev, i); 690 rte_memcpy(addr, pattern, user_pattern_size); 691 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 692 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 693 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 694 buf->entries++; 695 } 696 return 0; 697 } 698 699 enum mlx5_expansion { 700 MLX5_EXPANSION_ROOT, 701 MLX5_EXPANSION_ROOT_OUTER, 702 MLX5_EXPANSION_OUTER_ETH, 703 MLX5_EXPANSION_OUTER_VLAN, 704 MLX5_EXPANSION_OUTER_IPV4, 705 MLX5_EXPANSION_OUTER_IPV4_UDP, 706 MLX5_EXPANSION_OUTER_IPV4_TCP, 707 MLX5_EXPANSION_OUTER_IPV4_ESP, 708 MLX5_EXPANSION_OUTER_IPV4_ICMP, 709 MLX5_EXPANSION_OUTER_IPV6, 710 MLX5_EXPANSION_OUTER_IPV6_UDP, 711 MLX5_EXPANSION_OUTER_IPV6_TCP, 712 MLX5_EXPANSION_OUTER_IPV6_ESP, 713 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 714 MLX5_EXPANSION_VXLAN, 715 MLX5_EXPANSION_STD_VXLAN, 716 MLX5_EXPANSION_L3_VXLAN, 717 MLX5_EXPANSION_VXLAN_GPE, 718 MLX5_EXPANSION_GRE, 719 MLX5_EXPANSION_NVGRE, 720 MLX5_EXPANSION_GRE_KEY, 721 MLX5_EXPANSION_MPLS, 722 MLX5_EXPANSION_ETH, 723 MLX5_EXPANSION_VLAN, 724 MLX5_EXPANSION_IPV4, 725 MLX5_EXPANSION_IPV4_UDP, 726 MLX5_EXPANSION_IPV4_TCP, 727 MLX5_EXPANSION_IPV4_ESP, 728 MLX5_EXPANSION_IPV4_ICMP, 729 MLX5_EXPANSION_IPV6, 730 MLX5_EXPANSION_IPV6_UDP, 731 MLX5_EXPANSION_IPV6_TCP, 732 MLX5_EXPANSION_IPV6_ESP, 733 MLX5_EXPANSION_IPV6_ICMP6, 734 MLX5_EXPANSION_IPV6_FRAG_EXT, 735 MLX5_EXPANSION_GTP, 736 MLX5_EXPANSION_GENEVE, 737 }; 738 739 /** Supported expansion of items. */ 740 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 741 [MLX5_EXPANSION_ROOT] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 743 MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_END, 746 }, 747 [MLX5_EXPANSION_ROOT_OUTER] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 749 MLX5_EXPANSION_OUTER_IPV4, 750 MLX5_EXPANSION_OUTER_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_END, 752 }, 753 [MLX5_EXPANSION_OUTER_ETH] = { 754 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 755 .type = RTE_FLOW_ITEM_TYPE_ETH, 756 .rss_types = 0, 757 }, 758 [MLX5_EXPANSION_OUTER_VLAN] = { 759 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 760 MLX5_EXPANSION_OUTER_IPV6), 761 .type = RTE_FLOW_ITEM_TYPE_VLAN, 762 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 763 }, 764 [MLX5_EXPANSION_OUTER_IPV4] = { 765 .next = MLX5_FLOW_EXPAND_RSS_NEXT 766 (MLX5_EXPANSION_OUTER_IPV4_UDP, 767 MLX5_EXPANSION_OUTER_IPV4_TCP, 768 MLX5_EXPANSION_OUTER_IPV4_ESP, 769 MLX5_EXPANSION_OUTER_IPV4_ICMP, 770 MLX5_EXPANSION_GRE, 771 MLX5_EXPANSION_NVGRE, 772 MLX5_EXPANSION_IPV4, 773 MLX5_EXPANSION_IPV6), 774 .type = RTE_FLOW_ITEM_TYPE_IPV4, 775 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 776 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 777 }, 778 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 779 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 780 MLX5_EXPANSION_VXLAN_GPE, 781 MLX5_EXPANSION_MPLS, 782 MLX5_EXPANSION_GENEVE, 783 MLX5_EXPANSION_GTP), 784 .type = RTE_FLOW_ITEM_TYPE_UDP, 785 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 786 }, 787 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 788 .type = RTE_FLOW_ITEM_TYPE_TCP, 789 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 790 }, 791 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 792 .type = RTE_FLOW_ITEM_TYPE_ESP, 793 .rss_types = RTE_ETH_RSS_ESP, 794 }, 795 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 796 .type = RTE_FLOW_ITEM_TYPE_ICMP, 797 }, 798 [MLX5_EXPANSION_OUTER_IPV6] = { 799 .next = MLX5_FLOW_EXPAND_RSS_NEXT 800 (MLX5_EXPANSION_OUTER_IPV6_UDP, 801 MLX5_EXPANSION_OUTER_IPV6_TCP, 802 MLX5_EXPANSION_OUTER_IPV6_ESP, 803 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 804 MLX5_EXPANSION_IPV4, 805 MLX5_EXPANSION_IPV6, 806 MLX5_EXPANSION_GRE, 807 MLX5_EXPANSION_NVGRE), 808 .type = RTE_FLOW_ITEM_TYPE_IPV6, 809 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 810 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 811 }, 812 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 813 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 814 MLX5_EXPANSION_VXLAN_GPE, 815 MLX5_EXPANSION_MPLS, 816 MLX5_EXPANSION_GENEVE, 817 MLX5_EXPANSION_GTP), 818 .type = RTE_FLOW_ITEM_TYPE_UDP, 819 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 820 }, 821 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 822 .type = RTE_FLOW_ITEM_TYPE_TCP, 823 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 824 }, 825 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 826 .type = RTE_FLOW_ITEM_TYPE_ESP, 827 .rss_types = RTE_ETH_RSS_ESP, 828 }, 829 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 830 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 831 }, 832 [MLX5_EXPANSION_VXLAN] = { 833 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 834 MLX5_EXPANSION_IPV4, 835 MLX5_EXPANSION_IPV6), 836 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 837 }, 838 [MLX5_EXPANSION_STD_VXLAN] = { 839 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 840 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 841 }, 842 [MLX5_EXPANSION_L3_VXLAN] = { 843 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 844 MLX5_EXPANSION_IPV6), 845 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 846 }, 847 [MLX5_EXPANSION_VXLAN_GPE] = { 848 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 849 MLX5_EXPANSION_IPV4, 850 MLX5_EXPANSION_IPV6), 851 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 852 }, 853 [MLX5_EXPANSION_GRE] = { 854 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 855 MLX5_EXPANSION_IPV4, 856 MLX5_EXPANSION_IPV6, 857 MLX5_EXPANSION_GRE_KEY, 858 MLX5_EXPANSION_MPLS), 859 .type = RTE_FLOW_ITEM_TYPE_GRE, 860 }, 861 [MLX5_EXPANSION_GRE_KEY] = { 862 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 863 MLX5_EXPANSION_IPV6, 864 MLX5_EXPANSION_MPLS), 865 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 866 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 867 }, 868 [MLX5_EXPANSION_NVGRE] = { 869 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 870 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 871 }, 872 [MLX5_EXPANSION_MPLS] = { 873 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 874 MLX5_EXPANSION_IPV6, 875 MLX5_EXPANSION_ETH), 876 .type = RTE_FLOW_ITEM_TYPE_MPLS, 877 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 878 }, 879 [MLX5_EXPANSION_ETH] = { 880 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 881 .type = RTE_FLOW_ITEM_TYPE_ETH, 882 }, 883 [MLX5_EXPANSION_VLAN] = { 884 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 885 MLX5_EXPANSION_IPV6), 886 .type = RTE_FLOW_ITEM_TYPE_VLAN, 887 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 888 }, 889 [MLX5_EXPANSION_IPV4] = { 890 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 891 MLX5_EXPANSION_IPV4_TCP, 892 MLX5_EXPANSION_IPV4_ESP, 893 MLX5_EXPANSION_IPV4_ICMP), 894 .type = RTE_FLOW_ITEM_TYPE_IPV4, 895 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 896 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 897 }, 898 [MLX5_EXPANSION_IPV4_UDP] = { 899 .type = RTE_FLOW_ITEM_TYPE_UDP, 900 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 901 }, 902 [MLX5_EXPANSION_IPV4_TCP] = { 903 .type = RTE_FLOW_ITEM_TYPE_TCP, 904 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 905 }, 906 [MLX5_EXPANSION_IPV4_ESP] = { 907 .type = RTE_FLOW_ITEM_TYPE_ESP, 908 .rss_types = RTE_ETH_RSS_ESP, 909 }, 910 [MLX5_EXPANSION_IPV4_ICMP] = { 911 .type = RTE_FLOW_ITEM_TYPE_ICMP, 912 }, 913 [MLX5_EXPANSION_IPV6] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 915 MLX5_EXPANSION_IPV6_TCP, 916 MLX5_EXPANSION_IPV6_ESP, 917 MLX5_EXPANSION_IPV6_ICMP6, 918 MLX5_EXPANSION_IPV6_FRAG_EXT), 919 .type = RTE_FLOW_ITEM_TYPE_IPV6, 920 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 921 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 922 }, 923 [MLX5_EXPANSION_IPV6_UDP] = { 924 .type = RTE_FLOW_ITEM_TYPE_UDP, 925 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 926 }, 927 [MLX5_EXPANSION_IPV6_TCP] = { 928 .type = RTE_FLOW_ITEM_TYPE_TCP, 929 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 930 }, 931 [MLX5_EXPANSION_IPV6_ESP] = { 932 .type = RTE_FLOW_ITEM_TYPE_ESP, 933 .rss_types = RTE_ETH_RSS_ESP, 934 }, 935 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 936 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 937 }, 938 [MLX5_EXPANSION_IPV6_ICMP6] = { 939 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 940 }, 941 [MLX5_EXPANSION_GTP] = { 942 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 943 MLX5_EXPANSION_IPV6), 944 .type = RTE_FLOW_ITEM_TYPE_GTP, 945 }, 946 [MLX5_EXPANSION_GENEVE] = { 947 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 948 MLX5_EXPANSION_IPV4, 949 MLX5_EXPANSION_IPV6), 950 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 951 }, 952 }; 953 954 static struct rte_flow_action_handle * 955 mlx5_action_handle_create(struct rte_eth_dev *dev, 956 const struct rte_flow_indir_action_conf *conf, 957 const struct rte_flow_action *action, 958 struct rte_flow_error *error); 959 static int mlx5_action_handle_destroy 960 (struct rte_eth_dev *dev, 961 struct rte_flow_action_handle *handle, 962 struct rte_flow_error *error); 963 static int mlx5_action_handle_update 964 (struct rte_eth_dev *dev, 965 struct rte_flow_action_handle *handle, 966 const void *update, 967 struct rte_flow_error *error); 968 static int mlx5_action_handle_query 969 (struct rte_eth_dev *dev, 970 const struct rte_flow_action_handle *handle, 971 void *data, 972 struct rte_flow_error *error); 973 static int 974 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 975 struct rte_flow_tunnel *app_tunnel, 976 struct rte_flow_action **actions, 977 uint32_t *num_of_actions, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 981 struct rte_flow_tunnel *app_tunnel, 982 struct rte_flow_item **items, 983 uint32_t *num_of_items, 984 struct rte_flow_error *error); 985 static int 986 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 987 struct rte_flow_item *pmd_items, 988 uint32_t num_items, struct rte_flow_error *err); 989 static int 990 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 991 struct rte_flow_action *pmd_actions, 992 uint32_t num_actions, 993 struct rte_flow_error *err); 994 static int 995 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 996 struct rte_mbuf *m, 997 struct rte_flow_restore_info *info, 998 struct rte_flow_error *err); 999 static struct rte_flow_item_flex_handle * 1000 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 1001 const struct rte_flow_item_flex_conf *conf, 1002 struct rte_flow_error *error); 1003 static int 1004 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 1005 const struct rte_flow_item_flex_handle *handle, 1006 struct rte_flow_error *error); 1007 static int 1008 mlx5_flow_info_get(struct rte_eth_dev *dev, 1009 struct rte_flow_port_info *port_info, 1010 struct rte_flow_queue_info *queue_info, 1011 struct rte_flow_error *error); 1012 static int 1013 mlx5_flow_port_configure(struct rte_eth_dev *dev, 1014 const struct rte_flow_port_attr *port_attr, 1015 uint16_t nb_queue, 1016 const struct rte_flow_queue_attr *queue_attr[], 1017 struct rte_flow_error *err); 1018 1019 static struct rte_flow_pattern_template * 1020 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 1021 const struct rte_flow_pattern_template_attr *attr, 1022 const struct rte_flow_item items[], 1023 struct rte_flow_error *error); 1024 1025 static int 1026 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1027 struct rte_flow_pattern_template *template, 1028 struct rte_flow_error *error); 1029 static struct rte_flow_actions_template * 1030 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1031 const struct rte_flow_actions_template_attr *attr, 1032 const struct rte_flow_action actions[], 1033 const struct rte_flow_action masks[], 1034 struct rte_flow_error *error); 1035 static int 1036 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1037 struct rte_flow_actions_template *template, 1038 struct rte_flow_error *error); 1039 1040 static struct rte_flow_template_table * 1041 mlx5_flow_table_create(struct rte_eth_dev *dev, 1042 const struct rte_flow_template_table_attr *attr, 1043 struct rte_flow_pattern_template *item_templates[], 1044 uint8_t nb_item_templates, 1045 struct rte_flow_actions_template *action_templates[], 1046 uint8_t nb_action_templates, 1047 struct rte_flow_error *error); 1048 static int 1049 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1050 struct rte_flow_template_table *table, 1051 struct rte_flow_error *error); 1052 static int 1053 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 1054 uint32_t group_id, 1055 const struct rte_flow_group_attr *attr, 1056 const struct rte_flow_action actions[], 1057 struct rte_flow_error *error); 1058 1059 static int 1060 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1061 struct rte_flow_action_handle *handle, 1062 const void *update, void *query, 1063 enum rte_flow_query_update_mode qu_mode, 1064 struct rte_flow_error *error); 1065 1066 static struct rte_flow_action_list_handle * 1067 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 1068 const struct rte_flow_indir_action_conf *conf, 1069 const struct rte_flow_action *actions, 1070 struct rte_flow_error *error); 1071 1072 static int 1073 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 1074 struct rte_flow_action_list_handle *handle, 1075 struct rte_flow_error *error); 1076 1077 static int 1078 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 1079 const 1080 struct rte_flow_action_list_handle *handle, 1081 const void **update, void **query, 1082 enum rte_flow_query_update_mode mode, 1083 struct rte_flow_error *error); 1084 1085 static int 1086 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 1087 const struct rte_flow_template_table *table, 1088 const struct rte_flow_item pattern[], 1089 uint8_t pattern_template_index, 1090 uint32_t *hash, struct rte_flow_error *error); 1091 static int 1092 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 1093 const struct rte_flow_item pattern[], 1094 enum rte_flow_encap_hash_field dest_field, 1095 uint8_t *hash, 1096 struct rte_flow_error *error); 1097 1098 static const struct rte_flow_ops mlx5_flow_ops = { 1099 .validate = mlx5_flow_validate, 1100 .create = mlx5_flow_create, 1101 .destroy = mlx5_flow_destroy, 1102 .flush = mlx5_flow_flush, 1103 .isolate = mlx5_flow_isolate, 1104 .query = mlx5_flow_query, 1105 .dev_dump = mlx5_flow_dev_dump, 1106 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1107 .get_aged_flows = mlx5_flow_get_aged_flows, 1108 .action_handle_create = mlx5_action_handle_create, 1109 .action_handle_destroy = mlx5_action_handle_destroy, 1110 .action_handle_update = mlx5_action_handle_update, 1111 .action_handle_query = mlx5_action_handle_query, 1112 .action_handle_query_update = mlx5_action_handle_query_update, 1113 .action_list_handle_create = mlx5_action_list_handle_create, 1114 .action_list_handle_destroy = mlx5_action_list_handle_destroy, 1115 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1116 .tunnel_match = mlx5_flow_tunnel_match, 1117 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1118 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1119 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1120 .flex_item_create = mlx5_flow_flex_item_create, 1121 .flex_item_release = mlx5_flow_flex_item_release, 1122 .info_get = mlx5_flow_info_get, 1123 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1124 .configure = mlx5_flow_port_configure, 1125 .pattern_template_create = mlx5_flow_pattern_template_create, 1126 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1127 .actions_template_create = mlx5_flow_actions_template_create, 1128 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1129 .template_table_create = mlx5_flow_table_create, 1130 .template_table_destroy = mlx5_flow_table_destroy, 1131 .group_set_miss_actions = mlx5_flow_group_set_miss_actions, 1132 .action_list_handle_query_update = 1133 mlx5_flow_action_list_handle_query_update, 1134 .flow_calc_table_hash = mlx5_flow_calc_table_hash, 1135 .flow_calc_encap_hash = mlx5_flow_calc_encap_hash, 1136 }; 1137 1138 /* Tunnel information. */ 1139 struct mlx5_flow_tunnel_info { 1140 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1141 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1142 }; 1143 1144 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1145 { 1146 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1147 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1148 }, 1149 { 1150 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1151 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1152 }, 1153 { 1154 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1155 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1156 }, 1157 { 1158 .tunnel = MLX5_FLOW_LAYER_GRE, 1159 .ptype = RTE_PTYPE_TUNNEL_GRE, 1160 }, 1161 { 1162 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1163 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1164 }, 1165 { 1166 .tunnel = MLX5_FLOW_LAYER_MPLS, 1167 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1168 }, 1169 { 1170 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1171 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1172 }, 1173 { 1174 .tunnel = MLX5_FLOW_LAYER_IPIP, 1175 .ptype = RTE_PTYPE_TUNNEL_IP, 1176 }, 1177 { 1178 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1179 .ptype = RTE_PTYPE_TUNNEL_IP, 1180 }, 1181 { 1182 .tunnel = MLX5_FLOW_LAYER_GTP, 1183 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1184 }, 1185 }; 1186 1187 1188 1189 /** 1190 * Translate tag ID to register. 1191 * 1192 * @param[in] dev 1193 * Pointer to the Ethernet device structure. 1194 * @param[in] feature 1195 * The feature that request the register. 1196 * @param[in] id 1197 * The request register ID. 1198 * @param[out] error 1199 * Error description in case of any. 1200 * 1201 * @return 1202 * The request register on success, a negative errno 1203 * value otherwise and rte_errno is set. 1204 */ 1205 int 1206 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1207 enum mlx5_feature_name feature, 1208 uint32_t id, 1209 struct rte_flow_error *error) 1210 { 1211 struct mlx5_priv *priv = dev->data->dev_private; 1212 struct mlx5_sh_config *config = &priv->sh->config; 1213 struct mlx5_dev_registers *reg = &priv->sh->registers; 1214 enum modify_reg start_reg; 1215 bool skip_mtr_reg = false; 1216 1217 switch (feature) { 1218 case MLX5_HAIRPIN_RX: 1219 return REG_B; 1220 case MLX5_HAIRPIN_TX: 1221 return REG_A; 1222 case MLX5_METADATA_RX: 1223 switch (config->dv_xmeta_en) { 1224 case MLX5_XMETA_MODE_LEGACY: 1225 return REG_B; 1226 case MLX5_XMETA_MODE_META16: 1227 return REG_C_0; 1228 case MLX5_XMETA_MODE_META32: 1229 return REG_C_1; 1230 case MLX5_XMETA_MODE_META32_HWS: 1231 return REG_C_1; 1232 } 1233 break; 1234 case MLX5_METADATA_TX: 1235 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1236 return REG_C_1; 1237 } else { 1238 return REG_A; 1239 } 1240 case MLX5_METADATA_FDB: 1241 switch (config->dv_xmeta_en) { 1242 case MLX5_XMETA_MODE_LEGACY: 1243 return REG_NON; 1244 case MLX5_XMETA_MODE_META16: 1245 return REG_C_0; 1246 case MLX5_XMETA_MODE_META32: 1247 return REG_C_1; 1248 case MLX5_XMETA_MODE_META32_HWS: 1249 return REG_C_1; 1250 } 1251 break; 1252 case MLX5_FLOW_MARK: 1253 switch (config->dv_xmeta_en) { 1254 case MLX5_XMETA_MODE_LEGACY: 1255 case MLX5_XMETA_MODE_META32_HWS: 1256 return REG_NON; 1257 case MLX5_XMETA_MODE_META16: 1258 return REG_C_1; 1259 case MLX5_XMETA_MODE_META32: 1260 return REG_C_0; 1261 } 1262 break; 1263 case MLX5_MTR_ID: 1264 /* 1265 * If meter color and meter id share one register, flow match 1266 * should use the meter color register for match. 1267 */ 1268 if (priv->mtr_reg_share) 1269 return reg->aso_reg; 1270 else 1271 return reg->aso_reg != REG_C_2 ? REG_C_2 : 1272 REG_C_3; 1273 case MLX5_MTR_COLOR: 1274 case MLX5_ASO_FLOW_HIT: 1275 case MLX5_ASO_CONNTRACK: 1276 case MLX5_SAMPLE_ID: 1277 /* All features use the same REG_C. */ 1278 MLX5_ASSERT(reg->aso_reg != REG_NON); 1279 return reg->aso_reg; 1280 case MLX5_COPY_MARK: 1281 /* 1282 * Metadata COPY_MARK register using is in meter suffix sub 1283 * flow while with meter. It's safe to share the same register. 1284 */ 1285 return reg->aso_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1286 case MLX5_APP_TAG: 1287 /* 1288 * If meter is enable, it will engage the register for color 1289 * match and flow match. If meter color match is not using the 1290 * REG_C_2, need to skip the REG_C_x be used by meter color 1291 * match. 1292 * If meter is disable, free to use all available registers. 1293 */ 1294 start_reg = reg->aso_reg != REG_C_2 ? REG_C_2 : 1295 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1296 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1297 if (id > (uint32_t)(REG_C_7 - start_reg)) 1298 return rte_flow_error_set(error, EINVAL, 1299 RTE_FLOW_ERROR_TYPE_ITEM, 1300 NULL, "invalid tag id"); 1301 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1302 return rte_flow_error_set(error, ENOTSUP, 1303 RTE_FLOW_ERROR_TYPE_ITEM, 1304 NULL, "unsupported tag id"); 1305 /* 1306 * This case means meter is using the REG_C_x great than 2. 1307 * Take care not to conflict with meter color REG_C_x. 1308 * If the available index REG_C_y >= REG_C_x, skip the 1309 * color register. 1310 */ 1311 if (skip_mtr_reg && priv->sh->flow_mreg_c 1312 [id + start_reg - REG_C_0] >= reg->aso_reg) { 1313 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1314 return rte_flow_error_set(error, EINVAL, 1315 RTE_FLOW_ERROR_TYPE_ITEM, 1316 NULL, "invalid tag id"); 1317 if (priv->sh->flow_mreg_c 1318 [id + 1 + start_reg - REG_C_0] != REG_NON) 1319 return priv->sh->flow_mreg_c 1320 [id + 1 + start_reg - REG_C_0]; 1321 return rte_flow_error_set(error, ENOTSUP, 1322 RTE_FLOW_ERROR_TYPE_ITEM, 1323 NULL, "unsupported tag id"); 1324 } 1325 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1326 } 1327 MLX5_ASSERT(false); 1328 return rte_flow_error_set(error, EINVAL, 1329 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1330 NULL, "invalid feature name"); 1331 } 1332 1333 /** 1334 * Check extensive flow metadata register support. 1335 * 1336 * @param dev 1337 * Pointer to rte_eth_dev structure. 1338 * 1339 * @return 1340 * True if device supports extensive flow metadata register, otherwise false. 1341 */ 1342 bool 1343 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1344 { 1345 struct mlx5_priv *priv = dev->data->dev_private; 1346 1347 /* 1348 * Having available reg_c can be regarded inclusively as supporting 1349 * extensive flow metadata register, which could mean, 1350 * - metadata register copy action by modify header. 1351 * - 16 modify header actions is supported. 1352 * - reg_c's are preserved across different domain (FDB and NIC) on 1353 * packet loopback by flow lookup miss. 1354 */ 1355 return priv->sh->flow_mreg_c[2] != REG_NON; 1356 } 1357 1358 /** 1359 * Get the lowest priority. 1360 * 1361 * @param[in] dev 1362 * Pointer to the Ethernet device structure. 1363 * @param[in] attributes 1364 * Pointer to device flow rule attributes. 1365 * 1366 * @return 1367 * The value of lowest priority of flow. 1368 */ 1369 uint32_t 1370 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1371 const struct rte_flow_attr *attr) 1372 { 1373 struct mlx5_priv *priv = dev->data->dev_private; 1374 1375 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1376 return priv->sh->flow_max_priority - 2; 1377 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1378 } 1379 1380 /** 1381 * Calculate matcher priority of the flow. 1382 * 1383 * @param[in] dev 1384 * Pointer to the Ethernet device structure. 1385 * @param[in] attr 1386 * Pointer to device flow rule attributes. 1387 * @param[in] subpriority 1388 * The priority based on the items. 1389 * @param[in] external 1390 * Flow is user flow. 1391 * @return 1392 * The matcher priority of the flow. 1393 */ 1394 uint16_t 1395 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1396 const struct rte_flow_attr *attr, 1397 uint32_t subpriority, bool external) 1398 { 1399 uint16_t priority = (uint16_t)attr->priority; 1400 struct mlx5_priv *priv = dev->data->dev_private; 1401 1402 /* NIC root rules */ 1403 if (!attr->group && !attr->transfer) { 1404 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1405 priority = priv->sh->flow_max_priority - 1; 1406 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1407 /* FDB root rules */ 1408 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1409 attr->group == 0 && 1410 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1411 return (priv->sh->flow_max_priority - 1) * 3; 1412 } 1413 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1414 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1415 return priority * 3 + subpriority; 1416 } 1417 1418 /** 1419 * Verify the @p item specifications (spec, last, mask) are compatible with the 1420 * NIC capabilities. 1421 * 1422 * @param[in] item 1423 * Item specification. 1424 * @param[in] mask 1425 * @p item->mask or flow default bit-masks. 1426 * @param[in] nic_mask 1427 * Bit-masks covering supported fields by the NIC to compare with user mask. 1428 * @param[in] size 1429 * Bit-masks size in bytes. 1430 * @param[in] range_accepted 1431 * True if range of values is accepted for specific fields, false otherwise. 1432 * @param[out] error 1433 * Pointer to error structure. 1434 * 1435 * @return 1436 * 0 on success, a negative errno value otherwise and rte_errno is set. 1437 */ 1438 int 1439 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1440 const uint8_t *mask, 1441 const uint8_t *nic_mask, 1442 unsigned int size, 1443 bool range_accepted, 1444 struct rte_flow_error *error) 1445 { 1446 unsigned int i; 1447 1448 MLX5_ASSERT(nic_mask); 1449 for (i = 0; i < size; ++i) 1450 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1451 return rte_flow_error_set(error, ENOTSUP, 1452 RTE_FLOW_ERROR_TYPE_ITEM, 1453 item, 1454 "mask enables non supported" 1455 " bits"); 1456 if (!item->spec && (item->mask || item->last)) 1457 return rte_flow_error_set(error, EINVAL, 1458 RTE_FLOW_ERROR_TYPE_ITEM, item, 1459 "mask/last without a spec is not" 1460 " supported"); 1461 if (item->spec && item->last && !range_accepted) { 1462 uint8_t spec[size]; 1463 uint8_t last[size]; 1464 unsigned int i; 1465 int ret; 1466 1467 for (i = 0; i < size; ++i) { 1468 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1469 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1470 } 1471 ret = memcmp(spec, last, size); 1472 if (ret != 0) 1473 return rte_flow_error_set(error, EINVAL, 1474 RTE_FLOW_ERROR_TYPE_ITEM, 1475 item, 1476 "range is not valid"); 1477 } 1478 return 0; 1479 } 1480 1481 /** 1482 * Adjust the hash fields according to the @p flow information. 1483 * 1484 * @param[in] dev_flow. 1485 * Pointer to the mlx5_flow. 1486 * @param[in] tunnel 1487 * 1 when the hash field is for a tunnel item. 1488 * @param[in] layer_types 1489 * RTE_ETH_RSS_* types. 1490 * @param[in] hash_fields 1491 * Item hash fields. 1492 * 1493 * @return 1494 * The hash fields that should be used. 1495 */ 1496 uint64_t 1497 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1498 int tunnel __rte_unused, uint64_t layer_types, 1499 uint64_t hash_fields) 1500 { 1501 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1502 int rss_request_inner = rss_desc->level >= 2; 1503 1504 /* Check RSS hash level for tunnel. */ 1505 if (tunnel && rss_request_inner) 1506 hash_fields |= IBV_RX_HASH_INNER; 1507 else if (tunnel || rss_request_inner) 1508 return 0; 1509 #endif 1510 /* Check if requested layer matches RSS hash fields. */ 1511 if (!(rss_desc->types & layer_types)) 1512 return 0; 1513 return hash_fields; 1514 } 1515 1516 /** 1517 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1518 * if several tunnel rules are used on this queue, the tunnel ptype will be 1519 * cleared. 1520 * 1521 * @param rxq_ctrl 1522 * Rx queue to update. 1523 */ 1524 static void 1525 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1526 { 1527 unsigned int i; 1528 uint32_t tunnel_ptype = 0; 1529 1530 /* Look up for the ptype to use. */ 1531 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1532 if (!rxq_ctrl->flow_tunnels_n[i]) 1533 continue; 1534 if (!tunnel_ptype) { 1535 tunnel_ptype = tunnels_info[i].ptype; 1536 } else { 1537 tunnel_ptype = 0; 1538 break; 1539 } 1540 } 1541 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1542 } 1543 1544 /** 1545 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1546 * flow. 1547 * 1548 * @param[in] dev 1549 * Pointer to the Ethernet device structure. 1550 * @param[in] dev_handle 1551 * Pointer to device flow handle structure. 1552 */ 1553 void 1554 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1555 struct mlx5_flow_handle *dev_handle) 1556 { 1557 struct mlx5_priv *priv = dev->data->dev_private; 1558 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1559 struct mlx5_ind_table_obj *ind_tbl = NULL; 1560 unsigned int i; 1561 1562 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1563 struct mlx5_hrxq *hrxq; 1564 1565 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1566 dev_handle->rix_hrxq); 1567 if (hrxq) 1568 ind_tbl = hrxq->ind_table; 1569 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1570 struct mlx5_shared_action_rss *shared_rss; 1571 1572 shared_rss = mlx5_ipool_get 1573 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1574 dev_handle->rix_srss); 1575 if (shared_rss) 1576 ind_tbl = shared_rss->ind_tbl; 1577 } 1578 if (!ind_tbl) 1579 return; 1580 for (i = 0; i != ind_tbl->queues_n; ++i) { 1581 int idx = ind_tbl->queues[i]; 1582 struct mlx5_rxq_ctrl *rxq_ctrl; 1583 1584 if (mlx5_is_external_rxq(dev, idx)) 1585 continue; 1586 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1587 MLX5_ASSERT(rxq_ctrl != NULL); 1588 if (rxq_ctrl == NULL) 1589 continue; 1590 /* 1591 * To support metadata register copy on Tx loopback, 1592 * this must be always enabled (metadata may arive 1593 * from other port - not from local flows only. 1594 */ 1595 if (tunnel) { 1596 unsigned int j; 1597 1598 /* Increase the counter matching the flow. */ 1599 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1600 if ((tunnels_info[j].tunnel & 1601 dev_handle->layers) == 1602 tunnels_info[j].tunnel) { 1603 rxq_ctrl->flow_tunnels_n[j]++; 1604 break; 1605 } 1606 } 1607 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1608 } 1609 } 1610 } 1611 1612 static void 1613 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1614 { 1615 struct mlx5_priv *priv = dev->data->dev_private; 1616 struct mlx5_rxq_ctrl *rxq_ctrl; 1617 uint16_t port_id; 1618 1619 if (priv->sh->shared_mark_enabled) 1620 return; 1621 if (priv->master || priv->representor) { 1622 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1623 struct mlx5_priv *opriv = 1624 rte_eth_devices[port_id].data->dev_private; 1625 1626 if (!opriv || 1627 opriv->sh != priv->sh || 1628 opriv->domain_id != priv->domain_id || 1629 opriv->mark_enabled) 1630 continue; 1631 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1632 rxq_ctrl->rxq.mark = 1; 1633 } 1634 opriv->mark_enabled = 1; 1635 } 1636 } else { 1637 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1638 rxq_ctrl->rxq.mark = 1; 1639 } 1640 priv->mark_enabled = 1; 1641 } 1642 priv->sh->shared_mark_enabled = 1; 1643 } 1644 1645 /** 1646 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1647 * 1648 * @param[in] dev 1649 * Pointer to the Ethernet device structure. 1650 * @param[in] flow 1651 * Pointer to flow structure. 1652 */ 1653 static void 1654 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1655 { 1656 struct mlx5_priv *priv = dev->data->dev_private; 1657 uint32_t handle_idx; 1658 struct mlx5_flow_handle *dev_handle; 1659 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1660 1661 MLX5_ASSERT(wks); 1662 if (wks->mark) 1663 flow_rxq_mark_flag_set(dev); 1664 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1665 handle_idx, dev_handle, next) 1666 flow_drv_rxq_flags_set(dev, dev_handle); 1667 } 1668 1669 /** 1670 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1671 * device flow if no other flow uses it with the same kind of request. 1672 * 1673 * @param dev 1674 * Pointer to Ethernet device. 1675 * @param[in] dev_handle 1676 * Pointer to the device flow handle structure. 1677 */ 1678 static void 1679 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1680 struct mlx5_flow_handle *dev_handle) 1681 { 1682 struct mlx5_priv *priv = dev->data->dev_private; 1683 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1684 struct mlx5_ind_table_obj *ind_tbl = NULL; 1685 unsigned int i; 1686 1687 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1688 struct mlx5_hrxq *hrxq; 1689 1690 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1691 dev_handle->rix_hrxq); 1692 if (hrxq) 1693 ind_tbl = hrxq->ind_table; 1694 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1695 struct mlx5_shared_action_rss *shared_rss; 1696 1697 shared_rss = mlx5_ipool_get 1698 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1699 dev_handle->rix_srss); 1700 if (shared_rss) 1701 ind_tbl = shared_rss->ind_tbl; 1702 } 1703 if (!ind_tbl) 1704 return; 1705 MLX5_ASSERT(dev->data->dev_started); 1706 for (i = 0; i != ind_tbl->queues_n; ++i) { 1707 int idx = ind_tbl->queues[i]; 1708 struct mlx5_rxq_ctrl *rxq_ctrl; 1709 1710 if (mlx5_is_external_rxq(dev, idx)) 1711 continue; 1712 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1713 MLX5_ASSERT(rxq_ctrl != NULL); 1714 if (rxq_ctrl == NULL) 1715 continue; 1716 if (tunnel) { 1717 unsigned int j; 1718 1719 /* Decrease the counter matching the flow. */ 1720 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1721 if ((tunnels_info[j].tunnel & 1722 dev_handle->layers) == 1723 tunnels_info[j].tunnel) { 1724 rxq_ctrl->flow_tunnels_n[j]--; 1725 break; 1726 } 1727 } 1728 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1729 } 1730 } 1731 } 1732 1733 /** 1734 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1735 * @p flow if no other flow uses it with the same kind of request. 1736 * 1737 * @param dev 1738 * Pointer to Ethernet device. 1739 * @param[in] flow 1740 * Pointer to the flow. 1741 */ 1742 static void 1743 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1744 { 1745 struct mlx5_priv *priv = dev->data->dev_private; 1746 uint32_t handle_idx; 1747 struct mlx5_flow_handle *dev_handle; 1748 1749 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1750 handle_idx, dev_handle, next) 1751 flow_drv_rxq_flags_trim(dev, dev_handle); 1752 } 1753 1754 /** 1755 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1756 * 1757 * @param dev 1758 * Pointer to Ethernet device. 1759 */ 1760 static void 1761 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1762 { 1763 struct mlx5_priv *priv = dev->data->dev_private; 1764 unsigned int i; 1765 1766 for (i = 0; i != priv->rxqs_n; ++i) { 1767 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1768 unsigned int j; 1769 1770 if (rxq == NULL || rxq->ctrl == NULL) 1771 continue; 1772 rxq->ctrl->rxq.mark = 0; 1773 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1774 rxq->ctrl->flow_tunnels_n[j] = 0; 1775 rxq->ctrl->rxq.tunnel = 0; 1776 } 1777 priv->mark_enabled = 0; 1778 priv->sh->shared_mark_enabled = 0; 1779 } 1780 1781 static uint64_t mlx5_restore_info_dynflag; 1782 1783 int 1784 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 1785 { 1786 struct mlx5_priv *priv = dev->data->dev_private; 1787 uint64_t supported = 0; 1788 1789 if (!is_tunnel_offload_active(dev)) { 1790 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 1791 supported |= RTE_ETH_RX_METADATA_USER_MARK; 1792 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) { 1793 DRV_LOG(DEBUG, 1794 "tunnel offload was not activated, consider setting dv_xmeta_en=%d", 1795 MLX5_XMETA_MODE_MISS_INFO); 1796 } 1797 } else { 1798 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 1799 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 && 1800 mlx5_restore_info_dynflag == 0) 1801 mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag(); 1802 } 1803 1804 if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) 1805 priv->tunnel_enabled = 1; 1806 else 1807 priv->tunnel_enabled = 0; 1808 1809 *features &= supported; 1810 return 0; 1811 } 1812 1813 /** 1814 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1815 * 1816 * @param[in] dev 1817 * Pointer to the Ethernet device structure. 1818 */ 1819 void 1820 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) 1821 { 1822 struct mlx5_priv *priv = dev->data->dev_private; 1823 uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID; 1824 unsigned int i; 1825 1826 if (priv->tunnel_enabled) 1827 mark_flag |= mlx5_restore_info_dynflag; 1828 1829 for (i = 0; i != priv->rxqs_n; ++i) { 1830 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1831 struct mlx5_rxq_data *data; 1832 1833 if (rxq == NULL || rxq->ctrl == NULL) 1834 continue; 1835 data = &rxq->ctrl->rxq; 1836 if (!rte_flow_dynf_metadata_avail()) { 1837 data->dynf_meta = 0; 1838 data->flow_meta_mask = 0; 1839 data->flow_meta_offset = -1; 1840 data->flow_meta_port_mask = 0; 1841 } else { 1842 data->dynf_meta = 1; 1843 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1844 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1845 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1846 } 1847 data->mark_flag = mark_flag; 1848 } 1849 } 1850 1851 /* 1852 * return a pointer to the desired action in the list of actions. 1853 * 1854 * @param[in] actions 1855 * The list of actions to search the action in. 1856 * @param[in] action 1857 * The action to find. 1858 * 1859 * @return 1860 * Pointer to the action in the list, if found. NULL otherwise. 1861 */ 1862 const struct rte_flow_action * 1863 mlx5_flow_find_action(const struct rte_flow_action *actions, 1864 enum rte_flow_action_type action) 1865 { 1866 if (actions == NULL) 1867 return NULL; 1868 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1869 if (actions->type == action) 1870 return actions; 1871 return NULL; 1872 } 1873 1874 /* 1875 * Validate the flag action. 1876 * 1877 * @param[in] action_flags 1878 * Bit-fields that holds the actions detected until now. 1879 * @param[in] attr 1880 * Attributes of flow that includes this action. 1881 * @param[out] error 1882 * Pointer to error structure. 1883 * 1884 * @return 1885 * 0 on success, a negative errno value otherwise and rte_errno is set. 1886 */ 1887 int 1888 mlx5_flow_validate_action_flag(uint64_t action_flags, 1889 const struct rte_flow_attr *attr, 1890 struct rte_flow_error *error) 1891 { 1892 if (action_flags & MLX5_FLOW_ACTION_MARK) 1893 return rte_flow_error_set(error, EINVAL, 1894 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1895 "can't mark and flag in same flow"); 1896 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1897 return rte_flow_error_set(error, EINVAL, 1898 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1899 "can't have 2 flag" 1900 " actions in same flow"); 1901 if (attr->egress) 1902 return rte_flow_error_set(error, ENOTSUP, 1903 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1904 "flag action not supported for " 1905 "egress"); 1906 return 0; 1907 } 1908 1909 /* 1910 * Validate the mark action. 1911 * 1912 * @param[in] action 1913 * Pointer to the queue action. 1914 * @param[in] action_flags 1915 * Bit-fields that holds the actions detected until now. 1916 * @param[in] attr 1917 * Attributes of flow that includes this action. 1918 * @param[out] error 1919 * Pointer to error structure. 1920 * 1921 * @return 1922 * 0 on success, a negative errno value otherwise and rte_errno is set. 1923 */ 1924 int 1925 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1926 uint64_t action_flags, 1927 const struct rte_flow_attr *attr, 1928 struct rte_flow_error *error) 1929 { 1930 const struct rte_flow_action_mark *mark = action->conf; 1931 1932 if (!mark) 1933 return rte_flow_error_set(error, EINVAL, 1934 RTE_FLOW_ERROR_TYPE_ACTION, 1935 action, 1936 "configuration cannot be null"); 1937 if (mark->id >= MLX5_FLOW_MARK_MAX) 1938 return rte_flow_error_set(error, EINVAL, 1939 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1940 &mark->id, 1941 "mark id must in 0 <= id < " 1942 RTE_STR(MLX5_FLOW_MARK_MAX)); 1943 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1944 return rte_flow_error_set(error, EINVAL, 1945 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1946 "can't flag and mark in same flow"); 1947 if (action_flags & MLX5_FLOW_ACTION_MARK) 1948 return rte_flow_error_set(error, EINVAL, 1949 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1950 "can't have 2 mark actions in same" 1951 " flow"); 1952 if (attr->egress) 1953 return rte_flow_error_set(error, ENOTSUP, 1954 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1955 "mark action not supported for " 1956 "egress"); 1957 return 0; 1958 } 1959 1960 /* 1961 * Validate the drop action. 1962 * 1963 * @param[in] dev 1964 * Pointer to the Ethernet device structure. 1965 * @param[in] is_root 1966 * True if flow is validated for root table. False otherwise. 1967 * @param[in] attr 1968 * Attributes of flow that includes this action. 1969 * @param[out] error 1970 * Pointer to error structure. 1971 * 1972 * @return 1973 * 0 on success, a negative errno value otherwise and rte_errno is set. 1974 */ 1975 int 1976 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 1977 bool is_root, 1978 const struct rte_flow_attr *attr, 1979 struct rte_flow_error *error) 1980 { 1981 struct mlx5_priv *priv = dev->data->dev_private; 1982 1983 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 1984 return rte_flow_error_set(error, ENOTSUP, 1985 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1986 "drop action not supported for " 1987 "egress"); 1988 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 1989 !priv->sh->dr_root_drop_action_en) { 1990 return rte_flow_error_set(error, ENOTSUP, 1991 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 1992 "drop action not supported for " 1993 "egress and transfer on group 0"); 1994 } 1995 return 0; 1996 } 1997 1998 /* 1999 * Validate the queue action. 2000 * 2001 * @param[in] action 2002 * Pointer to the queue action. 2003 * @param[in] action_flags 2004 * Bit-fields that holds the actions detected until now. 2005 * @param[in] dev 2006 * Pointer to the Ethernet device structure. 2007 * @param[in] attr 2008 * Attributes of flow that includes this action. 2009 * @param[out] error 2010 * Pointer to error structure. 2011 * 2012 * @return 2013 * 0 on success, a negative errno value otherwise and rte_errno is set. 2014 */ 2015 int 2016 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 2017 uint64_t action_flags, 2018 struct rte_eth_dev *dev, 2019 const struct rte_flow_attr *attr, 2020 struct rte_flow_error *error) 2021 { 2022 struct mlx5_priv *priv = dev->data->dev_private; 2023 const struct rte_flow_action_queue *queue = action->conf; 2024 2025 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2026 return rte_flow_error_set(error, EINVAL, 2027 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2028 "can't have 2 fate actions in" 2029 " same flow"); 2030 if (attr->egress) 2031 return rte_flow_error_set(error, ENOTSUP, 2032 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2033 "queue action not supported for egress."); 2034 if (mlx5_is_external_rxq(dev, queue->index)) 2035 return 0; 2036 if (!priv->rxqs_n) 2037 return rte_flow_error_set(error, EINVAL, 2038 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2039 NULL, "No Rx queues configured"); 2040 if (queue->index >= priv->rxqs_n) 2041 return rte_flow_error_set(error, EINVAL, 2042 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2043 &queue->index, 2044 "queue index out of range"); 2045 if (mlx5_rxq_get(dev, queue->index) == NULL) 2046 return rte_flow_error_set(error, EINVAL, 2047 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2048 &queue->index, 2049 "queue is not configured"); 2050 return 0; 2051 } 2052 2053 /** 2054 * Validate queue numbers for device RSS. 2055 * 2056 * @param[in] dev 2057 * Configured device. 2058 * @param[in] queues 2059 * Array of queue numbers. 2060 * @param[in] queues_n 2061 * Size of the @p queues array. 2062 * @param[out] error 2063 * On error, filled with a textual error description. 2064 * @param[out] queue_idx 2065 * On error, filled with an offending queue index in @p queues array. 2066 * 2067 * @return 2068 * 0 on success, a negative errno code on error. 2069 */ 2070 static int 2071 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2072 const uint16_t *queues, uint32_t queues_n, 2073 const char **error, uint32_t *queue_idx) 2074 { 2075 const struct mlx5_priv *priv = dev->data->dev_private; 2076 bool is_hairpin = false; 2077 bool is_ext_rss = false; 2078 uint32_t i; 2079 2080 for (i = 0; i != queues_n; ++i) { 2081 struct mlx5_rxq_ctrl *rxq_ctrl; 2082 2083 if (mlx5_is_external_rxq(dev, queues[0])) { 2084 is_ext_rss = true; 2085 continue; 2086 } 2087 if (is_ext_rss) { 2088 *error = "Combining external and regular RSS queues is not supported"; 2089 *queue_idx = i; 2090 return -ENOTSUP; 2091 } 2092 if (queues[i] >= priv->rxqs_n) { 2093 *error = "queue index out of range"; 2094 *queue_idx = i; 2095 return -EINVAL; 2096 } 2097 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2098 if (rxq_ctrl == NULL) { 2099 *error = "queue is not configured"; 2100 *queue_idx = i; 2101 return -EINVAL; 2102 } 2103 if (i == 0 && rxq_ctrl->is_hairpin) 2104 is_hairpin = true; 2105 if (is_hairpin != rxq_ctrl->is_hairpin) { 2106 *error = "combining hairpin and regular RSS queues is not supported"; 2107 *queue_idx = i; 2108 return -ENOTSUP; 2109 } 2110 } 2111 return 0; 2112 } 2113 2114 /* 2115 * Validate the rss action. 2116 * 2117 * @param[in] dev 2118 * Pointer to the Ethernet device structure. 2119 * @param[in] action 2120 * Pointer to the queue action. 2121 * @param[out] error 2122 * Pointer to error structure. 2123 * 2124 * @return 2125 * 0 on success, a negative errno value otherwise and rte_errno is set. 2126 */ 2127 int 2128 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2129 const struct rte_flow_action *action, 2130 struct rte_flow_error *error) 2131 { 2132 struct mlx5_priv *priv = dev->data->dev_private; 2133 const struct rte_flow_action_rss *rss = action->conf; 2134 int ret; 2135 const char *message; 2136 uint32_t queue_idx; 2137 2138 if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 2139 DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT", 2140 dev->data->port_id); 2141 } else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2142 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2143 return rte_flow_error_set(error, ENOTSUP, 2144 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2145 &rss->func, 2146 "RSS hash function not supported"); 2147 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2148 if (rss->level > 2) 2149 #else 2150 if (rss->level > 1) 2151 #endif 2152 return rte_flow_error_set(error, ENOTSUP, 2153 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2154 &rss->level, 2155 "tunnel RSS is not supported"); 2156 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2157 if (rss->key_len == 0 && rss->key != NULL) 2158 return rte_flow_error_set(error, ENOTSUP, 2159 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2160 &rss->key_len, 2161 "RSS hash key length 0"); 2162 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2163 return rte_flow_error_set(error, ENOTSUP, 2164 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2165 &rss->key_len, 2166 "RSS hash key too small"); 2167 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2168 return rte_flow_error_set(error, ENOTSUP, 2169 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2170 &rss->key_len, 2171 "RSS hash key too large"); 2172 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2173 return rte_flow_error_set(error, ENOTSUP, 2174 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2175 &rss->queue_num, 2176 "number of queues too large"); 2177 if (rss->types & MLX5_RSS_HF_MASK) 2178 return rte_flow_error_set(error, ENOTSUP, 2179 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2180 &rss->types, 2181 "some RSS protocols are not" 2182 " supported"); 2183 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2184 !(rss->types & RTE_ETH_RSS_IP)) 2185 return rte_flow_error_set(error, EINVAL, 2186 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2187 "L3 partial RSS requested but L3 RSS" 2188 " type not specified"); 2189 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2190 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2191 return rte_flow_error_set(error, EINVAL, 2192 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2193 "L4 partial RSS requested but L4 RSS" 2194 " type not specified"); 2195 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2196 return rte_flow_error_set(error, EINVAL, 2197 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2198 NULL, "No Rx queues configured"); 2199 if (!rss->queue_num) 2200 return rte_flow_error_set(error, EINVAL, 2201 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2202 NULL, "No queues configured"); 2203 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2204 &message, &queue_idx); 2205 if (ret != 0) { 2206 return rte_flow_error_set(error, -ret, 2207 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2208 &rss->queue[queue_idx], message); 2209 } 2210 return 0; 2211 } 2212 2213 /* 2214 * Validate the rss action. 2215 * 2216 * @param[in] action 2217 * Pointer to the queue action. 2218 * @param[in] action_flags 2219 * Bit-fields that holds the actions detected until now. 2220 * @param[in] dev 2221 * Pointer to the Ethernet device structure. 2222 * @param[in] attr 2223 * Attributes of flow that includes this action. 2224 * @param[in] item_flags 2225 * Items that were detected. 2226 * @param[out] error 2227 * Pointer to error structure. 2228 * 2229 * @return 2230 * 0 on success, a negative errno value otherwise and rte_errno is set. 2231 */ 2232 int 2233 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2234 uint64_t action_flags, 2235 struct rte_eth_dev *dev, 2236 const struct rte_flow_attr *attr, 2237 uint64_t item_flags, 2238 struct rte_flow_error *error) 2239 { 2240 const struct rte_flow_action_rss *rss = action->conf; 2241 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2242 int ret; 2243 2244 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2245 return rte_flow_error_set(error, EINVAL, 2246 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2247 "can't have 2 fate actions" 2248 " in same flow"); 2249 ret = mlx5_validate_action_rss(dev, action, error); 2250 if (ret) 2251 return ret; 2252 if (attr->egress) 2253 return rte_flow_error_set(error, ENOTSUP, 2254 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2255 "rss action not supported for " 2256 "egress"); 2257 if (rss->level > 1 && !tunnel) 2258 return rte_flow_error_set(error, EINVAL, 2259 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2260 "inner RSS is not supported for " 2261 "non-tunnel flows"); 2262 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2263 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2266 "RSS on eCPRI is not supported now"); 2267 } 2268 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2269 !(item_flags & 2270 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2271 rss->level > 1) 2272 return rte_flow_error_set(error, EINVAL, 2273 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2274 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2275 return 0; 2276 } 2277 2278 /* 2279 * Validate the default miss action. 2280 * 2281 * @param[in] action_flags 2282 * Bit-fields that holds the actions detected until now. 2283 * @param[out] error 2284 * Pointer to error structure. 2285 * 2286 * @return 2287 * 0 on success, a negative errno value otherwise and rte_errno is set. 2288 */ 2289 int 2290 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2291 const struct rte_flow_attr *attr, 2292 struct rte_flow_error *error) 2293 { 2294 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2295 return rte_flow_error_set(error, EINVAL, 2296 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2297 "can't have 2 fate actions in" 2298 " same flow"); 2299 if (attr->egress) 2300 return rte_flow_error_set(error, ENOTSUP, 2301 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2302 "default miss action not supported " 2303 "for egress"); 2304 if (attr->group) 2305 return rte_flow_error_set(error, ENOTSUP, 2306 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2307 "only group 0 is supported"); 2308 if (attr->transfer) 2309 return rte_flow_error_set(error, ENOTSUP, 2310 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2311 NULL, "transfer is not supported"); 2312 return 0; 2313 } 2314 2315 /* 2316 * Validate the count action. 2317 * 2318 * @param[in] dev 2319 * Pointer to the Ethernet device structure. 2320 * @param[in] attr 2321 * Attributes of flow that includes this action. 2322 * @param[out] error 2323 * Pointer to error structure. 2324 * 2325 * @return 2326 * 0 on success, a negative errno value otherwise and rte_errno is set. 2327 */ 2328 int 2329 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2330 const struct rte_flow_attr *attr, 2331 struct rte_flow_error *error) 2332 { 2333 if (attr->egress) 2334 return rte_flow_error_set(error, ENOTSUP, 2335 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2336 "count action not supported for " 2337 "egress"); 2338 return 0; 2339 } 2340 2341 /* 2342 * Validate the ASO CT action. 2343 * 2344 * @param[in] dev 2345 * Pointer to the Ethernet device structure. 2346 * @param[in] conntrack 2347 * Pointer to the CT action profile. 2348 * @param[out] error 2349 * Pointer to error structure. 2350 * 2351 * @return 2352 * 0 on success, a negative errno value otherwise and rte_errno is set. 2353 */ 2354 int 2355 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2356 const struct rte_flow_action_conntrack *conntrack, 2357 struct rte_flow_error *error) 2358 { 2359 RTE_SET_USED(dev); 2360 2361 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2362 return rte_flow_error_set(error, EINVAL, 2363 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2364 "Invalid CT state"); 2365 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2366 return rte_flow_error_set(error, EINVAL, 2367 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2368 "Invalid last TCP packet flag"); 2369 return 0; 2370 } 2371 2372 /** 2373 * Validate the level value for modify field action. 2374 * 2375 * @param[in] data 2376 * Pointer to the rte_flow_field_data structure either src or dst. 2377 * @param[out] error 2378 * Pointer to error structure. 2379 * 2380 * @return 2381 * 0 on success, a negative errno value otherwise and rte_errno is set. 2382 */ 2383 int 2384 flow_validate_modify_field_level(const struct rte_flow_field_data *data, 2385 struct rte_flow_error *error) 2386 { 2387 if (data->level == 0) 2388 return 0; 2389 if (data->field != RTE_FLOW_FIELD_TAG && 2390 data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) 2391 return rte_flow_error_set(error, ENOTSUP, 2392 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2393 "inner header fields modification is not supported"); 2394 if (data->tag_index != 0) 2395 return rte_flow_error_set(error, EINVAL, 2396 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2397 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2398 /* 2399 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2400 * 'tag_index' field. In old API, it was provided using 'level' field 2401 * and it is still supported for backwards compatibility. 2402 */ 2403 DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field."); 2404 return 0; 2405 } 2406 2407 /** 2408 * Validate ICMP6 item. 2409 * 2410 * @param[in] item 2411 * Item specification. 2412 * @param[in] item_flags 2413 * Bit-fields that holds the items detected until now. 2414 * @param[in] ext_vlan_sup 2415 * Whether extended VLAN features are supported or not. 2416 * @param[out] error 2417 * Pointer to error structure. 2418 * 2419 * @return 2420 * 0 on success, a negative errno value otherwise and rte_errno is set. 2421 */ 2422 int 2423 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2424 uint64_t item_flags, 2425 uint8_t target_protocol, 2426 struct rte_flow_error *error) 2427 { 2428 const struct rte_flow_item_icmp6 *mask = item->mask; 2429 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2430 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2431 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2432 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2433 MLX5_FLOW_LAYER_OUTER_L4; 2434 int ret; 2435 2436 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2437 return rte_flow_error_set(error, EINVAL, 2438 RTE_FLOW_ERROR_TYPE_ITEM, item, 2439 "protocol filtering not compatible" 2440 " with ICMP6 layer"); 2441 if (!(item_flags & l3m)) 2442 return rte_flow_error_set(error, EINVAL, 2443 RTE_FLOW_ERROR_TYPE_ITEM, item, 2444 "IPv6 is mandatory to filter on" 2445 " ICMP6"); 2446 if (item_flags & l4m) 2447 return rte_flow_error_set(error, EINVAL, 2448 RTE_FLOW_ERROR_TYPE_ITEM, item, 2449 "multiple L4 layers not supported"); 2450 if (!mask) 2451 mask = &rte_flow_item_icmp6_mask; 2452 ret = mlx5_flow_item_acceptable 2453 (item, (const uint8_t *)mask, 2454 (const uint8_t *)&rte_flow_item_icmp6_mask, 2455 sizeof(struct rte_flow_item_icmp6), 2456 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2457 if (ret < 0) 2458 return ret; 2459 return 0; 2460 } 2461 2462 /** 2463 * Validate ICMP6 echo request/reply item. 2464 * 2465 * @param[in] item 2466 * Item specification. 2467 * @param[in] item_flags 2468 * Bit-fields that holds the items detected until now. 2469 * @param[in] ext_vlan_sup 2470 * Whether extended VLAN features are supported or not. 2471 * @param[out] error 2472 * Pointer to error structure. 2473 * 2474 * @return 2475 * 0 on success, a negative errno value otherwise and rte_errno is set. 2476 */ 2477 int 2478 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2479 uint64_t item_flags, 2480 uint8_t target_protocol, 2481 struct rte_flow_error *error) 2482 { 2483 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2484 const struct rte_flow_item_icmp6_echo nic_mask = { 2485 .hdr.base.type = 0xff, 2486 .hdr.base.code = 0xff, 2487 .hdr.identifier = RTE_BE16(0xffff), 2488 .hdr.sequence = RTE_BE16(0xffff), 2489 }; 2490 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2491 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2492 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2493 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2494 MLX5_FLOW_LAYER_OUTER_L4; 2495 int ret; 2496 2497 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2498 return rte_flow_error_set(error, EINVAL, 2499 RTE_FLOW_ERROR_TYPE_ITEM, item, 2500 "protocol filtering not compatible" 2501 " with ICMP6 layer"); 2502 if (!(item_flags & l3m)) 2503 return rte_flow_error_set(error, EINVAL, 2504 RTE_FLOW_ERROR_TYPE_ITEM, item, 2505 "IPv6 is mandatory to filter on" 2506 " ICMP6"); 2507 if (item_flags & l4m) 2508 return rte_flow_error_set(error, EINVAL, 2509 RTE_FLOW_ERROR_TYPE_ITEM, item, 2510 "multiple L4 layers not supported"); 2511 if (!mask) 2512 mask = &nic_mask; 2513 ret = mlx5_flow_item_acceptable 2514 (item, (const uint8_t *)mask, 2515 (const uint8_t *)&nic_mask, 2516 sizeof(struct rte_flow_item_icmp6_echo), 2517 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2518 if (ret < 0) 2519 return ret; 2520 return 0; 2521 } 2522 2523 /** 2524 * Validate ICMP item. 2525 * 2526 * @param[in] item 2527 * Item specification. 2528 * @param[in] item_flags 2529 * Bit-fields that holds the items detected until now. 2530 * @param[out] error 2531 * Pointer to error structure. 2532 * 2533 * @return 2534 * 0 on success, a negative errno value otherwise and rte_errno is set. 2535 */ 2536 int 2537 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2538 uint64_t item_flags, 2539 uint8_t target_protocol, 2540 struct rte_flow_error *error) 2541 { 2542 const struct rte_flow_item_icmp *mask = item->mask; 2543 const struct rte_flow_item_icmp nic_mask = { 2544 .hdr.icmp_type = 0xff, 2545 .hdr.icmp_code = 0xff, 2546 .hdr.icmp_ident = RTE_BE16(0xffff), 2547 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2548 }; 2549 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2550 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2551 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2552 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2553 MLX5_FLOW_LAYER_OUTER_L4; 2554 int ret; 2555 2556 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2557 return rte_flow_error_set(error, EINVAL, 2558 RTE_FLOW_ERROR_TYPE_ITEM, item, 2559 "protocol filtering not compatible" 2560 " with ICMP layer"); 2561 if (!(item_flags & l3m)) 2562 return rte_flow_error_set(error, EINVAL, 2563 RTE_FLOW_ERROR_TYPE_ITEM, item, 2564 "IPv4 is mandatory to filter" 2565 " on ICMP"); 2566 if (item_flags & l4m) 2567 return rte_flow_error_set(error, EINVAL, 2568 RTE_FLOW_ERROR_TYPE_ITEM, item, 2569 "multiple L4 layers not supported"); 2570 if (!mask) 2571 mask = &nic_mask; 2572 ret = mlx5_flow_item_acceptable 2573 (item, (const uint8_t *)mask, 2574 (const uint8_t *)&nic_mask, 2575 sizeof(struct rte_flow_item_icmp), 2576 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2577 if (ret < 0) 2578 return ret; 2579 return 0; 2580 } 2581 2582 /** 2583 * Validate Ethernet item. 2584 * 2585 * @param[in] item 2586 * Item specification. 2587 * @param[in] item_flags 2588 * Bit-fields that holds the items detected until now. 2589 * @param[out] error 2590 * Pointer to error structure. 2591 * 2592 * @return 2593 * 0 on success, a negative errno value otherwise and rte_errno is set. 2594 */ 2595 int 2596 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2597 uint64_t item_flags, bool ext_vlan_sup, 2598 struct rte_flow_error *error) 2599 { 2600 const struct rte_flow_item_eth *mask = item->mask; 2601 const struct rte_flow_item_eth nic_mask = { 2602 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2603 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2604 .hdr.ether_type = RTE_BE16(0xffff), 2605 .has_vlan = ext_vlan_sup ? 1 : 0, 2606 }; 2607 int ret; 2608 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2609 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2610 MLX5_FLOW_LAYER_OUTER_L2; 2611 2612 if (item_flags & ethm) 2613 return rte_flow_error_set(error, ENOTSUP, 2614 RTE_FLOW_ERROR_TYPE_ITEM, item, 2615 "multiple L2 layers not supported"); 2616 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2617 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2618 return rte_flow_error_set(error, EINVAL, 2619 RTE_FLOW_ERROR_TYPE_ITEM, item, 2620 "L2 layer should not follow " 2621 "L3 layers"); 2622 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2623 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2624 return rte_flow_error_set(error, EINVAL, 2625 RTE_FLOW_ERROR_TYPE_ITEM, item, 2626 "L2 layer should not follow VLAN"); 2627 if (item_flags & MLX5_FLOW_LAYER_GTP) 2628 return rte_flow_error_set(error, EINVAL, 2629 RTE_FLOW_ERROR_TYPE_ITEM, item, 2630 "L2 layer should not follow GTP"); 2631 if (!mask) 2632 mask = &rte_flow_item_eth_mask; 2633 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2634 (const uint8_t *)&nic_mask, 2635 sizeof(struct rte_flow_item_eth), 2636 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2637 return ret; 2638 } 2639 2640 /** 2641 * Validate VLAN item. 2642 * 2643 * @param[in] item 2644 * Item specification. 2645 * @param[in] item_flags 2646 * Bit-fields that holds the items detected until now. 2647 * @param[in] dev 2648 * Ethernet device flow is being created on. 2649 * @param[out] error 2650 * Pointer to error structure. 2651 * 2652 * @return 2653 * 0 on success, a negative errno value otherwise and rte_errno is set. 2654 */ 2655 int 2656 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2657 uint64_t item_flags, 2658 struct rte_eth_dev *dev, 2659 struct rte_flow_error *error) 2660 { 2661 const struct rte_flow_item_vlan *spec = item->spec; 2662 const struct rte_flow_item_vlan *mask = item->mask; 2663 const struct rte_flow_item_vlan nic_mask = { 2664 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2665 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2666 }; 2667 uint16_t vlan_tag = 0; 2668 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2669 int ret; 2670 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2671 MLX5_FLOW_LAYER_INNER_L4) : 2672 (MLX5_FLOW_LAYER_OUTER_L3 | 2673 MLX5_FLOW_LAYER_OUTER_L4); 2674 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2675 MLX5_FLOW_LAYER_OUTER_VLAN; 2676 2677 if (item_flags & vlanm) 2678 return rte_flow_error_set(error, EINVAL, 2679 RTE_FLOW_ERROR_TYPE_ITEM, item, 2680 "multiple VLAN layers not supported"); 2681 else if ((item_flags & l34m) != 0) 2682 return rte_flow_error_set(error, EINVAL, 2683 RTE_FLOW_ERROR_TYPE_ITEM, item, 2684 "VLAN cannot follow L3/L4 layer"); 2685 if (!mask) 2686 mask = &rte_flow_item_vlan_mask; 2687 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2688 (const uint8_t *)&nic_mask, 2689 sizeof(struct rte_flow_item_vlan), 2690 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2691 if (ret) 2692 return ret; 2693 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2694 struct mlx5_priv *priv = dev->data->dev_private; 2695 2696 if (priv->vmwa_context) { 2697 /* 2698 * Non-NULL context means we have a virtual machine 2699 * and SR-IOV enabled, we have to create VLAN interface 2700 * to make hypervisor to setup E-Switch vport 2701 * context correctly. We avoid creating the multiple 2702 * VLAN interfaces, so we cannot support VLAN tag mask. 2703 */ 2704 return rte_flow_error_set(error, EINVAL, 2705 RTE_FLOW_ERROR_TYPE_ITEM, 2706 item, 2707 "VLAN tag mask is not" 2708 " supported in virtual" 2709 " environment"); 2710 } 2711 } 2712 if (spec) { 2713 vlan_tag = spec->hdr.vlan_tci; 2714 vlan_tag &= mask->hdr.vlan_tci; 2715 } 2716 /* 2717 * From verbs perspective an empty VLAN is equivalent 2718 * to a packet without VLAN layer. 2719 */ 2720 if (!vlan_tag) 2721 return rte_flow_error_set(error, EINVAL, 2722 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2723 item->spec, 2724 "VLAN cannot be empty"); 2725 return 0; 2726 } 2727 2728 /** 2729 * Validate IPV4 item. 2730 * 2731 * @param[in] item 2732 * Item specification. 2733 * @param[in] item_flags 2734 * Bit-fields that holds the items detected until now. 2735 * @param[in] last_item 2736 * Previous validated item in the pattern items. 2737 * @param[in] ether_type 2738 * Type in the ethernet layer header (including dot1q). 2739 * @param[in] acc_mask 2740 * Acceptable mask, if NULL default internal default mask 2741 * will be used to check whether item fields are supported. 2742 * @param[in] range_accepted 2743 * True if range of values is accepted for specific fields, false otherwise. 2744 * @param[out] error 2745 * Pointer to error structure. 2746 * 2747 * @return 2748 * 0 on success, a negative errno value otherwise and rte_errno is set. 2749 */ 2750 int 2751 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2752 uint64_t item_flags, 2753 uint64_t last_item, 2754 uint16_t ether_type, 2755 const struct rte_flow_item_ipv4 *acc_mask, 2756 bool range_accepted, 2757 struct rte_flow_error *error) 2758 { 2759 const struct rte_flow_item_ipv4 *mask = item->mask; 2760 const struct rte_flow_item_ipv4 *spec = item->spec; 2761 const struct rte_flow_item_ipv4 nic_mask = { 2762 .hdr = { 2763 .src_addr = RTE_BE32(0xffffffff), 2764 .dst_addr = RTE_BE32(0xffffffff), 2765 .type_of_service = 0xff, 2766 .next_proto_id = 0xff, 2767 }, 2768 }; 2769 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2770 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2771 MLX5_FLOW_LAYER_OUTER_L3; 2772 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2773 MLX5_FLOW_LAYER_OUTER_L4; 2774 int ret; 2775 uint8_t next_proto = 0xFF; 2776 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2777 MLX5_FLOW_LAYER_OUTER_VLAN | 2778 MLX5_FLOW_LAYER_INNER_VLAN); 2779 2780 if ((last_item & l2_vlan) && ether_type && 2781 ether_type != RTE_ETHER_TYPE_IPV4) 2782 return rte_flow_error_set(error, EINVAL, 2783 RTE_FLOW_ERROR_TYPE_ITEM, item, 2784 "IPv4 cannot follow L2/VLAN layer " 2785 "which ether type is not IPv4"); 2786 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2787 if (mask && spec) 2788 next_proto = mask->hdr.next_proto_id & 2789 spec->hdr.next_proto_id; 2790 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2791 return rte_flow_error_set(error, EINVAL, 2792 RTE_FLOW_ERROR_TYPE_ITEM, 2793 item, 2794 "multiple tunnel " 2795 "not supported"); 2796 } 2797 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2798 return rte_flow_error_set(error, EINVAL, 2799 RTE_FLOW_ERROR_TYPE_ITEM, item, 2800 "wrong tunnel type - IPv6 specified " 2801 "but IPv4 item provided"); 2802 if (item_flags & l3m) 2803 return rte_flow_error_set(error, ENOTSUP, 2804 RTE_FLOW_ERROR_TYPE_ITEM, item, 2805 "multiple L3 layers not supported"); 2806 else if (item_flags & l4m) 2807 return rte_flow_error_set(error, EINVAL, 2808 RTE_FLOW_ERROR_TYPE_ITEM, item, 2809 "L3 cannot follow an L4 layer."); 2810 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2811 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2812 return rte_flow_error_set(error, EINVAL, 2813 RTE_FLOW_ERROR_TYPE_ITEM, item, 2814 "L3 cannot follow an NVGRE layer."); 2815 if (!mask) 2816 mask = &rte_flow_item_ipv4_mask; 2817 else if (mask->hdr.next_proto_id != 0 && 2818 mask->hdr.next_proto_id != 0xff) 2819 return rte_flow_error_set(error, EINVAL, 2820 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2821 "partial mask is not supported" 2822 " for protocol"); 2823 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2824 acc_mask ? (const uint8_t *)acc_mask 2825 : (const uint8_t *)&nic_mask, 2826 sizeof(struct rte_flow_item_ipv4), 2827 range_accepted, error); 2828 if (ret < 0) 2829 return ret; 2830 return 0; 2831 } 2832 2833 /** 2834 * Validate IPV6 item. 2835 * 2836 * @param[in] item 2837 * Item specification. 2838 * @param[in] item_flags 2839 * Bit-fields that holds the items detected until now. 2840 * @param[in] last_item 2841 * Previous validated item in the pattern items. 2842 * @param[in] ether_type 2843 * Type in the ethernet layer header (including dot1q). 2844 * @param[in] acc_mask 2845 * Acceptable mask, if NULL default internal default mask 2846 * will be used to check whether item fields are supported. 2847 * @param[out] error 2848 * Pointer to error structure. 2849 * 2850 * @return 2851 * 0 on success, a negative errno value otherwise and rte_errno is set. 2852 */ 2853 int 2854 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2855 uint64_t item_flags, 2856 uint64_t last_item, 2857 uint16_t ether_type, 2858 const struct rte_flow_item_ipv6 *acc_mask, 2859 struct rte_flow_error *error) 2860 { 2861 const struct rte_flow_item_ipv6 *mask = item->mask; 2862 const struct rte_flow_item_ipv6 *spec = item->spec; 2863 const struct rte_flow_item_ipv6 nic_mask = { 2864 .hdr = { 2865 .src_addr = 2866 "\xff\xff\xff\xff\xff\xff\xff\xff" 2867 "\xff\xff\xff\xff\xff\xff\xff\xff", 2868 .dst_addr = 2869 "\xff\xff\xff\xff\xff\xff\xff\xff" 2870 "\xff\xff\xff\xff\xff\xff\xff\xff", 2871 .vtc_flow = RTE_BE32(0xffffffff), 2872 .proto = 0xff, 2873 }, 2874 }; 2875 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2876 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2877 MLX5_FLOW_LAYER_OUTER_L3; 2878 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2879 MLX5_FLOW_LAYER_OUTER_L4; 2880 int ret; 2881 uint8_t next_proto = 0xFF; 2882 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2883 MLX5_FLOW_LAYER_OUTER_VLAN | 2884 MLX5_FLOW_LAYER_INNER_VLAN); 2885 2886 if ((last_item & l2_vlan) && ether_type && 2887 ether_type != RTE_ETHER_TYPE_IPV6) 2888 return rte_flow_error_set(error, EINVAL, 2889 RTE_FLOW_ERROR_TYPE_ITEM, item, 2890 "IPv6 cannot follow L2/VLAN layer " 2891 "which ether type is not IPv6"); 2892 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2893 next_proto = spec->hdr.proto; 2894 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2895 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2896 return rte_flow_error_set(error, EINVAL, 2897 RTE_FLOW_ERROR_TYPE_ITEM, 2898 item, 2899 "multiple tunnel " 2900 "not supported"); 2901 } 2902 if (next_proto == IPPROTO_HOPOPTS || 2903 next_proto == IPPROTO_ROUTING || 2904 next_proto == IPPROTO_FRAGMENT || 2905 next_proto == IPPROTO_ESP || 2906 next_proto == IPPROTO_AH || 2907 next_proto == IPPROTO_DSTOPTS) 2908 return rte_flow_error_set(error, EINVAL, 2909 RTE_FLOW_ERROR_TYPE_ITEM, item, 2910 "IPv6 proto (next header) should " 2911 "not be set as extension header"); 2912 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2913 return rte_flow_error_set(error, EINVAL, 2914 RTE_FLOW_ERROR_TYPE_ITEM, item, 2915 "wrong tunnel type - IPv4 specified " 2916 "but IPv6 item provided"); 2917 if (item_flags & l3m) 2918 return rte_flow_error_set(error, ENOTSUP, 2919 RTE_FLOW_ERROR_TYPE_ITEM, item, 2920 "multiple L3 layers not supported"); 2921 else if (item_flags & l4m) 2922 return rte_flow_error_set(error, EINVAL, 2923 RTE_FLOW_ERROR_TYPE_ITEM, item, 2924 "L3 cannot follow an L4 layer."); 2925 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2926 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2927 return rte_flow_error_set(error, EINVAL, 2928 RTE_FLOW_ERROR_TYPE_ITEM, item, 2929 "L3 cannot follow an NVGRE layer."); 2930 if (!mask) 2931 mask = &rte_flow_item_ipv6_mask; 2932 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2933 acc_mask ? (const uint8_t *)acc_mask 2934 : (const uint8_t *)&nic_mask, 2935 sizeof(struct rte_flow_item_ipv6), 2936 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2937 if (ret < 0) 2938 return ret; 2939 return 0; 2940 } 2941 2942 /** 2943 * Validate UDP item. 2944 * 2945 * @param[in] item 2946 * Item specification. 2947 * @param[in] item_flags 2948 * Bit-fields that holds the items detected until now. 2949 * @param[in] target_protocol 2950 * The next protocol in the previous item. 2951 * @param[in] flow_mask 2952 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2953 * @param[out] error 2954 * Pointer to error structure. 2955 * 2956 * @return 2957 * 0 on success, a negative errno value otherwise and rte_errno is set. 2958 */ 2959 int 2960 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2961 uint64_t item_flags, 2962 uint8_t target_protocol, 2963 struct rte_flow_error *error) 2964 { 2965 const struct rte_flow_item_udp *mask = item->mask; 2966 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2967 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2968 MLX5_FLOW_LAYER_OUTER_L3; 2969 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2970 MLX5_FLOW_LAYER_OUTER_L4; 2971 int ret; 2972 2973 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2974 return rte_flow_error_set(error, EINVAL, 2975 RTE_FLOW_ERROR_TYPE_ITEM, item, 2976 "protocol filtering not compatible" 2977 " with UDP layer"); 2978 if (!(item_flags & l3m)) 2979 return rte_flow_error_set(error, EINVAL, 2980 RTE_FLOW_ERROR_TYPE_ITEM, item, 2981 "L3 is mandatory to filter on L4"); 2982 if (item_flags & l4m) 2983 return rte_flow_error_set(error, EINVAL, 2984 RTE_FLOW_ERROR_TYPE_ITEM, item, 2985 "multiple L4 layers not supported"); 2986 if (!mask) 2987 mask = &rte_flow_item_udp_mask; 2988 ret = mlx5_flow_item_acceptable 2989 (item, (const uint8_t *)mask, 2990 (const uint8_t *)&rte_flow_item_udp_mask, 2991 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2992 error); 2993 if (ret < 0) 2994 return ret; 2995 return 0; 2996 } 2997 2998 /** 2999 * Validate TCP item. 3000 * 3001 * @param[in] item 3002 * Item specification. 3003 * @param[in] item_flags 3004 * Bit-fields that holds the items detected until now. 3005 * @param[in] target_protocol 3006 * The next protocol in the previous item. 3007 * @param[out] error 3008 * Pointer to error structure. 3009 * 3010 * @return 3011 * 0 on success, a negative errno value otherwise and rte_errno is set. 3012 */ 3013 int 3014 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 3015 uint64_t item_flags, 3016 uint8_t target_protocol, 3017 const struct rte_flow_item_tcp *flow_mask, 3018 struct rte_flow_error *error) 3019 { 3020 const struct rte_flow_item_tcp *mask = item->mask; 3021 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3022 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3023 MLX5_FLOW_LAYER_OUTER_L3; 3024 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3025 MLX5_FLOW_LAYER_OUTER_L4; 3026 int ret; 3027 3028 MLX5_ASSERT(flow_mask); 3029 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3030 return rte_flow_error_set(error, EINVAL, 3031 RTE_FLOW_ERROR_TYPE_ITEM, item, 3032 "protocol filtering not compatible" 3033 " with TCP layer"); 3034 if (!(item_flags & l3m)) 3035 return rte_flow_error_set(error, EINVAL, 3036 RTE_FLOW_ERROR_TYPE_ITEM, item, 3037 "L3 is mandatory to filter on L4"); 3038 if (item_flags & l4m) 3039 return rte_flow_error_set(error, EINVAL, 3040 RTE_FLOW_ERROR_TYPE_ITEM, item, 3041 "multiple L4 layers not supported"); 3042 if (!mask) 3043 mask = &rte_flow_item_tcp_mask; 3044 ret = mlx5_flow_item_acceptable 3045 (item, (const uint8_t *)mask, 3046 (const uint8_t *)flow_mask, 3047 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3048 error); 3049 if (ret < 0) 3050 return ret; 3051 return 0; 3052 } 3053 3054 /** 3055 * Validate VXLAN item. 3056 * 3057 * @param[in] dev 3058 * Pointer to the Ethernet device structure. 3059 * @param[in] udp_dport 3060 * UDP destination port 3061 * @param[in] item 3062 * Item specification. 3063 * @param[in] item_flags 3064 * Bit-fields that holds the items detected until now. 3065 * @param root 3066 * Whether action is on root table. 3067 * @param[out] error 3068 * Pointer to error structure. 3069 * 3070 * @return 3071 * 0 on success, a negative errno value otherwise and rte_errno is set. 3072 */ 3073 int 3074 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3075 uint16_t udp_dport, 3076 const struct rte_flow_item *item, 3077 uint64_t item_flags, 3078 bool root, 3079 struct rte_flow_error *error) 3080 { 3081 const struct rte_flow_item_vxlan *spec = item->spec; 3082 const struct rte_flow_item_vxlan *mask = item->mask; 3083 int ret; 3084 struct mlx5_priv *priv = dev->data->dev_private; 3085 union vni { 3086 uint32_t vlan_id; 3087 uint8_t vni[4]; 3088 } id = { .vlan_id = 0, }; 3089 const struct rte_flow_item_vxlan nic_mask = { 3090 .hdr.vni = "\xff\xff\xff", 3091 .hdr.rsvd1 = 0xff, 3092 }; 3093 const struct rte_flow_item_vxlan *valid_mask; 3094 3095 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3096 return rte_flow_error_set(error, ENOTSUP, 3097 RTE_FLOW_ERROR_TYPE_ITEM, item, 3098 "multiple tunnel layers not" 3099 " supported"); 3100 valid_mask = &rte_flow_item_vxlan_mask; 3101 /* 3102 * Verify only UDPv4 is present as defined in 3103 * https://tools.ietf.org/html/rfc7348 3104 */ 3105 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3106 return rte_flow_error_set(error, EINVAL, 3107 RTE_FLOW_ERROR_TYPE_ITEM, item, 3108 "no outer UDP layer found"); 3109 if (!mask) 3110 mask = &rte_flow_item_vxlan_mask; 3111 3112 if (priv->sh->steering_format_version != 3113 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3114 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3115 /* non-root table */ 3116 if (!root && priv->sh->misc5_cap) 3117 valid_mask = &nic_mask; 3118 /* Group zero in NIC domain */ 3119 if (!root && priv->sh->tunnel_header_0_1) 3120 valid_mask = &nic_mask; 3121 } 3122 ret = mlx5_flow_item_acceptable 3123 (item, (const uint8_t *)mask, 3124 (const uint8_t *)valid_mask, 3125 sizeof(struct rte_flow_item_vxlan), 3126 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3127 if (ret < 0) 3128 return ret; 3129 if (spec) { 3130 memcpy(&id.vni[1], spec->hdr.vni, 3); 3131 memcpy(&id.vni[1], mask->hdr.vni, 3); 3132 } 3133 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3134 return rte_flow_error_set(error, ENOTSUP, 3135 RTE_FLOW_ERROR_TYPE_ITEM, item, 3136 "VXLAN tunnel must be fully defined"); 3137 return 0; 3138 } 3139 3140 /** 3141 * Validate VXLAN_GPE item. 3142 * 3143 * @param[in] item 3144 * Item specification. 3145 * @param[in] item_flags 3146 * Bit-fields that holds the items detected until now. 3147 * @param[in] priv 3148 * Pointer to the private data structure. 3149 * @param[in] target_protocol 3150 * The next protocol in the previous item. 3151 * @param[out] error 3152 * Pointer to error structure. 3153 * 3154 * @return 3155 * 0 on success, a negative errno value otherwise and rte_errno is set. 3156 */ 3157 int 3158 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3159 uint64_t item_flags, 3160 struct rte_eth_dev *dev, 3161 struct rte_flow_error *error) 3162 { 3163 struct mlx5_priv *priv = dev->data->dev_private; 3164 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3165 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3166 int ret; 3167 union vni { 3168 uint32_t vlan_id; 3169 uint8_t vni[4]; 3170 } id = { .vlan_id = 0, }; 3171 3172 struct rte_flow_item_vxlan_gpe nic_mask = { 3173 .vni = "\xff\xff\xff", 3174 .protocol = 0xff, 3175 .flags = 0xff, 3176 }; 3177 3178 if (!priv->sh->config.l3_vxlan_en) 3179 return rte_flow_error_set(error, ENOTSUP, 3180 RTE_FLOW_ERROR_TYPE_ITEM, item, 3181 "L3 VXLAN is not enabled by device" 3182 " parameter and/or not configured in" 3183 " firmware"); 3184 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3185 return rte_flow_error_set(error, ENOTSUP, 3186 RTE_FLOW_ERROR_TYPE_ITEM, item, 3187 "multiple tunnel layers not" 3188 " supported"); 3189 /* 3190 * Verify only UDPv4 is present as defined in 3191 * https://tools.ietf.org/html/rfc7348 3192 */ 3193 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3194 return rte_flow_error_set(error, EINVAL, 3195 RTE_FLOW_ERROR_TYPE_ITEM, item, 3196 "no outer UDP layer found"); 3197 if (!mask) 3198 mask = &rte_flow_item_vxlan_gpe_mask; 3199 ret = mlx5_flow_item_acceptable 3200 (item, (const uint8_t *)mask, 3201 (const uint8_t *)&nic_mask, 3202 sizeof(struct rte_flow_item_vxlan_gpe), 3203 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3204 if (ret < 0) 3205 return ret; 3206 if (spec) { 3207 memcpy(&id.vni[1], spec->hdr.vni, 3); 3208 memcpy(&id.vni[1], mask->hdr.vni, 3); 3209 } 3210 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3211 return rte_flow_error_set(error, ENOTSUP, 3212 RTE_FLOW_ERROR_TYPE_ITEM, item, 3213 "VXLAN-GPE tunnel must be fully" 3214 " defined"); 3215 return 0; 3216 } 3217 /** 3218 * Validate GRE Key item. 3219 * 3220 * @param[in] item 3221 * Item specification. 3222 * @param[in] item_flags 3223 * Bit flags to mark detected items. 3224 * @param[in] gre_item 3225 * Pointer to gre_item 3226 * @param[out] error 3227 * Pointer to error structure. 3228 * 3229 * @return 3230 * 0 on success, a negative errno value otherwise and rte_errno is set. 3231 */ 3232 int 3233 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3234 uint64_t item_flags, 3235 const struct rte_flow_item *gre_item, 3236 struct rte_flow_error *error) 3237 { 3238 const rte_be32_t *mask = item->mask; 3239 int ret = 0; 3240 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3241 const struct rte_flow_item_gre *gre_spec; 3242 const struct rte_flow_item_gre *gre_mask; 3243 3244 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3245 return rte_flow_error_set(error, ENOTSUP, 3246 RTE_FLOW_ERROR_TYPE_ITEM, item, 3247 "Multiple GRE key not support"); 3248 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3249 return rte_flow_error_set(error, ENOTSUP, 3250 RTE_FLOW_ERROR_TYPE_ITEM, item, 3251 "No preceding GRE header"); 3252 if (item_flags & MLX5_FLOW_LAYER_INNER) 3253 return rte_flow_error_set(error, ENOTSUP, 3254 RTE_FLOW_ERROR_TYPE_ITEM, item, 3255 "GRE key following a wrong item"); 3256 gre_mask = gre_item->mask; 3257 if (!gre_mask) 3258 gre_mask = &rte_flow_item_gre_mask; 3259 gre_spec = gre_item->spec; 3260 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3261 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3262 return rte_flow_error_set(error, EINVAL, 3263 RTE_FLOW_ERROR_TYPE_ITEM, item, 3264 "Key bit must be on"); 3265 3266 if (!mask) 3267 mask = &gre_key_default_mask; 3268 ret = mlx5_flow_item_acceptable 3269 (item, (const uint8_t *)mask, 3270 (const uint8_t *)&gre_key_default_mask, 3271 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3272 return ret; 3273 } 3274 3275 /** 3276 * Validate GRE optional item. 3277 * 3278 * @param[in] dev 3279 * Pointer to the Ethernet device structure. 3280 * @param[in] item 3281 * Item specification. 3282 * @param[in] item_flags 3283 * Bit flags to mark detected items. 3284 * @param[in] attr 3285 * Flow rule attributes. 3286 * @param[in] gre_item 3287 * Pointer to gre_item 3288 * @param[out] error 3289 * Pointer to error structure. 3290 * 3291 * @return 3292 * 0 on success, a negative errno value otherwise and rte_errno is set. 3293 */ 3294 int 3295 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3296 const struct rte_flow_item *item, 3297 uint64_t item_flags, 3298 const struct rte_flow_attr *attr, 3299 const struct rte_flow_item *gre_item, 3300 struct rte_flow_error *error) 3301 { 3302 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3303 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3304 const struct rte_flow_item_gre_opt *spec = item->spec; 3305 const struct rte_flow_item_gre_opt *mask = item->mask; 3306 struct mlx5_priv *priv = dev->data->dev_private; 3307 int ret = 0; 3308 struct rte_flow_item_gre_opt nic_mask = { 3309 .checksum_rsvd = { 3310 .checksum = RTE_BE16(UINT16_MAX), 3311 .reserved1 = 0x0, 3312 }, 3313 .key = { 3314 .key = RTE_BE32(UINT32_MAX), 3315 }, 3316 .sequence = { 3317 .sequence = RTE_BE32(UINT32_MAX), 3318 }, 3319 }; 3320 3321 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3322 return rte_flow_error_set(error, ENOTSUP, 3323 RTE_FLOW_ERROR_TYPE_ITEM, item, 3324 "No preceding GRE header"); 3325 if (item_flags & MLX5_FLOW_LAYER_INNER) 3326 return rte_flow_error_set(error, ENOTSUP, 3327 RTE_FLOW_ERROR_TYPE_ITEM, item, 3328 "GRE option following a wrong item"); 3329 if (!spec || !mask) 3330 return rte_flow_error_set(error, EINVAL, 3331 RTE_FLOW_ERROR_TYPE_ITEM, item, 3332 "At least one field gre_option(checksum/key/sequence) must be specified"); 3333 if (!gre_mask) 3334 gre_mask = &rte_flow_item_gre_mask; 3335 if (mask->checksum_rsvd.checksum) 3336 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3337 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3338 return rte_flow_error_set(error, EINVAL, 3339 RTE_FLOW_ERROR_TYPE_ITEM, 3340 item, 3341 "Checksum bit must be on"); 3342 if (mask->key.key) 3343 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3344 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3345 return rte_flow_error_set(error, EINVAL, 3346 RTE_FLOW_ERROR_TYPE_ITEM, 3347 item, "Key bit must be on"); 3348 if (mask->sequence.sequence) 3349 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3350 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3351 return rte_flow_error_set(error, EINVAL, 3352 RTE_FLOW_ERROR_TYPE_ITEM, 3353 item, 3354 "Sequence bit must be on"); 3355 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3356 if (priv->sh->steering_format_version == 3357 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3358 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3359 !priv->sh->misc5_cap) || 3360 (!(priv->sh->tunnel_header_0_1 && 3361 priv->sh->tunnel_header_2_3) && 3362 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3363 return rte_flow_error_set(error, EINVAL, 3364 RTE_FLOW_ERROR_TYPE_ITEM, 3365 item, 3366 "Checksum/Sequence not supported"); 3367 } 3368 ret = mlx5_flow_item_acceptable 3369 (item, (const uint8_t *)mask, 3370 (const uint8_t *)&nic_mask, 3371 sizeof(struct rte_flow_item_gre_opt), 3372 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3373 return ret; 3374 } 3375 3376 /** 3377 * Validate GRE item. 3378 * 3379 * @param[in] item 3380 * Item specification. 3381 * @param[in] item_flags 3382 * Bit flags to mark detected items. 3383 * @param[in] target_protocol 3384 * The next protocol in the previous item. 3385 * @param[out] error 3386 * Pointer to error structure. 3387 * 3388 * @return 3389 * 0 on success, a negative errno value otherwise and rte_errno is set. 3390 */ 3391 int 3392 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3393 uint64_t item_flags, 3394 uint8_t target_protocol, 3395 struct rte_flow_error *error) 3396 { 3397 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3398 const struct rte_flow_item_gre *mask = item->mask; 3399 int ret; 3400 const struct rte_flow_item_gre nic_mask = { 3401 .c_rsvd0_ver = RTE_BE16(0xB000), 3402 .protocol = RTE_BE16(UINT16_MAX), 3403 }; 3404 3405 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3406 return rte_flow_error_set(error, EINVAL, 3407 RTE_FLOW_ERROR_TYPE_ITEM, item, 3408 "protocol filtering not compatible" 3409 " with this GRE layer"); 3410 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3411 return rte_flow_error_set(error, ENOTSUP, 3412 RTE_FLOW_ERROR_TYPE_ITEM, item, 3413 "multiple tunnel layers not" 3414 " supported"); 3415 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3416 return rte_flow_error_set(error, ENOTSUP, 3417 RTE_FLOW_ERROR_TYPE_ITEM, item, 3418 "L3 Layer is missing"); 3419 if (!mask) 3420 mask = &rte_flow_item_gre_mask; 3421 ret = mlx5_flow_item_acceptable 3422 (item, (const uint8_t *)mask, 3423 (const uint8_t *)&nic_mask, 3424 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3425 error); 3426 if (ret < 0) 3427 return ret; 3428 #ifndef HAVE_MLX5DV_DR 3429 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3430 if (spec && (spec->protocol & mask->protocol)) 3431 return rte_flow_error_set(error, ENOTSUP, 3432 RTE_FLOW_ERROR_TYPE_ITEM, item, 3433 "without MPLS support the" 3434 " specification cannot be used for" 3435 " filtering"); 3436 #endif 3437 #endif 3438 return 0; 3439 } 3440 3441 /** 3442 * Validate Geneve item. 3443 * 3444 * @param[in] item 3445 * Item specification. 3446 * @param[in] itemFlags 3447 * Bit-fields that holds the items detected until now. 3448 * @param[in] enPriv 3449 * Pointer to the private data structure. 3450 * @param[out] error 3451 * Pointer to error structure. 3452 * 3453 * @return 3454 * 0 on success, a negative errno value otherwise and rte_errno is set. 3455 */ 3456 3457 int 3458 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3459 uint64_t item_flags, 3460 struct rte_eth_dev *dev, 3461 struct rte_flow_error *error) 3462 { 3463 struct mlx5_priv *priv = dev->data->dev_private; 3464 const struct rte_flow_item_geneve *spec = item->spec; 3465 const struct rte_flow_item_geneve *mask = item->mask; 3466 int ret; 3467 uint16_t gbhdr; 3468 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3469 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3470 const struct rte_flow_item_geneve nic_mask = { 3471 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3472 .vni = "\xff\xff\xff", 3473 .protocol = RTE_BE16(UINT16_MAX), 3474 }; 3475 3476 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3477 return rte_flow_error_set(error, ENOTSUP, 3478 RTE_FLOW_ERROR_TYPE_ITEM, item, 3479 "L3 Geneve is not enabled by device" 3480 " parameter and/or not configured in" 3481 " firmware"); 3482 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3483 return rte_flow_error_set(error, ENOTSUP, 3484 RTE_FLOW_ERROR_TYPE_ITEM, item, 3485 "multiple tunnel layers not" 3486 " supported"); 3487 /* 3488 * Verify only UDPv4 is present as defined in 3489 * https://tools.ietf.org/html/rfc7348 3490 */ 3491 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3492 return rte_flow_error_set(error, EINVAL, 3493 RTE_FLOW_ERROR_TYPE_ITEM, item, 3494 "no outer UDP layer found"); 3495 if (!mask) 3496 mask = &rte_flow_item_geneve_mask; 3497 ret = mlx5_flow_item_acceptable 3498 (item, (const uint8_t *)mask, 3499 (const uint8_t *)&nic_mask, 3500 sizeof(struct rte_flow_item_geneve), 3501 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3502 if (ret) 3503 return ret; 3504 if (spec) { 3505 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3506 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3507 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3508 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3509 return rte_flow_error_set(error, ENOTSUP, 3510 RTE_FLOW_ERROR_TYPE_ITEM, 3511 item, 3512 "Geneve protocol unsupported" 3513 " fields are being used"); 3514 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3515 return rte_flow_error_set 3516 (error, ENOTSUP, 3517 RTE_FLOW_ERROR_TYPE_ITEM, 3518 item, 3519 "Unsupported Geneve options length"); 3520 } 3521 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3522 return rte_flow_error_set 3523 (error, ENOTSUP, 3524 RTE_FLOW_ERROR_TYPE_ITEM, item, 3525 "Geneve tunnel must be fully defined"); 3526 return 0; 3527 } 3528 3529 /** 3530 * Validate Geneve TLV option item. 3531 * 3532 * @param[in] item 3533 * Item specification. 3534 * @param[in] last_item 3535 * Previous validated item in the pattern items. 3536 * @param[in] geneve_item 3537 * Previous GENEVE item specification. 3538 * @param[in] dev 3539 * Pointer to the rte_eth_dev structure. 3540 * @param[out] error 3541 * Pointer to error structure. 3542 * 3543 * @return 3544 * 0 on success, a negative errno value otherwise and rte_errno is set. 3545 */ 3546 int 3547 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3548 uint64_t last_item, 3549 const struct rte_flow_item *geneve_item, 3550 struct rte_eth_dev *dev, 3551 struct rte_flow_error *error) 3552 { 3553 struct mlx5_priv *priv = dev->data->dev_private; 3554 struct mlx5_dev_ctx_shared *sh = priv->sh; 3555 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3556 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3557 uint8_t data_max_supported = 3558 hca_attr->max_geneve_tlv_option_data_len * 4; 3559 const struct rte_flow_item_geneve *geneve_spec; 3560 const struct rte_flow_item_geneve *geneve_mask; 3561 const struct rte_flow_item_geneve_opt *spec = item->spec; 3562 const struct rte_flow_item_geneve_opt *mask = item->mask; 3563 unsigned int i; 3564 unsigned int data_len; 3565 uint8_t tlv_option_len; 3566 uint16_t optlen_m, optlen_v; 3567 const struct rte_flow_item_geneve_opt full_mask = { 3568 .option_class = RTE_BE16(0xffff), 3569 .option_type = 0xff, 3570 .option_len = 0x1f, 3571 }; 3572 3573 if (!mask) 3574 mask = &rte_flow_item_geneve_opt_mask; 3575 if (!spec) 3576 return rte_flow_error_set 3577 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3578 "Geneve TLV opt class/type/length must be specified"); 3579 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3580 return rte_flow_error_set 3581 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3582 "Geneve TLV opt length exceeds the limit (31)"); 3583 /* Check if class type and length masks are full. */ 3584 if (full_mask.option_class != mask->option_class || 3585 full_mask.option_type != mask->option_type || 3586 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3587 return rte_flow_error_set 3588 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3589 "Geneve TLV opt class/type/length masks must be full"); 3590 /* Check if length is supported */ 3591 if ((uint32_t)spec->option_len > 3592 hca_attr->max_geneve_tlv_option_data_len) 3593 return rte_flow_error_set 3594 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3595 "Geneve TLV opt length not supported"); 3596 if (hca_attr->max_geneve_tlv_options > 1) 3597 DRV_LOG(DEBUG, 3598 "max_geneve_tlv_options supports more than 1 option"); 3599 /* Check GENEVE item preceding. */ 3600 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3601 return rte_flow_error_set 3602 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3603 "Geneve opt item must be preceded with Geneve item"); 3604 geneve_spec = geneve_item->spec; 3605 geneve_mask = geneve_item->mask ? geneve_item->mask : 3606 &rte_flow_item_geneve_mask; 3607 /* Check if GENEVE TLV option size doesn't exceed option length */ 3608 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3609 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3610 tlv_option_len = spec->option_len & mask->option_len; 3611 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3612 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3613 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3614 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3615 if ((optlen_v & optlen_m) <= tlv_option_len) 3616 return rte_flow_error_set 3617 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3618 "GENEVE TLV option length exceeds optlen"); 3619 } 3620 /* Check if length is 0 or data is 0. */ 3621 if (spec->data == NULL || spec->option_len == 0) 3622 return rte_flow_error_set 3623 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3624 "Geneve TLV opt with zero data/length not supported"); 3625 /* Check not all data & mask are 0. */ 3626 data_len = spec->option_len * 4; 3627 if (mask->data == NULL) { 3628 for (i = 0; i < data_len; i++) 3629 if (spec->data[i]) 3630 break; 3631 if (i == data_len) 3632 return rte_flow_error_set(error, ENOTSUP, 3633 RTE_FLOW_ERROR_TYPE_ITEM, item, 3634 "Can't match on Geneve option data 0"); 3635 } else { 3636 for (i = 0; i < data_len; i++) 3637 if (spec->data[i] & mask->data[i]) 3638 break; 3639 if (i == data_len) 3640 return rte_flow_error_set(error, ENOTSUP, 3641 RTE_FLOW_ERROR_TYPE_ITEM, item, 3642 "Can't match on Geneve option data and mask 0"); 3643 /* Check data mask supported. */ 3644 for (i = data_max_supported; i < data_len ; i++) 3645 if (mask->data[i]) 3646 return rte_flow_error_set(error, ENOTSUP, 3647 RTE_FLOW_ERROR_TYPE_ITEM, item, 3648 "Data mask is of unsupported size"); 3649 } 3650 /* Check GENEVE option is supported in NIC. */ 3651 if (!hca_attr->geneve_tlv_opt) 3652 return rte_flow_error_set 3653 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3654 "Geneve TLV opt not supported"); 3655 /* Check if we already have geneve option with different type/class. */ 3656 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3657 geneve_opt_resource = sh->geneve_tlv_option_resource; 3658 if (geneve_opt_resource != NULL) 3659 if (geneve_opt_resource->option_class != spec->option_class || 3660 geneve_opt_resource->option_type != spec->option_type || 3661 geneve_opt_resource->length != spec->option_len) { 3662 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3663 return rte_flow_error_set(error, ENOTSUP, 3664 RTE_FLOW_ERROR_TYPE_ITEM, item, 3665 "Only one Geneve TLV option supported"); 3666 } 3667 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3668 return 0; 3669 } 3670 3671 /** 3672 * Validate MPLS item. 3673 * 3674 * @param[in] dev 3675 * Pointer to the rte_eth_dev structure. 3676 * @param[in] item 3677 * Item specification. 3678 * @param[in] item_flags 3679 * Bit-fields that holds the items detected until now. 3680 * @param[in] prev_layer 3681 * The protocol layer indicated in previous item. 3682 * @param[out] error 3683 * Pointer to error structure. 3684 * 3685 * @return 3686 * 0 on success, a negative errno value otherwise and rte_errno is set. 3687 */ 3688 int 3689 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3690 const struct rte_flow_item *item __rte_unused, 3691 uint64_t item_flags __rte_unused, 3692 uint64_t prev_layer __rte_unused, 3693 struct rte_flow_error *error) 3694 { 3695 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3696 const struct rte_flow_item_mpls *mask = item->mask; 3697 struct mlx5_priv *priv = dev->data->dev_private; 3698 int ret; 3699 3700 if (!priv->sh->dev_cap.mpls_en) 3701 return rte_flow_error_set(error, ENOTSUP, 3702 RTE_FLOW_ERROR_TYPE_ITEM, item, 3703 "MPLS not supported or" 3704 " disabled in firmware" 3705 " configuration."); 3706 /* MPLS over UDP, GRE is allowed */ 3707 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3708 MLX5_FLOW_LAYER_GRE | 3709 MLX5_FLOW_LAYER_GRE_KEY))) 3710 return rte_flow_error_set(error, EINVAL, 3711 RTE_FLOW_ERROR_TYPE_ITEM, item, 3712 "protocol filtering not compatible" 3713 " with MPLS layer"); 3714 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3715 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3716 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3717 return rte_flow_error_set(error, ENOTSUP, 3718 RTE_FLOW_ERROR_TYPE_ITEM, item, 3719 "multiple tunnel layers not" 3720 " supported"); 3721 if (!mask) 3722 mask = &rte_flow_item_mpls_mask; 3723 ret = mlx5_flow_item_acceptable 3724 (item, (const uint8_t *)mask, 3725 (const uint8_t *)&rte_flow_item_mpls_mask, 3726 sizeof(struct rte_flow_item_mpls), 3727 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3728 if (ret < 0) 3729 return ret; 3730 return 0; 3731 #else 3732 return rte_flow_error_set(error, ENOTSUP, 3733 RTE_FLOW_ERROR_TYPE_ITEM, item, 3734 "MPLS is not supported by Verbs, please" 3735 " update."); 3736 #endif 3737 } 3738 3739 /** 3740 * Validate NVGRE item. 3741 * 3742 * @param[in] item 3743 * Item specification. 3744 * @param[in] item_flags 3745 * Bit flags to mark detected items. 3746 * @param[in] target_protocol 3747 * The next protocol in the previous item. 3748 * @param[out] error 3749 * Pointer to error structure. 3750 * 3751 * @return 3752 * 0 on success, a negative errno value otherwise and rte_errno is set. 3753 */ 3754 int 3755 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3756 uint64_t item_flags, 3757 uint8_t target_protocol, 3758 struct rte_flow_error *error) 3759 { 3760 const struct rte_flow_item_nvgre *mask = item->mask; 3761 int ret; 3762 3763 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3764 return rte_flow_error_set(error, EINVAL, 3765 RTE_FLOW_ERROR_TYPE_ITEM, item, 3766 "protocol filtering not compatible" 3767 " with this GRE layer"); 3768 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3769 return rte_flow_error_set(error, ENOTSUP, 3770 RTE_FLOW_ERROR_TYPE_ITEM, item, 3771 "multiple tunnel layers not" 3772 " supported"); 3773 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3774 return rte_flow_error_set(error, ENOTSUP, 3775 RTE_FLOW_ERROR_TYPE_ITEM, item, 3776 "L3 Layer is missing"); 3777 if (!mask) 3778 mask = &rte_flow_item_nvgre_mask; 3779 ret = mlx5_flow_item_acceptable 3780 (item, (const uint8_t *)mask, 3781 (const uint8_t *)&rte_flow_item_nvgre_mask, 3782 sizeof(struct rte_flow_item_nvgre), 3783 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3784 if (ret < 0) 3785 return ret; 3786 return 0; 3787 } 3788 3789 /** 3790 * Validate eCPRI item. 3791 * 3792 * @param[in] item 3793 * Item specification. 3794 * @param[in] item_flags 3795 * Bit-fields that holds the items detected until now. 3796 * @param[in] last_item 3797 * Previous validated item in the pattern items. 3798 * @param[in] ether_type 3799 * Type in the ethernet layer header (including dot1q). 3800 * @param[in] acc_mask 3801 * Acceptable mask, if NULL default internal default mask 3802 * will be used to check whether item fields are supported. 3803 * @param[out] error 3804 * Pointer to error structure. 3805 * 3806 * @return 3807 * 0 on success, a negative errno value otherwise and rte_errno is set. 3808 */ 3809 int 3810 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3811 uint64_t item_flags, 3812 uint64_t last_item, 3813 uint16_t ether_type, 3814 const struct rte_flow_item_ecpri *acc_mask, 3815 struct rte_flow_error *error) 3816 { 3817 const struct rte_flow_item_ecpri *mask = item->mask; 3818 const struct rte_flow_item_ecpri nic_mask = { 3819 .hdr = { 3820 .common = { 3821 .u32 = 3822 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3823 .type = 0xFF, 3824 }).u32), 3825 }, 3826 .dummy[0] = 0xFFFFFFFF, 3827 }, 3828 }; 3829 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3830 MLX5_FLOW_LAYER_OUTER_VLAN); 3831 struct rte_flow_item_ecpri mask_lo; 3832 3833 if (!(last_item & outer_l2_vlan) && 3834 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3835 return rte_flow_error_set(error, EINVAL, 3836 RTE_FLOW_ERROR_TYPE_ITEM, item, 3837 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3838 if ((last_item & outer_l2_vlan) && ether_type && 3839 ether_type != RTE_ETHER_TYPE_ECPRI) 3840 return rte_flow_error_set(error, EINVAL, 3841 RTE_FLOW_ERROR_TYPE_ITEM, item, 3842 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3843 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3844 return rte_flow_error_set(error, EINVAL, 3845 RTE_FLOW_ERROR_TYPE_ITEM, item, 3846 "eCPRI with tunnel is not supported right now"); 3847 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3848 return rte_flow_error_set(error, ENOTSUP, 3849 RTE_FLOW_ERROR_TYPE_ITEM, item, 3850 "multiple L3 layers not supported"); 3851 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3852 return rte_flow_error_set(error, EINVAL, 3853 RTE_FLOW_ERROR_TYPE_ITEM, item, 3854 "eCPRI cannot coexist with a TCP layer"); 3855 /* In specification, eCPRI could be over UDP layer. */ 3856 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3857 return rte_flow_error_set(error, EINVAL, 3858 RTE_FLOW_ERROR_TYPE_ITEM, item, 3859 "eCPRI over UDP layer is not yet supported right now"); 3860 /* Mask for type field in common header could be zero. */ 3861 if (!mask) 3862 mask = &rte_flow_item_ecpri_mask; 3863 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3864 /* Input mask is in big-endian format. */ 3865 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3866 return rte_flow_error_set(error, EINVAL, 3867 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3868 "partial mask is not supported for protocol"); 3869 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3870 return rte_flow_error_set(error, EINVAL, 3871 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3872 "message header mask must be after a type mask"); 3873 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3874 acc_mask ? (const uint8_t *)acc_mask 3875 : (const uint8_t *)&nic_mask, 3876 sizeof(struct rte_flow_item_ecpri), 3877 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3878 } 3879 3880 /** 3881 * Validate the NSH item. 3882 * 3883 * @param[in] dev 3884 * Pointer to Ethernet device on which flow rule is being created on. 3885 * @param[out] error 3886 * Pointer to error structure. 3887 * 3888 * @return 3889 * 0 on success, a negative errno value otherwise and rte_errno is set. 3890 */ 3891 int 3892 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev, 3893 const struct rte_flow_item *item, 3894 struct rte_flow_error *error) 3895 { 3896 struct mlx5_priv *priv = dev->data->dev_private; 3897 3898 if (item->mask) { 3899 return rte_flow_error_set(error, ENOTSUP, 3900 RTE_FLOW_ERROR_TYPE_ITEM, item, 3901 "NSH fields matching is not supported"); 3902 } 3903 3904 if (!priv->sh->config.dv_flow_en) { 3905 return rte_flow_error_set(error, ENOTSUP, 3906 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3907 NULL, "NSH support requires DV flow interface"); 3908 } 3909 3910 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) { 3911 return rte_flow_error_set(error, ENOTSUP, 3912 RTE_FLOW_ERROR_TYPE_ITEM, item, 3913 "Current FW does not support matching on NSH"); 3914 } 3915 3916 return 0; 3917 } 3918 3919 static int 3920 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3921 const struct rte_flow_attr *attr __rte_unused, 3922 const struct rte_flow_item items[] __rte_unused, 3923 const struct rte_flow_action actions[] __rte_unused, 3924 bool external __rte_unused, 3925 int hairpin __rte_unused, 3926 struct rte_flow_error *error) 3927 { 3928 return rte_flow_error_set(error, ENOTSUP, 3929 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3930 } 3931 3932 static struct mlx5_flow * 3933 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3934 const struct rte_flow_attr *attr __rte_unused, 3935 const struct rte_flow_item items[] __rte_unused, 3936 const struct rte_flow_action actions[] __rte_unused, 3937 struct rte_flow_error *error) 3938 { 3939 rte_flow_error_set(error, ENOTSUP, 3940 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3941 return NULL; 3942 } 3943 3944 static int 3945 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3946 struct mlx5_flow *dev_flow __rte_unused, 3947 const struct rte_flow_attr *attr __rte_unused, 3948 const struct rte_flow_item items[] __rte_unused, 3949 const struct rte_flow_action actions[] __rte_unused, 3950 struct rte_flow_error *error) 3951 { 3952 return rte_flow_error_set(error, ENOTSUP, 3953 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3954 } 3955 3956 static int 3957 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3958 struct rte_flow *flow __rte_unused, 3959 struct rte_flow_error *error) 3960 { 3961 return rte_flow_error_set(error, ENOTSUP, 3962 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3963 } 3964 3965 static void 3966 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3967 struct rte_flow *flow __rte_unused) 3968 { 3969 } 3970 3971 static void 3972 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3973 struct rte_flow *flow __rte_unused) 3974 { 3975 } 3976 3977 static int 3978 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3979 struct rte_flow *flow __rte_unused, 3980 const struct rte_flow_action *actions __rte_unused, 3981 void *data __rte_unused, 3982 struct rte_flow_error *error) 3983 { 3984 return rte_flow_error_set(error, ENOTSUP, 3985 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3986 } 3987 3988 static int 3989 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3990 uint32_t domains __rte_unused, 3991 uint32_t flags __rte_unused) 3992 { 3993 return 0; 3994 } 3995 3996 int 3997 flow_null_get_aged_flows(struct rte_eth_dev *dev, 3998 void **context __rte_unused, 3999 uint32_t nb_contexts __rte_unused, 4000 struct rte_flow_error *error __rte_unused) 4001 { 4002 DRV_LOG(ERR, "port %u get aged flows is not supported.", 4003 dev->data->port_id); 4004 return -ENOTSUP; 4005 } 4006 4007 uint32_t 4008 flow_null_counter_allocate(struct rte_eth_dev *dev) 4009 { 4010 DRV_LOG(ERR, "port %u counter allocate is not supported.", 4011 dev->data->port_id); 4012 return 0; 4013 } 4014 4015 void 4016 flow_null_counter_free(struct rte_eth_dev *dev, 4017 uint32_t counter __rte_unused) 4018 { 4019 DRV_LOG(ERR, "port %u counter free is not supported.", 4020 dev->data->port_id); 4021 } 4022 4023 int 4024 flow_null_counter_query(struct rte_eth_dev *dev, 4025 uint32_t counter __rte_unused, 4026 bool clear __rte_unused, 4027 uint64_t *pkts __rte_unused, 4028 uint64_t *bytes __rte_unused, 4029 void **action __rte_unused) 4030 { 4031 DRV_LOG(ERR, "port %u counter query is not supported.", 4032 dev->data->port_id); 4033 return -ENOTSUP; 4034 } 4035 4036 /* Void driver to protect from null pointer reference. */ 4037 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 4038 .validate = flow_null_validate, 4039 .prepare = flow_null_prepare, 4040 .translate = flow_null_translate, 4041 .apply = flow_null_apply, 4042 .remove = flow_null_remove, 4043 .destroy = flow_null_destroy, 4044 .query = flow_null_query, 4045 .sync_domain = flow_null_sync_domain, 4046 .get_aged_flows = flow_null_get_aged_flows, 4047 .counter_alloc = flow_null_counter_allocate, 4048 .counter_free = flow_null_counter_free, 4049 .counter_query = flow_null_counter_query 4050 }; 4051 4052 /** 4053 * Select flow driver type according to flow attributes and device 4054 * configuration. 4055 * 4056 * @param[in] dev 4057 * Pointer to the dev structure. 4058 * @param[in] attr 4059 * Pointer to the flow attributes. 4060 * 4061 * @return 4062 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4063 */ 4064 static enum mlx5_flow_drv_type 4065 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4066 { 4067 struct mlx5_priv *priv = dev->data->dev_private; 4068 /* The OS can determine first a specific flow type (DV, VERBS) */ 4069 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4070 4071 if (type != MLX5_FLOW_TYPE_MAX) 4072 return type; 4073 /* 4074 * Currently when dv_flow_en == 2, only HW steering engine is 4075 * supported. New engines can also be chosen here if ready. 4076 */ 4077 if (priv->sh->config.dv_flow_en == 2) 4078 return MLX5_FLOW_TYPE_HW; 4079 if (!attr) 4080 return MLX5_FLOW_TYPE_MIN; 4081 /* If no OS specific type - continue with DV/VERBS selection */ 4082 if (attr->transfer && priv->sh->config.dv_esw_en) 4083 type = MLX5_FLOW_TYPE_DV; 4084 if (!attr->transfer) 4085 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4086 MLX5_FLOW_TYPE_VERBS; 4087 return type; 4088 } 4089 4090 #define flow_get_drv_ops(type) flow_drv_ops[type] 4091 4092 /** 4093 * Flow driver validation API. This abstracts calling driver specific functions. 4094 * The type of flow driver is determined according to flow attributes. 4095 * 4096 * @param[in] dev 4097 * Pointer to the dev structure. 4098 * @param[in] attr 4099 * Pointer to the flow attributes. 4100 * @param[in] items 4101 * Pointer to the list of items. 4102 * @param[in] actions 4103 * Pointer to the list of actions. 4104 * @param[in] external 4105 * This flow rule is created by request external to PMD. 4106 * @param[in] hairpin 4107 * Number of hairpin TX actions, 0 means classic flow. 4108 * @param[out] error 4109 * Pointer to the error structure. 4110 * 4111 * @return 4112 * 0 on success, a negative errno value otherwise and rte_errno is set. 4113 */ 4114 static inline int 4115 flow_drv_validate(struct rte_eth_dev *dev, 4116 const struct rte_flow_attr *attr, 4117 const struct rte_flow_item items[], 4118 const struct rte_flow_action actions[], 4119 bool external, int hairpin, struct rte_flow_error *error) 4120 { 4121 const struct mlx5_flow_driver_ops *fops; 4122 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4123 4124 fops = flow_get_drv_ops(type); 4125 return fops->validate(dev, attr, items, actions, external, 4126 hairpin, error); 4127 } 4128 4129 /** 4130 * Flow driver preparation API. This abstracts calling driver specific 4131 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4132 * calculates the size of memory required for device flow, allocates the memory, 4133 * initializes the device flow and returns the pointer. 4134 * 4135 * @note 4136 * This function initializes device flow structure such as dv or verbs in 4137 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4138 * rest. For example, adding returning device flow to flow->dev_flow list and 4139 * setting backward reference to the flow should be done out of this function. 4140 * layers field is not filled either. 4141 * 4142 * @param[in] dev 4143 * Pointer to the dev structure. 4144 * @param[in] attr 4145 * Pointer to the flow attributes. 4146 * @param[in] items 4147 * Pointer to the list of items. 4148 * @param[in] actions 4149 * Pointer to the list of actions. 4150 * @param[in] flow_idx 4151 * This memory pool index to the flow. 4152 * @param[out] error 4153 * Pointer to the error structure. 4154 * 4155 * @return 4156 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4157 */ 4158 static inline struct mlx5_flow * 4159 flow_drv_prepare(struct rte_eth_dev *dev, 4160 const struct rte_flow *flow, 4161 const struct rte_flow_attr *attr, 4162 const struct rte_flow_item items[], 4163 const struct rte_flow_action actions[], 4164 uint32_t flow_idx, 4165 struct rte_flow_error *error) 4166 { 4167 const struct mlx5_flow_driver_ops *fops; 4168 enum mlx5_flow_drv_type type = flow->drv_type; 4169 struct mlx5_flow *mlx5_flow = NULL; 4170 4171 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4172 fops = flow_get_drv_ops(type); 4173 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4174 if (mlx5_flow) 4175 mlx5_flow->flow_idx = flow_idx; 4176 return mlx5_flow; 4177 } 4178 4179 /** 4180 * Flow driver translation API. This abstracts calling driver specific 4181 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4182 * translates a generic flow into a driver flow. flow_drv_prepare() must 4183 * precede. 4184 * 4185 * @note 4186 * dev_flow->layers could be filled as a result of parsing during translation 4187 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4188 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4189 * flow->actions could be overwritten even though all the expanded dev_flows 4190 * have the same actions. 4191 * 4192 * @param[in] dev 4193 * Pointer to the rte dev structure. 4194 * @param[in, out] dev_flow 4195 * Pointer to the mlx5 flow. 4196 * @param[in] attr 4197 * Pointer to the flow attributes. 4198 * @param[in] items 4199 * Pointer to the list of items. 4200 * @param[in] actions 4201 * Pointer to the list of actions. 4202 * @param[out] error 4203 * Pointer to the error structure. 4204 * 4205 * @return 4206 * 0 on success, a negative errno value otherwise and rte_errno is set. 4207 */ 4208 static inline int 4209 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4210 const struct rte_flow_attr *attr, 4211 const struct rte_flow_item items[], 4212 const struct rte_flow_action actions[], 4213 struct rte_flow_error *error) 4214 { 4215 const struct mlx5_flow_driver_ops *fops; 4216 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4217 4218 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4219 fops = flow_get_drv_ops(type); 4220 return fops->translate(dev, dev_flow, attr, items, actions, error); 4221 } 4222 4223 /** 4224 * Flow driver apply API. This abstracts calling driver specific functions. 4225 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4226 * translated driver flows on to device. flow_drv_translate() must precede. 4227 * 4228 * @param[in] dev 4229 * Pointer to Ethernet device structure. 4230 * @param[in, out] flow 4231 * Pointer to flow structure. 4232 * @param[out] error 4233 * Pointer to error structure. 4234 * 4235 * @return 4236 * 0 on success, a negative errno value otherwise and rte_errno is set. 4237 */ 4238 static inline int 4239 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4240 struct rte_flow_error *error) 4241 { 4242 const struct mlx5_flow_driver_ops *fops; 4243 enum mlx5_flow_drv_type type = flow->drv_type; 4244 4245 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4246 fops = flow_get_drv_ops(type); 4247 return fops->apply(dev, flow, error); 4248 } 4249 4250 /** 4251 * Flow driver destroy API. This abstracts calling driver specific functions. 4252 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4253 * on device and releases resources of the flow. 4254 * 4255 * @param[in] dev 4256 * Pointer to Ethernet device. 4257 * @param[in, out] flow 4258 * Pointer to flow structure. 4259 */ 4260 static inline void 4261 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4262 { 4263 const struct mlx5_flow_driver_ops *fops; 4264 enum mlx5_flow_drv_type type = flow->drv_type; 4265 4266 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4267 fops = flow_get_drv_ops(type); 4268 fops->destroy(dev, flow); 4269 } 4270 4271 /** 4272 * Flow driver find RSS policy tbl API. This abstracts calling driver 4273 * specific functions. Parent flow (rte_flow) should have driver 4274 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4275 * 4276 * @param[in] dev 4277 * Pointer to Ethernet device. 4278 * @param[in, out] flow 4279 * Pointer to flow structure. 4280 * @param[in] policy 4281 * Pointer to meter policy table. 4282 * @param[in] rss_desc 4283 * Pointer to rss_desc 4284 */ 4285 static struct mlx5_flow_meter_sub_policy * 4286 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4287 struct rte_flow *flow, 4288 struct mlx5_flow_meter_policy *policy, 4289 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4290 { 4291 const struct mlx5_flow_driver_ops *fops; 4292 enum mlx5_flow_drv_type type = flow->drv_type; 4293 4294 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4295 fops = flow_get_drv_ops(type); 4296 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4297 } 4298 4299 /** 4300 * Flow driver color tag rule API. This abstracts calling driver 4301 * specific functions. Parent flow (rte_flow) should have driver 4302 * type (drv_type). It will create the color tag rules in hierarchy meter. 4303 * 4304 * @param[in] dev 4305 * Pointer to Ethernet device. 4306 * @param[in, out] flow 4307 * Pointer to flow structure. 4308 * @param[in] fm 4309 * Pointer to flow meter structure. 4310 * @param[in] src_port 4311 * The src port this extra rule should use. 4312 * @param[in] item 4313 * The src port id match item. 4314 * @param[out] error 4315 * Pointer to error structure. 4316 */ 4317 static int 4318 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4319 struct rte_flow *flow, 4320 struct mlx5_flow_meter_info *fm, 4321 int32_t src_port, 4322 const struct rte_flow_item *item, 4323 struct rte_flow_error *error) 4324 { 4325 const struct mlx5_flow_driver_ops *fops; 4326 enum mlx5_flow_drv_type type = flow->drv_type; 4327 4328 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4329 fops = flow_get_drv_ops(type); 4330 return fops->meter_hierarchy_rule_create(dev, fm, 4331 src_port, item, error); 4332 } 4333 4334 /** 4335 * Get RSS action from the action list. 4336 * 4337 * @param[in] dev 4338 * Pointer to Ethernet device. 4339 * @param[in] actions 4340 * Pointer to the list of actions. 4341 * @param[in] flow 4342 * Parent flow structure pointer. 4343 * 4344 * @return 4345 * Pointer to the RSS action if exist, else return NULL. 4346 */ 4347 static const struct rte_flow_action_rss* 4348 flow_get_rss_action(struct rte_eth_dev *dev, 4349 const struct rte_flow_action actions[]) 4350 { 4351 struct mlx5_priv *priv = dev->data->dev_private; 4352 const struct rte_flow_action_rss *rss = NULL; 4353 struct mlx5_meter_policy_action_container *acg; 4354 struct mlx5_meter_policy_action_container *acy; 4355 4356 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4357 switch (actions->type) { 4358 case RTE_FLOW_ACTION_TYPE_RSS: 4359 rss = actions->conf; 4360 break; 4361 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4362 { 4363 const struct rte_flow_action_sample *sample = 4364 actions->conf; 4365 const struct rte_flow_action *act = sample->actions; 4366 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4367 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4368 rss = act->conf; 4369 break; 4370 } 4371 case RTE_FLOW_ACTION_TYPE_METER: 4372 { 4373 uint32_t mtr_idx; 4374 struct mlx5_flow_meter_info *fm; 4375 struct mlx5_flow_meter_policy *policy; 4376 const struct rte_flow_action_meter *mtr = actions->conf; 4377 4378 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4379 if (fm && !fm->def_policy) { 4380 policy = mlx5_flow_meter_policy_find(dev, 4381 fm->policy_id, NULL); 4382 MLX5_ASSERT(policy); 4383 if (policy->is_hierarchy) { 4384 policy = 4385 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4386 policy); 4387 if (!policy) 4388 return NULL; 4389 } 4390 if (policy->is_rss) { 4391 acg = 4392 &policy->act_cnt[RTE_COLOR_GREEN]; 4393 acy = 4394 &policy->act_cnt[RTE_COLOR_YELLOW]; 4395 if (acg->fate_action == 4396 MLX5_FLOW_FATE_SHARED_RSS) 4397 rss = acg->rss->conf; 4398 else if (acy->fate_action == 4399 MLX5_FLOW_FATE_SHARED_RSS) 4400 rss = acy->rss->conf; 4401 } 4402 } 4403 break; 4404 } 4405 default: 4406 break; 4407 } 4408 } 4409 return rss; 4410 } 4411 4412 /** 4413 * Get ASO age action by index. 4414 * 4415 * @param[in] dev 4416 * Pointer to the Ethernet device structure. 4417 * @param[in] age_idx 4418 * Index to the ASO age action. 4419 * 4420 * @return 4421 * The specified ASO age action. 4422 */ 4423 struct mlx5_aso_age_action* 4424 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4425 { 4426 uint16_t pool_idx = age_idx & UINT16_MAX; 4427 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4428 struct mlx5_priv *priv = dev->data->dev_private; 4429 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4430 struct mlx5_aso_age_pool *pool; 4431 4432 rte_rwlock_read_lock(&mng->resize_rwl); 4433 pool = mng->pools[pool_idx]; 4434 rte_rwlock_read_unlock(&mng->resize_rwl); 4435 return &pool->actions[offset - 1]; 4436 } 4437 4438 /* maps indirect action to translated direct in some actions array */ 4439 struct mlx5_translated_action_handle { 4440 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4441 int index; /**< Index in related array of rte_flow_action. */ 4442 }; 4443 4444 /** 4445 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4446 * direct action if translation possible. 4447 * This functionality used to run same execution path for both direct and 4448 * indirect actions on flow create. All necessary preparations for indirect 4449 * action handling should be performed on *handle* actions list returned 4450 * from this call. 4451 * 4452 * @param[in] dev 4453 * Pointer to Ethernet device. 4454 * @param[in] actions 4455 * List of actions to translate. 4456 * @param[out] handle 4457 * List to store translated indirect action object handles. 4458 * @param[in, out] indir_n 4459 * Size of *handle* array. On return should be updated with number of 4460 * indirect actions retrieved from the *actions* list. 4461 * @param[out] translated_actions 4462 * List of actions where all indirect actions were translated to direct 4463 * if possible. NULL if no translation took place. 4464 * @param[out] error 4465 * Pointer to the error structure. 4466 * 4467 * @return 4468 * 0 on success, a negative errno value otherwise and rte_errno is set. 4469 */ 4470 static int 4471 flow_action_handles_translate(struct rte_eth_dev *dev, 4472 const struct rte_flow_action actions[], 4473 struct mlx5_translated_action_handle *handle, 4474 int *indir_n, 4475 struct rte_flow_action **translated_actions, 4476 struct rte_flow_error *error) 4477 { 4478 struct mlx5_priv *priv = dev->data->dev_private; 4479 struct rte_flow_action *translated = NULL; 4480 size_t actions_size; 4481 int n; 4482 int copied_n = 0; 4483 struct mlx5_translated_action_handle *handle_end = NULL; 4484 4485 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4486 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4487 continue; 4488 if (copied_n == *indir_n) { 4489 return rte_flow_error_set 4490 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4491 NULL, "too many shared actions"); 4492 } 4493 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4494 sizeof(actions[n].conf)); 4495 handle[copied_n].index = n; 4496 copied_n++; 4497 } 4498 n++; 4499 *indir_n = copied_n; 4500 if (!copied_n) 4501 return 0; 4502 actions_size = sizeof(struct rte_flow_action) * n; 4503 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4504 if (!translated) { 4505 rte_errno = ENOMEM; 4506 return -ENOMEM; 4507 } 4508 memcpy(translated, actions, actions_size); 4509 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4510 struct mlx5_shared_action_rss *shared_rss; 4511 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4512 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4513 uint32_t idx = act_idx & 4514 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4515 4516 switch (type) { 4517 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4518 shared_rss = mlx5_ipool_get 4519 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4520 translated[handle->index].type = 4521 RTE_FLOW_ACTION_TYPE_RSS; 4522 translated[handle->index].conf = 4523 &shared_rss->origin; 4524 break; 4525 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4526 translated[handle->index].type = 4527 (enum rte_flow_action_type) 4528 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4529 translated[handle->index].conf = (void *)(uintptr_t)idx; 4530 break; 4531 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4532 translated[handle->index].type = 4533 (enum rte_flow_action_type) 4534 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4535 translated[handle->index].conf = (void *)(uintptr_t)idx; 4536 break; 4537 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4538 if (priv->sh->flow_hit_aso_en) { 4539 translated[handle->index].type = 4540 (enum rte_flow_action_type) 4541 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4542 translated[handle->index].conf = 4543 (void *)(uintptr_t)idx; 4544 break; 4545 } 4546 /* Fall-through */ 4547 case MLX5_INDIRECT_ACTION_TYPE_CT: 4548 if (priv->sh->ct_aso_en) { 4549 translated[handle->index].type = 4550 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4551 translated[handle->index].conf = 4552 (void *)(uintptr_t)idx; 4553 break; 4554 } 4555 /* Fall-through */ 4556 default: 4557 mlx5_free(translated); 4558 return rte_flow_error_set 4559 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4560 NULL, "invalid indirect action type"); 4561 } 4562 } 4563 *translated_actions = translated; 4564 return 0; 4565 } 4566 4567 /** 4568 * Get Shared RSS action from the action list. 4569 * 4570 * @param[in] dev 4571 * Pointer to Ethernet device. 4572 * @param[in] shared 4573 * Pointer to the list of actions. 4574 * @param[in] shared_n 4575 * Actions list length. 4576 * 4577 * @return 4578 * The MLX5 RSS action ID if exists, otherwise return 0. 4579 */ 4580 static uint32_t 4581 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4582 struct mlx5_translated_action_handle *handle, 4583 int shared_n) 4584 { 4585 struct mlx5_translated_action_handle *handle_end; 4586 struct mlx5_priv *priv = dev->data->dev_private; 4587 struct mlx5_shared_action_rss *shared_rss; 4588 4589 4590 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4591 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4592 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4593 uint32_t idx = act_idx & 4594 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4595 switch (type) { 4596 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4597 shared_rss = mlx5_ipool_get 4598 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4599 idx); 4600 __atomic_fetch_add(&shared_rss->refcnt, 1, 4601 __ATOMIC_RELAXED); 4602 return idx; 4603 default: 4604 break; 4605 } 4606 } 4607 return 0; 4608 } 4609 4610 static unsigned int 4611 find_graph_root(uint32_t rss_level) 4612 { 4613 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4614 MLX5_EXPANSION_ROOT_OUTER; 4615 } 4616 4617 /** 4618 * Get layer flags from the prefix flow. 4619 * 4620 * Some flows may be split to several subflows, the prefix subflow gets the 4621 * match items and the suffix sub flow gets the actions. 4622 * Some actions need the user defined match item flags to get the detail for 4623 * the action. 4624 * This function helps the suffix flow to get the item layer flags from prefix 4625 * subflow. 4626 * 4627 * @param[in] dev_flow 4628 * Pointer the created prefix subflow. 4629 * 4630 * @return 4631 * The layers get from prefix subflow. 4632 */ 4633 static inline uint64_t 4634 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4635 { 4636 uint64_t layers = 0; 4637 4638 /* 4639 * Layers bits could be localization, but usually the compiler will 4640 * help to do the optimization work for source code. 4641 * If no decap actions, use the layers directly. 4642 */ 4643 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4644 return dev_flow->handle->layers; 4645 /* Convert L3 layers with decap action. */ 4646 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4647 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4648 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4649 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4650 /* Convert L4 layers with decap action. */ 4651 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4652 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4653 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4654 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4655 return layers; 4656 } 4657 4658 /** 4659 * Get metadata split action information. 4660 * 4661 * @param[in] actions 4662 * Pointer to the list of actions. 4663 * @param[out] qrss 4664 * Pointer to the return pointer. 4665 * @param[out] qrss_type 4666 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4667 * if no QUEUE/RSS is found. 4668 * @param[out] encap_idx 4669 * Pointer to the index of the encap action if exists, otherwise the last 4670 * action index. 4671 * 4672 * @return 4673 * Total number of actions. 4674 */ 4675 static int 4676 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4677 const struct rte_flow_action **qrss, 4678 int *encap_idx) 4679 { 4680 const struct rte_flow_action_raw_encap *raw_encap; 4681 int actions_n = 0; 4682 int raw_decap_idx = -1; 4683 4684 *encap_idx = -1; 4685 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4686 switch (actions->type) { 4687 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4688 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4689 *encap_idx = actions_n; 4690 break; 4691 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4692 raw_decap_idx = actions_n; 4693 break; 4694 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4695 raw_encap = actions->conf; 4696 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4697 *encap_idx = raw_decap_idx != -1 ? 4698 raw_decap_idx : actions_n; 4699 break; 4700 case RTE_FLOW_ACTION_TYPE_QUEUE: 4701 case RTE_FLOW_ACTION_TYPE_RSS: 4702 *qrss = actions; 4703 break; 4704 default: 4705 break; 4706 } 4707 actions_n++; 4708 } 4709 if (*encap_idx == -1) 4710 *encap_idx = actions_n; 4711 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4712 return actions_n + 1; 4713 } 4714 4715 /** 4716 * Check if the action will change packet. 4717 * 4718 * @param dev 4719 * Pointer to Ethernet device. 4720 * @param[in] type 4721 * action type. 4722 * 4723 * @return 4724 * true if action will change packet, false otherwise. 4725 */ 4726 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4727 enum rte_flow_action_type type) 4728 { 4729 struct mlx5_priv *priv = dev->data->dev_private; 4730 4731 switch (type) { 4732 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4733 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4734 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4735 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4736 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4737 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4738 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4739 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4740 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4741 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4742 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4743 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4744 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4745 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4746 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4747 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4748 case RTE_FLOW_ACTION_TYPE_SET_META: 4749 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4750 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4751 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4752 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4753 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4754 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4755 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4756 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4757 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4758 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4759 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4760 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4761 return true; 4762 case RTE_FLOW_ACTION_TYPE_FLAG: 4763 case RTE_FLOW_ACTION_TYPE_MARK: 4764 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4765 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4766 return true; 4767 else 4768 return false; 4769 default: 4770 return false; 4771 } 4772 } 4773 4774 /** 4775 * Check meter action from the action list. 4776 * 4777 * @param dev 4778 * Pointer to Ethernet device. 4779 * @param[in] actions 4780 * Pointer to the list of actions. 4781 * @param[out] has_mtr 4782 * Pointer to the meter exist flag. 4783 * @param[out] has_modify 4784 * Pointer to the flag showing there's packet change action. 4785 * @param[out] meter_id 4786 * Pointer to the meter id. 4787 * 4788 * @return 4789 * Total number of actions. 4790 */ 4791 static int 4792 flow_check_meter_action(struct rte_eth_dev *dev, 4793 const struct rte_flow_action actions[], 4794 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4795 { 4796 const struct rte_flow_action_meter *mtr = NULL; 4797 int actions_n = 0; 4798 4799 MLX5_ASSERT(has_mtr); 4800 *has_mtr = false; 4801 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4802 switch (actions->type) { 4803 case RTE_FLOW_ACTION_TYPE_METER: 4804 mtr = actions->conf; 4805 *meter_id = mtr->mtr_id; 4806 *has_mtr = true; 4807 break; 4808 default: 4809 break; 4810 } 4811 if (!*has_mtr) 4812 *has_modify |= flow_check_modify_action_type(dev, 4813 actions->type); 4814 actions_n++; 4815 } 4816 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4817 return actions_n + 1; 4818 } 4819 4820 /** 4821 * Check if the flow should be split due to hairpin. 4822 * The reason for the split is that in current HW we can't 4823 * support encap and push-vlan on Rx, so if a flow contains 4824 * these actions we move it to Tx. 4825 * 4826 * @param dev 4827 * Pointer to Ethernet device. 4828 * @param[in] attr 4829 * Flow rule attributes. 4830 * @param[in] actions 4831 * Associated actions (list terminated by the END action). 4832 * 4833 * @return 4834 * > 0 the number of actions and the flow should be split, 4835 * 0 when no split required. 4836 */ 4837 static int 4838 flow_check_hairpin_split(struct rte_eth_dev *dev, 4839 const struct rte_flow_attr *attr, 4840 const struct rte_flow_action actions[]) 4841 { 4842 int queue_action = 0; 4843 int action_n = 0; 4844 int split = 0; 4845 int push_vlan = 0; 4846 const struct rte_flow_action_queue *queue; 4847 const struct rte_flow_action_rss *rss; 4848 const struct rte_flow_action_raw_encap *raw_encap; 4849 const struct rte_eth_hairpin_conf *conf; 4850 4851 if (!attr->ingress) 4852 return 0; 4853 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4854 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4855 push_vlan = 1; 4856 switch (actions->type) { 4857 case RTE_FLOW_ACTION_TYPE_QUEUE: 4858 queue = actions->conf; 4859 if (queue == NULL) 4860 return 0; 4861 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4862 if (conf == NULL || conf->tx_explicit != 0) 4863 return 0; 4864 queue_action = 1; 4865 action_n++; 4866 break; 4867 case RTE_FLOW_ACTION_TYPE_RSS: 4868 rss = actions->conf; 4869 if (rss == NULL || rss->queue_num == 0) 4870 return 0; 4871 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4872 if (conf == NULL || conf->tx_explicit != 0) 4873 return 0; 4874 queue_action = 1; 4875 action_n++; 4876 break; 4877 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4878 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4879 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4880 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4881 split++; 4882 action_n++; 4883 break; 4884 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4885 if (push_vlan) 4886 split++; 4887 action_n++; 4888 break; 4889 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4890 raw_encap = actions->conf; 4891 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4892 split++; 4893 action_n++; 4894 break; 4895 default: 4896 action_n++; 4897 break; 4898 } 4899 } 4900 if (split && queue_action) 4901 return action_n; 4902 return 0; 4903 } 4904 4905 /* Declare flow create/destroy prototype in advance. */ 4906 static uint32_t 4907 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4908 const struct rte_flow_attr *attr, 4909 const struct rte_flow_item items[], 4910 const struct rte_flow_action actions[], 4911 bool external, struct rte_flow_error *error); 4912 4913 static void 4914 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4915 uint32_t flow_idx); 4916 4917 int 4918 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4919 struct mlx5_list_entry *entry, void *cb_ctx) 4920 { 4921 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4922 struct mlx5_flow_mreg_copy_resource *mcp_res = 4923 container_of(entry, typeof(*mcp_res), hlist_ent); 4924 4925 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4926 } 4927 4928 struct mlx5_list_entry * 4929 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4930 { 4931 struct rte_eth_dev *dev = tool_ctx; 4932 struct mlx5_priv *priv = dev->data->dev_private; 4933 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4934 struct mlx5_flow_mreg_copy_resource *mcp_res; 4935 struct rte_flow_error *error = ctx->error; 4936 uint32_t idx = 0; 4937 int ret; 4938 uint32_t mark_id = *(uint32_t *)(ctx->data); 4939 struct rte_flow_attr attr = { 4940 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4941 .ingress = 1, 4942 }; 4943 struct mlx5_rte_flow_item_tag tag_spec = { 4944 .data = mark_id, 4945 }; 4946 struct rte_flow_item items[] = { 4947 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4948 }; 4949 struct rte_flow_action_mark ftag = { 4950 .id = mark_id, 4951 }; 4952 struct mlx5_flow_action_copy_mreg cp_mreg = { 4953 .dst = REG_B, 4954 .src = REG_NON, 4955 }; 4956 struct rte_flow_action_jump jump = { 4957 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4958 }; 4959 struct rte_flow_action actions[] = { 4960 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4961 }; 4962 4963 /* Fill the register fields in the flow. */ 4964 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4965 if (ret < 0) 4966 return NULL; 4967 tag_spec.id = ret; 4968 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4969 if (ret < 0) 4970 return NULL; 4971 cp_mreg.src = ret; 4972 /* Provide the full width of FLAG specific value. */ 4973 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4974 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4975 /* Build a new flow. */ 4976 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4977 items[0] = (struct rte_flow_item){ 4978 .type = (enum rte_flow_item_type) 4979 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4980 .spec = &tag_spec, 4981 }; 4982 items[1] = (struct rte_flow_item){ 4983 .type = RTE_FLOW_ITEM_TYPE_END, 4984 }; 4985 actions[0] = (struct rte_flow_action){ 4986 .type = (enum rte_flow_action_type) 4987 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4988 .conf = &ftag, 4989 }; 4990 actions[1] = (struct rte_flow_action){ 4991 .type = (enum rte_flow_action_type) 4992 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4993 .conf = &cp_mreg, 4994 }; 4995 actions[2] = (struct rte_flow_action){ 4996 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4997 .conf = &jump, 4998 }; 4999 actions[3] = (struct rte_flow_action){ 5000 .type = RTE_FLOW_ACTION_TYPE_END, 5001 }; 5002 } else { 5003 /* Default rule, wildcard match. */ 5004 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 5005 items[0] = (struct rte_flow_item){ 5006 .type = RTE_FLOW_ITEM_TYPE_END, 5007 }; 5008 actions[0] = (struct rte_flow_action){ 5009 .type = (enum rte_flow_action_type) 5010 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5011 .conf = &cp_mreg, 5012 }; 5013 actions[1] = (struct rte_flow_action){ 5014 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5015 .conf = &jump, 5016 }; 5017 actions[2] = (struct rte_flow_action){ 5018 .type = RTE_FLOW_ACTION_TYPE_END, 5019 }; 5020 } 5021 /* Build a new entry. */ 5022 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5023 if (!mcp_res) { 5024 rte_errno = ENOMEM; 5025 return NULL; 5026 } 5027 mcp_res->idx = idx; 5028 mcp_res->mark_id = mark_id; 5029 /* 5030 * The copy Flows are not included in any list. There 5031 * ones are referenced from other Flows and can not 5032 * be applied, removed, deleted in arbitrary order 5033 * by list traversing. 5034 */ 5035 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 5036 &attr, items, actions, false, error); 5037 if (!mcp_res->rix_flow) { 5038 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 5039 return NULL; 5040 } 5041 return &mcp_res->hlist_ent; 5042 } 5043 5044 struct mlx5_list_entry * 5045 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5046 void *cb_ctx __rte_unused) 5047 { 5048 struct rte_eth_dev *dev = tool_ctx; 5049 struct mlx5_priv *priv = dev->data->dev_private; 5050 struct mlx5_flow_mreg_copy_resource *mcp_res; 5051 uint32_t idx = 0; 5052 5053 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5054 if (!mcp_res) { 5055 rte_errno = ENOMEM; 5056 return NULL; 5057 } 5058 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 5059 mcp_res->idx = idx; 5060 return &mcp_res->hlist_ent; 5061 } 5062 5063 void 5064 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5065 { 5066 struct mlx5_flow_mreg_copy_resource *mcp_res = 5067 container_of(entry, typeof(*mcp_res), hlist_ent); 5068 struct rte_eth_dev *dev = tool_ctx; 5069 struct mlx5_priv *priv = dev->data->dev_private; 5070 5071 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5072 } 5073 5074 /** 5075 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5076 * 5077 * As mark_id is unique, if there's already a registered flow for the mark_id, 5078 * return by increasing the reference counter of the resource. Otherwise, create 5079 * the resource (mcp_res) and flow. 5080 * 5081 * Flow looks like, 5082 * - If ingress port is ANY and reg_c[1] is mark_id, 5083 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5084 * 5085 * For default flow (zero mark_id), flow is like, 5086 * - If ingress port is ANY, 5087 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5088 * 5089 * @param dev 5090 * Pointer to Ethernet device. 5091 * @param mark_id 5092 * ID of MARK action, zero means default flow for META. 5093 * @param[out] error 5094 * Perform verbose error reporting if not NULL. 5095 * 5096 * @return 5097 * Associated resource on success, NULL otherwise and rte_errno is set. 5098 */ 5099 static struct mlx5_flow_mreg_copy_resource * 5100 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5101 struct rte_flow_error *error) 5102 { 5103 struct mlx5_priv *priv = dev->data->dev_private; 5104 struct mlx5_list_entry *entry; 5105 struct mlx5_flow_cb_ctx ctx = { 5106 .dev = dev, 5107 .error = error, 5108 .data = &mark_id, 5109 }; 5110 5111 /* Check if already registered. */ 5112 MLX5_ASSERT(priv->mreg_cp_tbl); 5113 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5114 if (!entry) 5115 return NULL; 5116 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5117 hlist_ent); 5118 } 5119 5120 void 5121 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5122 { 5123 struct mlx5_flow_mreg_copy_resource *mcp_res = 5124 container_of(entry, typeof(*mcp_res), hlist_ent); 5125 struct rte_eth_dev *dev = tool_ctx; 5126 struct mlx5_priv *priv = dev->data->dev_private; 5127 5128 MLX5_ASSERT(mcp_res->rix_flow); 5129 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5130 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5131 } 5132 5133 /** 5134 * Release flow in RX_CP_TBL. 5135 * 5136 * @param dev 5137 * Pointer to Ethernet device. 5138 * @flow 5139 * Parent flow for wich copying is provided. 5140 */ 5141 static void 5142 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5143 struct rte_flow *flow) 5144 { 5145 struct mlx5_flow_mreg_copy_resource *mcp_res; 5146 struct mlx5_priv *priv = dev->data->dev_private; 5147 5148 if (!flow->rix_mreg_copy) 5149 return; 5150 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5151 flow->rix_mreg_copy); 5152 if (!mcp_res || !priv->mreg_cp_tbl) 5153 return; 5154 MLX5_ASSERT(mcp_res->rix_flow); 5155 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5156 flow->rix_mreg_copy = 0; 5157 } 5158 5159 /** 5160 * Remove the default copy action from RX_CP_TBL. 5161 * 5162 * This functions is called in the mlx5_dev_start(). No thread safe 5163 * is guaranteed. 5164 * 5165 * @param dev 5166 * Pointer to Ethernet device. 5167 */ 5168 static void 5169 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5170 { 5171 struct mlx5_list_entry *entry; 5172 struct mlx5_priv *priv = dev->data->dev_private; 5173 struct mlx5_flow_cb_ctx ctx; 5174 uint32_t mark_id; 5175 5176 /* Check if default flow is registered. */ 5177 if (!priv->mreg_cp_tbl) 5178 return; 5179 mark_id = MLX5_DEFAULT_COPY_ID; 5180 ctx.data = &mark_id; 5181 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5182 if (!entry) 5183 return; 5184 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5185 } 5186 5187 /** 5188 * Add the default copy action in RX_CP_TBL. 5189 * 5190 * This functions is called in the mlx5_dev_start(). No thread safe 5191 * is guaranteed. 5192 * 5193 * @param dev 5194 * Pointer to Ethernet device. 5195 * @param[out] error 5196 * Perform verbose error reporting if not NULL. 5197 * 5198 * @return 5199 * 0 for success, negative value otherwise and rte_errno is set. 5200 */ 5201 static int 5202 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5203 struct rte_flow_error *error) 5204 { 5205 struct mlx5_priv *priv = dev->data->dev_private; 5206 struct mlx5_flow_mreg_copy_resource *mcp_res; 5207 struct mlx5_flow_cb_ctx ctx; 5208 uint32_t mark_id; 5209 5210 /* Check whether extensive metadata feature is engaged. */ 5211 if (!priv->sh->config.dv_flow_en || 5212 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5213 !mlx5_flow_ext_mreg_supported(dev) || 5214 !priv->sh->dv_regc0_mask) 5215 return 0; 5216 /* 5217 * Add default mreg copy flow may be called multiple time, but 5218 * only be called once in stop. Avoid register it twice. 5219 */ 5220 mark_id = MLX5_DEFAULT_COPY_ID; 5221 ctx.data = &mark_id; 5222 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5223 return 0; 5224 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5225 if (!mcp_res) 5226 return -rte_errno; 5227 return 0; 5228 } 5229 5230 /** 5231 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5232 * 5233 * All the flow having Q/RSS action should be split by 5234 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5235 * performs the following, 5236 * - CQE->flow_tag := reg_c[1] (MARK) 5237 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5238 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5239 * but there should be a flow per each MARK ID set by MARK action. 5240 * 5241 * For the aforementioned reason, if there's a MARK action in flow's action 5242 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5243 * the MARK ID to CQE's flow_tag like, 5244 * - If reg_c[1] is mark_id, 5245 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5246 * 5247 * For SET_META action which stores value in reg_c[0], as the destination is 5248 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5249 * MARK ID means the default flow. The default flow looks like, 5250 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5251 * 5252 * @param dev 5253 * Pointer to Ethernet device. 5254 * @param flow 5255 * Pointer to flow structure. 5256 * @param[in] actions 5257 * Pointer to the list of actions. 5258 * @param[out] error 5259 * Perform verbose error reporting if not NULL. 5260 * 5261 * @return 5262 * 0 on success, negative value otherwise and rte_errno is set. 5263 */ 5264 static int 5265 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5266 struct rte_flow *flow, 5267 const struct rte_flow_action *actions, 5268 struct rte_flow_error *error) 5269 { 5270 struct mlx5_priv *priv = dev->data->dev_private; 5271 struct mlx5_sh_config *config = &priv->sh->config; 5272 struct mlx5_flow_mreg_copy_resource *mcp_res; 5273 const struct rte_flow_action_mark *mark; 5274 5275 /* Check whether extensive metadata feature is engaged. */ 5276 if (!config->dv_flow_en || 5277 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5278 !mlx5_flow_ext_mreg_supported(dev) || 5279 !priv->sh->dv_regc0_mask) 5280 return 0; 5281 /* Find MARK action. */ 5282 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5283 switch (actions->type) { 5284 case RTE_FLOW_ACTION_TYPE_FLAG: 5285 mcp_res = flow_mreg_add_copy_action 5286 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5287 if (!mcp_res) 5288 return -rte_errno; 5289 flow->rix_mreg_copy = mcp_res->idx; 5290 return 0; 5291 case RTE_FLOW_ACTION_TYPE_MARK: 5292 mark = (const struct rte_flow_action_mark *) 5293 actions->conf; 5294 mcp_res = 5295 flow_mreg_add_copy_action(dev, mark->id, error); 5296 if (!mcp_res) 5297 return -rte_errno; 5298 flow->rix_mreg_copy = mcp_res->idx; 5299 return 0; 5300 default: 5301 break; 5302 } 5303 } 5304 return 0; 5305 } 5306 5307 #define MLX5_MAX_SPLIT_ACTIONS 24 5308 #define MLX5_MAX_SPLIT_ITEMS 24 5309 5310 /** 5311 * Split the hairpin flow. 5312 * Since HW can't support encap and push-vlan on Rx, we move these 5313 * actions to Tx. 5314 * If the count action is after the encap then we also 5315 * move the count action. in this case the count will also measure 5316 * the outer bytes. 5317 * 5318 * @param dev 5319 * Pointer to Ethernet device. 5320 * @param[in] actions 5321 * Associated actions (list terminated by the END action). 5322 * @param[out] actions_rx 5323 * Rx flow actions. 5324 * @param[out] actions_tx 5325 * Tx flow actions.. 5326 * @param[out] pattern_tx 5327 * The pattern items for the Tx flow. 5328 * @param[out] flow_id 5329 * The flow ID connected to this flow. 5330 * 5331 * @return 5332 * 0 on success. 5333 */ 5334 static int 5335 flow_hairpin_split(struct rte_eth_dev *dev, 5336 const struct rte_flow_action actions[], 5337 struct rte_flow_action actions_rx[], 5338 struct rte_flow_action actions_tx[], 5339 struct rte_flow_item pattern_tx[], 5340 uint32_t flow_id) 5341 { 5342 const struct rte_flow_action_raw_encap *raw_encap; 5343 const struct rte_flow_action_raw_decap *raw_decap; 5344 struct mlx5_rte_flow_action_set_tag *set_tag; 5345 struct rte_flow_action *tag_action; 5346 struct mlx5_rte_flow_item_tag *tag_item; 5347 struct rte_flow_item *item; 5348 char *addr; 5349 int push_vlan = 0; 5350 int encap = 0; 5351 5352 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5353 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5354 push_vlan = 1; 5355 switch (actions->type) { 5356 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5357 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5358 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5359 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5360 rte_memcpy(actions_tx, actions, 5361 sizeof(struct rte_flow_action)); 5362 actions_tx++; 5363 break; 5364 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5365 if (push_vlan) { 5366 rte_memcpy(actions_tx, actions, 5367 sizeof(struct rte_flow_action)); 5368 actions_tx++; 5369 } else { 5370 rte_memcpy(actions_rx, actions, 5371 sizeof(struct rte_flow_action)); 5372 actions_rx++; 5373 } 5374 break; 5375 case RTE_FLOW_ACTION_TYPE_COUNT: 5376 if (encap) { 5377 rte_memcpy(actions_tx, actions, 5378 sizeof(struct rte_flow_action)); 5379 actions_tx++; 5380 } else { 5381 rte_memcpy(actions_rx, actions, 5382 sizeof(struct rte_flow_action)); 5383 actions_rx++; 5384 } 5385 break; 5386 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5387 raw_encap = actions->conf; 5388 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5389 memcpy(actions_tx, actions, 5390 sizeof(struct rte_flow_action)); 5391 actions_tx++; 5392 encap = 1; 5393 } else { 5394 rte_memcpy(actions_rx, actions, 5395 sizeof(struct rte_flow_action)); 5396 actions_rx++; 5397 } 5398 break; 5399 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5400 raw_decap = actions->conf; 5401 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5402 memcpy(actions_tx, actions, 5403 sizeof(struct rte_flow_action)); 5404 actions_tx++; 5405 } else { 5406 rte_memcpy(actions_rx, actions, 5407 sizeof(struct rte_flow_action)); 5408 actions_rx++; 5409 } 5410 break; 5411 default: 5412 rte_memcpy(actions_rx, actions, 5413 sizeof(struct rte_flow_action)); 5414 actions_rx++; 5415 break; 5416 } 5417 } 5418 /* Add set meta action and end action for the Rx flow. */ 5419 tag_action = actions_rx; 5420 tag_action->type = (enum rte_flow_action_type) 5421 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5422 actions_rx++; 5423 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5424 actions_rx++; 5425 set_tag = (void *)actions_rx; 5426 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5427 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5428 .data = flow_id, 5429 }; 5430 MLX5_ASSERT(set_tag->id > REG_NON); 5431 tag_action->conf = set_tag; 5432 /* Create Tx item list. */ 5433 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5434 addr = (void *)&pattern_tx[2]; 5435 item = pattern_tx; 5436 item->type = (enum rte_flow_item_type) 5437 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5438 tag_item = (void *)addr; 5439 tag_item->data = flow_id; 5440 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5441 MLX5_ASSERT(set_tag->id > REG_NON); 5442 item->spec = tag_item; 5443 addr += sizeof(struct mlx5_rte_flow_item_tag); 5444 tag_item = (void *)addr; 5445 tag_item->data = UINT32_MAX; 5446 tag_item->id = UINT16_MAX; 5447 item->mask = tag_item; 5448 item->last = NULL; 5449 item++; 5450 item->type = RTE_FLOW_ITEM_TYPE_END; 5451 return 0; 5452 } 5453 5454 /** 5455 * The last stage of splitting chain, just creates the subflow 5456 * without any modification. 5457 * 5458 * @param[in] dev 5459 * Pointer to Ethernet device. 5460 * @param[in] flow 5461 * Parent flow structure pointer. 5462 * @param[in, out] sub_flow 5463 * Pointer to return the created subflow, may be NULL. 5464 * @param[in] attr 5465 * Flow rule attributes. 5466 * @param[in] items 5467 * Pattern specification (list terminated by the END pattern item). 5468 * @param[in] actions 5469 * Associated actions (list terminated by the END action). 5470 * @param[in] flow_split_info 5471 * Pointer to flow split info structure. 5472 * @param[out] error 5473 * Perform verbose error reporting if not NULL. 5474 * @return 5475 * 0 on success, negative value otherwise 5476 */ 5477 static int 5478 flow_create_split_inner(struct rte_eth_dev *dev, 5479 struct rte_flow *flow, 5480 struct mlx5_flow **sub_flow, 5481 const struct rte_flow_attr *attr, 5482 const struct rte_flow_item items[], 5483 const struct rte_flow_action actions[], 5484 struct mlx5_flow_split_info *flow_split_info, 5485 struct rte_flow_error *error) 5486 { 5487 struct mlx5_flow *dev_flow; 5488 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5489 5490 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5491 flow_split_info->flow_idx, error); 5492 if (!dev_flow) 5493 return -rte_errno; 5494 dev_flow->flow = flow; 5495 dev_flow->external = flow_split_info->external; 5496 dev_flow->skip_scale = flow_split_info->skip_scale; 5497 /* Subflow object was created, we must include one in the list. */ 5498 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5499 dev_flow->handle, next); 5500 /* 5501 * If dev_flow is as one of the suffix flow, some actions in suffix 5502 * flow may need some user defined item layer flags, and pass the 5503 * Metadata rxq mark flag to suffix flow as well. 5504 */ 5505 if (flow_split_info->prefix_layers) 5506 dev_flow->handle->layers = flow_split_info->prefix_layers; 5507 if (flow_split_info->prefix_mark) { 5508 MLX5_ASSERT(wks); 5509 wks->mark = 1; 5510 } 5511 if (sub_flow) 5512 *sub_flow = dev_flow; 5513 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5514 dev_flow->dv.table_id = flow_split_info->table_id; 5515 #endif 5516 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5517 } 5518 5519 /** 5520 * Get the sub policy of a meter. 5521 * 5522 * @param[in] dev 5523 * Pointer to Ethernet device. 5524 * @param[in] flow 5525 * Parent flow structure pointer. 5526 * @param wks 5527 * Pointer to thread flow work space. 5528 * @param[in] attr 5529 * Flow rule attributes. 5530 * @param[in] items 5531 * Pattern specification (list terminated by the END pattern item). 5532 * @param[out] error 5533 * Perform verbose error reporting if not NULL. 5534 * 5535 * @return 5536 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5537 */ 5538 static struct mlx5_flow_meter_sub_policy * 5539 get_meter_sub_policy(struct rte_eth_dev *dev, 5540 struct rte_flow *flow, 5541 struct mlx5_flow_workspace *wks, 5542 const struct rte_flow_attr *attr, 5543 const struct rte_flow_item items[], 5544 struct rte_flow_error *error) 5545 { 5546 struct mlx5_flow_meter_policy *policy; 5547 struct mlx5_flow_meter_policy *final_policy; 5548 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5549 5550 policy = wks->policy; 5551 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5552 if (final_policy->is_rss || final_policy->is_queue) { 5553 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5554 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5555 uint32_t i; 5556 5557 /* 5558 * This is a tmp dev_flow, 5559 * no need to register any matcher for it in translate. 5560 */ 5561 wks->skip_matcher_reg = 1; 5562 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5563 struct mlx5_flow dev_flow = {0}; 5564 struct mlx5_flow_handle dev_handle = { {0} }; 5565 uint8_t fate = final_policy->act_cnt[i].fate_action; 5566 5567 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5568 const struct rte_flow_action_rss *rss_act = 5569 final_policy->act_cnt[i].rss->conf; 5570 struct rte_flow_action rss_actions[2] = { 5571 [0] = { 5572 .type = RTE_FLOW_ACTION_TYPE_RSS, 5573 .conf = rss_act, 5574 }, 5575 [1] = { 5576 .type = RTE_FLOW_ACTION_TYPE_END, 5577 .conf = NULL, 5578 } 5579 }; 5580 5581 dev_flow.handle = &dev_handle; 5582 dev_flow.ingress = attr->ingress; 5583 dev_flow.flow = flow; 5584 dev_flow.external = 0; 5585 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5586 dev_flow.dv.transfer = attr->transfer; 5587 #endif 5588 /** 5589 * Translate RSS action to get rss hash fields. 5590 */ 5591 if (flow_drv_translate(dev, &dev_flow, attr, 5592 items, rss_actions, error)) 5593 goto exit; 5594 rss_desc_v[i] = wks->rss_desc; 5595 rss_desc_v[i].symmetric_hash_function = 5596 dev_flow.symmetric_hash_function; 5597 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5598 rss_desc_v[i].hash_fields = 5599 dev_flow.hash_fields; 5600 rss_desc_v[i].queue_num = 5601 rss_desc_v[i].hash_fields ? 5602 rss_desc_v[i].queue_num : 1; 5603 rss_desc_v[i].tunnel = 5604 !!(dev_flow.handle->layers & 5605 MLX5_FLOW_LAYER_TUNNEL); 5606 /* Use the RSS queues in the containers. */ 5607 rss_desc_v[i].queue = 5608 (uint16_t *)(uintptr_t)rss_act->queue; 5609 rss_desc[i] = &rss_desc_v[i]; 5610 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5611 /* This is queue action. */ 5612 rss_desc_v[i] = wks->rss_desc; 5613 rss_desc_v[i].key_len = 0; 5614 rss_desc_v[i].hash_fields = 0; 5615 rss_desc_v[i].queue = 5616 &final_policy->act_cnt[i].queue; 5617 rss_desc_v[i].queue_num = 1; 5618 rss_desc[i] = &rss_desc_v[i]; 5619 } else { 5620 rss_desc[i] = NULL; 5621 } 5622 } 5623 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5624 flow, policy, rss_desc); 5625 } else { 5626 enum mlx5_meter_domain mtr_domain = 5627 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5628 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5629 MLX5_MTR_DOMAIN_INGRESS); 5630 sub_policy = policy->sub_policys[mtr_domain][0]; 5631 } 5632 if (!sub_policy) 5633 rte_flow_error_set(error, EINVAL, 5634 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5635 "Failed to get meter sub-policy."); 5636 exit: 5637 return sub_policy; 5638 } 5639 5640 /** 5641 * Split the meter flow. 5642 * 5643 * As meter flow will split to three sub flow, other than meter 5644 * action, the other actions make sense to only meter accepts 5645 * the packet. If it need to be dropped, no other additional 5646 * actions should be take. 5647 * 5648 * One kind of special action which decapsulates the L3 tunnel 5649 * header will be in the prefix sub flow, as not to take the 5650 * L3 tunnel header into account. 5651 * 5652 * @param[in] dev 5653 * Pointer to Ethernet device. 5654 * @param[in] flow 5655 * Parent flow structure pointer. 5656 * @param wks 5657 * Pointer to thread flow work space. 5658 * @param[in] attr 5659 * Flow rule attributes. 5660 * @param[in] items 5661 * Pattern specification (list terminated by the END pattern item). 5662 * @param[out] sfx_items 5663 * Suffix flow match items (list terminated by the END pattern item). 5664 * @param[in] actions 5665 * Associated actions (list terminated by the END action). 5666 * @param[out] actions_sfx 5667 * Suffix flow actions. 5668 * @param[out] actions_pre 5669 * Prefix flow actions. 5670 * @param[out] mtr_flow_id 5671 * Pointer to meter flow id. 5672 * @param[out] error 5673 * Perform verbose error reporting if not NULL. 5674 * 5675 * @return 5676 * 0 on success, a negative errno value otherwise and rte_errno is set. 5677 */ 5678 static int 5679 flow_meter_split_prep(struct rte_eth_dev *dev, 5680 struct rte_flow *flow, 5681 struct mlx5_flow_workspace *wks, 5682 const struct rte_flow_attr *attr, 5683 const struct rte_flow_item items[], 5684 struct rte_flow_item sfx_items[], 5685 const struct rte_flow_action actions[], 5686 struct rte_flow_action actions_sfx[], 5687 struct rte_flow_action actions_pre[], 5688 uint32_t *mtr_flow_id, 5689 struct rte_flow_error *error) 5690 { 5691 struct mlx5_priv *priv = dev->data->dev_private; 5692 struct mlx5_flow_meter_info *fm = wks->fm; 5693 struct rte_flow_action *tag_action = NULL; 5694 struct rte_flow_item *tag_item; 5695 struct mlx5_rte_flow_action_set_tag *set_tag; 5696 const struct rte_flow_action_raw_encap *raw_encap; 5697 const struct rte_flow_action_raw_decap *raw_decap; 5698 struct mlx5_rte_flow_item_tag *tag_item_spec; 5699 struct mlx5_rte_flow_item_tag *tag_item_mask; 5700 uint32_t tag_id = 0; 5701 struct rte_flow_item *vlan_item_dst = NULL; 5702 const struct rte_flow_item *vlan_item_src = NULL; 5703 const struct rte_flow_item *orig_items = items; 5704 struct rte_flow_action *hw_mtr_action; 5705 struct rte_flow_action *action_pre_head = NULL; 5706 uint16_t flow_src_port = priv->representor_id; 5707 bool mtr_first; 5708 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5709 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5710 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5711 uint32_t flow_id = 0; 5712 uint32_t flow_id_reversed = 0; 5713 uint8_t flow_id_bits = 0; 5714 bool after_meter = false; 5715 int shift; 5716 5717 /* Prepare the suffix subflow items. */ 5718 tag_item = sfx_items++; 5719 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5720 int item_type = items->type; 5721 5722 switch (item_type) { 5723 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5724 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5725 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5726 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5727 return -rte_errno; 5728 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5729 flow_src_port != priv->representor_id) { 5730 if (flow_drv_mtr_hierarchy_rule_create(dev, 5731 flow, fm, 5732 flow_src_port, 5733 items, 5734 error)) 5735 return -rte_errno; 5736 } 5737 memcpy(sfx_items, items, sizeof(*sfx_items)); 5738 sfx_items++; 5739 break; 5740 case RTE_FLOW_ITEM_TYPE_VLAN: 5741 /* Determine if copy vlan item below. */ 5742 vlan_item_src = items; 5743 vlan_item_dst = sfx_items++; 5744 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5745 break; 5746 default: 5747 break; 5748 } 5749 } 5750 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5751 sfx_items++; 5752 mtr_first = priv->sh->meter_aso_en && 5753 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5754 /* For ASO meter, meter must be before tag in TX direction. */ 5755 if (mtr_first) { 5756 action_pre_head = actions_pre++; 5757 /* Leave space for tag action. */ 5758 tag_action = actions_pre++; 5759 } 5760 /* Prepare the actions for prefix and suffix flow. */ 5761 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5762 struct rte_flow_action *action_cur = NULL; 5763 5764 switch (actions->type) { 5765 case RTE_FLOW_ACTION_TYPE_METER: 5766 if (mtr_first) { 5767 action_cur = action_pre_head; 5768 } else { 5769 /* Leave space for tag action. */ 5770 tag_action = actions_pre++; 5771 action_cur = actions_pre++; 5772 } 5773 after_meter = true; 5774 break; 5775 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5776 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5777 action_cur = actions_pre++; 5778 break; 5779 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5780 raw_encap = actions->conf; 5781 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5782 action_cur = actions_pre++; 5783 break; 5784 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5785 raw_decap = actions->conf; 5786 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5787 action_cur = actions_pre++; 5788 break; 5789 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5790 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5791 if (vlan_item_dst && vlan_item_src) { 5792 memcpy(vlan_item_dst, vlan_item_src, 5793 sizeof(*vlan_item_dst)); 5794 /* 5795 * Convert to internal match item, it is used 5796 * for vlan push and set vid. 5797 */ 5798 vlan_item_dst->type = (enum rte_flow_item_type) 5799 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5800 } 5801 break; 5802 case RTE_FLOW_ACTION_TYPE_COUNT: 5803 if (fm->def_policy) 5804 action_cur = after_meter ? 5805 actions_sfx++ : actions_pre++; 5806 break; 5807 default: 5808 break; 5809 } 5810 if (!action_cur) 5811 action_cur = (fm->def_policy) ? 5812 actions_sfx++ : actions_pre++; 5813 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5814 } 5815 /* Add end action to the actions. */ 5816 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5817 if (priv->sh->meter_aso_en) { 5818 /** 5819 * For ASO meter, need to add an extra jump action explicitly, 5820 * to jump from meter to policer table. 5821 */ 5822 struct mlx5_flow_meter_sub_policy *sub_policy; 5823 struct mlx5_flow_tbl_data_entry *tbl_data; 5824 5825 if (!fm->def_policy) { 5826 sub_policy = get_meter_sub_policy(dev, flow, wks, 5827 attr, orig_items, 5828 error); 5829 if (!sub_policy) 5830 return -rte_errno; 5831 } else { 5832 enum mlx5_meter_domain mtr_domain = 5833 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5834 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5835 MLX5_MTR_DOMAIN_INGRESS); 5836 5837 sub_policy = 5838 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5839 } 5840 tbl_data = container_of(sub_policy->tbl_rsc, 5841 struct mlx5_flow_tbl_data_entry, tbl); 5842 hw_mtr_action = actions_pre++; 5843 hw_mtr_action->type = (enum rte_flow_action_type) 5844 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5845 hw_mtr_action->conf = tbl_data->jump.action; 5846 } 5847 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5848 actions_pre++; 5849 if (!tag_action) 5850 return rte_flow_error_set(error, ENOMEM, 5851 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5852 NULL, "No tag action space."); 5853 if (!mtr_flow_id) { 5854 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5855 goto exit; 5856 } 5857 /* Only default-policy Meter creates mtr flow id. */ 5858 if (fm->def_policy) { 5859 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5860 if (!tag_id) 5861 return rte_flow_error_set(error, ENOMEM, 5862 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5863 "Failed to allocate meter flow id."); 5864 flow_id = tag_id - 1; 5865 flow_id_bits = (!flow_id) ? 1 : 5866 (MLX5_REG_BITS - rte_clz32(flow_id)); 5867 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5868 mtr_reg_bits) { 5869 mlx5_ipool_free(fm->flow_ipool, tag_id); 5870 return rte_flow_error_set(error, EINVAL, 5871 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5872 "Meter flow id exceeds max limit."); 5873 } 5874 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5875 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5876 } 5877 /* Build tag actions and items for meter_id/meter flow_id. */ 5878 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5879 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5880 tag_item_mask = tag_item_spec + 1; 5881 /* Both flow_id and meter_id share the same register. */ 5882 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5883 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5884 0, error), 5885 .offset = mtr_id_offset, 5886 .length = mtr_reg_bits, 5887 .data = flow->meter, 5888 }; 5889 /* 5890 * The color Reg bits used by flow_id are growing from 5891 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5892 */ 5893 for (shift = 0; shift < flow_id_bits; shift++) 5894 flow_id_reversed = (flow_id_reversed << 1) | 5895 ((flow_id >> shift) & 0x1); 5896 set_tag->data |= 5897 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5898 tag_item_spec->id = set_tag->id; 5899 tag_item_spec->data = set_tag->data << mtr_id_offset; 5900 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5901 tag_action->type = (enum rte_flow_action_type) 5902 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5903 tag_action->conf = set_tag; 5904 tag_item->type = (enum rte_flow_item_type) 5905 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5906 tag_item->spec = tag_item_spec; 5907 tag_item->last = NULL; 5908 tag_item->mask = tag_item_mask; 5909 exit: 5910 if (mtr_flow_id) 5911 *mtr_flow_id = tag_id; 5912 return 0; 5913 } 5914 5915 /** 5916 * Split action list having QUEUE/RSS for metadata register copy. 5917 * 5918 * Once Q/RSS action is detected in user's action list, the flow action 5919 * should be split in order to copy metadata registers, which will happen in 5920 * RX_CP_TBL like, 5921 * - CQE->flow_tag := reg_c[1] (MARK) 5922 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5923 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5924 * This is because the last action of each flow must be a terminal action 5925 * (QUEUE, RSS or DROP). 5926 * 5927 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5928 * stored and kept in the mlx5_flow structure per each sub_flow. 5929 * 5930 * The Q/RSS action is replaced with, 5931 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5932 * And the following JUMP action is added at the end, 5933 * - JUMP, to RX_CP_TBL. 5934 * 5935 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5936 * flow_create_split_metadata() routine. The flow will look like, 5937 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5938 * 5939 * @param dev 5940 * Pointer to Ethernet device. 5941 * @param[out] split_actions 5942 * Pointer to store split actions to jump to CP_TBL. 5943 * @param[in] actions 5944 * Pointer to the list of original flow actions. 5945 * @param[in] qrss 5946 * Pointer to the Q/RSS action. 5947 * @param[in] actions_n 5948 * Number of original actions. 5949 * @param[in] mtr_sfx 5950 * Check if it is in meter suffix table. 5951 * @param[out] error 5952 * Perform verbose error reporting if not NULL. 5953 * 5954 * @return 5955 * non-zero unique flow_id on success, otherwise 0 and 5956 * error/rte_error are set. 5957 */ 5958 static uint32_t 5959 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5960 struct rte_flow_action *split_actions, 5961 const struct rte_flow_action *actions, 5962 const struct rte_flow_action *qrss, 5963 int actions_n, int mtr_sfx, 5964 struct rte_flow_error *error) 5965 { 5966 struct mlx5_priv *priv = dev->data->dev_private; 5967 struct mlx5_rte_flow_action_set_tag *set_tag; 5968 struct rte_flow_action_jump *jump; 5969 const int qrss_idx = qrss - actions; 5970 uint32_t flow_id = 0; 5971 int ret = 0; 5972 5973 /* 5974 * Given actions will be split 5975 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5976 * - Add jump to mreg CP_TBL. 5977 * As a result, there will be one more action. 5978 */ 5979 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5980 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5981 ++actions_n; 5982 set_tag = (void *)(split_actions + actions_n); 5983 /* 5984 * If we are not the meter suffix flow, add the tag action. 5985 * Since meter suffix flow already has the tag added. 5986 */ 5987 if (!mtr_sfx) { 5988 /* 5989 * Allocate the new subflow ID. This one is unique within 5990 * device and not shared with representors. Otherwise, 5991 * we would have to resolve multi-thread access synch 5992 * issue. Each flow on the shared device is appended 5993 * with source vport identifier, so the resulting 5994 * flows will be unique in the shared (by master and 5995 * representors) domain even if they have coinciding 5996 * IDs. 5997 */ 5998 mlx5_ipool_malloc(priv->sh->ipool 5999 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 6000 if (!flow_id) 6001 return rte_flow_error_set(error, ENOMEM, 6002 RTE_FLOW_ERROR_TYPE_ACTION, 6003 NULL, "can't allocate id " 6004 "for split Q/RSS subflow"); 6005 /* Internal SET_TAG action to set flow ID. */ 6006 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 6007 .data = flow_id, 6008 }; 6009 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 6010 if (ret < 0) 6011 return ret; 6012 set_tag->id = ret; 6013 /* Construct new actions array. */ 6014 /* Replace QUEUE/RSS action. */ 6015 split_actions[qrss_idx] = (struct rte_flow_action){ 6016 .type = (enum rte_flow_action_type) 6017 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6018 .conf = set_tag, 6019 }; 6020 } else { 6021 /* 6022 * If we are the suffix flow of meter, tag already exist. 6023 * Set the QUEUE/RSS action to void. 6024 */ 6025 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 6026 } 6027 /* JUMP action to jump to mreg copy table (CP_TBL). */ 6028 jump = (void *)(set_tag + 1); 6029 *jump = (struct rte_flow_action_jump){ 6030 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6031 }; 6032 split_actions[actions_n - 2] = (struct rte_flow_action){ 6033 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6034 .conf = jump, 6035 }; 6036 split_actions[actions_n - 1] = (struct rte_flow_action){ 6037 .type = RTE_FLOW_ACTION_TYPE_END, 6038 }; 6039 return flow_id; 6040 } 6041 6042 /** 6043 * Extend the given action list for Tx metadata copy. 6044 * 6045 * Copy the given action list to the ext_actions and add flow metadata register 6046 * copy action in order to copy reg_a set by WQE to reg_c[0]. 6047 * 6048 * @param[out] ext_actions 6049 * Pointer to the extended action list. 6050 * @param[in] actions 6051 * Pointer to the list of actions. 6052 * @param[in] actions_n 6053 * Number of actions in the list. 6054 * @param[out] error 6055 * Perform verbose error reporting if not NULL. 6056 * @param[in] encap_idx 6057 * The encap action index. 6058 * 6059 * @return 6060 * 0 on success, negative value otherwise 6061 */ 6062 static int 6063 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6064 struct rte_flow_action *ext_actions, 6065 const struct rte_flow_action *actions, 6066 int actions_n, struct rte_flow_error *error, 6067 int encap_idx) 6068 { 6069 struct mlx5_flow_action_copy_mreg *cp_mreg = 6070 (struct mlx5_flow_action_copy_mreg *) 6071 (ext_actions + actions_n + 1); 6072 int ret; 6073 6074 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6075 if (ret < 0) 6076 return ret; 6077 cp_mreg->dst = ret; 6078 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6079 if (ret < 0) 6080 return ret; 6081 cp_mreg->src = ret; 6082 if (encap_idx != 0) 6083 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6084 if (encap_idx == actions_n - 1) { 6085 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6086 .type = (enum rte_flow_action_type) 6087 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6088 .conf = cp_mreg, 6089 }; 6090 ext_actions[actions_n] = (struct rte_flow_action){ 6091 .type = RTE_FLOW_ACTION_TYPE_END, 6092 }; 6093 } else { 6094 ext_actions[encap_idx] = (struct rte_flow_action){ 6095 .type = (enum rte_flow_action_type) 6096 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6097 .conf = cp_mreg, 6098 }; 6099 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6100 sizeof(*ext_actions) * (actions_n - encap_idx)); 6101 } 6102 return 0; 6103 } 6104 6105 /** 6106 * Check the match action from the action list. 6107 * 6108 * @param[in] actions 6109 * Pointer to the list of actions. 6110 * @param[in] attr 6111 * Flow rule attributes. 6112 * @param[in] action 6113 * The action to be check if exist. 6114 * @param[out] match_action_pos 6115 * Pointer to the position of the matched action if exists, otherwise is -1. 6116 * @param[out] qrss_action_pos 6117 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6118 * @param[out] modify_after_mirror 6119 * Pointer to the flag of modify action after FDB mirroring. 6120 * 6121 * @return 6122 * > 0 the total number of actions. 6123 * 0 if not found match action in action list. 6124 */ 6125 static int 6126 flow_check_match_action(const struct rte_flow_action actions[], 6127 const struct rte_flow_attr *attr, 6128 enum rte_flow_action_type action, 6129 int *match_action_pos, int *qrss_action_pos, 6130 int *modify_after_mirror) 6131 { 6132 const struct rte_flow_action_sample *sample; 6133 const struct rte_flow_action_raw_decap *decap; 6134 const struct rte_flow_action *action_cur = NULL; 6135 int actions_n = 0; 6136 uint32_t ratio = 0; 6137 int sub_type = 0; 6138 int flag = 0; 6139 int fdb_mirror = 0; 6140 6141 *match_action_pos = -1; 6142 *qrss_action_pos = -1; 6143 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6144 if (actions->type == action) { 6145 flag = 1; 6146 *match_action_pos = actions_n; 6147 } 6148 switch (actions->type) { 6149 case RTE_FLOW_ACTION_TYPE_QUEUE: 6150 case RTE_FLOW_ACTION_TYPE_RSS: 6151 *qrss_action_pos = actions_n; 6152 break; 6153 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6154 sample = actions->conf; 6155 ratio = sample->ratio; 6156 sub_type = ((const struct rte_flow_action *) 6157 (sample->actions))->type; 6158 if (ratio == 1 && attr->transfer && 6159 sub_type != RTE_FLOW_ACTION_TYPE_END) 6160 fdb_mirror = 1; 6161 break; 6162 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6163 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6164 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6165 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6166 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6167 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6168 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6169 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6170 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6171 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6172 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6173 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6174 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6175 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6176 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6177 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6178 case RTE_FLOW_ACTION_TYPE_FLAG: 6179 case RTE_FLOW_ACTION_TYPE_MARK: 6180 case RTE_FLOW_ACTION_TYPE_SET_META: 6181 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6182 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6183 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6184 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6185 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6186 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6187 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6188 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6189 case RTE_FLOW_ACTION_TYPE_METER: 6190 if (fdb_mirror) 6191 *modify_after_mirror = 1; 6192 break; 6193 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6194 decap = actions->conf; 6195 action_cur = actions; 6196 while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID) 6197 ; 6198 if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6199 const struct rte_flow_action_raw_encap *encap = 6200 action_cur->conf; 6201 if (decap->size <= 6202 MLX5_ENCAPSULATION_DECISION_SIZE && 6203 encap->size > 6204 MLX5_ENCAPSULATION_DECISION_SIZE) 6205 /* L3 encap. */ 6206 break; 6207 } 6208 if (fdb_mirror) 6209 *modify_after_mirror = 1; 6210 break; 6211 default: 6212 break; 6213 } 6214 actions_n++; 6215 } 6216 if (flag && fdb_mirror && !*modify_after_mirror) { 6217 /* FDB mirroring uses the destination array to implement 6218 * instead of FLOW_SAMPLER object. 6219 */ 6220 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6221 flag = 0; 6222 } 6223 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6224 return flag ? actions_n + 1 : 0; 6225 } 6226 6227 #define SAMPLE_SUFFIX_ITEM 3 6228 6229 /** 6230 * Split the sample flow. 6231 * 6232 * As sample flow will split to two sub flow, sample flow with 6233 * sample action, the other actions will move to new suffix flow. 6234 * 6235 * Also add unique tag id with tag action in the sample flow, 6236 * the same tag id will be as match in the suffix flow. 6237 * 6238 * @param dev 6239 * Pointer to Ethernet device. 6240 * @param[in] add_tag 6241 * Add extra tag action flag. 6242 * @param[out] sfx_items 6243 * Suffix flow match items (list terminated by the END pattern item). 6244 * @param[in] actions 6245 * Associated actions (list terminated by the END action). 6246 * @param[out] actions_sfx 6247 * Suffix flow actions. 6248 * @param[out] actions_pre 6249 * Prefix flow actions. 6250 * @param[in] actions_n 6251 * The total number of actions. 6252 * @param[in] sample_action_pos 6253 * The sample action position. 6254 * @param[in] qrss_action_pos 6255 * The Queue/RSS action position. 6256 * @param[in] jump_table 6257 * Add extra jump action flag. 6258 * @param[out] error 6259 * Perform verbose error reporting if not NULL. 6260 * 6261 * @return 6262 * 0 on success, or unique flow_id, a negative errno value 6263 * otherwise and rte_errno is set. 6264 */ 6265 static int 6266 flow_sample_split_prep(struct rte_eth_dev *dev, 6267 int add_tag, 6268 const struct rte_flow_item items[], 6269 struct rte_flow_item sfx_items[], 6270 const struct rte_flow_action actions[], 6271 struct rte_flow_action actions_sfx[], 6272 struct rte_flow_action actions_pre[], 6273 int actions_n, 6274 int sample_action_pos, 6275 int qrss_action_pos, 6276 int jump_table, 6277 struct rte_flow_error *error) 6278 { 6279 struct mlx5_priv *priv = dev->data->dev_private; 6280 struct mlx5_rte_flow_action_set_tag *set_tag; 6281 struct mlx5_rte_flow_item_tag *tag_spec; 6282 struct mlx5_rte_flow_item_tag *tag_mask; 6283 struct rte_flow_action_jump *jump_action; 6284 uint32_t tag_id = 0; 6285 int append_index = 0; 6286 int set_tag_idx = -1; 6287 int index; 6288 int ret; 6289 6290 if (sample_action_pos < 0) 6291 return rte_flow_error_set(error, EINVAL, 6292 RTE_FLOW_ERROR_TYPE_ACTION, 6293 NULL, "invalid position of sample " 6294 "action in list"); 6295 /* Prepare the actions for prefix and suffix flow. */ 6296 if (add_tag) { 6297 /* Update the new added tag action index preceding 6298 * the PUSH_VLAN or ENCAP action. 6299 */ 6300 const struct rte_flow_action_raw_encap *raw_encap; 6301 const struct rte_flow_action *action = actions; 6302 int encap_idx; 6303 int action_idx = 0; 6304 int raw_decap_idx = -1; 6305 int push_vlan_idx = -1; 6306 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6307 switch (action->type) { 6308 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6309 raw_decap_idx = action_idx; 6310 break; 6311 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6312 raw_encap = action->conf; 6313 if (raw_encap->size > 6314 MLX5_ENCAPSULATION_DECISION_SIZE) { 6315 encap_idx = raw_decap_idx != -1 ? 6316 raw_decap_idx : action_idx; 6317 if (encap_idx < sample_action_pos && 6318 push_vlan_idx == -1) 6319 set_tag_idx = encap_idx; 6320 } 6321 break; 6322 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6323 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6324 encap_idx = action_idx; 6325 if (encap_idx < sample_action_pos && 6326 push_vlan_idx == -1) 6327 set_tag_idx = encap_idx; 6328 break; 6329 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6330 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6331 if (action_idx < sample_action_pos && 6332 push_vlan_idx == -1) { 6333 set_tag_idx = action_idx; 6334 push_vlan_idx = action_idx; 6335 } 6336 break; 6337 default: 6338 break; 6339 } 6340 action_idx++; 6341 } 6342 } 6343 /* Prepare the actions for prefix and suffix flow. */ 6344 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6345 index = qrss_action_pos; 6346 /* Put the preceding the Queue/RSS action into prefix flow. */ 6347 if (index != 0) 6348 memcpy(actions_pre, actions, 6349 sizeof(struct rte_flow_action) * index); 6350 /* Put others preceding the sample action into prefix flow. */ 6351 if (sample_action_pos > index + 1) 6352 memcpy(actions_pre + index, actions + index + 1, 6353 sizeof(struct rte_flow_action) * 6354 (sample_action_pos - index - 1)); 6355 index = sample_action_pos - 1; 6356 /* Put Queue/RSS action into Suffix flow. */ 6357 memcpy(actions_sfx, actions + qrss_action_pos, 6358 sizeof(struct rte_flow_action)); 6359 actions_sfx++; 6360 } else if (add_tag && set_tag_idx >= 0) { 6361 if (set_tag_idx > 0) 6362 memcpy(actions_pre, actions, 6363 sizeof(struct rte_flow_action) * set_tag_idx); 6364 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6365 sizeof(struct rte_flow_action) * 6366 (sample_action_pos - set_tag_idx)); 6367 index = sample_action_pos; 6368 } else { 6369 index = sample_action_pos; 6370 if (index != 0) 6371 memcpy(actions_pre, actions, 6372 sizeof(struct rte_flow_action) * index); 6373 } 6374 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6375 * For CX6DX and above, metadata registers Cx preserve their value, 6376 * add an extra tag action for NIC-RX and E-Switch Domain. 6377 */ 6378 if (add_tag) { 6379 /* Prepare the prefix tag action. */ 6380 append_index++; 6381 set_tag = (void *)(actions_pre + actions_n + append_index); 6382 /* Trust VF/SF on CX5 not supported meter so that the reserved 6383 * metadata regC is REG_NON, back to use application tag 6384 * index 0. 6385 */ 6386 if (unlikely(priv->sh->registers.aso_reg == REG_NON)) 6387 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6388 else 6389 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6390 if (ret < 0) 6391 return ret; 6392 mlx5_ipool_malloc(priv->sh->ipool 6393 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6394 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6395 .id = ret, 6396 .data = tag_id, 6397 }; 6398 /* Prepare the suffix subflow items. */ 6399 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6400 tag_spec->data = tag_id; 6401 tag_spec->id = set_tag->id; 6402 tag_mask = tag_spec + 1; 6403 tag_mask->data = UINT32_MAX; 6404 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6405 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6406 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6407 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6408 memcpy(sfx_items, items, sizeof(*sfx_items)); 6409 sfx_items++; 6410 break; 6411 } 6412 } 6413 sfx_items[0] = (struct rte_flow_item){ 6414 .type = (enum rte_flow_item_type) 6415 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6416 .spec = tag_spec, 6417 .last = NULL, 6418 .mask = tag_mask, 6419 }; 6420 sfx_items[1] = (struct rte_flow_item){ 6421 .type = (enum rte_flow_item_type) 6422 RTE_FLOW_ITEM_TYPE_END, 6423 }; 6424 /* Prepare the tag action in prefix subflow. */ 6425 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6426 actions_pre[set_tag_idx] = 6427 (struct rte_flow_action){ 6428 .type = (enum rte_flow_action_type) 6429 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6430 .conf = set_tag, 6431 }; 6432 /* Update next sample position due to add one tag action */ 6433 index += 1; 6434 } 6435 /* Copy the sample action into prefix flow. */ 6436 memcpy(actions_pre + index, actions + sample_action_pos, 6437 sizeof(struct rte_flow_action)); 6438 index += 1; 6439 /* For the modify action after the sample action in E-Switch mirroring, 6440 * Add the extra jump action in prefix subflow and jump into the next 6441 * table, then do the modify action in the new table. 6442 */ 6443 if (jump_table) { 6444 /* Prepare the prefix jump action. */ 6445 append_index++; 6446 jump_action = (void *)(actions_pre + actions_n + append_index); 6447 jump_action->group = jump_table; 6448 actions_pre[index++] = 6449 (struct rte_flow_action){ 6450 .type = (enum rte_flow_action_type) 6451 RTE_FLOW_ACTION_TYPE_JUMP, 6452 .conf = jump_action, 6453 }; 6454 } 6455 actions_pre[index] = (struct rte_flow_action){ 6456 .type = (enum rte_flow_action_type) 6457 RTE_FLOW_ACTION_TYPE_END, 6458 }; 6459 /* Put the actions after sample into Suffix flow. */ 6460 memcpy(actions_sfx, actions + sample_action_pos + 1, 6461 sizeof(struct rte_flow_action) * 6462 (actions_n - sample_action_pos - 1)); 6463 return tag_id; 6464 } 6465 6466 /** 6467 * The splitting for metadata feature. 6468 * 6469 * - Q/RSS action on NIC Rx should be split in order to pass by 6470 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6471 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6472 * 6473 * - All the actions on NIC Tx should have a mreg copy action to 6474 * copy reg_a from WQE to reg_c[0]. 6475 * 6476 * @param dev 6477 * Pointer to Ethernet device. 6478 * @param[in] flow 6479 * Parent flow structure pointer. 6480 * @param[in] attr 6481 * Flow rule attributes. 6482 * @param[in] items 6483 * Pattern specification (list terminated by the END pattern item). 6484 * @param[in] actions 6485 * Associated actions (list terminated by the END action). 6486 * @param[in] flow_split_info 6487 * Pointer to flow split info structure. 6488 * @param[out] error 6489 * Perform verbose error reporting if not NULL. 6490 * @return 6491 * 0 on success, negative value otherwise 6492 */ 6493 static int 6494 flow_create_split_metadata(struct rte_eth_dev *dev, 6495 struct rte_flow *flow, 6496 const struct rte_flow_attr *attr, 6497 const struct rte_flow_item items[], 6498 const struct rte_flow_action actions[], 6499 struct mlx5_flow_split_info *flow_split_info, 6500 struct rte_flow_error *error) 6501 { 6502 struct mlx5_priv *priv = dev->data->dev_private; 6503 struct mlx5_sh_config *config = &priv->sh->config; 6504 const struct rte_flow_action *qrss = NULL; 6505 struct rte_flow_action *ext_actions = NULL; 6506 struct mlx5_flow *dev_flow = NULL; 6507 uint32_t qrss_id = 0; 6508 int mtr_sfx = 0; 6509 size_t act_size; 6510 int actions_n; 6511 int encap_idx; 6512 int ret; 6513 6514 /* Check whether extensive metadata feature is engaged. */ 6515 if (!config->dv_flow_en || 6516 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6517 !mlx5_flow_ext_mreg_supported(dev)) 6518 return flow_create_split_inner(dev, flow, NULL, attr, items, 6519 actions, flow_split_info, error); 6520 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6521 &encap_idx); 6522 if (qrss) { 6523 /* Exclude hairpin flows from splitting. */ 6524 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6525 const struct rte_flow_action_queue *queue; 6526 6527 queue = qrss->conf; 6528 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6529 qrss = NULL; 6530 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6531 const struct rte_flow_action_rss *rss; 6532 6533 rss = qrss->conf; 6534 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6535 qrss = NULL; 6536 } 6537 } 6538 if (qrss) { 6539 /* Check if it is in meter suffix table. */ 6540 mtr_sfx = attr->group == 6541 ((attr->transfer && priv->fdb_def_rule) ? 6542 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6543 MLX5_FLOW_TABLE_LEVEL_METER); 6544 /* 6545 * Q/RSS action on NIC Rx should be split in order to pass by 6546 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6547 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6548 */ 6549 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6550 sizeof(struct rte_flow_action_set_tag) + 6551 sizeof(struct rte_flow_action_jump); 6552 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6553 SOCKET_ID_ANY); 6554 if (!ext_actions) 6555 return rte_flow_error_set(error, ENOMEM, 6556 RTE_FLOW_ERROR_TYPE_ACTION, 6557 NULL, "no memory to split " 6558 "metadata flow"); 6559 /* 6560 * Create the new actions list with removed Q/RSS action 6561 * and appended set tag and jump to register copy table 6562 * (RX_CP_TBL). We should preallocate unique tag ID here 6563 * in advance, because it is needed for set tag action. 6564 */ 6565 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6566 qrss, actions_n, 6567 mtr_sfx, error); 6568 if (!mtr_sfx && !qrss_id) { 6569 ret = -rte_errno; 6570 goto exit; 6571 } 6572 } else if (attr->egress) { 6573 /* 6574 * All the actions on NIC Tx should have a metadata register 6575 * copy action to copy reg_a from WQE to reg_c[meta] 6576 */ 6577 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6578 sizeof(struct mlx5_flow_action_copy_mreg); 6579 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6580 SOCKET_ID_ANY); 6581 if (!ext_actions) 6582 return rte_flow_error_set(error, ENOMEM, 6583 RTE_FLOW_ERROR_TYPE_ACTION, 6584 NULL, "no memory to split " 6585 "metadata flow"); 6586 /* Create the action list appended with copy register. */ 6587 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6588 actions_n, error, encap_idx); 6589 if (ret < 0) 6590 goto exit; 6591 } 6592 /* Add the unmodified original or prefix subflow. */ 6593 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6594 items, ext_actions ? ext_actions : 6595 actions, flow_split_info, error); 6596 if (ret < 0) 6597 goto exit; 6598 MLX5_ASSERT(dev_flow); 6599 if (qrss) { 6600 const struct rte_flow_attr q_attr = { 6601 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6602 .ingress = 1, 6603 }; 6604 /* Internal PMD action to set register. */ 6605 struct mlx5_rte_flow_item_tag q_tag_spec = { 6606 .data = qrss_id, 6607 .id = REG_NON, 6608 }; 6609 struct rte_flow_item q_items[] = { 6610 { 6611 .type = (enum rte_flow_item_type) 6612 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6613 .spec = &q_tag_spec, 6614 .last = NULL, 6615 .mask = NULL, 6616 }, 6617 { 6618 .type = RTE_FLOW_ITEM_TYPE_END, 6619 }, 6620 }; 6621 struct rte_flow_action q_actions[] = { 6622 { 6623 .type = qrss->type, 6624 .conf = qrss->conf, 6625 }, 6626 { 6627 .type = RTE_FLOW_ACTION_TYPE_END, 6628 }, 6629 }; 6630 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6631 6632 /* 6633 * Configure the tag item only if there is no meter subflow. 6634 * Since tag is already marked in the meter suffix subflow 6635 * we can just use the meter suffix items as is. 6636 */ 6637 if (qrss_id) { 6638 /* Not meter subflow. */ 6639 MLX5_ASSERT(!mtr_sfx); 6640 /* 6641 * Put unique id in prefix flow due to it is destroyed 6642 * after suffix flow and id will be freed after there 6643 * is no actual flows with this id and identifier 6644 * reallocation becomes possible (for example, for 6645 * other flows in other threads). 6646 */ 6647 dev_flow->handle->split_flow_id = qrss_id; 6648 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6649 error); 6650 if (ret < 0) 6651 goto exit; 6652 q_tag_spec.id = ret; 6653 } 6654 dev_flow = NULL; 6655 /* Add suffix subflow to execute Q/RSS. */ 6656 flow_split_info->prefix_layers = layers; 6657 flow_split_info->prefix_mark = 0; 6658 flow_split_info->table_id = 0; 6659 ret = flow_create_split_inner(dev, flow, &dev_flow, 6660 &q_attr, mtr_sfx ? items : 6661 q_items, q_actions, 6662 flow_split_info, error); 6663 if (ret < 0) 6664 goto exit; 6665 /* qrss ID should be freed if failed. */ 6666 qrss_id = 0; 6667 MLX5_ASSERT(dev_flow); 6668 } 6669 6670 exit: 6671 /* 6672 * We do not destroy the partially created sub_flows in case of error. 6673 * These ones are included into parent flow list and will be destroyed 6674 * by flow_drv_destroy. 6675 */ 6676 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6677 qrss_id); 6678 mlx5_free(ext_actions); 6679 return ret; 6680 } 6681 6682 /** 6683 * Create meter internal drop flow with the original pattern. 6684 * 6685 * @param dev 6686 * Pointer to Ethernet device. 6687 * @param[in] flow 6688 * Parent flow structure pointer. 6689 * @param[in] attr 6690 * Flow rule attributes. 6691 * @param[in] items 6692 * Pattern specification (list terminated by the END pattern item). 6693 * @param[in] flow_split_info 6694 * Pointer to flow split info structure. 6695 * @param[in] fm 6696 * Pointer to flow meter structure. 6697 * @param[out] error 6698 * Perform verbose error reporting if not NULL. 6699 * @return 6700 * 0 on success, negative value otherwise 6701 */ 6702 static uint32_t 6703 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6704 struct rte_flow *flow, 6705 const struct rte_flow_attr *attr, 6706 const struct rte_flow_item items[], 6707 struct mlx5_flow_split_info *flow_split_info, 6708 struct mlx5_flow_meter_info *fm, 6709 struct rte_flow_error *error) 6710 { 6711 struct mlx5_flow *dev_flow = NULL; 6712 struct rte_flow_attr drop_attr = *attr; 6713 struct rte_flow_action drop_actions[3]; 6714 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6715 6716 MLX5_ASSERT(fm->drop_cnt); 6717 drop_actions[0].type = 6718 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6719 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6720 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6721 drop_actions[1].conf = NULL; 6722 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6723 drop_actions[2].conf = NULL; 6724 drop_split_info.external = false; 6725 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6726 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6727 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6728 return flow_create_split_inner(dev, flow, &dev_flow, 6729 &drop_attr, items, drop_actions, 6730 &drop_split_info, error); 6731 } 6732 6733 /** 6734 * The splitting for meter feature. 6735 * 6736 * - The meter flow will be split to two flows as prefix and 6737 * suffix flow. The packets make sense only it pass the prefix 6738 * meter action. 6739 * 6740 * - Reg_C_5 is used for the packet to match betweend prefix and 6741 * suffix flow. 6742 * 6743 * @param dev 6744 * Pointer to Ethernet device. 6745 * @param[in] flow 6746 * Parent flow structure pointer. 6747 * @param[in] attr 6748 * Flow rule attributes. 6749 * @param[in] items 6750 * Pattern specification (list terminated by the END pattern item). 6751 * @param[in] actions 6752 * Associated actions (list terminated by the END action). 6753 * @param[in] flow_split_info 6754 * Pointer to flow split info structure. 6755 * @param[out] error 6756 * Perform verbose error reporting if not NULL. 6757 * @return 6758 * 0 on success, negative value otherwise 6759 */ 6760 static int 6761 flow_create_split_meter(struct rte_eth_dev *dev, 6762 struct rte_flow *flow, 6763 const struct rte_flow_attr *attr, 6764 const struct rte_flow_item items[], 6765 const struct rte_flow_action actions[], 6766 struct mlx5_flow_split_info *flow_split_info, 6767 struct rte_flow_error *error) 6768 { 6769 struct mlx5_priv *priv = dev->data->dev_private; 6770 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6771 struct rte_flow_action *sfx_actions = NULL; 6772 struct rte_flow_action *pre_actions = NULL; 6773 struct rte_flow_item *sfx_items = NULL; 6774 struct mlx5_flow *dev_flow = NULL; 6775 struct rte_flow_attr sfx_attr = *attr; 6776 struct mlx5_flow_meter_info *fm = NULL; 6777 uint8_t skip_scale_restore; 6778 bool has_mtr = false; 6779 bool has_modify = false; 6780 bool set_mtr_reg = true; 6781 bool is_mtr_hierarchy = false; 6782 uint32_t meter_id = 0; 6783 uint32_t mtr_idx = 0; 6784 uint32_t mtr_flow_id = 0; 6785 size_t act_size; 6786 size_t item_size; 6787 int actions_n = 0; 6788 int ret = 0; 6789 6790 if (priv->mtr_en) 6791 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6792 &has_modify, &meter_id); 6793 if (has_mtr) { 6794 if (flow->meter) { 6795 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6796 if (!fm) 6797 return rte_flow_error_set(error, EINVAL, 6798 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6799 NULL, "Meter not found."); 6800 } else { 6801 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6802 if (!fm) 6803 return rte_flow_error_set(error, EINVAL, 6804 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6805 NULL, "Meter not found."); 6806 ret = mlx5_flow_meter_attach(priv, fm, 6807 &sfx_attr, error); 6808 if (ret) 6809 return -rte_errno; 6810 flow->meter = mtr_idx; 6811 } 6812 MLX5_ASSERT(wks); 6813 wks->fm = fm; 6814 if (!fm->def_policy) { 6815 wks->policy = mlx5_flow_meter_policy_find(dev, 6816 fm->policy_id, 6817 NULL); 6818 MLX5_ASSERT(wks->policy); 6819 if (wks->policy->mark) 6820 wks->mark = 1; 6821 if (wks->policy->is_hierarchy) { 6822 wks->final_policy = 6823 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6824 wks->policy); 6825 if (!wks->final_policy) 6826 return rte_flow_error_set(error, 6827 EINVAL, 6828 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6829 "Failed to find terminal policy of hierarchy."); 6830 is_mtr_hierarchy = true; 6831 } 6832 } 6833 /* 6834 * If it isn't default-policy Meter, and 6835 * 1. Not meter hierarchy and there's no action in flow to change 6836 * packet (modify/encap/decap etc.), OR 6837 * 2. No drop count needed for this meter. 6838 * Then no need to use regC to save meter id anymore. 6839 */ 6840 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6841 set_mtr_reg = false; 6842 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6843 #define METER_PREFIX_ACTION 7 6844 act_size = (sizeof(struct rte_flow_action) * 6845 (actions_n + METER_PREFIX_ACTION)) + 6846 sizeof(struct mlx5_rte_flow_action_set_tag); 6847 /* Suffix items: tag, vlan, port id, end. */ 6848 #define METER_SUFFIX_ITEM 4 6849 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6850 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6851 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6852 0, SOCKET_ID_ANY); 6853 if (!sfx_actions) 6854 return rte_flow_error_set(error, ENOMEM, 6855 RTE_FLOW_ERROR_TYPE_ACTION, 6856 NULL, "no memory to split " 6857 "meter flow"); 6858 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6859 act_size); 6860 /* There's no suffix flow for meter of non-default policy. */ 6861 if (!fm->def_policy) 6862 pre_actions = sfx_actions + 1; 6863 else 6864 pre_actions = sfx_actions + actions_n; 6865 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6866 items, sfx_items, actions, 6867 sfx_actions, pre_actions, 6868 (set_mtr_reg ? &mtr_flow_id : NULL), 6869 error); 6870 if (ret) { 6871 ret = -rte_errno; 6872 goto exit; 6873 } 6874 /* Add the prefix subflow. */ 6875 skip_scale_restore = flow_split_info->skip_scale; 6876 flow_split_info->skip_scale |= 6877 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6878 ret = flow_create_split_inner(dev, flow, &dev_flow, 6879 attr, items, pre_actions, 6880 flow_split_info, error); 6881 flow_split_info->skip_scale = skip_scale_restore; 6882 if (ret) { 6883 if (mtr_flow_id) 6884 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6885 ret = -rte_errno; 6886 goto exit; 6887 } 6888 if (mtr_flow_id) { 6889 dev_flow->handle->split_flow_id = mtr_flow_id; 6890 dev_flow->handle->is_meter_flow_id = 1; 6891 } 6892 if (!fm->def_policy) { 6893 if (!set_mtr_reg && fm->drop_cnt) 6894 ret = 6895 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6896 &sfx_attr, items, 6897 flow_split_info, 6898 fm, error); 6899 goto exit; 6900 } 6901 /* Setting the sfx group atrr. */ 6902 sfx_attr.group = sfx_attr.transfer ? 6903 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6904 MLX5_FLOW_TABLE_LEVEL_METER; 6905 flow_split_info->prefix_layers = 6906 flow_get_prefix_layer_flags(dev_flow); 6907 flow_split_info->prefix_mark |= wks->mark; 6908 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6909 } 6910 /* Add the prefix subflow. */ 6911 ret = flow_create_split_metadata(dev, flow, 6912 &sfx_attr, sfx_items ? 6913 sfx_items : items, 6914 sfx_actions ? sfx_actions : actions, 6915 flow_split_info, error); 6916 exit: 6917 if (sfx_actions) 6918 mlx5_free(sfx_actions); 6919 return ret; 6920 } 6921 6922 /** 6923 * The splitting for sample feature. 6924 * 6925 * Once Sample action is detected in the action list, the flow actions should 6926 * be split into prefix sub flow and suffix sub flow. 6927 * 6928 * The original items remain in the prefix sub flow, all actions preceding the 6929 * sample action and the sample action itself will be copied to the prefix 6930 * sub flow, the actions following the sample action will be copied to the 6931 * suffix sub flow, Queue action always be located in the suffix sub flow. 6932 * 6933 * In order to make the packet from prefix sub flow matches with suffix sub 6934 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6935 * flow uses tag item with the unique flow id. 6936 * 6937 * @param dev 6938 * Pointer to Ethernet device. 6939 * @param[in] flow 6940 * Parent flow structure pointer. 6941 * @param[in] attr 6942 * Flow rule attributes. 6943 * @param[in] items 6944 * Pattern specification (list terminated by the END pattern item). 6945 * @param[in] actions 6946 * Associated actions (list terminated by the END action). 6947 * @param[in] flow_split_info 6948 * Pointer to flow split info structure. 6949 * @param[out] error 6950 * Perform verbose error reporting if not NULL. 6951 * @return 6952 * 0 on success, negative value otherwise 6953 */ 6954 static int 6955 flow_create_split_sample(struct rte_eth_dev *dev, 6956 struct rte_flow *flow, 6957 const struct rte_flow_attr *attr, 6958 const struct rte_flow_item items[], 6959 const struct rte_flow_action actions[], 6960 struct mlx5_flow_split_info *flow_split_info, 6961 struct rte_flow_error *error) 6962 { 6963 struct mlx5_priv *priv = dev->data->dev_private; 6964 struct rte_flow_action *sfx_actions = NULL; 6965 struct rte_flow_action *pre_actions = NULL; 6966 struct rte_flow_item *sfx_items = NULL; 6967 struct mlx5_flow *dev_flow = NULL; 6968 struct rte_flow_attr sfx_attr = *attr; 6969 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6970 struct mlx5_flow_dv_sample_resource *sample_res; 6971 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6972 struct mlx5_flow_tbl_resource *sfx_tbl; 6973 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6974 #endif 6975 size_t act_size; 6976 size_t item_size; 6977 uint32_t fdb_tx = 0; 6978 int32_t tag_id = 0; 6979 int actions_n = 0; 6980 int sample_action_pos; 6981 int qrss_action_pos; 6982 int add_tag = 0; 6983 int modify_after_mirror = 0; 6984 uint16_t jump_table = 0; 6985 const uint32_t next_ft_step = 1; 6986 int ret = 0; 6987 struct mlx5_priv *item_port_priv = NULL; 6988 const struct rte_flow_item *item; 6989 6990 if (priv->sampler_en) 6991 actions_n = flow_check_match_action(actions, attr, 6992 RTE_FLOW_ACTION_TYPE_SAMPLE, 6993 &sample_action_pos, &qrss_action_pos, 6994 &modify_after_mirror); 6995 if (actions_n) { 6996 /* The prefix actions must includes sample, tag, end. */ 6997 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6998 + sizeof(struct mlx5_rte_flow_action_set_tag); 6999 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 7000 sizeof(struct mlx5_rte_flow_item_tag) * 2; 7001 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 7002 item_size), 0, SOCKET_ID_ANY); 7003 if (!sfx_actions) 7004 return rte_flow_error_set(error, ENOMEM, 7005 RTE_FLOW_ERROR_TYPE_ACTION, 7006 NULL, "no memory to split " 7007 "sample flow"); 7008 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 7009 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 7010 const struct rte_flow_item_port_id *spec; 7011 7012 spec = (const struct rte_flow_item_port_id *)item->spec; 7013 if (spec) 7014 item_port_priv = 7015 mlx5_port_to_eswitch_info(spec->id, true); 7016 break; 7017 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 7018 const struct rte_flow_item_ethdev *spec; 7019 7020 spec = (const struct rte_flow_item_ethdev *)item->spec; 7021 if (spec) 7022 item_port_priv = 7023 mlx5_port_to_eswitch_info(spec->port_id, true); 7024 break; 7025 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 7026 const struct rte_flow_item_ethdev *spec; 7027 7028 spec = (const struct rte_flow_item_ethdev *)item->spec; 7029 if (spec) 7030 item_port_priv = 7031 mlx5_port_to_eswitch_info(spec->port_id, true); 7032 break; 7033 } 7034 } 7035 /* The representor_id is UINT16_MAX for uplink. */ 7036 fdb_tx = (attr->transfer && 7037 flow_source_vport_representor(priv, item_port_priv)); 7038 /* 7039 * When reg_c_preserve is set, metadata registers Cx preserve 7040 * their value even through packet duplication. 7041 */ 7042 add_tag = (!fdb_tx || 7043 priv->sh->cdev->config.hca_attr.reg_c_preserve); 7044 if (add_tag) 7045 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 7046 + act_size); 7047 if (modify_after_mirror) 7048 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 7049 next_ft_step; 7050 pre_actions = sfx_actions + actions_n; 7051 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 7052 actions, sfx_actions, 7053 pre_actions, actions_n, 7054 sample_action_pos, 7055 qrss_action_pos, jump_table, 7056 error); 7057 if (tag_id < 0 || (add_tag && !tag_id)) { 7058 ret = -rte_errno; 7059 goto exit; 7060 } 7061 if (modify_after_mirror) 7062 flow_split_info->skip_scale = 7063 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 7064 /* Add the prefix subflow. */ 7065 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7066 items, pre_actions, 7067 flow_split_info, error); 7068 if (ret) { 7069 ret = -rte_errno; 7070 goto exit; 7071 } 7072 dev_flow->handle->split_flow_id = tag_id; 7073 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7074 if (!modify_after_mirror) { 7075 /* Set the sfx group attr. */ 7076 sample_res = (struct mlx5_flow_dv_sample_resource *) 7077 dev_flow->dv.sample_res; 7078 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7079 sample_res->normal_path_tbl; 7080 sfx_tbl_data = container_of(sfx_tbl, 7081 struct mlx5_flow_tbl_data_entry, 7082 tbl); 7083 sfx_attr.group = sfx_attr.transfer ? 7084 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7085 } else { 7086 MLX5_ASSERT(attr->transfer); 7087 sfx_attr.group = jump_table; 7088 } 7089 flow_split_info->prefix_layers = 7090 flow_get_prefix_layer_flags(dev_flow); 7091 MLX5_ASSERT(wks); 7092 flow_split_info->prefix_mark |= wks->mark; 7093 /* Suffix group level already be scaled with factor, set 7094 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7095 * again in translation. 7096 */ 7097 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7098 #endif 7099 } 7100 /* Add the suffix subflow. */ 7101 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7102 sfx_items ? sfx_items : items, 7103 sfx_actions ? sfx_actions : actions, 7104 flow_split_info, error); 7105 exit: 7106 if (sfx_actions) 7107 mlx5_free(sfx_actions); 7108 return ret; 7109 } 7110 7111 /** 7112 * Split the flow to subflow set. The splitters might be linked 7113 * in the chain, like this: 7114 * flow_create_split_outer() calls: 7115 * flow_create_split_meter() calls: 7116 * flow_create_split_metadata(meter_subflow_0) calls: 7117 * flow_create_split_inner(metadata_subflow_0) 7118 * flow_create_split_inner(metadata_subflow_1) 7119 * flow_create_split_inner(metadata_subflow_2) 7120 * flow_create_split_metadata(meter_subflow_1) calls: 7121 * flow_create_split_inner(metadata_subflow_0) 7122 * flow_create_split_inner(metadata_subflow_1) 7123 * flow_create_split_inner(metadata_subflow_2) 7124 * 7125 * This provide flexible way to add new levels of flow splitting. 7126 * The all of successfully created subflows are included to the 7127 * parent flow dev_flow list. 7128 * 7129 * @param dev 7130 * Pointer to Ethernet device. 7131 * @param[in] flow 7132 * Parent flow structure pointer. 7133 * @param[in] attr 7134 * Flow rule attributes. 7135 * @param[in] items 7136 * Pattern specification (list terminated by the END pattern item). 7137 * @param[in] actions 7138 * Associated actions (list terminated by the END action). 7139 * @param[in] flow_split_info 7140 * Pointer to flow split info structure. 7141 * @param[out] error 7142 * Perform verbose error reporting if not NULL. 7143 * @return 7144 * 0 on success, negative value otherwise 7145 */ 7146 static int 7147 flow_create_split_outer(struct rte_eth_dev *dev, 7148 struct rte_flow *flow, 7149 const struct rte_flow_attr *attr, 7150 const struct rte_flow_item items[], 7151 const struct rte_flow_action actions[], 7152 struct mlx5_flow_split_info *flow_split_info, 7153 struct rte_flow_error *error) 7154 { 7155 int ret; 7156 7157 ret = flow_create_split_sample(dev, flow, attr, items, 7158 actions, flow_split_info, error); 7159 MLX5_ASSERT(ret <= 0); 7160 return ret; 7161 } 7162 7163 static inline struct mlx5_flow_tunnel * 7164 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7165 { 7166 struct mlx5_flow_tunnel *tunnel; 7167 7168 #pragma GCC diagnostic push 7169 #pragma GCC diagnostic ignored "-Wcast-qual" 7170 tunnel = (typeof(tunnel))flow->tunnel; 7171 #pragma GCC diagnostic pop 7172 7173 return tunnel; 7174 } 7175 7176 /** 7177 * Create a flow and add it to @p list. 7178 * 7179 * @param dev 7180 * Pointer to Ethernet device. 7181 * @param list 7182 * Pointer to a TAILQ flow list. If this parameter NULL, 7183 * no list insertion occurred, flow is just created, 7184 * this is caller's responsibility to track the 7185 * created flow. 7186 * @param[in] attr 7187 * Flow rule attributes. 7188 * @param[in] items 7189 * Pattern specification (list terminated by the END pattern item). 7190 * @param[in] actions 7191 * Associated actions (list terminated by the END action). 7192 * @param[in] external 7193 * This flow rule is created by request external to PMD. 7194 * @param[out] error 7195 * Perform verbose error reporting if not NULL. 7196 * 7197 * @return 7198 * A flow index on success, 0 otherwise and rte_errno is set. 7199 */ 7200 static uint32_t 7201 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7202 const struct rte_flow_attr *attr, 7203 const struct rte_flow_item items[], 7204 const struct rte_flow_action original_actions[], 7205 bool external, struct rte_flow_error *error) 7206 { 7207 struct mlx5_priv *priv = dev->data->dev_private; 7208 struct rte_flow *flow = NULL; 7209 struct mlx5_flow *dev_flow; 7210 const struct rte_flow_action_rss *rss = NULL; 7211 struct mlx5_translated_action_handle 7212 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7213 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7214 union { 7215 struct mlx5_flow_expand_rss buf; 7216 uint8_t buffer[8192]; 7217 } expand_buffer; 7218 union { 7219 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7220 uint8_t buffer[2048]; 7221 } actions_rx; 7222 union { 7223 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7224 uint8_t buffer[2048]; 7225 } actions_hairpin_tx; 7226 union { 7227 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7228 uint8_t buffer[2048]; 7229 } items_tx; 7230 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7231 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7232 struct mlx5_flow_rss_desc *rss_desc; 7233 const struct rte_flow_action *p_actions_rx; 7234 uint32_t i; 7235 uint32_t idx = 0; 7236 int hairpin_flow; 7237 struct rte_flow_attr attr_tx = { .priority = 0 }; 7238 const struct rte_flow_action *actions; 7239 struct rte_flow_action *translated_actions = NULL; 7240 struct mlx5_flow_tunnel *tunnel; 7241 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7242 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7243 struct mlx5_flow_split_info flow_split_info = { 7244 .external = !!external, 7245 .skip_scale = 0, 7246 .flow_idx = 0, 7247 .prefix_mark = 0, 7248 .prefix_layers = 0, 7249 .table_id = 0 7250 }; 7251 int ret; 7252 7253 MLX5_ASSERT(wks); 7254 rss_desc = &wks->rss_desc; 7255 ret = flow_action_handles_translate(dev, original_actions, 7256 indir_actions, 7257 &indir_actions_n, 7258 &translated_actions, error); 7259 if (ret < 0) { 7260 MLX5_ASSERT(translated_actions == NULL); 7261 return 0; 7262 } 7263 actions = translated_actions ? translated_actions : original_actions; 7264 p_actions_rx = actions; 7265 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7266 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7267 external, hairpin_flow, error); 7268 if (ret < 0) 7269 goto error_before_hairpin_split; 7270 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7271 if (!flow) { 7272 rte_errno = ENOMEM; 7273 goto error_before_hairpin_split; 7274 } 7275 if (hairpin_flow > 0) { 7276 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7277 rte_errno = EINVAL; 7278 goto error_before_hairpin_split; 7279 } 7280 flow_hairpin_split(dev, actions, actions_rx.actions, 7281 actions_hairpin_tx.actions, items_tx.items, 7282 idx); 7283 p_actions_rx = actions_rx.actions; 7284 } 7285 flow_split_info.flow_idx = idx; 7286 flow->drv_type = flow_get_drv_type(dev, attr); 7287 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7288 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7289 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7290 /* RSS Action only works on NIC RX domain */ 7291 if (attr->ingress) 7292 rss = flow_get_rss_action(dev, p_actions_rx); 7293 if (rss) { 7294 MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); 7295 rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func); 7296 /* 7297 * The following information is required by 7298 * mlx5_flow_hashfields_adjust() in advance. 7299 */ 7300 rss_desc->level = rss->level; 7301 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7302 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7303 } 7304 flow->dev_handles = 0; 7305 if (rss && rss->types) { 7306 unsigned int graph_root; 7307 7308 graph_root = find_graph_root(rss->level); 7309 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7310 items, rss->types, 7311 mlx5_support_expansion, graph_root); 7312 MLX5_ASSERT(ret > 0 && 7313 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7314 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7315 for (i = 0; i < buf->entries; ++i) 7316 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7317 } 7318 } else { 7319 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7320 sizeof(expand_buffer.buffer), 7321 items, sq_specs); 7322 if (ret) { 7323 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7324 NULL, "not enough memory for rte_flow"); 7325 goto error; 7326 } 7327 if (buf->entries == 0) { 7328 buf->entries = 1; 7329 buf->entry[0].pattern = (void *)(uintptr_t)items; 7330 } 7331 } 7332 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7333 indir_actions_n); 7334 for (i = 0; i < buf->entries; ++i) { 7335 /* Initialize flow split data. */ 7336 flow_split_info.prefix_layers = 0; 7337 flow_split_info.prefix_mark = 0; 7338 flow_split_info.skip_scale = 0; 7339 /* 7340 * The splitter may create multiple dev_flows, 7341 * depending on configuration. In the simplest 7342 * case it just creates unmodified original flow. 7343 */ 7344 ret = flow_create_split_outer(dev, flow, attr, 7345 buf->entry[i].pattern, 7346 p_actions_rx, &flow_split_info, 7347 error); 7348 if (ret < 0) 7349 goto error; 7350 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7351 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7352 p_actions_rx, 7353 idx, 7354 wks->flows[0].tunnel, 7355 &default_miss_ctx, 7356 error); 7357 if (ret < 0) { 7358 mlx5_free(default_miss_ctx.queue); 7359 goto error; 7360 } 7361 } 7362 } 7363 /* Create the tx flow. */ 7364 if (hairpin_flow) { 7365 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7366 attr_tx.ingress = 0; 7367 attr_tx.egress = 1; 7368 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7369 actions_hairpin_tx.actions, 7370 idx, error); 7371 if (!dev_flow) 7372 goto error; 7373 dev_flow->flow = flow; 7374 dev_flow->external = 0; 7375 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7376 dev_flow->handle, next); 7377 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7378 items_tx.items, 7379 actions_hairpin_tx.actions, error); 7380 if (ret < 0) 7381 goto error; 7382 } 7383 /* 7384 * Update the metadata register copy table. If extensive 7385 * metadata feature is enabled and registers are supported 7386 * we might create the extra rte_flow for each unique 7387 * MARK/FLAG action ID. 7388 * 7389 * The table is updated for ingress and transfer flows only, because 7390 * the egress Flows belong to the different device and 7391 * copy table should be updated in peer NIC Rx domain. 7392 */ 7393 if ((attr->ingress || attr->transfer) && 7394 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7395 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7396 if (ret) 7397 goto error; 7398 } 7399 /* 7400 * If the flow is external (from application) OR device is started, 7401 * OR mreg discover, then apply immediately. 7402 */ 7403 if (external || dev->data->dev_started || 7404 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7405 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7406 ret = flow_drv_apply(dev, flow, error); 7407 if (ret < 0) 7408 goto error; 7409 } 7410 flow->type = type; 7411 flow_rxq_flags_set(dev, flow); 7412 rte_free(translated_actions); 7413 tunnel = flow_tunnel_from_rule(wks->flows); 7414 if (tunnel) { 7415 flow->tunnel = 1; 7416 flow->tunnel_id = tunnel->tunnel_id; 7417 __atomic_fetch_add(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7418 mlx5_free(default_miss_ctx.queue); 7419 } 7420 mlx5_flow_pop_thread_workspace(); 7421 return idx; 7422 error: 7423 MLX5_ASSERT(flow); 7424 ret = rte_errno; /* Save rte_errno before cleanup. */ 7425 flow_mreg_del_copy_action(dev, flow); 7426 flow_drv_destroy(dev, flow); 7427 if (rss_desc->shared_rss) 7428 __atomic_fetch_sub(&((struct mlx5_shared_action_rss *) 7429 mlx5_ipool_get 7430 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7431 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7432 mlx5_ipool_free(priv->flows[type], idx); 7433 rte_errno = ret; /* Restore rte_errno. */ 7434 ret = rte_errno; 7435 rte_errno = ret; 7436 error_before_hairpin_split: 7437 mlx5_flow_pop_thread_workspace(); 7438 rte_free(translated_actions); 7439 return 0; 7440 } 7441 7442 /** 7443 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7444 * incoming packets to table 1. 7445 * 7446 * Other flow rules, requested for group n, will be created in 7447 * e-switch table n+1. 7448 * Jump action to e-switch group n will be created to group n+1. 7449 * 7450 * Used when working in switchdev mode, to utilise advantages of table 1 7451 * and above. 7452 * 7453 * @param dev 7454 * Pointer to Ethernet device. 7455 * 7456 * @return 7457 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7458 */ 7459 struct rte_flow * 7460 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7461 { 7462 const struct rte_flow_attr attr = { 7463 .group = 0, 7464 .priority = 0, 7465 .ingress = 0, 7466 .egress = 0, 7467 .transfer = 1, 7468 }; 7469 const struct rte_flow_item pattern = { 7470 .type = RTE_FLOW_ITEM_TYPE_END, 7471 }; 7472 struct rte_flow_action_jump jump = { 7473 .group = 1, 7474 }; 7475 const struct rte_flow_action actions[] = { 7476 { 7477 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7478 .conf = &jump, 7479 }, 7480 { 7481 .type = RTE_FLOW_ACTION_TYPE_END, 7482 }, 7483 }; 7484 struct rte_flow_error error; 7485 7486 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7487 &attr, &pattern, 7488 actions, false, &error); 7489 } 7490 7491 /** 7492 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7493 * and sq number, directs all packets to peer vport. 7494 * 7495 * @param dev 7496 * Pointer to Ethernet device. 7497 * @param sq_num 7498 * SQ number. 7499 * 7500 * @return 7501 * Flow ID on success, 0 otherwise and rte_errno is set. 7502 */ 7503 uint32_t 7504 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7505 { 7506 struct rte_flow_attr attr = { 7507 .group = 0, 7508 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7509 .ingress = 0, 7510 .egress = 0, 7511 .transfer = 1, 7512 }; 7513 struct rte_flow_item_port_id port_spec = { 7514 .id = MLX5_PORT_ESW_MGR, 7515 }; 7516 struct mlx5_rte_flow_item_sq sq_spec = { 7517 .queue = sq_num, 7518 }; 7519 struct rte_flow_item pattern[] = { 7520 { 7521 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7522 .spec = &port_spec, 7523 }, 7524 { 7525 .type = (enum rte_flow_item_type) 7526 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7527 .spec = &sq_spec, 7528 }, 7529 { 7530 .type = RTE_FLOW_ITEM_TYPE_END, 7531 }, 7532 }; 7533 struct rte_flow_action_jump jump = { 7534 .group = 1, 7535 }; 7536 struct rte_flow_action_port_id port = { 7537 .id = dev->data->port_id, 7538 }; 7539 struct rte_flow_action actions[] = { 7540 { 7541 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7542 .conf = &jump, 7543 }, 7544 { 7545 .type = RTE_FLOW_ACTION_TYPE_END, 7546 }, 7547 }; 7548 struct rte_flow_error error; 7549 7550 /* 7551 * Creates group 0, highest priority jump flow. 7552 * Matches txq to bypass kernel packets. 7553 */ 7554 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7555 false, &error) == 0) 7556 return 0; 7557 /* Create group 1, lowest priority redirect flow for txq. */ 7558 attr.group = 1; 7559 actions[0].conf = &port; 7560 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7561 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7562 actions, false, &error); 7563 } 7564 7565 /** 7566 * Validate a flow supported by the NIC. 7567 * 7568 * @see rte_flow_validate() 7569 * @see rte_flow_ops 7570 */ 7571 int 7572 mlx5_flow_validate(struct rte_eth_dev *dev, 7573 const struct rte_flow_attr *attr, 7574 const struct rte_flow_item items[], 7575 const struct rte_flow_action original_actions[], 7576 struct rte_flow_error *error) 7577 { 7578 int hairpin_flow; 7579 struct mlx5_translated_action_handle 7580 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7581 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7582 const struct rte_flow_action *actions; 7583 struct rte_flow_action *translated_actions = NULL; 7584 int ret = flow_action_handles_translate(dev, original_actions, 7585 indir_actions, 7586 &indir_actions_n, 7587 &translated_actions, error); 7588 7589 if (ret) 7590 return ret; 7591 actions = translated_actions ? translated_actions : original_actions; 7592 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7593 ret = flow_drv_validate(dev, attr, items, actions, 7594 true, hairpin_flow, error); 7595 rte_free(translated_actions); 7596 return ret; 7597 } 7598 7599 static int 7600 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7601 const struct rte_flow_attr *attr, 7602 const uint32_t orig_prio, 7603 const struct rte_flow_item *items, 7604 const struct rte_flow_action *actions, 7605 uint32_t flow_idx) 7606 { 7607 struct mlx5_priv *priv = dev->data->dev_private; 7608 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7609 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7610 struct rte_flow_error error; 7611 int len, ret; 7612 7613 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7614 if (!flow_info) { 7615 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7616 return -1; 7617 } 7618 flow_info->orig_prio = orig_prio; 7619 flow_info->attr = *attr; 7620 /* Standby mode rule awlays saves it in low priority entry. */ 7621 flow_info->flow_idx_low_prio = flow_idx; 7622 7623 /* Store matching items. */ 7624 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7625 if (ret <= 0) { 7626 DRV_LOG(ERR, "Can't get items length."); 7627 goto end; 7628 } 7629 len = RTE_ALIGN(ret, 16); 7630 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7631 if (!flow_info->items) { 7632 DRV_LOG(ERR, "No enough memory for items caching."); 7633 goto end; 7634 } 7635 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7636 if (ret <= 0) { 7637 DRV_LOG(ERR, "Can't duplicate items."); 7638 goto end; 7639 } 7640 7641 /* Store flow actions. */ 7642 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7643 if (ret <= 0) { 7644 DRV_LOG(ERR, "Can't get actions length."); 7645 goto end; 7646 } 7647 len = RTE_ALIGN(ret, 16); 7648 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7649 if (!flow_info->actions) { 7650 DRV_LOG(ERR, "No enough memory for actions caching."); 7651 goto end; 7652 } 7653 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7654 if (ret <= 0) { 7655 DRV_LOG(ERR, "Can't duplicate actions."); 7656 goto end; 7657 } 7658 7659 /* Insert to the list end. */ 7660 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7661 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7662 } else { 7663 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7664 while (LIST_NEXT(tmp_info, next)) 7665 tmp_info = LIST_NEXT(tmp_info, next); 7666 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7667 } 7668 return 0; 7669 end: 7670 if (flow_info->items) 7671 mlx5_free(flow_info->items); 7672 if (flow_info->actions) 7673 mlx5_free(flow_info->actions); 7674 mlx5_free(flow_info); 7675 return -1; 7676 } 7677 7678 static int 7679 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7680 { 7681 struct mlx5_priv *priv = dev->data->dev_private; 7682 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7683 struct mlx5_dv_flow_info *flow_info; 7684 struct rte_flow_attr attr; 7685 struct rte_flow_error error; 7686 struct rte_flow *high, *low; 7687 7688 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7689 while (flow_info) { 7690 /* DUP flow may have the same priority. */ 7691 if (flow_info->orig_prio != flow_info->attr.priority) { 7692 attr = flow_info->attr; 7693 if (orig_prio) 7694 attr.priority = flow_info->orig_prio; 7695 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7696 &attr, flow_info->items, flow_info->actions, 7697 true, &error); 7698 if (!flow_info->flow_idx_high_prio) { 7699 DRV_LOG(ERR, "Priority toggle failed internally."); 7700 goto err; 7701 } 7702 } 7703 flow_info = LIST_NEXT(flow_info, next); 7704 } 7705 /* Delete the low priority rules and swap the flow handle. */ 7706 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7707 while (flow_info) { 7708 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7709 if (flow_info->orig_prio != flow_info->attr.priority) { 7710 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7711 flow_info->flow_idx_high_prio); 7712 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7713 flow_info->flow_idx_low_prio); 7714 if (high && low) { 7715 RTE_SWAP(*low, *high); 7716 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7717 flow_info->flow_idx_low_prio); 7718 flow_info->flow_idx_high_prio = 0; 7719 } 7720 } 7721 flow_info = LIST_NEXT(flow_info, next); 7722 } 7723 return 0; 7724 err: 7725 /* Destroy preceding successful high priority rules. */ 7726 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7727 while (flow_info) { 7728 if (flow_info->orig_prio != flow_info->attr.priority) { 7729 if (flow_info->flow_idx_high_prio) 7730 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7731 flow_info->flow_idx_high_prio); 7732 else 7733 break; 7734 flow_info->flow_idx_high_prio = 0; 7735 } 7736 flow_info = LIST_NEXT(flow_info, next); 7737 } 7738 return -1; 7739 } 7740 7741 /** 7742 * Set the mode of the flow engine of a process to active or standby during live migration. 7743 * 7744 * @param[in] mode 7745 * MLX5 flow engine mode, @see `enum rte_pmd_mlx5_flow_engine_mode`. 7746 * @param[in] flags 7747 * Flow engine mode specific flags. 7748 * 7749 * @return 7750 * Negative value on error, positive on success. 7751 */ 7752 int 7753 rte_pmd_mlx5_flow_engine_set_mode(enum rte_pmd_mlx5_flow_engine_mode mode, uint32_t flags) 7754 { 7755 struct mlx5_priv *priv; 7756 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info; 7757 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7758 uint16_t port, port_id; 7759 uint16_t toggle_num = 0; 7760 struct rte_eth_dev *dev; 7761 enum rte_pmd_mlx5_flow_engine_mode orig_mode; 7762 uint32_t orig_flags; 7763 bool need_toggle = false; 7764 7765 /* Check if flags combinations are supported. */ 7766 if (flags && flags != RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7767 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7768 return -1; 7769 } 7770 MLX5_ETH_FOREACH_DEV(port, NULL) { 7771 dev = &rte_eth_devices[port]; 7772 priv = dev->data->dev_private; 7773 mode_info = &priv->mode_info; 7774 /* No mode change. Assume all devices hold the same mode. */ 7775 if (mode_info->mode == mode) { 7776 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7777 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7778 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7779 port, mode_info->mode_flag); 7780 orig_mode = mode_info->mode; 7781 orig_flags = mode_info->mode_flag; 7782 goto err; 7783 } 7784 mode_info->mode_flag = flags; 7785 toggle_num++; 7786 continue; 7787 } 7788 /* Active -> standby. */ 7789 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY) { 7790 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7791 DRV_LOG(ERR, "Cached rule existed"); 7792 orig_mode = mode_info->mode; 7793 orig_flags = mode_info->mode_flag; 7794 goto err; 7795 } 7796 mode_info->mode_flag = flags; 7797 mode_info->mode = mode; 7798 toggle_num++; 7799 /* Standby -> active. */ 7800 } else if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7801 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7802 DRV_LOG(INFO, "No cached rule existed"); 7803 } else { 7804 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7805 orig_mode = mode_info->mode; 7806 orig_flags = mode_info->mode_flag; 7807 need_toggle = true; 7808 goto err; 7809 } 7810 } 7811 toggle_num++; 7812 } 7813 } 7814 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7815 /* Clear cache flow rules. */ 7816 MLX5_ETH_FOREACH_DEV(port, NULL) { 7817 priv = rte_eth_devices[port].data->dev_private; 7818 mode_info = &priv->mode_info; 7819 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7820 while (flow_info) { 7821 tmp_info = LIST_NEXT(flow_info, next); 7822 LIST_REMOVE(flow_info, next); 7823 mlx5_free(flow_info->actions); 7824 mlx5_free(flow_info->items); 7825 mlx5_free(flow_info); 7826 flow_info = tmp_info; 7827 } 7828 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7829 } 7830 } 7831 return toggle_num; 7832 err: 7833 /* Rollback all preceding successful ports. */ 7834 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7835 if (port_id == port) 7836 break; 7837 priv = rte_eth_devices[port_id].data->dev_private; 7838 mode_info = &priv->mode_info; 7839 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7840 mlx5_flow_cache_flow_toggle(dev, false)) 7841 return -EPERM; 7842 mode_info->mode = orig_mode; 7843 mode_info->mode_flag = orig_flags; 7844 } 7845 return -EINVAL; 7846 } 7847 /** 7848 * Create a flow. 7849 * 7850 * @see rte_flow_create() 7851 * @see rte_flow_ops 7852 */ 7853 struct rte_flow * 7854 mlx5_flow_create(struct rte_eth_dev *dev, 7855 const struct rte_flow_attr *attr, 7856 const struct rte_flow_item items[], 7857 const struct rte_flow_action actions[], 7858 struct rte_flow_error *error) 7859 { 7860 struct mlx5_priv *priv = dev->data->dev_private; 7861 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7862 uint32_t prio = attr->priority; 7863 uint32_t flow_idx; 7864 7865 if (priv->sh->config.dv_flow_en == 2) { 7866 rte_flow_error_set(error, ENOTSUP, 7867 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7868 NULL, 7869 "Flow non-Q creation not supported"); 7870 return NULL; 7871 } 7872 /* 7873 * If the device is not started yet, it is not allowed to created a 7874 * flow from application. PMD default flows and traffic control flows 7875 * are not affected. 7876 */ 7877 if (unlikely(!dev->data->dev_started)) { 7878 DRV_LOG(DEBUG, "port %u is not started when " 7879 "inserting a flow", dev->data->port_id); 7880 rte_flow_error_set(error, ENODEV, 7881 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7882 NULL, 7883 "port not started"); 7884 return NULL; 7885 } 7886 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7887 if (attr->transfer || 7888 (attr->ingress && !(priv->mode_info.mode_flag & 7889 RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 7890 new_attr->priority += 1; 7891 } 7892 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 7893 if (!flow_idx) 7894 return NULL; 7895 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7896 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 7897 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7898 flow_idx = 0; 7899 } 7900 } 7901 return (void *)(uintptr_t)flow_idx; 7902 } 7903 7904 /** 7905 * Destroy a flow in a list. 7906 * 7907 * @param dev 7908 * Pointer to Ethernet device. 7909 * @param[in] flow_idx 7910 * Index of flow to destroy. 7911 */ 7912 static void 7913 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7914 uint32_t flow_idx) 7915 { 7916 struct mlx5_priv *priv = dev->data->dev_private; 7917 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7918 7919 if (!flow) 7920 return; 7921 MLX5_ASSERT(flow->type == type); 7922 /* 7923 * Update RX queue flags only if port is started, otherwise it is 7924 * already clean. 7925 */ 7926 if (dev->data->dev_started) 7927 flow_rxq_flags_trim(dev, flow); 7928 flow_drv_destroy(dev, flow); 7929 if (flow->tunnel) { 7930 struct mlx5_flow_tunnel *tunnel; 7931 7932 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7933 RTE_VERIFY(tunnel); 7934 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 7935 mlx5_flow_tunnel_free(dev, tunnel); 7936 } 7937 flow_mreg_del_copy_action(dev, flow); 7938 mlx5_ipool_free(priv->flows[type], flow_idx); 7939 } 7940 7941 /** 7942 * Destroy all flows. 7943 * 7944 * @param dev 7945 * Pointer to Ethernet device. 7946 * @param type 7947 * Flow type to be flushed. 7948 * @param active 7949 * If flushing is called actively. 7950 */ 7951 void 7952 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7953 bool active) 7954 { 7955 struct mlx5_priv *priv = dev->data->dev_private; 7956 uint32_t num_flushed = 0, fidx = 1; 7957 struct rte_flow *flow; 7958 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7959 struct mlx5_dv_flow_info *flow_info; 7960 7961 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7962 if (priv->sh->config.dv_flow_en == 2 && 7963 type == MLX5_FLOW_TYPE_GEN) { 7964 flow_hw_q_flow_flush(dev, NULL); 7965 return; 7966 } 7967 #endif 7968 7969 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7970 flow_list_destroy(dev, type, fidx); 7971 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 7972 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7973 while (flow_info) { 7974 /* Romove the cache flow info. */ 7975 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 7976 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 7977 LIST_REMOVE(flow_info, next); 7978 mlx5_free(flow_info->items); 7979 mlx5_free(flow_info->actions); 7980 mlx5_free(flow_info); 7981 break; 7982 } 7983 flow_info = LIST_NEXT(flow_info, next); 7984 } 7985 } 7986 num_flushed++; 7987 } 7988 if (active) { 7989 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7990 dev->data->port_id, num_flushed); 7991 } 7992 } 7993 7994 /** 7995 * Stop all default actions for flows. 7996 * 7997 * @param dev 7998 * Pointer to Ethernet device. 7999 */ 8000 void 8001 mlx5_flow_stop_default(struct rte_eth_dev *dev) 8002 { 8003 flow_mreg_del_default_copy_action(dev); 8004 flow_rxq_flags_clear(dev); 8005 } 8006 8007 /** 8008 * Set rxq flag. 8009 * 8010 * @param[in] dev 8011 * Pointer to the rte_eth_dev structure. 8012 * @param[in] enable 8013 * Flag to enable or not. 8014 */ 8015 void 8016 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 8017 { 8018 struct mlx5_priv *priv = dev->data->dev_private; 8019 unsigned int i; 8020 8021 if ((!priv->mark_enabled && !enable) || 8022 (priv->mark_enabled && enable)) 8023 return; 8024 for (i = 0; i < priv->rxqs_n; ++i) { 8025 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 8026 8027 /* With RXQ start/stop feature, RXQ might be stopped. */ 8028 if (!rxq_ctrl) 8029 continue; 8030 rxq_ctrl->rxq.mark = enable; 8031 } 8032 priv->mark_enabled = enable; 8033 } 8034 8035 /** 8036 * Start all default actions for flows. 8037 * 8038 * @param dev 8039 * Pointer to Ethernet device. 8040 * @return 8041 * 0 on success, a negative errno value otherwise and rte_errno is set. 8042 */ 8043 int 8044 mlx5_flow_start_default(struct rte_eth_dev *dev) 8045 { 8046 struct rte_flow_error error; 8047 8048 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8049 return flow_mreg_add_default_copy_action(dev, &error); 8050 } 8051 8052 /** 8053 * Release key of thread specific flow workspace data. 8054 */ 8055 void 8056 flow_release_workspace(void *data) 8057 { 8058 struct mlx5_flow_workspace *wks = data; 8059 struct mlx5_flow_workspace *next; 8060 8061 while (wks) { 8062 next = wks->next; 8063 free(wks); 8064 wks = next; 8065 } 8066 } 8067 8068 /** 8069 * Get thread specific current flow workspace. 8070 * 8071 * @return pointer to thread specific flow workspace data, NULL on error. 8072 */ 8073 struct mlx5_flow_workspace* 8074 mlx5_flow_get_thread_workspace(void) 8075 { 8076 struct mlx5_flow_workspace *data; 8077 8078 data = mlx5_flow_os_get_specific_workspace(); 8079 MLX5_ASSERT(data && data->inuse); 8080 if (!data || !data->inuse) 8081 DRV_LOG(ERR, "flow workspace not initialized."); 8082 return data; 8083 } 8084 8085 /** 8086 * Allocate and init new flow workspace. 8087 * 8088 * @return pointer to flow workspace data, NULL on error. 8089 */ 8090 static struct mlx5_flow_workspace* 8091 flow_alloc_thread_workspace(void) 8092 { 8093 size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); 8094 size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; 8095 struct mlx5_flow_workspace *data = calloc(1, data_size + 8096 rss_queue_array_size); 8097 8098 if (!data) { 8099 DRV_LOG(ERR, "Failed to allocate flow workspace memory."); 8100 return NULL; 8101 } 8102 data->rss_desc.queue = RTE_PTR_ADD(data, data_size); 8103 return data; 8104 } 8105 8106 /** 8107 * Get new thread specific flow workspace. 8108 * 8109 * If current workspace inuse, create new one and set as current. 8110 * 8111 * @return pointer to thread specific flow workspace data, NULL on error. 8112 */ 8113 struct mlx5_flow_workspace* 8114 mlx5_flow_push_thread_workspace(void) 8115 { 8116 struct mlx5_flow_workspace *curr; 8117 struct mlx5_flow_workspace *data; 8118 8119 curr = mlx5_flow_os_get_specific_workspace(); 8120 if (!curr) { 8121 data = flow_alloc_thread_workspace(); 8122 if (!data) 8123 return NULL; 8124 mlx5_flow_os_workspace_gc_add(data); 8125 } else if (!curr->inuse) { 8126 data = curr; 8127 } else if (curr->next) { 8128 data = curr->next; 8129 } else { 8130 data = flow_alloc_thread_workspace(); 8131 if (!data) 8132 return NULL; 8133 curr->next = data; 8134 data->prev = curr; 8135 } 8136 data->inuse = 1; 8137 data->flow_idx = 0; 8138 /* Set as current workspace */ 8139 if (mlx5_flow_os_set_specific_workspace(data)) 8140 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8141 return data; 8142 } 8143 8144 /** 8145 * Close current thread specific flow workspace. 8146 * 8147 * If previous workspace available, set it as current. 8148 * 8149 * @return pointer to thread specific flow workspace data, NULL on error. 8150 */ 8151 void 8152 mlx5_flow_pop_thread_workspace(void) 8153 { 8154 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8155 8156 if (!data) 8157 return; 8158 if (!data->inuse) { 8159 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8160 return; 8161 } 8162 data->inuse = 0; 8163 if (!data->prev) 8164 return; 8165 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8166 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8167 } 8168 8169 /** 8170 * Verify the flow list is empty 8171 * 8172 * @param dev 8173 * Pointer to Ethernet device. 8174 * 8175 * @return the number of flows not released. 8176 */ 8177 int 8178 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8179 { 8180 struct mlx5_priv *priv = dev->data->dev_private; 8181 struct rte_flow *flow; 8182 uint32_t idx = 0; 8183 int ret = 0, i; 8184 8185 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8186 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8187 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8188 dev->data->port_id, (void *)flow); 8189 ret++; 8190 } 8191 } 8192 return ret; 8193 } 8194 8195 /** 8196 * Enable default hairpin egress flow. 8197 * 8198 * @param dev 8199 * Pointer to Ethernet device. 8200 * @param sq_num 8201 * The SQ hw number. 8202 * 8203 * @return 8204 * 0 on success, a negative errno value otherwise and rte_errno is set. 8205 */ 8206 int 8207 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8208 uint32_t sq_num) 8209 { 8210 const struct rte_flow_attr attr = { 8211 .egress = 1, 8212 .priority = 0, 8213 }; 8214 struct mlx5_rte_flow_item_sq queue_spec = { 8215 .queue = sq_num, 8216 }; 8217 struct mlx5_rte_flow_item_sq queue_mask = { 8218 .queue = UINT32_MAX, 8219 }; 8220 struct rte_flow_item items[] = { 8221 { 8222 .type = (enum rte_flow_item_type) 8223 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8224 .spec = &queue_spec, 8225 .last = NULL, 8226 .mask = &queue_mask, 8227 }, 8228 { 8229 .type = RTE_FLOW_ITEM_TYPE_END, 8230 }, 8231 }; 8232 struct rte_flow_action_jump jump = { 8233 .group = MLX5_HAIRPIN_TX_TABLE, 8234 }; 8235 struct rte_flow_action actions[2]; 8236 uint32_t flow_idx; 8237 struct rte_flow_error error; 8238 8239 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8240 actions[0].conf = &jump; 8241 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8242 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8243 &attr, items, actions, false, &error); 8244 if (!flow_idx) { 8245 DRV_LOG(DEBUG, 8246 "Failed to create ctrl flow: rte_errno(%d)," 8247 " type(%d), message(%s)", 8248 rte_errno, error.type, 8249 error.message ? error.message : " (no stated reason)"); 8250 return -rte_errno; 8251 } 8252 return 0; 8253 } 8254 8255 /** 8256 * Enable a control flow configured from the control plane. 8257 * 8258 * @param dev 8259 * Pointer to Ethernet device. 8260 * @param eth_spec 8261 * An Ethernet flow spec to apply. 8262 * @param eth_mask 8263 * An Ethernet flow mask to apply. 8264 * @param vlan_spec 8265 * A VLAN flow spec to apply. 8266 * @param vlan_mask 8267 * A VLAN flow mask to apply. 8268 * 8269 * @return 8270 * 0 on success, a negative errno value otherwise and rte_errno is set. 8271 */ 8272 int 8273 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8274 struct rte_flow_item_eth *eth_spec, 8275 struct rte_flow_item_eth *eth_mask, 8276 struct rte_flow_item_vlan *vlan_spec, 8277 struct rte_flow_item_vlan *vlan_mask) 8278 { 8279 struct mlx5_priv *priv = dev->data->dev_private; 8280 const struct rte_flow_attr attr = { 8281 .ingress = 1, 8282 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8283 }; 8284 struct rte_flow_item items[] = { 8285 { 8286 .type = RTE_FLOW_ITEM_TYPE_ETH, 8287 .spec = eth_spec, 8288 .last = NULL, 8289 .mask = eth_mask, 8290 }, 8291 { 8292 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8293 RTE_FLOW_ITEM_TYPE_END, 8294 .spec = vlan_spec, 8295 .last = NULL, 8296 .mask = vlan_mask, 8297 }, 8298 { 8299 .type = RTE_FLOW_ITEM_TYPE_END, 8300 }, 8301 }; 8302 uint16_t queue[priv->reta_idx_n]; 8303 struct rte_flow_action_rss action_rss = { 8304 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8305 .level = 0, 8306 .types = priv->rss_conf.rss_hf, 8307 .key_len = priv->rss_conf.rss_key_len, 8308 .queue_num = priv->reta_idx_n, 8309 .key = priv->rss_conf.rss_key, 8310 .queue = queue, 8311 }; 8312 struct rte_flow_action actions[] = { 8313 { 8314 .type = RTE_FLOW_ACTION_TYPE_RSS, 8315 .conf = &action_rss, 8316 }, 8317 { 8318 .type = RTE_FLOW_ACTION_TYPE_END, 8319 }, 8320 }; 8321 uint32_t flow_idx; 8322 struct rte_flow_error error; 8323 unsigned int i; 8324 8325 if (!priv->reta_idx_n || !priv->rxqs_n) { 8326 return 0; 8327 } 8328 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8329 action_rss.types = 0; 8330 for (i = 0; i != priv->reta_idx_n; ++i) 8331 queue[i] = (*priv->reta_idx)[i]; 8332 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8333 &attr, items, actions, false, &error); 8334 if (!flow_idx) 8335 return -rte_errno; 8336 return 0; 8337 } 8338 8339 /** 8340 * Enable a flow control configured from the control plane. 8341 * 8342 * @param dev 8343 * Pointer to Ethernet device. 8344 * @param eth_spec 8345 * An Ethernet flow spec to apply. 8346 * @param eth_mask 8347 * An Ethernet flow mask to apply. 8348 * 8349 * @return 8350 * 0 on success, a negative errno value otherwise and rte_errno is set. 8351 */ 8352 int 8353 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8354 struct rte_flow_item_eth *eth_spec, 8355 struct rte_flow_item_eth *eth_mask) 8356 { 8357 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8358 } 8359 8360 /** 8361 * Create default miss flow rule matching lacp traffic 8362 * 8363 * @param dev 8364 * Pointer to Ethernet device. 8365 * @param eth_spec 8366 * An Ethernet flow spec to apply. 8367 * 8368 * @return 8369 * 0 on success, a negative errno value otherwise and rte_errno is set. 8370 */ 8371 int 8372 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8373 { 8374 /* 8375 * The LACP matching is done by only using ether type since using 8376 * a multicast dst mac causes kernel to give low priority to this flow. 8377 */ 8378 static const struct rte_flow_item_eth lacp_spec = { 8379 .hdr.ether_type = RTE_BE16(0x8809), 8380 }; 8381 static const struct rte_flow_item_eth lacp_mask = { 8382 .hdr.ether_type = 0xffff, 8383 }; 8384 const struct rte_flow_attr attr = { 8385 .ingress = 1, 8386 }; 8387 struct rte_flow_item items[] = { 8388 { 8389 .type = RTE_FLOW_ITEM_TYPE_ETH, 8390 .spec = &lacp_spec, 8391 .mask = &lacp_mask, 8392 }, 8393 { 8394 .type = RTE_FLOW_ITEM_TYPE_END, 8395 }, 8396 }; 8397 struct rte_flow_action actions[] = { 8398 { 8399 .type = (enum rte_flow_action_type) 8400 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8401 }, 8402 { 8403 .type = RTE_FLOW_ACTION_TYPE_END, 8404 }, 8405 }; 8406 struct rte_flow_error error; 8407 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8408 &attr, items, actions, 8409 false, &error); 8410 8411 if (!flow_idx) 8412 return -rte_errno; 8413 return 0; 8414 } 8415 8416 /** 8417 * Destroy a flow. 8418 * 8419 * @see rte_flow_destroy() 8420 * @see rte_flow_ops 8421 */ 8422 int 8423 mlx5_flow_destroy(struct rte_eth_dev *dev, 8424 struct rte_flow *flow, 8425 struct rte_flow_error *error __rte_unused) 8426 { 8427 struct mlx5_priv *priv = dev->data->dev_private; 8428 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8429 struct mlx5_dv_flow_info *flow_info; 8430 8431 if (priv->sh->config.dv_flow_en == 2) 8432 return rte_flow_error_set(error, ENOTSUP, 8433 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8434 NULL, 8435 "Flow non-Q destruction not supported"); 8436 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8437 (uintptr_t)(void *)flow); 8438 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8439 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8440 while (flow_info) { 8441 /* Romove the cache flow info. */ 8442 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8443 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8444 LIST_REMOVE(flow_info, next); 8445 mlx5_free(flow_info->items); 8446 mlx5_free(flow_info->actions); 8447 mlx5_free(flow_info); 8448 break; 8449 } 8450 flow_info = LIST_NEXT(flow_info, next); 8451 } 8452 } 8453 return 0; 8454 } 8455 8456 /** 8457 * Destroy all flows. 8458 * 8459 * @see rte_flow_flush() 8460 * @see rte_flow_ops 8461 */ 8462 int 8463 mlx5_flow_flush(struct rte_eth_dev *dev, 8464 struct rte_flow_error *error __rte_unused) 8465 { 8466 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8467 return 0; 8468 } 8469 8470 /** 8471 * Isolated mode. 8472 * 8473 * @see rte_flow_isolate() 8474 * @see rte_flow_ops 8475 */ 8476 int 8477 mlx5_flow_isolate(struct rte_eth_dev *dev, 8478 int enable, 8479 struct rte_flow_error *error) 8480 { 8481 struct mlx5_priv *priv = dev->data->dev_private; 8482 8483 if (dev->data->dev_started) { 8484 rte_flow_error_set(error, EBUSY, 8485 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8486 NULL, 8487 "port must be stopped first"); 8488 return -rte_errno; 8489 } 8490 if (!enable && !priv->sh->config.repr_matching) 8491 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8492 "isolated mode cannot be disabled when " 8493 "representor matching is disabled"); 8494 priv->isolated = !!enable; 8495 if (enable) 8496 dev->dev_ops = &mlx5_dev_ops_isolate; 8497 else 8498 dev->dev_ops = &mlx5_dev_ops; 8499 8500 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8501 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8502 8503 return 0; 8504 } 8505 8506 /** 8507 * Query a flow. 8508 * 8509 * @see rte_flow_query() 8510 * @see rte_flow_ops 8511 */ 8512 static int 8513 flow_drv_query(struct rte_eth_dev *dev, 8514 struct rte_flow *eflow, 8515 const struct rte_flow_action *actions, 8516 void *data, 8517 struct rte_flow_error *error) 8518 { 8519 struct mlx5_priv *priv = dev->data->dev_private; 8520 const struct mlx5_flow_driver_ops *fops; 8521 struct rte_flow *flow = NULL; 8522 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8523 8524 if (priv->sh->config.dv_flow_en == 2) { 8525 #ifdef HAVE_MLX5_HWS_SUPPORT 8526 flow = eflow; 8527 ftype = MLX5_FLOW_TYPE_HW; 8528 #endif 8529 } else { 8530 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8531 (uintptr_t)(void *)eflow); 8532 } 8533 if (!flow) { 8534 return rte_flow_error_set(error, ENOENT, 8535 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8536 NULL, 8537 "invalid flow handle"); 8538 } 8539 if (ftype == MLX5_FLOW_TYPE_MIN) 8540 ftype = flow->drv_type; 8541 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8542 fops = flow_get_drv_ops(ftype); 8543 8544 return fops->query(dev, flow, actions, data, error); 8545 } 8546 8547 /** 8548 * Query a flow. 8549 * 8550 * @see rte_flow_query() 8551 * @see rte_flow_ops 8552 */ 8553 int 8554 mlx5_flow_query(struct rte_eth_dev *dev, 8555 struct rte_flow *flow, 8556 const struct rte_flow_action *actions, 8557 void *data, 8558 struct rte_flow_error *error) 8559 { 8560 int ret; 8561 8562 ret = flow_drv_query(dev, flow, actions, data, 8563 error); 8564 if (ret < 0) 8565 return ret; 8566 return 0; 8567 } 8568 8569 /** 8570 * Get rte_flow callbacks. 8571 * 8572 * @param dev 8573 * Pointer to Ethernet device structure. 8574 * @param ops 8575 * Pointer to operation-specific structure. 8576 * 8577 * @return 0 8578 */ 8579 int 8580 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8581 const struct rte_flow_ops **ops) 8582 { 8583 *ops = &mlx5_flow_ops; 8584 return 0; 8585 } 8586 8587 /** 8588 * Validate meter policy actions. 8589 * Dispatcher for action type specific validation. 8590 * 8591 * @param[in] dev 8592 * Pointer to the Ethernet device structure. 8593 * @param[in] action 8594 * The meter policy action object to validate. 8595 * @param[in] attr 8596 * Attributes of flow to determine steering domain. 8597 * @param[out] is_rss 8598 * Is RSS or not. 8599 * @param[out] domain_bitmap 8600 * Domain bitmap. 8601 * @param[out] is_def_policy 8602 * Is default policy or not. 8603 * @param[out] error 8604 * Perform verbose error reporting if not NULL. Initialized in case of 8605 * error only. 8606 * 8607 * @return 8608 * 0 on success, otherwise negative errno value. 8609 */ 8610 int 8611 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8612 const struct rte_flow_action *actions[RTE_COLORS], 8613 struct rte_flow_attr *attr, 8614 bool *is_rss, 8615 uint8_t *domain_bitmap, 8616 uint8_t *policy_mode, 8617 struct rte_mtr_error *error) 8618 { 8619 const struct mlx5_flow_driver_ops *fops; 8620 8621 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8622 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8623 domain_bitmap, policy_mode, error); 8624 } 8625 8626 /** 8627 * Destroy the meter table set. 8628 * 8629 * @param[in] dev 8630 * Pointer to Ethernet device. 8631 * @param[in] mtr_policy 8632 * Meter policy struct. 8633 */ 8634 void 8635 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8636 struct mlx5_flow_meter_policy *mtr_policy) 8637 { 8638 const struct mlx5_flow_driver_ops *fops; 8639 8640 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8641 fops->destroy_mtr_acts(dev, mtr_policy); 8642 } 8643 8644 /** 8645 * Create policy action, lock free, 8646 * (mutex should be acquired by caller). 8647 * Dispatcher for action type specific call. 8648 * 8649 * @param[in] dev 8650 * Pointer to the Ethernet device structure. 8651 * @param[in] mtr_policy 8652 * Meter policy struct. 8653 * @param[in] action 8654 * Action specification used to create meter actions. 8655 * @param[in] attr 8656 * Flow rule attributes. 8657 * @param[out] error 8658 * Perform verbose error reporting if not NULL. Initialized in case of 8659 * error only. 8660 * 8661 * @return 8662 * 0 on success, otherwise negative errno value. 8663 */ 8664 int 8665 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8666 struct mlx5_flow_meter_policy *mtr_policy, 8667 const struct rte_flow_action *actions[RTE_COLORS], 8668 struct rte_flow_attr *attr, 8669 struct rte_mtr_error *error) 8670 { 8671 const struct mlx5_flow_driver_ops *fops; 8672 8673 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8674 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8675 } 8676 8677 /** 8678 * Create policy rules, lock free, 8679 * (mutex should be acquired by caller). 8680 * Dispatcher for action type specific call. 8681 * 8682 * @param[in] dev 8683 * Pointer to the Ethernet device structure. 8684 * @param[in] mtr_policy 8685 * Meter policy struct. 8686 * 8687 * @return 8688 * 0 on success, -1 otherwise. 8689 */ 8690 int 8691 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8692 struct mlx5_flow_meter_policy *mtr_policy) 8693 { 8694 const struct mlx5_flow_driver_ops *fops; 8695 8696 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8697 return fops->create_policy_rules(dev, mtr_policy); 8698 } 8699 8700 /** 8701 * Destroy policy rules, lock free, 8702 * (mutex should be acquired by caller). 8703 * Dispatcher for action type specific call. 8704 * 8705 * @param[in] dev 8706 * Pointer to the Ethernet device structure. 8707 * @param[in] mtr_policy 8708 * Meter policy struct. 8709 */ 8710 void 8711 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8712 struct mlx5_flow_meter_policy *mtr_policy) 8713 { 8714 const struct mlx5_flow_driver_ops *fops; 8715 8716 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8717 fops->destroy_policy_rules(dev, mtr_policy); 8718 } 8719 8720 /** 8721 * Destroy the default policy table set. 8722 * 8723 * @param[in] dev 8724 * Pointer to Ethernet device. 8725 */ 8726 void 8727 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8728 { 8729 const struct mlx5_flow_driver_ops *fops; 8730 8731 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8732 fops->destroy_def_policy(dev); 8733 } 8734 8735 /** 8736 * Destroy the default policy table set. 8737 * 8738 * @param[in] dev 8739 * Pointer to Ethernet device. 8740 * 8741 * @return 8742 * 0 on success, -1 otherwise. 8743 */ 8744 int 8745 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8746 { 8747 const struct mlx5_flow_driver_ops *fops; 8748 8749 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8750 return fops->create_def_policy(dev); 8751 } 8752 8753 /** 8754 * Create the needed meter and suffix tables. 8755 * 8756 * @param[in] dev 8757 * Pointer to Ethernet device. 8758 * 8759 * @return 8760 * 0 on success, -1 otherwise. 8761 */ 8762 int 8763 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8764 struct mlx5_flow_meter_info *fm, 8765 uint32_t mtr_idx, 8766 uint8_t domain_bitmap) 8767 { 8768 const struct mlx5_flow_driver_ops *fops; 8769 8770 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8771 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8772 } 8773 8774 /** 8775 * Destroy the meter table set. 8776 * 8777 * @param[in] dev 8778 * Pointer to Ethernet device. 8779 * @param[in] tbl 8780 * Pointer to the meter table set. 8781 */ 8782 void 8783 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8784 struct mlx5_flow_meter_info *fm) 8785 { 8786 const struct mlx5_flow_driver_ops *fops; 8787 8788 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8789 fops->destroy_mtr_tbls(dev, fm); 8790 } 8791 8792 /** 8793 * Destroy the global meter drop table. 8794 * 8795 * @param[in] dev 8796 * Pointer to Ethernet device. 8797 */ 8798 void 8799 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8800 { 8801 const struct mlx5_flow_driver_ops *fops; 8802 8803 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8804 fops->destroy_mtr_drop_tbls(dev); 8805 } 8806 8807 /** 8808 * Destroy the sub policy table with RX queue. 8809 * 8810 * @param[in] dev 8811 * Pointer to Ethernet device. 8812 * @param[in] mtr_policy 8813 * Pointer to meter policy table. 8814 */ 8815 void 8816 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8817 struct mlx5_flow_meter_policy *mtr_policy) 8818 { 8819 const struct mlx5_flow_driver_ops *fops; 8820 8821 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8822 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8823 } 8824 8825 /** 8826 * Allocate the needed aso flow meter id. 8827 * 8828 * @param[in] dev 8829 * Pointer to Ethernet device. 8830 * 8831 * @return 8832 * Index to aso flow meter on success, NULL otherwise. 8833 */ 8834 uint32_t 8835 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8836 { 8837 const struct mlx5_flow_driver_ops *fops; 8838 8839 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8840 return fops->create_meter(dev); 8841 } 8842 8843 /** 8844 * Free the aso flow meter id. 8845 * 8846 * @param[in] dev 8847 * Pointer to Ethernet device. 8848 * @param[in] mtr_idx 8849 * Index to aso flow meter to be free. 8850 * 8851 * @return 8852 * 0 on success. 8853 */ 8854 void 8855 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8856 { 8857 const struct mlx5_flow_driver_ops *fops; 8858 8859 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8860 fops->free_meter(dev, mtr_idx); 8861 } 8862 8863 /** 8864 * Allocate a counter. 8865 * 8866 * @param[in] dev 8867 * Pointer to Ethernet device structure. 8868 * 8869 * @return 8870 * Index to allocated counter on success, 0 otherwise. 8871 */ 8872 uint32_t 8873 mlx5_counter_alloc(struct rte_eth_dev *dev) 8874 { 8875 struct rte_flow_attr attr = { .transfer = 0 }; 8876 8877 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8878 (dev); 8879 } 8880 8881 /** 8882 * Free a counter. 8883 * 8884 * @param[in] dev 8885 * Pointer to Ethernet device structure. 8886 * @param[in] cnt 8887 * Index to counter to be free. 8888 */ 8889 void 8890 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8891 { 8892 struct rte_flow_attr attr = { .transfer = 0 }; 8893 8894 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8895 } 8896 8897 /** 8898 * Query counter statistics. 8899 * 8900 * @param[in] dev 8901 * Pointer to Ethernet device structure. 8902 * @param[in] cnt 8903 * Index to counter to query. 8904 * @param[in] clear 8905 * Set to clear counter statistics. 8906 * @param[out] pkts 8907 * The counter hits packets number to save. 8908 * @param[out] bytes 8909 * The counter hits bytes number to save. 8910 * 8911 * @return 8912 * 0 on success, a negative errno value otherwise. 8913 */ 8914 int 8915 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8916 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8917 { 8918 struct rte_flow_attr attr = { .transfer = 0 }; 8919 8920 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8921 (dev, cnt, clear, pkts, bytes, action); 8922 } 8923 8924 /** 8925 * Get information about HWS pre-configurable resources. 8926 * 8927 * @param[in] dev 8928 * Pointer to the rte_eth_dev structure. 8929 * @param[out] port_info 8930 * Pointer to port information. 8931 * @param[out] queue_info 8932 * Pointer to queue information. 8933 * @param[out] error 8934 * Pointer to error structure. 8935 * 8936 * @return 8937 * 0 on success, a negative errno value otherwise and rte_errno is set. 8938 */ 8939 static int 8940 mlx5_flow_info_get(struct rte_eth_dev *dev, 8941 struct rte_flow_port_info *port_info, 8942 struct rte_flow_queue_info *queue_info, 8943 struct rte_flow_error *error) 8944 { 8945 const struct mlx5_flow_driver_ops *fops; 8946 struct rte_flow_attr attr = {0}; 8947 8948 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8949 return rte_flow_error_set(error, ENOTSUP, 8950 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8951 NULL, 8952 "info get with incorrect steering mode"); 8953 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8954 return fops->info_get(dev, port_info, queue_info, error); 8955 } 8956 8957 /** 8958 * Configure port HWS resources. 8959 * 8960 * @param[in] dev 8961 * Pointer to the rte_eth_dev structure. 8962 * @param[in] port_attr 8963 * Port configuration attributes. 8964 * @param[in] nb_queue 8965 * Number of queue. 8966 * @param[in] queue_attr 8967 * Array that holds attributes for each flow queue. 8968 * @param[out] error 8969 * Pointer to error structure. 8970 * 8971 * @return 8972 * 0 on success, a negative errno value otherwise and rte_errno is set. 8973 */ 8974 static int 8975 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8976 const struct rte_flow_port_attr *port_attr, 8977 uint16_t nb_queue, 8978 const struct rte_flow_queue_attr *queue_attr[], 8979 struct rte_flow_error *error) 8980 { 8981 const struct mlx5_flow_driver_ops *fops; 8982 struct rte_flow_attr attr = {0}; 8983 8984 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8985 return rte_flow_error_set(error, ENOTSUP, 8986 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8987 NULL, 8988 "port configure with incorrect steering mode"); 8989 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8990 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8991 } 8992 8993 /** 8994 * Validate item template. 8995 * 8996 * @param[in] dev 8997 * Pointer to the rte_eth_dev structure. 8998 * @param[in] attr 8999 * Pointer to the item template attributes. 9000 * @param[in] items 9001 * The template item pattern. 9002 * @param[out] error 9003 * Pointer to error structure. 9004 * 9005 * @return 9006 * 0 on success, a negative errno value otherwise and rte_errno is set. 9007 */ 9008 int 9009 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 9010 const struct rte_flow_pattern_template_attr *attr, 9011 const struct rte_flow_item items[], 9012 struct rte_flow_error *error) 9013 { 9014 const struct mlx5_flow_driver_ops *fops; 9015 struct rte_flow_attr fattr = {0}; 9016 9017 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9018 rte_flow_error_set(error, ENOTSUP, 9019 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9020 "pattern validate with incorrect steering mode"); 9021 return -ENOTSUP; 9022 } 9023 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9024 return fops->pattern_validate(dev, attr, items, error); 9025 } 9026 9027 /** 9028 * Create flow item template. 9029 * 9030 * @param[in] dev 9031 * Pointer to the rte_eth_dev structure. 9032 * @param[in] attr 9033 * Pointer to the item template attributes. 9034 * @param[in] items 9035 * The template item pattern. 9036 * @param[out] error 9037 * Pointer to error structure. 9038 * 9039 * @return 9040 * 0 on success, a negative errno value otherwise and rte_errno is set. 9041 */ 9042 static struct rte_flow_pattern_template * 9043 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9044 const struct rte_flow_pattern_template_attr *attr, 9045 const struct rte_flow_item items[], 9046 struct rte_flow_error *error) 9047 { 9048 const struct mlx5_flow_driver_ops *fops; 9049 struct rte_flow_attr fattr = {0}; 9050 9051 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9052 rte_flow_error_set(error, ENOTSUP, 9053 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9054 NULL, 9055 "pattern create with incorrect steering mode"); 9056 return NULL; 9057 } 9058 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9059 return fops->pattern_template_create(dev, attr, items, error); 9060 } 9061 9062 /** 9063 * Destroy flow item template. 9064 * 9065 * @param[in] dev 9066 * Pointer to the rte_eth_dev structure. 9067 * @param[in] template 9068 * Pointer to the item template to be destroyed. 9069 * @param[out] error 9070 * Pointer to error structure. 9071 * 9072 * @return 9073 * 0 on success, a negative errno value otherwise and rte_errno is set. 9074 */ 9075 static int 9076 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9077 struct rte_flow_pattern_template *template, 9078 struct rte_flow_error *error) 9079 { 9080 const struct mlx5_flow_driver_ops *fops; 9081 struct rte_flow_attr attr = {0}; 9082 9083 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9084 return rte_flow_error_set(error, ENOTSUP, 9085 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9086 NULL, 9087 "pattern destroy with incorrect steering mode"); 9088 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9089 return fops->pattern_template_destroy(dev, template, error); 9090 } 9091 9092 /** 9093 * Validate flow actions template. 9094 * 9095 * @param[in] dev 9096 * Pointer to the rte_eth_dev structure. 9097 * @param[in] attr 9098 * Pointer to the action template attributes. 9099 * @param[in] actions 9100 * Associated actions (list terminated by the END action). 9101 * @param[in] masks 9102 * List of actions that marks which of the action's member is constant. 9103 * @param[out] error 9104 * Pointer to error structure. 9105 * 9106 * @return 9107 * 0 on success, a negative errno value otherwise and rte_errno is set. 9108 */ 9109 int 9110 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9111 const struct rte_flow_actions_template_attr *attr, 9112 const struct rte_flow_action actions[], 9113 const struct rte_flow_action masks[], 9114 struct rte_flow_error *error) 9115 { 9116 const struct mlx5_flow_driver_ops *fops; 9117 struct rte_flow_attr fattr = {0}; 9118 9119 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9120 rte_flow_error_set(error, ENOTSUP, 9121 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9122 "actions validate with incorrect steering mode"); 9123 return -ENOTSUP; 9124 } 9125 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9126 return fops->actions_validate(dev, attr, actions, masks, error); 9127 } 9128 9129 /** 9130 * Create flow item template. 9131 * 9132 * @param[in] dev 9133 * Pointer to the rte_eth_dev structure. 9134 * @param[in] attr 9135 * Pointer to the action template attributes. 9136 * @param[in] actions 9137 * Associated actions (list terminated by the END action). 9138 * @param[in] masks 9139 * List of actions that marks which of the action's member is constant. 9140 * @param[out] error 9141 * Pointer to error structure. 9142 * 9143 * @return 9144 * 0 on success, a negative errno value otherwise and rte_errno is set. 9145 */ 9146 static struct rte_flow_actions_template * 9147 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9148 const struct rte_flow_actions_template_attr *attr, 9149 const struct rte_flow_action actions[], 9150 const struct rte_flow_action masks[], 9151 struct rte_flow_error *error) 9152 { 9153 const struct mlx5_flow_driver_ops *fops; 9154 struct rte_flow_attr fattr = {0}; 9155 9156 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9157 rte_flow_error_set(error, ENOTSUP, 9158 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9159 NULL, 9160 "action create with incorrect steering mode"); 9161 return NULL; 9162 } 9163 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9164 return fops->actions_template_create(dev, attr, actions, masks, error); 9165 } 9166 9167 /** 9168 * Destroy flow action template. 9169 * 9170 * @param[in] dev 9171 * Pointer to the rte_eth_dev structure. 9172 * @param[in] template 9173 * Pointer to the action template to be destroyed. 9174 * @param[out] error 9175 * Pointer to error structure. 9176 * 9177 * @return 9178 * 0 on success, a negative errno value otherwise and rte_errno is set. 9179 */ 9180 static int 9181 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9182 struct rte_flow_actions_template *template, 9183 struct rte_flow_error *error) 9184 { 9185 const struct mlx5_flow_driver_ops *fops; 9186 struct rte_flow_attr attr = {0}; 9187 9188 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9189 return rte_flow_error_set(error, ENOTSUP, 9190 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9191 NULL, 9192 "action destroy with incorrect steering mode"); 9193 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9194 return fops->actions_template_destroy(dev, template, error); 9195 } 9196 9197 /** 9198 * Create flow table. 9199 * 9200 * @param[in] dev 9201 * Pointer to the rte_eth_dev structure. 9202 * @param[in] attr 9203 * Pointer to the table attributes. 9204 * @param[in] item_templates 9205 * Item template array to be binded to the table. 9206 * @param[in] nb_item_templates 9207 * Number of item template. 9208 * @param[in] action_templates 9209 * Action template array to be binded to the table. 9210 * @param[in] nb_action_templates 9211 * Number of action template. 9212 * @param[out] error 9213 * Pointer to error structure. 9214 * 9215 * @return 9216 * Table on success, NULL otherwise and rte_errno is set. 9217 */ 9218 static struct rte_flow_template_table * 9219 mlx5_flow_table_create(struct rte_eth_dev *dev, 9220 const struct rte_flow_template_table_attr *attr, 9221 struct rte_flow_pattern_template *item_templates[], 9222 uint8_t nb_item_templates, 9223 struct rte_flow_actions_template *action_templates[], 9224 uint8_t nb_action_templates, 9225 struct rte_flow_error *error) 9226 { 9227 const struct mlx5_flow_driver_ops *fops; 9228 struct rte_flow_attr fattr = {0}; 9229 9230 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9231 rte_flow_error_set(error, ENOTSUP, 9232 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9233 NULL, 9234 "table create with incorrect steering mode"); 9235 return NULL; 9236 } 9237 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9238 return fops->template_table_create(dev, 9239 attr, 9240 item_templates, 9241 nb_item_templates, 9242 action_templates, 9243 nb_action_templates, 9244 error); 9245 } 9246 9247 /** 9248 * PMD destroy flow table. 9249 * 9250 * @param[in] dev 9251 * Pointer to the rte_eth_dev structure. 9252 * @param[in] table 9253 * Pointer to the table to be destroyed. 9254 * @param[out] error 9255 * Pointer to error structure. 9256 * 9257 * @return 9258 * 0 on success, a negative errno value otherwise and rte_errno is set. 9259 */ 9260 static int 9261 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9262 struct rte_flow_template_table *table, 9263 struct rte_flow_error *error) 9264 { 9265 const struct mlx5_flow_driver_ops *fops; 9266 struct rte_flow_attr attr = {0}; 9267 9268 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9269 return rte_flow_error_set(error, ENOTSUP, 9270 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9271 NULL, 9272 "table destroy with incorrect steering mode"); 9273 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9274 return fops->template_table_destroy(dev, table, error); 9275 } 9276 9277 /** 9278 * PMD group set miss actions. 9279 * 9280 * @param[in] dev 9281 * Pointer to the rte_eth_dev structure. 9282 * @param[in] attr 9283 * Pointer to group attributes 9284 * @param[in] actions 9285 * Array of actions 9286 * @param[out] error 9287 * Pointer to error structure. 9288 * 9289 * @return 9290 * 0 on success, a negative errno value otherwise and rte_errno is set. 9291 */ 9292 static int 9293 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 9294 uint32_t group_id, 9295 const struct rte_flow_group_attr *attr, 9296 const struct rte_flow_action actions[], 9297 struct rte_flow_error *error) 9298 { 9299 const struct mlx5_flow_driver_ops *fops; 9300 struct rte_flow_attr fattr = {0}; 9301 9302 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9303 return rte_flow_error_set(error, ENOTSUP, 9304 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9305 NULL, 9306 "group set miss actions with incorrect steering mode"); 9307 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9308 return fops->group_set_miss_actions(dev, group_id, attr, actions, error); 9309 } 9310 9311 /** 9312 * Allocate a new memory for the counter values wrapped by all the needed 9313 * management. 9314 * 9315 * @param[in] sh 9316 * Pointer to mlx5_dev_ctx_shared object. 9317 * 9318 * @return 9319 * 0 on success, a negative errno value otherwise. 9320 */ 9321 static int 9322 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9323 { 9324 struct mlx5_counter_stats_mem_mng *mem_mng; 9325 volatile struct flow_counter_stats *raw_data; 9326 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9327 int size = (sizeof(struct flow_counter_stats) * 9328 MLX5_COUNTERS_PER_POOL + 9329 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9330 sizeof(struct mlx5_counter_stats_mem_mng); 9331 size_t pgsize = rte_mem_page_size(); 9332 uint8_t *mem; 9333 int ret; 9334 int i; 9335 9336 if (pgsize == (size_t)-1) { 9337 DRV_LOG(ERR, "Failed to get mem page size"); 9338 rte_errno = ENOMEM; 9339 return -ENOMEM; 9340 } 9341 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9342 if (!mem) { 9343 rte_errno = ENOMEM; 9344 return -ENOMEM; 9345 } 9346 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9347 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9348 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9349 sh->cdev->pdn, mem, size, 9350 &mem_mng->wm); 9351 if (ret) { 9352 rte_errno = errno; 9353 mlx5_free(mem); 9354 return -rte_errno; 9355 } 9356 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9357 raw_data = (volatile struct flow_counter_stats *)mem; 9358 for (i = 0; i < raws_n; ++i) { 9359 mem_mng->raws[i].mem_mng = mem_mng; 9360 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9361 } 9362 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9363 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9364 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9365 next); 9366 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9367 sh->sws_cmng.mem_mng = mem_mng; 9368 return 0; 9369 } 9370 9371 /** 9372 * Set the statistic memory to the new counter pool. 9373 * 9374 * @param[in] sh 9375 * Pointer to mlx5_dev_ctx_shared object. 9376 * @param[in] pool 9377 * Pointer to the pool to set the statistic memory. 9378 * 9379 * @return 9380 * 0 on success, a negative errno value otherwise. 9381 */ 9382 static int 9383 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9384 struct mlx5_flow_counter_pool *pool) 9385 { 9386 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9387 /* Resize statistic memory once used out. */ 9388 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9389 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9390 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9391 return -1; 9392 } 9393 rte_spinlock_lock(&pool->sl); 9394 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9395 rte_spinlock_unlock(&pool->sl); 9396 pool->raw_hw = NULL; 9397 return 0; 9398 } 9399 9400 #define MLX5_POOL_QUERY_FREQ_US 1000000 9401 9402 /** 9403 * Set the periodic procedure for triggering asynchronous batch queries for all 9404 * the counter pools. 9405 * 9406 * @param[in] sh 9407 * Pointer to mlx5_dev_ctx_shared object. 9408 */ 9409 void 9410 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9411 { 9412 uint32_t pools_n, us; 9413 9414 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9415 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9416 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9417 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9418 sh->sws_cmng.query_thread_on = 0; 9419 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9420 } else { 9421 sh->sws_cmng.query_thread_on = 1; 9422 } 9423 } 9424 9425 /** 9426 * The periodic procedure for triggering asynchronous batch queries for all the 9427 * counter pools. This function is probably called by the host thread. 9428 * 9429 * @param[in] arg 9430 * The parameter for the alarm process. 9431 */ 9432 void 9433 mlx5_flow_query_alarm(void *arg) 9434 { 9435 struct mlx5_dev_ctx_shared *sh = arg; 9436 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9437 uint16_t pool_index = cmng->pool_index; 9438 struct mlx5_flow_counter_pool *pool; 9439 uint16_t n_valid; 9440 int ret; 9441 9442 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9443 goto set_alarm; 9444 rte_spinlock_lock(&cmng->pool_update_sl); 9445 pool = cmng->pools[pool_index]; 9446 n_valid = cmng->n_valid; 9447 rte_spinlock_unlock(&cmng->pool_update_sl); 9448 /* Set the statistic memory to the new created pool. */ 9449 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9450 goto set_alarm; 9451 if (pool->raw_hw) 9452 /* There is a pool query in progress. */ 9453 goto set_alarm; 9454 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9455 if (!pool->raw_hw) 9456 /* No free counter statistics raw memory. */ 9457 goto set_alarm; 9458 /* 9459 * Identify the counters released between query trigger and query 9460 * handle more efficiently. The counter released in this gap period 9461 * should wait for a new round of query as the new arrived packets 9462 * will not be taken into account. 9463 */ 9464 pool->query_gen++; 9465 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9466 MLX5_COUNTERS_PER_POOL, 9467 NULL, NULL, 9468 pool->raw_hw->mem_mng->wm.lkey, 9469 (void *)(uintptr_t) 9470 pool->raw_hw->data, 9471 sh->devx_comp, 9472 (uint64_t)(uintptr_t)pool); 9473 if (ret) { 9474 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9475 " %d", pool->min_dcs->id); 9476 pool->raw_hw = NULL; 9477 goto set_alarm; 9478 } 9479 LIST_REMOVE(pool->raw_hw, next); 9480 cmng->pending_queries++; 9481 pool_index++; 9482 if (pool_index >= n_valid) 9483 pool_index = 0; 9484 set_alarm: 9485 cmng->pool_index = pool_index; 9486 mlx5_set_query_alarm(sh); 9487 } 9488 9489 /** 9490 * Check and callback event for new aged flow in the counter pool 9491 * 9492 * @param[in] sh 9493 * Pointer to mlx5_dev_ctx_shared object. 9494 * @param[in] pool 9495 * Pointer to Current counter pool. 9496 */ 9497 static void 9498 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9499 struct mlx5_flow_counter_pool *pool) 9500 { 9501 struct mlx5_priv *priv; 9502 struct mlx5_flow_counter *cnt; 9503 struct mlx5_age_info *age_info; 9504 struct mlx5_age_param *age_param; 9505 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9506 struct mlx5_counter_stats_raw *prev = pool->raw; 9507 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9508 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9509 uint16_t expected = AGE_CANDIDATE; 9510 uint32_t i; 9511 9512 pool->time_of_last_age_check = curr_time; 9513 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9514 cnt = MLX5_POOL_GET_CNT(pool, i); 9515 age_param = MLX5_CNT_TO_AGE(cnt); 9516 if (__atomic_load_n(&age_param->state, 9517 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9518 continue; 9519 if (cur->data[i].hits != prev->data[i].hits) { 9520 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9521 __ATOMIC_RELAXED); 9522 continue; 9523 } 9524 if (__atomic_fetch_add(&age_param->sec_since_last_hit, 9525 time_delta, 9526 __ATOMIC_RELAXED) + time_delta <= age_param->timeout) 9527 continue; 9528 /** 9529 * Hold the lock first, or if between the 9530 * state AGE_TMOUT and tailq operation the 9531 * release happened, the release procedure 9532 * may delete a non-existent tailq node. 9533 */ 9534 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9535 age_info = GET_PORT_AGE_INFO(priv); 9536 rte_spinlock_lock(&age_info->aged_sl); 9537 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9538 AGE_TMOUT, false, 9539 __ATOMIC_RELAXED, 9540 __ATOMIC_RELAXED)) { 9541 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9542 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9543 } 9544 rte_spinlock_unlock(&age_info->aged_sl); 9545 } 9546 mlx5_age_event_prepare(sh); 9547 } 9548 9549 /** 9550 * Handler for the HW respond about ready values from an asynchronous batch 9551 * query. This function is probably called by the host thread. 9552 * 9553 * @param[in] sh 9554 * The pointer to the shared device context. 9555 * @param[in] async_id 9556 * The Devx async ID. 9557 * @param[in] status 9558 * The status of the completion. 9559 */ 9560 void 9561 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9562 uint64_t async_id, int status) 9563 { 9564 struct mlx5_flow_counter_pool *pool = 9565 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9566 struct mlx5_counter_stats_raw *raw_to_free; 9567 uint8_t query_gen = pool->query_gen ^ 1; 9568 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9569 enum mlx5_counter_type cnt_type = 9570 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9571 MLX5_COUNTER_TYPE_ORIGIN; 9572 9573 if (unlikely(status)) { 9574 raw_to_free = pool->raw_hw; 9575 } else { 9576 raw_to_free = pool->raw; 9577 if (pool->is_aged) 9578 mlx5_flow_aging_check(sh, pool); 9579 rte_spinlock_lock(&pool->sl); 9580 pool->raw = pool->raw_hw; 9581 rte_spinlock_unlock(&pool->sl); 9582 /* Be sure the new raw counters data is updated in memory. */ 9583 rte_io_wmb(); 9584 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9585 rte_spinlock_lock(&cmng->csl[cnt_type]); 9586 TAILQ_CONCAT(&cmng->counters[cnt_type], 9587 &pool->counters[query_gen], next); 9588 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9589 } 9590 } 9591 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9592 pool->raw_hw = NULL; 9593 sh->sws_cmng.pending_queries--; 9594 } 9595 9596 static int 9597 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9598 const struct flow_grp_info *grp_info, 9599 struct rte_flow_error *error) 9600 { 9601 if (grp_info->transfer && grp_info->external && 9602 grp_info->fdb_def_rule) { 9603 if (group == UINT32_MAX) 9604 return rte_flow_error_set 9605 (error, EINVAL, 9606 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9607 NULL, 9608 "group index not supported"); 9609 *table = group + 1; 9610 } else { 9611 *table = group; 9612 } 9613 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9614 return 0; 9615 } 9616 9617 /** 9618 * Translate the rte_flow group index to HW table value. 9619 * 9620 * If tunnel offload is disabled, all group ids converted to flow table 9621 * id using the standard method. 9622 * If tunnel offload is enabled, group id can be converted using the 9623 * standard or tunnel conversion method. Group conversion method 9624 * selection depends on flags in `grp_info` parameter: 9625 * - Internal (grp_info.external == 0) groups conversion uses the 9626 * standard method. 9627 * - Group ids in JUMP action converted with the tunnel conversion. 9628 * - Group id in rule attribute conversion depends on a rule type and 9629 * group id value: 9630 * ** non zero group attributes converted with the tunnel method 9631 * ** zero group attribute in non-tunnel rule is converted using the 9632 * standard method - there's only one root table 9633 * ** zero group attribute in steer tunnel rule is converted with the 9634 * standard method - single root table 9635 * ** zero group attribute in match tunnel rule is a special OvS 9636 * case: that value is used for portability reasons. That group 9637 * id is converted with the tunnel conversion method. 9638 * 9639 * @param[in] dev 9640 * Port device 9641 * @param[in] tunnel 9642 * PMD tunnel offload object 9643 * @param[in] group 9644 * rte_flow group index value. 9645 * @param[out] table 9646 * HW table value. 9647 * @param[in] grp_info 9648 * flags used for conversion 9649 * @param[out] error 9650 * Pointer to error structure. 9651 * 9652 * @return 9653 * 0 on success, a negative errno value otherwise and rte_errno is set. 9654 */ 9655 int 9656 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9657 const struct mlx5_flow_tunnel *tunnel, 9658 uint32_t group, uint32_t *table, 9659 const struct flow_grp_info *grp_info, 9660 struct rte_flow_error *error) 9661 { 9662 int ret; 9663 bool standard_translation; 9664 9665 if (!grp_info->skip_scale && grp_info->external && 9666 group < MLX5_MAX_TABLES_EXTERNAL) 9667 group *= MLX5_FLOW_TABLE_FACTOR; 9668 if (is_tunnel_offload_active(dev)) { 9669 standard_translation = !grp_info->external || 9670 grp_info->std_tbl_fix; 9671 } else { 9672 standard_translation = true; 9673 } 9674 DRV_LOG(DEBUG, 9675 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9676 dev->data->port_id, group, grp_info->transfer, 9677 grp_info->external, grp_info->fdb_def_rule, 9678 standard_translation ? "STANDARD" : "TUNNEL"); 9679 if (standard_translation) 9680 ret = flow_group_to_table(dev->data->port_id, group, table, 9681 grp_info, error); 9682 else 9683 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9684 table, error); 9685 9686 return ret; 9687 } 9688 9689 /** 9690 * Discover availability of metadata reg_c's. 9691 * 9692 * Iteratively use test flows to check availability. 9693 * 9694 * @param[in] dev 9695 * Pointer to the Ethernet device structure. 9696 * 9697 * @return 9698 * 0 on success, a negative errno value otherwise and rte_errno is set. 9699 */ 9700 int 9701 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9702 { 9703 struct mlx5_priv *priv = dev->data->dev_private; 9704 enum modify_reg idx; 9705 int n = 0; 9706 9707 /* reg_c[0] and reg_c[1] are reserved. */ 9708 priv->sh->flow_mreg_c[n++] = REG_C_0; 9709 priv->sh->flow_mreg_c[n++] = REG_C_1; 9710 /* Discover availability of other reg_c's. */ 9711 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9712 struct rte_flow_attr attr = { 9713 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9714 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9715 .ingress = 1, 9716 }; 9717 struct rte_flow_item items[] = { 9718 [0] = { 9719 .type = RTE_FLOW_ITEM_TYPE_END, 9720 }, 9721 }; 9722 struct rte_flow_action actions[] = { 9723 [0] = { 9724 .type = (enum rte_flow_action_type) 9725 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9726 .conf = &(struct mlx5_flow_action_copy_mreg){ 9727 .src = REG_C_1, 9728 .dst = idx, 9729 }, 9730 }, 9731 [1] = { 9732 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9733 .conf = &(struct rte_flow_action_jump){ 9734 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9735 }, 9736 }, 9737 [2] = { 9738 .type = RTE_FLOW_ACTION_TYPE_END, 9739 }, 9740 }; 9741 uint32_t flow_idx; 9742 struct rte_flow *flow; 9743 struct rte_flow_error error; 9744 9745 if (!priv->sh->config.dv_flow_en) 9746 break; 9747 /* Create internal flow, validation skips copy action. */ 9748 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9749 items, actions, false, &error); 9750 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9751 flow_idx); 9752 if (!flow) 9753 continue; 9754 priv->sh->flow_mreg_c[n++] = idx; 9755 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9756 } 9757 for (; n < MLX5_MREG_C_NUM; ++n) 9758 priv->sh->flow_mreg_c[n] = REG_NON; 9759 priv->sh->metadata_regc_check_flag = 1; 9760 return 0; 9761 } 9762 9763 int 9764 save_dump_file(const uint8_t *data, uint32_t size, 9765 uint32_t type, uint64_t id, void *arg, FILE *file) 9766 { 9767 char line[BUF_SIZE]; 9768 uint32_t out = 0; 9769 uint32_t k; 9770 uint32_t actions_num; 9771 struct rte_flow_query_count *count; 9772 9773 memset(line, 0, BUF_SIZE); 9774 switch (type) { 9775 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9776 actions_num = *(uint32_t *)(arg); 9777 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9778 type, id, actions_num); 9779 break; 9780 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9781 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9782 type, id); 9783 break; 9784 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9785 count = (struct rte_flow_query_count *)arg; 9786 fprintf(file, 9787 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9788 type, id, count->hits, count->bytes); 9789 return 0; 9790 default: 9791 return -1; 9792 } 9793 9794 for (k = 0; k < size; k++) { 9795 /* Make sure we do not overrun the line buffer length. */ 9796 if (out >= BUF_SIZE - 4) { 9797 line[out] = '\0'; 9798 break; 9799 } 9800 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9801 (data[k]) & 0xff); 9802 } 9803 fprintf(file, "%s\n", line); 9804 return 0; 9805 } 9806 9807 int 9808 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9809 struct rte_flow_query_count *count, struct rte_flow_error *error) 9810 { 9811 struct rte_flow_action action[2]; 9812 enum mlx5_flow_drv_type ftype; 9813 const struct mlx5_flow_driver_ops *fops; 9814 9815 if (!flow) { 9816 return rte_flow_error_set(error, ENOENT, 9817 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9818 NULL, 9819 "invalid flow handle"); 9820 } 9821 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9822 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9823 if (flow->counter) { 9824 memset(count, 0, sizeof(struct rte_flow_query_count)); 9825 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9826 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9827 ftype < MLX5_FLOW_TYPE_MAX); 9828 fops = flow_get_drv_ops(ftype); 9829 return fops->query(dev, flow, action, count, error); 9830 } 9831 return -1; 9832 } 9833 9834 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9835 /** 9836 * Dump flow ipool data to file 9837 * 9838 * @param[in] dev 9839 * The pointer to Ethernet device. 9840 * @param[in] file 9841 * A pointer to a file for output. 9842 * @param[out] error 9843 * Perform verbose error reporting if not NULL. PMDs initialize this 9844 * structure in case of error only. 9845 * @return 9846 * 0 on success, a negative value otherwise. 9847 */ 9848 int 9849 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9850 struct rte_flow *flow, FILE *file, 9851 struct rte_flow_error *error) 9852 { 9853 struct mlx5_priv *priv = dev->data->dev_private; 9854 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9855 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9856 uint32_t handle_idx; 9857 struct mlx5_flow_handle *dh; 9858 struct rte_flow_query_count count; 9859 uint32_t actions_num; 9860 const uint8_t *data; 9861 size_t size; 9862 uint64_t id; 9863 uint32_t type; 9864 void *action = NULL; 9865 9866 if (!flow) { 9867 return rte_flow_error_set(error, ENOENT, 9868 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9869 NULL, 9870 "invalid flow handle"); 9871 } 9872 handle_idx = flow->dev_handles; 9873 /* query counter */ 9874 if (flow->counter && 9875 (!mlx5_counter_query(dev, flow->counter, false, 9876 &count.hits, &count.bytes, &action)) && action) { 9877 id = (uint64_t)(uintptr_t)action; 9878 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9879 save_dump_file(NULL, 0, type, 9880 id, (void *)&count, file); 9881 } 9882 9883 while (handle_idx) { 9884 dh = mlx5_ipool_get(priv->sh->ipool 9885 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9886 if (!dh) 9887 continue; 9888 handle_idx = dh->next.next; 9889 9890 /* Get modify_hdr and encap_decap buf from ipools. */ 9891 encap_decap = NULL; 9892 modify_hdr = dh->dvh.modify_hdr; 9893 9894 if (dh->dvh.rix_encap_decap) { 9895 encap_decap = mlx5_ipool_get(priv->sh->ipool 9896 [MLX5_IPOOL_DECAP_ENCAP], 9897 dh->dvh.rix_encap_decap); 9898 } 9899 if (modify_hdr) { 9900 data = (const uint8_t *)modify_hdr->actions; 9901 size = (size_t)(modify_hdr->actions_num) * 8; 9902 id = (uint64_t)(uintptr_t)modify_hdr->action; 9903 actions_num = modify_hdr->actions_num; 9904 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9905 save_dump_file(data, size, type, id, 9906 (void *)(&actions_num), file); 9907 } 9908 if (encap_decap) { 9909 data = encap_decap->buf; 9910 size = encap_decap->size; 9911 id = (uint64_t)(uintptr_t)encap_decap->action; 9912 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9913 save_dump_file(data, size, type, 9914 id, NULL, file); 9915 } 9916 } 9917 return 0; 9918 } 9919 9920 /** 9921 * Dump all flow's encap_decap/modify_hdr/counter data to file 9922 * 9923 * @param[in] dev 9924 * The pointer to Ethernet device. 9925 * @param[in] file 9926 * A pointer to a file for output. 9927 * @param[out] error 9928 * Perform verbose error reporting if not NULL. PMDs initialize this 9929 * structure in case of error only. 9930 * @return 9931 * 0 on success, a negative value otherwise. 9932 */ 9933 static int 9934 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9935 FILE *file, struct rte_flow_error *error __rte_unused) 9936 { 9937 struct mlx5_priv *priv = dev->data->dev_private; 9938 struct mlx5_dev_ctx_shared *sh = priv->sh; 9939 struct mlx5_hlist *h; 9940 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9941 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9942 struct rte_flow_query_count count; 9943 uint32_t actions_num; 9944 const uint8_t *data; 9945 size_t size; 9946 uint64_t id; 9947 uint32_t type; 9948 uint32_t i; 9949 uint32_t j; 9950 struct mlx5_list_inconst *l_inconst; 9951 struct mlx5_list_entry *e; 9952 int lcore_index; 9953 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9954 uint32_t max; 9955 void *action; 9956 9957 /* encap_decap hlist is lcore_share, get global core cache. */ 9958 i = MLX5_LIST_GLOBAL; 9959 h = sh->encaps_decaps; 9960 if (h) { 9961 for (j = 0; j <= h->mask; j++) { 9962 l_inconst = &h->buckets[j].l; 9963 if (!l_inconst || !l_inconst->cache[i]) 9964 continue; 9965 9966 e = LIST_FIRST(&l_inconst->cache[i]->h); 9967 while (e) { 9968 encap_decap = 9969 (struct mlx5_flow_dv_encap_decap_resource *)e; 9970 data = encap_decap->buf; 9971 size = encap_decap->size; 9972 id = (uint64_t)(uintptr_t)encap_decap->action; 9973 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9974 save_dump_file(data, size, type, 9975 id, NULL, file); 9976 e = LIST_NEXT(e, next); 9977 } 9978 } 9979 } 9980 9981 /* get modify_hdr */ 9982 h = sh->modify_cmds; 9983 if (h) { 9984 lcore_index = rte_lcore_index(rte_lcore_id()); 9985 if (unlikely(lcore_index == -1)) { 9986 lcore_index = MLX5_LIST_NLCORE; 9987 rte_spinlock_lock(&h->l_const.lcore_lock); 9988 } 9989 i = lcore_index; 9990 9991 if (lcore_index == MLX5_LIST_NLCORE) { 9992 for (i = 0; i <= (uint32_t)lcore_index; i++) { 9993 for (j = 0; j <= h->mask; j++) { 9994 l_inconst = &h->buckets[j].l; 9995 if (!l_inconst || !l_inconst->cache[i]) 9996 continue; 9997 9998 e = LIST_FIRST(&l_inconst->cache[i]->h); 9999 while (e) { 10000 modify_hdr = 10001 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10002 data = (const uint8_t *)modify_hdr->actions; 10003 size = (size_t)(modify_hdr->actions_num) * 8; 10004 actions_num = modify_hdr->actions_num; 10005 id = (uint64_t)(uintptr_t)modify_hdr->action; 10006 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10007 save_dump_file(data, size, type, id, 10008 (void *)(&actions_num), file); 10009 e = LIST_NEXT(e, next); 10010 } 10011 } 10012 } 10013 } else { 10014 for (j = 0; j <= h->mask; j++) { 10015 l_inconst = &h->buckets[j].l; 10016 if (!l_inconst || !l_inconst->cache[i]) 10017 continue; 10018 10019 e = LIST_FIRST(&l_inconst->cache[i]->h); 10020 while (e) { 10021 modify_hdr = 10022 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10023 data = (const uint8_t *)modify_hdr->actions; 10024 size = (size_t)(modify_hdr->actions_num) * 8; 10025 actions_num = modify_hdr->actions_num; 10026 id = (uint64_t)(uintptr_t)modify_hdr->action; 10027 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10028 save_dump_file(data, size, type, id, 10029 (void *)(&actions_num), file); 10030 e = LIST_NEXT(e, next); 10031 } 10032 } 10033 } 10034 10035 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10036 rte_spinlock_unlock(&h->l_const.lcore_lock); 10037 } 10038 10039 /* get counter */ 10040 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10041 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10042 for (j = 1; j <= max; j++) { 10043 action = NULL; 10044 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10045 &count.bytes, &action)) && action) { 10046 id = (uint64_t)(uintptr_t)action; 10047 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10048 save_dump_file(NULL, 0, type, 10049 id, (void *)&count, file); 10050 } 10051 } 10052 return 0; 10053 } 10054 #endif 10055 10056 /** 10057 * Dump flow raw hw data to file 10058 * 10059 * @param[in] dev 10060 * The pointer to Ethernet device. 10061 * @param[in] file 10062 * A pointer to a file for output. 10063 * @param[out] error 10064 * Perform verbose error reporting if not NULL. PMDs initialize this 10065 * structure in case of error only. 10066 * @return 10067 * 0 on success, a negative value otherwise. 10068 */ 10069 int 10070 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10071 FILE *file, 10072 struct rte_flow_error *error __rte_unused) 10073 { 10074 struct mlx5_priv *priv = dev->data->dev_private; 10075 struct mlx5_dev_ctx_shared *sh = priv->sh; 10076 uint32_t handle_idx; 10077 int ret; 10078 struct mlx5_flow_handle *dh; 10079 struct rte_flow *flow; 10080 10081 if (!sh->config.dv_flow_en) { 10082 if (fputs("device dv flow disabled\n", file) <= 0) 10083 return -errno; 10084 return -ENOTSUP; 10085 } 10086 10087 /* dump all */ 10088 if (!flow_idx) { 10089 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10090 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10091 return -EINVAL; 10092 10093 if (sh->config.dv_flow_en == 2) 10094 return mlx5dr_debug_dump(priv->dr_ctx, file); 10095 #endif 10096 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10097 sh->rx_domain, 10098 sh->tx_domain, file); 10099 } 10100 /* dump one */ 10101 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10102 (uintptr_t)(void *)flow_idx); 10103 if (!flow) 10104 return -EINVAL; 10105 10106 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10107 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10108 #endif 10109 handle_idx = flow->dev_handles; 10110 while (handle_idx) { 10111 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10112 handle_idx); 10113 if (!dh) 10114 return -ENOENT; 10115 if (dh->drv_flow) { 10116 if (sh->config.dv_flow_en == 2) 10117 return -ENOTSUP; 10118 10119 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10120 file); 10121 if (ret) 10122 return -ENOENT; 10123 } 10124 handle_idx = dh->next.next; 10125 } 10126 return 0; 10127 } 10128 10129 /** 10130 * Get aged-out flows. 10131 * 10132 * @param[in] dev 10133 * Pointer to the Ethernet device structure. 10134 * @param[in] context 10135 * The address of an array of pointers to the aged-out flows contexts. 10136 * @param[in] nb_countexts 10137 * The length of context array pointers. 10138 * @param[out] error 10139 * Perform verbose error reporting if not NULL. Initialized in case of 10140 * error only. 10141 * 10142 * @return 10143 * how many contexts get in success, otherwise negative errno value. 10144 * if nb_contexts is 0, return the amount of all aged contexts. 10145 * if nb_contexts is not 0 , return the amount of aged flows reported 10146 * in the context array. 10147 */ 10148 int 10149 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10150 uint32_t nb_contexts, struct rte_flow_error *error) 10151 { 10152 struct rte_flow_attr attr = { .transfer = 0 }; 10153 10154 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10155 (dev, contexts, nb_contexts, error); 10156 } 10157 10158 /** 10159 * Get aged-out flows per HWS queue. 10160 * 10161 * @param[in] dev 10162 * Pointer to the Ethernet device structure. 10163 * @param[in] queue_id 10164 * Flow queue to query. 10165 * @param[in] context 10166 * The address of an array of pointers to the aged-out flows contexts. 10167 * @param[in] nb_countexts 10168 * The length of context array pointers. 10169 * @param[out] error 10170 * Perform verbose error reporting if not NULL. Initialized in case of 10171 * error only. 10172 * 10173 * @return 10174 * how many contexts get in success, otherwise negative errno value. 10175 * if nb_contexts is 0, return the amount of all aged contexts. 10176 * if nb_contexts is not 0 , return the amount of aged flows reported 10177 * in the context array. 10178 */ 10179 int 10180 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10181 void **contexts, uint32_t nb_contexts, 10182 struct rte_flow_error *error) 10183 { 10184 const struct mlx5_flow_driver_ops *fops; 10185 struct rte_flow_attr attr = { 0 }; 10186 10187 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10188 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10189 return fops->get_q_aged_flows(dev, queue_id, contexts, 10190 nb_contexts, error); 10191 } 10192 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10193 dev->data->port_id, queue_id); 10194 return rte_flow_error_set(error, ENOTSUP, 10195 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10196 "get Q aged flows with incorrect steering mode"); 10197 } 10198 10199 /* Wrapper for driver action_validate op callback */ 10200 static int 10201 flow_drv_action_validate(struct rte_eth_dev *dev, 10202 const struct rte_flow_indir_action_conf *conf, 10203 const struct rte_flow_action *action, 10204 const struct mlx5_flow_driver_ops *fops, 10205 struct rte_flow_error *error) 10206 { 10207 static const char err_msg[] = "indirect action validation unsupported"; 10208 10209 if (!fops->action_validate) { 10210 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10211 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10212 NULL, err_msg); 10213 return -rte_errno; 10214 } 10215 return fops->action_validate(dev, conf, action, error); 10216 } 10217 10218 /** 10219 * Destroys the shared action by handle. 10220 * 10221 * @param dev 10222 * Pointer to Ethernet device structure. 10223 * @param[in] handle 10224 * Handle for the indirect action object to be destroyed. 10225 * @param[out] error 10226 * Perform verbose error reporting if not NULL. PMDs initialize this 10227 * structure in case of error only. 10228 * 10229 * @return 10230 * 0 on success, a negative errno value otherwise and rte_errno is set. 10231 * 10232 * @note: wrapper for driver action_create op callback. 10233 */ 10234 static int 10235 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10236 struct rte_flow_action_handle *handle, 10237 struct rte_flow_error *error) 10238 { 10239 static const char err_msg[] = "indirect action destruction unsupported"; 10240 struct rte_flow_attr attr = { .transfer = 0 }; 10241 const struct mlx5_flow_driver_ops *fops = 10242 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10243 10244 if (!fops->action_destroy) { 10245 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10246 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10247 NULL, err_msg); 10248 return -rte_errno; 10249 } 10250 return fops->action_destroy(dev, handle, error); 10251 } 10252 10253 /* Wrapper for driver action_destroy op callback */ 10254 static int 10255 flow_drv_action_update(struct rte_eth_dev *dev, 10256 struct rte_flow_action_handle *handle, 10257 const void *update, 10258 const struct mlx5_flow_driver_ops *fops, 10259 struct rte_flow_error *error) 10260 { 10261 static const char err_msg[] = "indirect action update unsupported"; 10262 10263 if (!fops->action_update) { 10264 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10265 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10266 NULL, err_msg); 10267 return -rte_errno; 10268 } 10269 return fops->action_update(dev, handle, update, error); 10270 } 10271 10272 /* Wrapper for driver action_destroy op callback */ 10273 static int 10274 flow_drv_action_query(struct rte_eth_dev *dev, 10275 const struct rte_flow_action_handle *handle, 10276 void *data, 10277 const struct mlx5_flow_driver_ops *fops, 10278 struct rte_flow_error *error) 10279 { 10280 static const char err_msg[] = "indirect action query unsupported"; 10281 10282 if (!fops->action_query) { 10283 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10284 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10285 NULL, err_msg); 10286 return -rte_errno; 10287 } 10288 return fops->action_query(dev, handle, data, error); 10289 } 10290 10291 /** 10292 * Create indirect action for reuse in multiple flow rules. 10293 * 10294 * @param dev 10295 * Pointer to Ethernet device structure. 10296 * @param conf 10297 * Pointer to indirect action object configuration. 10298 * @param[in] action 10299 * Action configuration for indirect action object creation. 10300 * @param[out] error 10301 * Perform verbose error reporting if not NULL. PMDs initialize this 10302 * structure in case of error only. 10303 * @return 10304 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10305 */ 10306 static struct rte_flow_action_handle * 10307 mlx5_action_handle_create(struct rte_eth_dev *dev, 10308 const struct rte_flow_indir_action_conf *conf, 10309 const struct rte_flow_action *action, 10310 struct rte_flow_error *error) 10311 { 10312 static const char err_msg[] = "indirect action creation unsupported"; 10313 struct rte_flow_attr attr = { .transfer = 0 }; 10314 const struct mlx5_flow_driver_ops *fops = 10315 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10316 10317 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10318 return NULL; 10319 if (!fops->action_create) { 10320 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10321 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10322 NULL, err_msg); 10323 return NULL; 10324 } 10325 return fops->action_create(dev, conf, action, error); 10326 } 10327 10328 /** 10329 * Updates inplace the indirect action configuration pointed by *handle* 10330 * with the configuration provided as *update* argument. 10331 * The update of the indirect action configuration effects all flow rules 10332 * reusing the action via handle. 10333 * 10334 * @param dev 10335 * Pointer to Ethernet device structure. 10336 * @param[in] handle 10337 * Handle for the indirect action to be updated. 10338 * @param[in] update 10339 * Action specification used to modify the action pointed by handle. 10340 * *update* could be of same type with the action pointed by the *handle* 10341 * handle argument, or some other structures like a wrapper, depending on 10342 * the indirect action type. 10343 * @param[out] error 10344 * Perform verbose error reporting if not NULL. PMDs initialize this 10345 * structure in case of error only. 10346 * 10347 * @return 10348 * 0 on success, a negative errno value otherwise and rte_errno is set. 10349 */ 10350 static int 10351 mlx5_action_handle_update(struct rte_eth_dev *dev, 10352 struct rte_flow_action_handle *handle, 10353 const void *update, 10354 struct rte_flow_error *error) 10355 { 10356 struct rte_flow_attr attr = { .transfer = 0 }; 10357 const struct mlx5_flow_driver_ops *fops = 10358 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10359 int ret; 10360 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 10361 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 10362 10363 switch (type) { 10364 case MLX5_INDIRECT_ACTION_TYPE_CT: 10365 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 10366 ret = 0; 10367 break; 10368 default: 10369 ret = flow_drv_action_validate(dev, NULL, 10370 (const struct rte_flow_action *)update, 10371 fops, error); 10372 } 10373 if (ret) 10374 return ret; 10375 return flow_drv_action_update(dev, handle, update, fops, 10376 error); 10377 } 10378 10379 /** 10380 * Query the indirect action by handle. 10381 * 10382 * This function allows retrieving action-specific data such as counters. 10383 * Data is gathered by special action which may be present/referenced in 10384 * more than one flow rule definition. 10385 * 10386 * see @RTE_FLOW_ACTION_TYPE_COUNT 10387 * 10388 * @param dev 10389 * Pointer to Ethernet device structure. 10390 * @param[in] handle 10391 * Handle for the indirect action to query. 10392 * @param[in, out] data 10393 * Pointer to storage for the associated query data type. 10394 * @param[out] error 10395 * Perform verbose error reporting if not NULL. PMDs initialize this 10396 * structure in case of error only. 10397 * 10398 * @return 10399 * 0 on success, a negative errno value otherwise and rte_errno is set. 10400 */ 10401 static int 10402 mlx5_action_handle_query(struct rte_eth_dev *dev, 10403 const struct rte_flow_action_handle *handle, 10404 void *data, 10405 struct rte_flow_error *error) 10406 { 10407 struct rte_flow_attr attr = { .transfer = 0 }; 10408 const struct mlx5_flow_driver_ops *fops = 10409 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10410 10411 return flow_drv_action_query(dev, handle, data, fops, error); 10412 } 10413 10414 static int 10415 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10416 struct rte_flow_action_handle *handle, 10417 const void *update, void *query, 10418 enum rte_flow_query_update_mode qu_mode, 10419 struct rte_flow_error *error) 10420 { 10421 struct rte_flow_attr attr = { .transfer = 0 }; 10422 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10423 const struct mlx5_flow_driver_ops *fops; 10424 10425 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10426 return rte_flow_error_set(error, ENOTSUP, 10427 RTE_FLOW_ERROR_TYPE_ACTION, 10428 NULL, "invalid driver type"); 10429 fops = flow_get_drv_ops(drv_type); 10430 if (!fops || !fops->action_query_update) 10431 return rte_flow_error_set(error, ENOTSUP, 10432 RTE_FLOW_ERROR_TYPE_ACTION, 10433 NULL, "no query_update handler"); 10434 return fops->action_query_update(dev, handle, update, 10435 query, qu_mode, error); 10436 } 10437 10438 10439 #define MLX5_DRV_FOPS_OR_ERR(dev, fops, drv_cb, ret) \ 10440 { \ 10441 struct rte_flow_attr attr = { .transfer = 0 }; \ 10442 enum mlx5_flow_drv_type drv_type = flow_get_drv_type((dev), &attr); \ 10443 if (drv_type == MLX5_FLOW_TYPE_MIN || \ 10444 drv_type == MLX5_FLOW_TYPE_MAX) { \ 10445 rte_flow_error_set(error, ENOTSUP, \ 10446 RTE_FLOW_ERROR_TYPE_ACTION, \ 10447 NULL, "invalid driver type"); \ 10448 return ret; \ 10449 } \ 10450 (fops) = flow_get_drv_ops(drv_type); \ 10451 if (!(fops) || !(fops)->drv_cb) { \ 10452 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, \ 10453 NULL, "no action_list handler"); \ 10454 return ret; \ 10455 } \ 10456 } 10457 10458 static struct rte_flow_action_list_handle * 10459 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 10460 const struct rte_flow_indir_action_conf *conf, 10461 const struct rte_flow_action *actions, 10462 struct rte_flow_error *error) 10463 { 10464 const struct mlx5_flow_driver_ops *fops; 10465 10466 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_create, NULL); 10467 return fops->action_list_handle_create(dev, conf, actions, error); 10468 } 10469 10470 static int 10471 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 10472 struct rte_flow_action_list_handle *handle, 10473 struct rte_flow_error *error) 10474 { 10475 const struct mlx5_flow_driver_ops *fops; 10476 10477 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_destroy, ENOTSUP); 10478 return fops->action_list_handle_destroy(dev, handle, error); 10479 } 10480 10481 static int 10482 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 10483 const 10484 struct rte_flow_action_list_handle *handle, 10485 const void **update, void **query, 10486 enum rte_flow_query_update_mode mode, 10487 struct rte_flow_error *error) 10488 { 10489 const struct mlx5_flow_driver_ops *fops; 10490 10491 MLX5_DRV_FOPS_OR_ERR(dev, fops, 10492 action_list_handle_query_update, ENOTSUP); 10493 return fops->action_list_handle_query_update(dev, handle, update, query, 10494 mode, error); 10495 } 10496 static int 10497 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 10498 const struct rte_flow_template_table *table, 10499 const struct rte_flow_item pattern[], 10500 uint8_t pattern_template_index, 10501 uint32_t *hash, struct rte_flow_error *error) 10502 { 10503 struct rte_flow_attr attr = { .transfer = 0 }; 10504 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10505 const struct mlx5_flow_driver_ops *fops; 10506 10507 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10508 return rte_flow_error_set(error, ENOTSUP, 10509 RTE_FLOW_ERROR_TYPE_ACTION, 10510 NULL, "invalid driver type"); 10511 fops = flow_get_drv_ops(drv_type); 10512 if (!fops || !fops->action_query_update) 10513 return rte_flow_error_set(error, ENOTSUP, 10514 RTE_FLOW_ERROR_TYPE_ACTION, 10515 NULL, "no query_update handler"); 10516 return fops->flow_calc_table_hash(dev, table, pattern, pattern_template_index, 10517 hash, error); 10518 } 10519 10520 static int 10521 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 10522 const struct rte_flow_item pattern[], 10523 enum rte_flow_encap_hash_field dest_field, 10524 uint8_t *hash, 10525 struct rte_flow_error *error) 10526 { 10527 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, NULL); 10528 const struct mlx5_flow_driver_ops *fops; 10529 10530 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10531 return rte_flow_error_set(error, ENOTSUP, 10532 RTE_FLOW_ERROR_TYPE_ACTION, 10533 NULL, "invalid driver type"); 10534 fops = flow_get_drv_ops(drv_type); 10535 if (!fops || !fops->flow_calc_encap_hash) 10536 return rte_flow_error_set(error, ENOTSUP, 10537 RTE_FLOW_ERROR_TYPE_ACTION, 10538 NULL, "no calc encap hash handler"); 10539 return fops->flow_calc_encap_hash(dev, pattern, dest_field, hash, error); 10540 } 10541 10542 /** 10543 * Destroy all indirect actions (shared RSS). 10544 * 10545 * @param dev 10546 * Pointer to Ethernet device. 10547 * 10548 * @return 10549 * 0 on success, a negative errno value otherwise and rte_errno is set. 10550 */ 10551 int 10552 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10553 { 10554 struct rte_flow_error error; 10555 struct mlx5_priv *priv = dev->data->dev_private; 10556 struct mlx5_shared_action_rss *shared_rss; 10557 int ret = 0; 10558 uint32_t idx; 10559 10560 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10561 priv->rss_shared_actions, idx, shared_rss, next) { 10562 ret |= mlx5_action_handle_destroy(dev, 10563 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10564 } 10565 return ret; 10566 } 10567 10568 /** 10569 * Validate existing indirect actions against current device configuration 10570 * and attach them to device resources. 10571 * 10572 * @param dev 10573 * Pointer to Ethernet device. 10574 * 10575 * @return 10576 * 0 on success, a negative errno value otherwise and rte_errno is set. 10577 */ 10578 int 10579 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10580 { 10581 struct mlx5_priv *priv = dev->data->dev_private; 10582 int ret = 0; 10583 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10584 10585 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10586 const char *message; 10587 uint32_t queue_idx; 10588 10589 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10590 ind_tbl->queues_n, 10591 &message, &queue_idx); 10592 if (ret != 0) { 10593 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10594 dev->data->port_id, ind_tbl->queues[queue_idx], 10595 message); 10596 break; 10597 } 10598 } 10599 if (ret != 0) 10600 return ret; 10601 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10602 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10603 if (ret != 0) { 10604 DRV_LOG(ERR, "Port %u could not attach " 10605 "indirection table obj %p", 10606 dev->data->port_id, (void *)ind_tbl); 10607 goto error; 10608 } 10609 } 10610 10611 return 0; 10612 error: 10613 ind_tbl_last = ind_tbl; 10614 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10615 if (ind_tbl == ind_tbl_last) 10616 break; 10617 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10618 DRV_LOG(CRIT, "Port %u could not detach " 10619 "indirection table obj %p on rollback", 10620 dev->data->port_id, (void *)ind_tbl); 10621 } 10622 return ret; 10623 } 10624 10625 /** 10626 * Detach indirect actions of the device from its resources. 10627 * 10628 * @param dev 10629 * Pointer to Ethernet device. 10630 * 10631 * @return 10632 * 0 on success, a negative errno value otherwise and rte_errno is set. 10633 */ 10634 int 10635 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10636 { 10637 struct mlx5_priv *priv = dev->data->dev_private; 10638 int ret = 0; 10639 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10640 10641 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10642 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10643 if (ret != 0) { 10644 DRV_LOG(ERR, "Port %u could not detach " 10645 "indirection table obj %p", 10646 dev->data->port_id, (void *)ind_tbl); 10647 goto error; 10648 } 10649 } 10650 return 0; 10651 error: 10652 ind_tbl_last = ind_tbl; 10653 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10654 if (ind_tbl == ind_tbl_last) 10655 break; 10656 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10657 DRV_LOG(CRIT, "Port %u could not attach " 10658 "indirection table obj %p on rollback", 10659 dev->data->port_id, (void *)ind_tbl); 10660 } 10661 return ret; 10662 } 10663 10664 #ifndef HAVE_MLX5DV_DR 10665 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10666 #else 10667 #define MLX5_DOMAIN_SYNC_FLOW \ 10668 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10669 #endif 10670 10671 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10672 { 10673 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10674 const struct mlx5_flow_driver_ops *fops; 10675 int ret; 10676 struct rte_flow_attr attr = { .transfer = 0 }; 10677 10678 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10679 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10680 if (ret > 0) 10681 ret = -ret; 10682 return ret; 10683 } 10684 10685 const struct mlx5_flow_tunnel * 10686 mlx5_get_tof(const struct rte_flow_item *item, 10687 const struct rte_flow_action *action, 10688 enum mlx5_tof_rule_type *rule_type) 10689 { 10690 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10691 if (item->type == (typeof(item->type)) 10692 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10693 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10694 return flow_items_to_tunnel(item); 10695 } 10696 } 10697 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10698 if (action->type == (typeof(action->type)) 10699 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10700 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10701 return flow_actions_to_tunnel(action); 10702 } 10703 } 10704 return NULL; 10705 } 10706 10707 /** 10708 * tunnel offload functionality is defined for DV environment only 10709 */ 10710 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10711 __extension__ 10712 union tunnel_offload_mark { 10713 uint32_t val; 10714 struct { 10715 uint32_t app_reserve:8; 10716 uint32_t table_id:15; 10717 uint32_t transfer:1; 10718 uint32_t _unused_:8; 10719 }; 10720 }; 10721 10722 static bool 10723 mlx5_access_tunnel_offload_db 10724 (struct rte_eth_dev *dev, 10725 bool (*match)(struct rte_eth_dev *, 10726 struct mlx5_flow_tunnel *, const void *), 10727 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10728 void (*miss)(struct rte_eth_dev *, void *), 10729 void *ctx, bool lock_op); 10730 10731 static int 10732 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10733 struct rte_flow *flow, 10734 const struct rte_flow_attr *attr, 10735 const struct rte_flow_action *app_actions, 10736 uint32_t flow_idx, 10737 const struct mlx5_flow_tunnel *tunnel, 10738 struct tunnel_default_miss_ctx *ctx, 10739 struct rte_flow_error *error) 10740 { 10741 struct mlx5_priv *priv = dev->data->dev_private; 10742 struct mlx5_flow *dev_flow; 10743 struct rte_flow_attr miss_attr = *attr; 10744 const struct rte_flow_item miss_items[2] = { 10745 { 10746 .type = RTE_FLOW_ITEM_TYPE_ETH, 10747 .spec = NULL, 10748 .last = NULL, 10749 .mask = NULL 10750 }, 10751 { 10752 .type = RTE_FLOW_ITEM_TYPE_END, 10753 .spec = NULL, 10754 .last = NULL, 10755 .mask = NULL 10756 } 10757 }; 10758 union tunnel_offload_mark mark_id; 10759 struct rte_flow_action_mark miss_mark; 10760 struct rte_flow_action miss_actions[3] = { 10761 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10762 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10763 }; 10764 const struct rte_flow_action_jump *jump_data; 10765 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10766 struct flow_grp_info grp_info = { 10767 .external = 1, 10768 .transfer = attr->transfer, 10769 .fdb_def_rule = !!priv->fdb_def_rule, 10770 .std_tbl_fix = 0, 10771 }; 10772 int ret; 10773 10774 if (!attr->transfer) { 10775 uint32_t q_size; 10776 10777 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10778 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10779 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10780 0, SOCKET_ID_ANY); 10781 if (!ctx->queue) 10782 return rte_flow_error_set 10783 (error, ENOMEM, 10784 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10785 NULL, "invalid default miss RSS"); 10786 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10787 ctx->action_rss.level = 0, 10788 ctx->action_rss.types = priv->rss_conf.rss_hf, 10789 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10790 ctx->action_rss.queue_num = priv->reta_idx_n, 10791 ctx->action_rss.key = priv->rss_conf.rss_key, 10792 ctx->action_rss.queue = ctx->queue; 10793 if (!priv->reta_idx_n || !priv->rxqs_n) 10794 return rte_flow_error_set 10795 (error, EINVAL, 10796 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10797 NULL, "invalid port configuration"); 10798 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10799 ctx->action_rss.types = 0; 10800 for (i = 0; i != priv->reta_idx_n; ++i) 10801 ctx->queue[i] = (*priv->reta_idx)[i]; 10802 } else { 10803 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10804 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10805 } 10806 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10807 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10808 jump_data = app_actions->conf; 10809 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10810 miss_attr.group = jump_data->group; 10811 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10812 &flow_table, &grp_info, error); 10813 if (ret) 10814 return rte_flow_error_set(error, EINVAL, 10815 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10816 NULL, "invalid tunnel id"); 10817 mark_id.app_reserve = 0; 10818 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10819 mark_id.transfer = !!attr->transfer; 10820 mark_id._unused_ = 0; 10821 miss_mark.id = mark_id.val; 10822 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10823 miss_items, miss_actions, flow_idx, error); 10824 if (!dev_flow) 10825 return -rte_errno; 10826 dev_flow->flow = flow; 10827 dev_flow->external = true; 10828 dev_flow->tunnel = tunnel; 10829 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10830 /* Subflow object was created, we must include one in the list. */ 10831 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10832 dev_flow->handle, next); 10833 DRV_LOG(DEBUG, 10834 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10835 dev->data->port_id, tunnel->app_tunnel.type, 10836 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10837 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10838 miss_actions, error); 10839 if (!ret) 10840 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10841 error); 10842 10843 return ret; 10844 } 10845 10846 static const struct mlx5_flow_tbl_data_entry * 10847 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10848 { 10849 struct mlx5_priv *priv = dev->data->dev_private; 10850 struct mlx5_dev_ctx_shared *sh = priv->sh; 10851 struct mlx5_list_entry *he; 10852 union tunnel_offload_mark mbits = { .val = mark }; 10853 union mlx5_flow_tbl_key table_key = { 10854 { 10855 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10856 .id = 0, 10857 .reserved = 0, 10858 .dummy = 0, 10859 .is_fdb = !!mbits.transfer, 10860 .is_egress = 0, 10861 } 10862 }; 10863 struct mlx5_flow_cb_ctx ctx = { 10864 .data = &table_key.v64, 10865 }; 10866 10867 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10868 return he ? 10869 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10870 } 10871 10872 static void 10873 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10874 struct mlx5_list_entry *entry) 10875 { 10876 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10877 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10878 10879 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10880 tunnel_flow_tbl_to_id(tte->flow_table)); 10881 mlx5_free(tte); 10882 } 10883 10884 static int 10885 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10886 struct mlx5_list_entry *entry, void *cb_ctx) 10887 { 10888 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10889 union tunnel_tbl_key tbl = { 10890 .val = *(uint64_t *)(ctx->data), 10891 }; 10892 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10893 10894 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10895 } 10896 10897 static struct mlx5_list_entry * 10898 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10899 { 10900 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10901 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10902 struct tunnel_tbl_entry *tte; 10903 union tunnel_tbl_key tbl = { 10904 .val = *(uint64_t *)(ctx->data), 10905 }; 10906 10907 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10908 sizeof(*tte), 0, 10909 SOCKET_ID_ANY); 10910 if (!tte) 10911 goto err; 10912 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10913 &tte->flow_table); 10914 if (tte->flow_table >= MLX5_MAX_TABLES) { 10915 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10916 tte->flow_table); 10917 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10918 tte->flow_table); 10919 goto err; 10920 } else if (!tte->flow_table) { 10921 goto err; 10922 } 10923 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10924 tte->tunnel_id = tbl.tunnel_id; 10925 tte->group = tbl.group; 10926 return &tte->hash; 10927 err: 10928 if (tte) 10929 mlx5_free(tte); 10930 return NULL; 10931 } 10932 10933 static struct mlx5_list_entry * 10934 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10935 struct mlx5_list_entry *oentry, 10936 void *cb_ctx __rte_unused) 10937 { 10938 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10939 0, SOCKET_ID_ANY); 10940 10941 if (!tte) 10942 return NULL; 10943 memcpy(tte, oentry, sizeof(*tte)); 10944 return &tte->hash; 10945 } 10946 10947 static void 10948 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10949 struct mlx5_list_entry *entry) 10950 { 10951 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10952 10953 mlx5_free(tte); 10954 } 10955 10956 static uint32_t 10957 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10958 const struct mlx5_flow_tunnel *tunnel, 10959 uint32_t group, uint32_t *table, 10960 struct rte_flow_error *error) 10961 { 10962 struct mlx5_list_entry *he; 10963 struct tunnel_tbl_entry *tte; 10964 union tunnel_tbl_key key = { 10965 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10966 .group = group 10967 }; 10968 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10969 struct mlx5_hlist *group_hash; 10970 struct mlx5_flow_cb_ctx ctx = { 10971 .data = &key.val, 10972 }; 10973 10974 group_hash = tunnel ? tunnel->groups : thub->groups; 10975 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10976 if (!he) 10977 return rte_flow_error_set(error, EINVAL, 10978 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10979 NULL, 10980 "tunnel group index not supported"); 10981 tte = container_of(he, typeof(*tte), hash); 10982 *table = tte->flow_table; 10983 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10984 dev->data->port_id, key.tunnel_id, group, *table); 10985 return 0; 10986 } 10987 10988 static void 10989 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10990 struct mlx5_flow_tunnel *tunnel) 10991 { 10992 struct mlx5_priv *priv = dev->data->dev_private; 10993 struct mlx5_indexed_pool *ipool; 10994 10995 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 10996 dev->data->port_id, tunnel->tunnel_id); 10997 LIST_REMOVE(tunnel, chain); 10998 mlx5_hlist_destroy(tunnel->groups); 10999 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11000 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11001 } 11002 11003 static bool 11004 mlx5_access_tunnel_offload_db 11005 (struct rte_eth_dev *dev, 11006 bool (*match)(struct rte_eth_dev *, 11007 struct mlx5_flow_tunnel *, const void *), 11008 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11009 void (*miss)(struct rte_eth_dev *, void *), 11010 void *ctx, bool lock_op) 11011 { 11012 bool verdict = false; 11013 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11014 struct mlx5_flow_tunnel *tunnel; 11015 11016 rte_spinlock_lock(&thub->sl); 11017 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11018 verdict = match(dev, tunnel, (const void *)ctx); 11019 if (verdict) 11020 break; 11021 } 11022 if (!lock_op) 11023 rte_spinlock_unlock(&thub->sl); 11024 if (verdict && hit) 11025 hit(dev, tunnel, ctx); 11026 if (!verdict && miss) 11027 miss(dev, ctx); 11028 if (lock_op) 11029 rte_spinlock_unlock(&thub->sl); 11030 11031 return verdict; 11032 } 11033 11034 struct tunnel_db_find_tunnel_id_ctx { 11035 uint32_t tunnel_id; 11036 struct mlx5_flow_tunnel *tunnel; 11037 }; 11038 11039 static bool 11040 find_tunnel_id_match(struct rte_eth_dev *dev, 11041 struct mlx5_flow_tunnel *tunnel, const void *x) 11042 { 11043 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11044 11045 RTE_SET_USED(dev); 11046 return tunnel->tunnel_id == ctx->tunnel_id; 11047 } 11048 11049 static void 11050 find_tunnel_id_hit(struct rte_eth_dev *dev, 11051 struct mlx5_flow_tunnel *tunnel, void *x) 11052 { 11053 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11054 RTE_SET_USED(dev); 11055 ctx->tunnel = tunnel; 11056 } 11057 11058 static struct mlx5_flow_tunnel * 11059 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11060 { 11061 struct tunnel_db_find_tunnel_id_ctx ctx = { 11062 .tunnel_id = id, 11063 }; 11064 11065 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11066 find_tunnel_id_hit, NULL, &ctx, true); 11067 11068 return ctx.tunnel; 11069 } 11070 11071 static struct mlx5_flow_tunnel * 11072 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11073 const struct rte_flow_tunnel *app_tunnel) 11074 { 11075 struct mlx5_priv *priv = dev->data->dev_private; 11076 struct mlx5_indexed_pool *ipool; 11077 struct mlx5_flow_tunnel *tunnel; 11078 uint32_t id; 11079 11080 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11081 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11082 if (!tunnel) 11083 return NULL; 11084 if (id >= MLX5_MAX_TUNNELS) { 11085 mlx5_ipool_free(ipool, id); 11086 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11087 return NULL; 11088 } 11089 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11090 priv->sh, 11091 mlx5_flow_tunnel_grp2tbl_create_cb, 11092 mlx5_flow_tunnel_grp2tbl_match_cb, 11093 mlx5_flow_tunnel_grp2tbl_remove_cb, 11094 mlx5_flow_tunnel_grp2tbl_clone_cb, 11095 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11096 if (!tunnel->groups) { 11097 mlx5_ipool_free(ipool, id); 11098 return NULL; 11099 } 11100 /* initiate new PMD tunnel */ 11101 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11102 tunnel->tunnel_id = id; 11103 tunnel->action.type = (typeof(tunnel->action.type)) 11104 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11105 tunnel->action.conf = tunnel; 11106 tunnel->item.type = (typeof(tunnel->item.type)) 11107 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11108 tunnel->item.spec = tunnel; 11109 tunnel->item.last = NULL; 11110 tunnel->item.mask = NULL; 11111 11112 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11113 dev->data->port_id, tunnel->tunnel_id); 11114 11115 return tunnel; 11116 } 11117 11118 struct tunnel_db_get_tunnel_ctx { 11119 const struct rte_flow_tunnel *app_tunnel; 11120 struct mlx5_flow_tunnel *tunnel; 11121 }; 11122 11123 static bool get_tunnel_match(struct rte_eth_dev *dev, 11124 struct mlx5_flow_tunnel *tunnel, const void *x) 11125 { 11126 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11127 11128 RTE_SET_USED(dev); 11129 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11130 sizeof(*ctx->app_tunnel)); 11131 } 11132 11133 static void get_tunnel_hit(struct rte_eth_dev *dev, 11134 struct mlx5_flow_tunnel *tunnel, void *x) 11135 { 11136 /* called under tunnel spinlock protection */ 11137 struct tunnel_db_get_tunnel_ctx *ctx = x; 11138 11139 RTE_SET_USED(dev); 11140 tunnel->refctn++; 11141 ctx->tunnel = tunnel; 11142 } 11143 11144 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11145 { 11146 /* called under tunnel spinlock protection */ 11147 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11148 struct tunnel_db_get_tunnel_ctx *ctx = x; 11149 11150 rte_spinlock_unlock(&thub->sl); 11151 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11152 rte_spinlock_lock(&thub->sl); 11153 if (ctx->tunnel) { 11154 ctx->tunnel->refctn = 1; 11155 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11156 } 11157 } 11158 11159 11160 static int 11161 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11162 const struct rte_flow_tunnel *app_tunnel, 11163 struct mlx5_flow_tunnel **tunnel) 11164 { 11165 struct tunnel_db_get_tunnel_ctx ctx = { 11166 .app_tunnel = app_tunnel, 11167 }; 11168 11169 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11170 get_tunnel_miss, &ctx, true); 11171 *tunnel = ctx.tunnel; 11172 return ctx.tunnel ? 0 : -ENOMEM; 11173 } 11174 11175 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11176 { 11177 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11178 11179 if (!thub) 11180 return; 11181 if (!LIST_EMPTY(&thub->tunnels)) 11182 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11183 mlx5_hlist_destroy(thub->groups); 11184 mlx5_free(thub); 11185 } 11186 11187 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11188 { 11189 int err; 11190 struct mlx5_flow_tunnel_hub *thub; 11191 11192 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11193 0, SOCKET_ID_ANY); 11194 if (!thub) 11195 return -ENOMEM; 11196 LIST_INIT(&thub->tunnels); 11197 rte_spinlock_init(&thub->sl); 11198 thub->groups = mlx5_hlist_create("flow groups", 64, 11199 false, true, sh, 11200 mlx5_flow_tunnel_grp2tbl_create_cb, 11201 mlx5_flow_tunnel_grp2tbl_match_cb, 11202 mlx5_flow_tunnel_grp2tbl_remove_cb, 11203 mlx5_flow_tunnel_grp2tbl_clone_cb, 11204 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11205 if (!thub->groups) { 11206 err = -rte_errno; 11207 goto err; 11208 } 11209 sh->tunnel_hub = thub; 11210 11211 return 0; 11212 11213 err: 11214 if (thub->groups) 11215 mlx5_hlist_destroy(thub->groups); 11216 if (thub) 11217 mlx5_free(thub); 11218 return err; 11219 } 11220 11221 static inline int 11222 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11223 struct rte_flow_tunnel *tunnel, 11224 struct rte_flow_error *error) 11225 { 11226 struct mlx5_priv *priv = dev->data->dev_private; 11227 11228 if (!priv->sh->config.dv_flow_en) 11229 return rte_flow_error_set(error, ENOTSUP, 11230 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11231 "flow DV interface is off"); 11232 if (!is_tunnel_offload_active(dev)) 11233 return rte_flow_error_set(error, ENOTSUP, 11234 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11235 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11236 if (!tunnel) 11237 return rte_flow_error_set(error, EINVAL, 11238 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11239 "no application tunnel"); 11240 switch (tunnel->type) { 11241 default: 11242 return rte_flow_error_set(error, EINVAL, 11243 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11244 "unsupported tunnel type"); 11245 case RTE_FLOW_ITEM_TYPE_VXLAN: 11246 case RTE_FLOW_ITEM_TYPE_GRE: 11247 case RTE_FLOW_ITEM_TYPE_NVGRE: 11248 case RTE_FLOW_ITEM_TYPE_GENEVE: 11249 break; 11250 } 11251 return 0; 11252 } 11253 11254 static int 11255 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11256 struct rte_flow_tunnel *app_tunnel, 11257 struct rte_flow_action **actions, 11258 uint32_t *num_of_actions, 11259 struct rte_flow_error *error) 11260 { 11261 struct mlx5_flow_tunnel *tunnel; 11262 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11263 11264 if (ret) 11265 return ret; 11266 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11267 if (ret < 0) { 11268 return rte_flow_error_set(error, ret, 11269 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11270 "failed to initialize pmd tunnel"); 11271 } 11272 *actions = &tunnel->action; 11273 *num_of_actions = 1; 11274 return 0; 11275 } 11276 11277 static int 11278 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11279 struct rte_flow_tunnel *app_tunnel, 11280 struct rte_flow_item **items, 11281 uint32_t *num_of_items, 11282 struct rte_flow_error *error) 11283 { 11284 struct mlx5_flow_tunnel *tunnel; 11285 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11286 11287 if (ret) 11288 return ret; 11289 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11290 if (ret < 0) { 11291 return rte_flow_error_set(error, ret, 11292 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11293 "failed to initialize pmd tunnel"); 11294 } 11295 *items = &tunnel->item; 11296 *num_of_items = 1; 11297 return 0; 11298 } 11299 11300 struct tunnel_db_element_release_ctx { 11301 struct rte_flow_item *items; 11302 struct rte_flow_action *actions; 11303 uint32_t num_elements; 11304 struct rte_flow_error *error; 11305 int ret; 11306 }; 11307 11308 static bool 11309 tunnel_element_release_match(struct rte_eth_dev *dev, 11310 struct mlx5_flow_tunnel *tunnel, const void *x) 11311 { 11312 const struct tunnel_db_element_release_ctx *ctx = x; 11313 11314 RTE_SET_USED(dev); 11315 if (ctx->num_elements != 1) 11316 return false; 11317 else if (ctx->items) 11318 return ctx->items == &tunnel->item; 11319 else if (ctx->actions) 11320 return ctx->actions == &tunnel->action; 11321 11322 return false; 11323 } 11324 11325 static void 11326 tunnel_element_release_hit(struct rte_eth_dev *dev, 11327 struct mlx5_flow_tunnel *tunnel, void *x) 11328 { 11329 struct tunnel_db_element_release_ctx *ctx = x; 11330 ctx->ret = 0; 11331 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 11332 mlx5_flow_tunnel_free(dev, tunnel); 11333 } 11334 11335 static void 11336 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11337 { 11338 struct tunnel_db_element_release_ctx *ctx = x; 11339 RTE_SET_USED(dev); 11340 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11341 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11342 "invalid argument"); 11343 } 11344 11345 static int 11346 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11347 struct rte_flow_item *pmd_items, 11348 uint32_t num_items, struct rte_flow_error *err) 11349 { 11350 struct tunnel_db_element_release_ctx ctx = { 11351 .items = pmd_items, 11352 .actions = NULL, 11353 .num_elements = num_items, 11354 .error = err, 11355 }; 11356 11357 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11358 tunnel_element_release_hit, 11359 tunnel_element_release_miss, &ctx, false); 11360 11361 return ctx.ret; 11362 } 11363 11364 static int 11365 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11366 struct rte_flow_action *pmd_actions, 11367 uint32_t num_actions, struct rte_flow_error *err) 11368 { 11369 struct tunnel_db_element_release_ctx ctx = { 11370 .items = NULL, 11371 .actions = pmd_actions, 11372 .num_elements = num_actions, 11373 .error = err, 11374 }; 11375 11376 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11377 tunnel_element_release_hit, 11378 tunnel_element_release_miss, &ctx, false); 11379 11380 return ctx.ret; 11381 } 11382 11383 static int 11384 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11385 struct rte_mbuf *m, 11386 struct rte_flow_restore_info *info, 11387 struct rte_flow_error *err) 11388 { 11389 uint64_t ol_flags = m->ol_flags; 11390 const struct mlx5_flow_tbl_data_entry *tble; 11391 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11392 struct mlx5_priv *priv = dev->data->dev_private; 11393 11394 if (priv->tunnel_enabled == 0) 11395 goto err; 11396 if ((ol_flags & mask) != mask) 11397 goto err; 11398 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11399 if (!tble) { 11400 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11401 dev->data->port_id, m->hash.fdir.hi); 11402 goto err; 11403 } 11404 MLX5_ASSERT(tble->tunnel); 11405 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11406 info->group_id = tble->group_id; 11407 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11408 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11409 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11410 11411 return 0; 11412 11413 err: 11414 return rte_flow_error_set(err, EINVAL, 11415 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11416 "failed to get restore info"); 11417 } 11418 11419 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11420 static int 11421 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11422 __rte_unused struct rte_flow_tunnel *app_tunnel, 11423 __rte_unused struct rte_flow_action **actions, 11424 __rte_unused uint32_t *num_of_actions, 11425 __rte_unused struct rte_flow_error *error) 11426 { 11427 return -ENOTSUP; 11428 } 11429 11430 static int 11431 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11432 __rte_unused struct rte_flow_tunnel *app_tunnel, 11433 __rte_unused struct rte_flow_item **items, 11434 __rte_unused uint32_t *num_of_items, 11435 __rte_unused struct rte_flow_error *error) 11436 { 11437 return -ENOTSUP; 11438 } 11439 11440 static int 11441 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11442 __rte_unused struct rte_flow_item *pmd_items, 11443 __rte_unused uint32_t num_items, 11444 __rte_unused struct rte_flow_error *err) 11445 { 11446 return -ENOTSUP; 11447 } 11448 11449 static int 11450 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11451 __rte_unused struct rte_flow_action *pmd_action, 11452 __rte_unused uint32_t num_actions, 11453 __rte_unused struct rte_flow_error *err) 11454 { 11455 return -ENOTSUP; 11456 } 11457 11458 static int 11459 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11460 __rte_unused struct rte_mbuf *m, 11461 __rte_unused struct rte_flow_restore_info *i, 11462 __rte_unused struct rte_flow_error *err) 11463 { 11464 return -ENOTSUP; 11465 } 11466 11467 static int 11468 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11469 __rte_unused struct rte_flow *flow, 11470 __rte_unused const struct rte_flow_attr *attr, 11471 __rte_unused const struct rte_flow_action *actions, 11472 __rte_unused uint32_t flow_idx, 11473 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11474 __rte_unused struct tunnel_default_miss_ctx *ctx, 11475 __rte_unused struct rte_flow_error *error) 11476 { 11477 return -ENOTSUP; 11478 } 11479 11480 static struct mlx5_flow_tunnel * 11481 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11482 __rte_unused uint32_t id) 11483 { 11484 return NULL; 11485 } 11486 11487 static void 11488 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11489 __rte_unused struct mlx5_flow_tunnel *tunnel) 11490 { 11491 } 11492 11493 static uint32_t 11494 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11495 __rte_unused const struct mlx5_flow_tunnel *t, 11496 __rte_unused uint32_t group, 11497 __rte_unused uint32_t *table, 11498 struct rte_flow_error *error) 11499 { 11500 return rte_flow_error_set(error, ENOTSUP, 11501 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11502 "tunnel offload requires DV support"); 11503 } 11504 11505 void 11506 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11507 __rte_unused uint16_t port_id) 11508 { 11509 } 11510 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11511 11512 /* Flex flow item API */ 11513 static struct rte_flow_item_flex_handle * 11514 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11515 const struct rte_flow_item_flex_conf *conf, 11516 struct rte_flow_error *error) 11517 { 11518 static const char err_msg[] = "flex item creation unsupported"; 11519 struct mlx5_priv *priv = dev->data->dev_private; 11520 struct rte_flow_attr attr = { .transfer = 0 }; 11521 const struct mlx5_flow_driver_ops *fops = 11522 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11523 11524 if (!priv->pci_dev) { 11525 rte_flow_error_set(error, ENOTSUP, 11526 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11527 "create flex item on PF only"); 11528 return NULL; 11529 } 11530 switch (priv->pci_dev->id.device_id) { 11531 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11532 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11533 break; 11534 default: 11535 rte_flow_error_set(error, ENOTSUP, 11536 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11537 "flex item available on BlueField ports only"); 11538 return NULL; 11539 } 11540 if (!fops->item_create) { 11541 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11542 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11543 NULL, err_msg); 11544 return NULL; 11545 } 11546 return fops->item_create(dev, conf, error); 11547 } 11548 11549 static int 11550 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11551 const struct rte_flow_item_flex_handle *handle, 11552 struct rte_flow_error *error) 11553 { 11554 static const char err_msg[] = "flex item release unsupported"; 11555 struct rte_flow_attr attr = { .transfer = 0 }; 11556 const struct mlx5_flow_driver_ops *fops = 11557 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11558 11559 if (!fops->item_release) { 11560 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11561 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11562 NULL, err_msg); 11563 return -rte_errno; 11564 } 11565 return fops->item_release(dev, handle, error); 11566 } 11567 11568 static void 11569 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11570 { 11571 int ret; 11572 struct rte_flow_error error; 11573 11574 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11575 char *item_name; 11576 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11577 sizeof(item_name), 11578 (void *)(uintptr_t)item->type, &error); 11579 if (ret > 0) 11580 printf("%s ", item_name); 11581 else 11582 printf("%d\n", (int)item->type); 11583 } 11584 printf("END\n"); 11585 } 11586 11587 static int 11588 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11589 { 11590 const struct rte_flow_item_udp *spec = udp_item->spec; 11591 const struct rte_flow_item_udp *mask = udp_item->mask; 11592 uint16_t udp_dport = 0; 11593 11594 if (spec != NULL) { 11595 if (!mask) 11596 mask = &rte_flow_item_udp_mask; 11597 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11598 mask->hdr.dst_port); 11599 } 11600 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11601 } 11602 11603 static const struct mlx5_flow_expand_node * 11604 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11605 unsigned int item_idx, 11606 const struct mlx5_flow_expand_node graph[], 11607 const struct mlx5_flow_expand_node *node) 11608 { 11609 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11610 11611 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11612 node != NULL && 11613 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11614 /* 11615 * The expansion node is VXLAN and it is also the last 11616 * expandable item in the pattern, so need to continue 11617 * expansion of the inner tunnel. 11618 */ 11619 MLX5_ASSERT(item_idx > 0); 11620 prev_item = pattern + item_idx - 1; 11621 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11622 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11623 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11624 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11625 } 11626 return node; 11627 } 11628 11629 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11630 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11631 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11632 }; 11633 11634 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11635 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11636 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11637 { 9, 10, 11 }, { 12, 13, 14 }, 11638 }; 11639 11640 /** 11641 * Discover the number of available flow priorities. 11642 * 11643 * @param dev 11644 * Ethernet device. 11645 * 11646 * @return 11647 * On success, number of available flow priorities. 11648 * On failure, a negative errno-style code and rte_errno is set. 11649 */ 11650 int 11651 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11652 { 11653 static const uint16_t vprio[] = {8, 16}; 11654 const struct mlx5_priv *priv = dev->data->dev_private; 11655 const struct mlx5_flow_driver_ops *fops; 11656 enum mlx5_flow_drv_type type; 11657 int ret; 11658 11659 type = mlx5_flow_os_get_type(); 11660 if (type == MLX5_FLOW_TYPE_MAX) { 11661 type = MLX5_FLOW_TYPE_VERBS; 11662 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11663 type = MLX5_FLOW_TYPE_DV; 11664 } 11665 fops = flow_get_drv_ops(type); 11666 if (fops->discover_priorities == NULL) { 11667 DRV_LOG(ERR, "Priority discovery not supported"); 11668 rte_errno = ENOTSUP; 11669 return -rte_errno; 11670 } 11671 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11672 if (ret < 0) 11673 return ret; 11674 switch (ret) { 11675 case 8: 11676 ret = RTE_DIM(priority_map_3); 11677 break; 11678 case 16: 11679 ret = RTE_DIM(priority_map_5); 11680 break; 11681 default: 11682 rte_errno = ENOTSUP; 11683 DRV_LOG(ERR, 11684 "port %u maximum priority: %d expected 8/16", 11685 dev->data->port_id, ret); 11686 return -rte_errno; 11687 } 11688 DRV_LOG(INFO, "port %u supported flow priorities:" 11689 " 0-%d for ingress or egress root table," 11690 " 0-%d for non-root table or transfer root table.", 11691 dev->data->port_id, ret - 2, 11692 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11693 return ret; 11694 } 11695 11696 /** 11697 * Adjust flow priority based on the highest layer and the request priority. 11698 * 11699 * @param[in] dev 11700 * Pointer to the Ethernet device structure. 11701 * @param[in] priority 11702 * The rule base priority. 11703 * @param[in] subpriority 11704 * The priority based on the items. 11705 * 11706 * @return 11707 * The new priority. 11708 */ 11709 uint32_t 11710 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11711 uint32_t subpriority) 11712 { 11713 uint32_t res = 0; 11714 struct mlx5_priv *priv = dev->data->dev_private; 11715 11716 switch (priv->sh->flow_max_priority) { 11717 case RTE_DIM(priority_map_3): 11718 res = priority_map_3[priority][subpriority]; 11719 break; 11720 case RTE_DIM(priority_map_5): 11721 res = priority_map_5[priority][subpriority]; 11722 break; 11723 } 11724 return res; 11725 } 11726 11727 /** 11728 * Get the priority for sending traffic to kernel table. 11729 * 11730 * @param[in] dev 11731 * Pointer to the Ethernet device structure. 11732 * 11733 * @return 11734 * On success: the value of priority for sending traffic to kernel table 11735 * On failure: -1 11736 */ 11737 uint32_t 11738 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11739 { 11740 struct mlx5_priv *priv = dev->data->dev_private; 11741 uint32_t res; 11742 11743 switch (priv->sh->flow_max_priority) { 11744 case RTE_DIM(priority_map_5): 11745 res = 15; 11746 break; 11747 case RTE_DIM(priority_map_3): 11748 res = 7; 11749 break; 11750 default: 11751 DRV_LOG(ERR, 11752 "port %u maximum priority: %d expected 8/16", 11753 dev->data->port_id, priv->sh->flow_max_priority); 11754 res = (uint32_t)-1; 11755 } 11756 return res; 11757 } 11758 11759 /** 11760 * Get the E-Switch Manager vport id. 11761 * 11762 * @param[in] dev 11763 * Pointer to the Ethernet device structure. 11764 * 11765 * @return 11766 * The vport id. 11767 */ 11768 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11769 { 11770 struct mlx5_priv *priv = dev->data->dev_private; 11771 struct mlx5_common_device *cdev = priv->sh->cdev; 11772 11773 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11774 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11775 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11776 11777 if (priv->pci_dev == NULL) 11778 return 0; 11779 switch (priv->pci_dev->id.device_id) { 11780 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 11781 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11782 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11783 /* 11784 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11785 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11786 * ECPF vport ID is selected and not the host port (0) in any BF case. 11787 */ 11788 return (int16_t)MLX5_ECPF_VPORT_ID; 11789 default: 11790 return MLX5_PF_VPORT_ID; 11791 } 11792 } 11793 11794 /** 11795 * Parse item to get the vport id. 11796 * 11797 * @param[in] dev 11798 * Pointer to the Ethernet device structure. 11799 * @param[in] item 11800 * The src port id match item. 11801 * @param[out] vport_id 11802 * Pointer to put the vport id. 11803 * @param[out] all_ports 11804 * Indicate if the item matches all ports. 11805 * @param[out] error 11806 * Pointer to error structure. 11807 * 11808 * @return 11809 * 0 on success, a negative errno value otherwise and rte_errno is set. 11810 */ 11811 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11812 const struct rte_flow_item *item, 11813 uint16_t *vport_id, 11814 bool *all_ports, 11815 struct rte_flow_error *error) 11816 { 11817 struct mlx5_priv *port_priv; 11818 const struct rte_flow_item_port_id *pid_v = NULL; 11819 const struct rte_flow_item_ethdev *dev_v = NULL; 11820 uint32_t esw_mgr_port; 11821 uint32_t src_port; 11822 11823 if (all_ports) 11824 *all_ports = false; 11825 switch (item->type) { 11826 case RTE_FLOW_ITEM_TYPE_PORT_ID: 11827 pid_v = item->spec; 11828 if (!pid_v) 11829 return 0; 11830 src_port = pid_v->id; 11831 esw_mgr_port = MLX5_PORT_ESW_MGR; 11832 break; 11833 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 11834 dev_v = item->spec; 11835 if (!dev_v) { 11836 if (all_ports) 11837 *all_ports = true; 11838 return 0; 11839 } 11840 src_port = dev_v->port_id; 11841 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11842 break; 11843 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 11844 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11845 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11846 break; 11847 default: 11848 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11849 NULL, "Incorrect item type."); 11850 } 11851 if (src_port == esw_mgr_port) { 11852 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11853 } else { 11854 port_priv = mlx5_port_to_eswitch_info(src_port, false); 11855 if (!port_priv) 11856 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11857 NULL, "Failed to get port info."); 11858 *vport_id = port_priv->representor_id; 11859 } 11860 11861 return 0; 11862 } 11863 11864 int 11865 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11866 uint16_t *proxy_port_id, 11867 struct rte_flow_error *error) 11868 { 11869 const struct mlx5_priv *priv = dev->data->dev_private; 11870 uint16_t port_id; 11871 11872 if (!priv->sh->config.dv_esw_en) 11873 return rte_flow_error_set(error, EINVAL, 11874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11875 NULL, 11876 "unable to provide a proxy port" 11877 " without E-Switch configured"); 11878 if (!priv->master && !priv->representor) 11879 return rte_flow_error_set(error, EINVAL, 11880 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11881 NULL, 11882 "unable to provide a proxy port" 11883 " for port which is not a master" 11884 " or a representor port"); 11885 if (priv->master) { 11886 *proxy_port_id = dev->data->port_id; 11887 return 0; 11888 } 11889 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11890 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11891 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11892 11893 if (port_priv->master && 11894 port_priv->domain_id == priv->domain_id) { 11895 *proxy_port_id = port_id; 11896 return 0; 11897 } 11898 } 11899 return rte_flow_error_set(error, ENODEV, 11900 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11901 NULL, "unable to find a proxy port"); 11902 } 11903 11904 /** 11905 * Discover IPv6 traffic class ID support in rdma-core and firmware. 11906 * 11907 * @param dev 11908 * Ethernet device. 11909 * 11910 * @return 11911 * 0, rdma-core is good to work with firmware. 11912 * -EOPNOTSUPP, rdma-core could not work with new IPv6 TC ID. 11913 */ 11914 int 11915 mlx5_flow_discover_ipv6_tc_support(struct rte_eth_dev *dev) 11916 { 11917 struct rte_flow_action_set_dscp set_dscp; 11918 struct rte_flow_attr attr; 11919 struct rte_flow_action actions[2]; 11920 struct rte_flow_item items[3]; 11921 struct rte_flow_error error; 11922 uint32_t flow_idx; 11923 11924 memset(&attr, 0, sizeof(attr)); 11925 memset(actions, 0, sizeof(actions)); 11926 memset(items, 0, sizeof(items)); 11927 attr.group = 1; 11928 attr.egress = 1; 11929 items[0].type = RTE_FLOW_ITEM_TYPE_ETH; 11930 items[1].type = RTE_FLOW_ITEM_TYPE_IPV6; 11931 items[2].type = RTE_FLOW_ITEM_TYPE_END; 11932 /* Random value */ 11933 set_dscp.dscp = 9; 11934 actions[0].type = RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP; 11935 actions[0].conf = &set_dscp; 11936 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 11937 11938 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, items, actions, true, &error); 11939 if (!flow_idx) 11940 return -EOPNOTSUPP; 11941 11942 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 11943 return 0; 11944 } 11945