1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 struct tunnel_default_miss_ctx { 43 uint16_t *queue; 44 __extension__ 45 union { 46 struct rte_flow_action_rss action_rss; 47 struct rte_flow_action_queue miss_queue; 48 struct rte_flow_action_jump miss_jump; 49 uint8_t raw[0]; 50 }; 51 }; 52 53 void 54 mlx5_indirect_list_handles_release(struct rte_eth_dev *dev) 55 { 56 struct mlx5_priv *priv = dev->data->dev_private; 57 #ifdef HAVE_MLX5_HWS_SUPPORT 58 struct rte_flow_error error; 59 #endif 60 61 while (!LIST_EMPTY(&priv->indirect_list_head)) { 62 struct mlx5_indirect_list *e = 63 LIST_FIRST(&priv->indirect_list_head); 64 65 LIST_REMOVE(e, entry); 66 switch (e->type) { 67 #ifdef HAVE_MLX5_HWS_SUPPORT 68 case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR: 69 mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)e); 70 break; 71 case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY: 72 mlx5_destroy_legacy_indirect(dev, e); 73 break; 74 case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT: 75 mlx5_reformat_action_destroy(dev, 76 (struct rte_flow_action_list_handle *)e, &error); 77 break; 78 #endif 79 default: 80 DRV_LOG(ERR, "invalid indirect list type"); 81 MLX5_ASSERT(false); 82 break; 83 } 84 } 85 } 86 87 static int 88 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 89 struct rte_flow *flow, 90 const struct rte_flow_attr *attr, 91 const struct rte_flow_action *app_actions, 92 uint32_t flow_idx, 93 const struct mlx5_flow_tunnel *tunnel, 94 struct tunnel_default_miss_ctx *ctx, 95 struct rte_flow_error *error); 96 static struct mlx5_flow_tunnel * 97 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 98 static void 99 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 100 static uint32_t 101 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 102 const struct mlx5_flow_tunnel *tunnel, 103 uint32_t group, uint32_t *table, 104 struct rte_flow_error *error); 105 106 /** Device flow drivers. */ 107 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 108 109 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 110 111 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 112 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 113 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 114 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 115 #endif 116 #ifdef HAVE_MLX5_HWS_SUPPORT 117 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 118 #endif 119 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 120 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 121 }; 122 123 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 124 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 125 (const int []){ \ 126 __VA_ARGS__, 0, \ 127 } 128 129 /** Node object of input graph for mlx5_flow_expand_rss(). */ 130 struct mlx5_flow_expand_node { 131 const int *const next; 132 /**< 133 * List of next node indexes. Index 0 is interpreted as a terminator. 134 */ 135 const enum rte_flow_item_type type; 136 /**< Pattern item type of current node. */ 137 uint64_t rss_types; 138 /**< 139 * RSS types bit-field associated with this node 140 * (see RTE_ETH_RSS_* definitions). 141 */ 142 uint64_t node_flags; 143 /**< 144 * Bit-fields that define how the node is used in the expansion. 145 * (see MLX5_EXPANSION_NODE_* definitions). 146 */ 147 }; 148 149 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 150 struct mlx5_flow_expand_sqn { 151 uint32_t entries; /** Number of entries */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 /* Optional expand field. The expansion alg will not go deeper. */ 159 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 160 161 /* The node is not added implicitly as expansion to the flow pattern. 162 * If the node type does not match the flow pattern item type, the 163 * expansion alg will go deeper to its next items. 164 * In the current implementation, the list of next nodes indexes can 165 * have up to one node with this flag set and it has to be the last 166 * node index (before the list terminator). 167 */ 168 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 169 170 /** Object returned by mlx5_flow_expand_rss(). */ 171 struct mlx5_flow_expand_rss { 172 uint32_t entries; 173 /**< Number of entries @p patterns and @p priorities. */ 174 struct { 175 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 176 uint32_t priority; /**< Priority offset for each expansion. */ 177 } entry[]; 178 }; 179 180 static void 181 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 182 183 static const struct mlx5_flow_expand_node * 184 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 185 unsigned int item_idx, 186 const struct mlx5_flow_expand_node graph[], 187 const struct mlx5_flow_expand_node *node); 188 189 static __rte_always_inline int 190 mlx5_need_cache_flow(const struct mlx5_priv *priv, 191 const struct rte_flow_attr *attr) 192 { 193 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 194 (attr ? !attr->group : true) && 195 priv->mode_info.mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY && 196 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 197 } 198 199 static bool 200 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 201 { 202 switch (item->type) { 203 case RTE_FLOW_ITEM_TYPE_ETH: 204 case RTE_FLOW_ITEM_TYPE_VLAN: 205 case RTE_FLOW_ITEM_TYPE_IPV4: 206 case RTE_FLOW_ITEM_TYPE_IPV6: 207 case RTE_FLOW_ITEM_TYPE_UDP: 208 case RTE_FLOW_ITEM_TYPE_TCP: 209 case RTE_FLOW_ITEM_TYPE_ESP: 210 case RTE_FLOW_ITEM_TYPE_ICMP: 211 case RTE_FLOW_ITEM_TYPE_ICMP6: 212 case RTE_FLOW_ITEM_TYPE_VXLAN: 213 case RTE_FLOW_ITEM_TYPE_NVGRE: 214 case RTE_FLOW_ITEM_TYPE_GRE: 215 case RTE_FLOW_ITEM_TYPE_GENEVE: 216 case RTE_FLOW_ITEM_TYPE_MPLS: 217 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 218 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 219 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 220 case RTE_FLOW_ITEM_TYPE_GTP: 221 return true; 222 default: 223 break; 224 } 225 return false; 226 } 227 228 /** 229 * Network Service Header (NSH) and its next protocol values 230 * are described in RFC-8393. 231 */ 232 static enum rte_flow_item_type 233 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 234 { 235 enum rte_flow_item_type type; 236 237 switch (proto_mask & proto_spec) { 238 case 0: 239 type = RTE_FLOW_ITEM_TYPE_VOID; 240 break; 241 case RTE_VXLAN_GPE_TYPE_IPV4: 242 type = RTE_FLOW_ITEM_TYPE_IPV4; 243 break; 244 case RTE_VXLAN_GPE_TYPE_IPV6: 245 type = RTE_VXLAN_GPE_TYPE_IPV6; 246 break; 247 case RTE_VXLAN_GPE_TYPE_ETH: 248 type = RTE_FLOW_ITEM_TYPE_ETH; 249 break; 250 default: 251 type = RTE_FLOW_ITEM_TYPE_END; 252 } 253 return type; 254 } 255 256 static enum rte_flow_item_type 257 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 258 { 259 enum rte_flow_item_type type; 260 261 switch (proto_mask & proto_spec) { 262 case 0: 263 type = RTE_FLOW_ITEM_TYPE_VOID; 264 break; 265 case IPPROTO_UDP: 266 type = RTE_FLOW_ITEM_TYPE_UDP; 267 break; 268 case IPPROTO_TCP: 269 type = RTE_FLOW_ITEM_TYPE_TCP; 270 break; 271 case IPPROTO_IPIP: 272 type = RTE_FLOW_ITEM_TYPE_IPV4; 273 break; 274 case IPPROTO_IPV6: 275 type = RTE_FLOW_ITEM_TYPE_IPV6; 276 break; 277 case IPPROTO_ESP: 278 type = RTE_FLOW_ITEM_TYPE_ESP; 279 break; 280 default: 281 type = RTE_FLOW_ITEM_TYPE_END; 282 } 283 return type; 284 } 285 286 static enum rte_flow_item_type 287 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 288 rte_be16_t type_mask, bool is_tunnel) 289 { 290 enum rte_flow_item_type type; 291 292 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 293 case 0: 294 type = RTE_FLOW_ITEM_TYPE_VOID; 295 break; 296 case RTE_ETHER_TYPE_TEB: 297 type = is_tunnel ? 298 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 299 break; 300 case RTE_ETHER_TYPE_VLAN: 301 type = !is_tunnel ? 302 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 303 break; 304 case RTE_ETHER_TYPE_IPV4: 305 type = RTE_FLOW_ITEM_TYPE_IPV4; 306 break; 307 case RTE_ETHER_TYPE_IPV6: 308 type = RTE_FLOW_ITEM_TYPE_IPV6; 309 break; 310 default: 311 type = RTE_FLOW_ITEM_TYPE_END; 312 } 313 return type; 314 } 315 316 static enum rte_flow_item_type 317 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 318 { 319 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 320 do { \ 321 const void *m = item->mask; \ 322 const void *s = item->spec; \ 323 mask = m ? \ 324 ((const struct rte_flow_item_##type *)m)->fld : \ 325 rte_flow_item_##type##_mask.fld; \ 326 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 327 } while (0) 328 329 enum rte_flow_item_type ret; 330 uint16_t spec, mask; 331 332 if (item == NULL || item->spec == NULL) 333 return RTE_FLOW_ITEM_TYPE_VOID; 334 switch (item->type) { 335 case RTE_FLOW_ITEM_TYPE_ETH: 336 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 337 if (!mask) 338 return RTE_FLOW_ITEM_TYPE_VOID; 339 ret = mlx5_ethertype_to_item_type(spec, mask, false); 340 break; 341 case RTE_FLOW_ITEM_TYPE_VLAN: 342 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 343 if (!mask) 344 return RTE_FLOW_ITEM_TYPE_VOID; 345 ret = mlx5_ethertype_to_item_type(spec, mask, false); 346 break; 347 case RTE_FLOW_ITEM_TYPE_IPV4: 348 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 349 if (!mask) 350 return RTE_FLOW_ITEM_TYPE_VOID; 351 ret = mlx5_inet_proto_to_item_type(spec, mask); 352 break; 353 case RTE_FLOW_ITEM_TYPE_IPV6: 354 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 355 if (!mask) 356 return RTE_FLOW_ITEM_TYPE_VOID; 357 ret = mlx5_inet_proto_to_item_type(spec, mask); 358 break; 359 case RTE_FLOW_ITEM_TYPE_GENEVE: 360 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 361 ret = mlx5_ethertype_to_item_type(spec, mask, true); 362 break; 363 case RTE_FLOW_ITEM_TYPE_GRE: 364 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 365 ret = mlx5_ethertype_to_item_type(spec, mask, true); 366 break; 367 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 368 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 369 ret = mlx5_nsh_proto_to_item_type(spec, mask); 370 break; 371 default: 372 ret = RTE_FLOW_ITEM_TYPE_VOID; 373 break; 374 } 375 return ret; 376 #undef MLX5_XSET_ITEM_MASK_SPEC 377 } 378 379 static const int * 380 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 381 const int *next_node) 382 { 383 const struct mlx5_flow_expand_node *node = NULL; 384 const int *next = next_node; 385 386 while (next && *next) { 387 /* 388 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 389 * flag set, because they were not found in the flow pattern. 390 */ 391 node = &graph[*next]; 392 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 393 break; 394 next = node->next; 395 } 396 return next; 397 } 398 399 #define MLX5_RSS_EXP_ELT_N 32 400 401 /** 402 * Expand RSS flows into several possible flows according to the RSS hash 403 * fields requested and the driver capabilities. 404 * 405 * @param[out] buf 406 * Buffer to store the result expansion. 407 * @param[in] size 408 * Buffer size in bytes. If 0, @p buf can be NULL. 409 * @param[in] pattern 410 * User flow pattern. 411 * @param[in] types 412 * RSS types to expand (see RTE_ETH_RSS_* definitions). 413 * @param[in] graph 414 * Input graph to expand @p pattern according to @p types. 415 * @param[in] graph_root_index 416 * Index of root node in @p graph, typically 0. 417 * 418 * @return 419 * A positive value representing the size of @p buf in bytes regardless of 420 * @p size on success, a negative errno value otherwise and rte_errno is 421 * set, the following errors are defined: 422 * 423 * -E2BIG: graph-depth @p graph is too deep. 424 * -EINVAL: @p size has not enough space for expanded pattern. 425 */ 426 static int 427 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 428 const struct rte_flow_item *pattern, uint64_t types, 429 const struct mlx5_flow_expand_node graph[], 430 int graph_root_index) 431 { 432 const struct rte_flow_item *item; 433 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 434 const int *next_node; 435 const int *stack[MLX5_RSS_EXP_ELT_N]; 436 int stack_pos = 0; 437 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 438 unsigned int i, item_idx, last_expand_item_idx = 0; 439 size_t lsize; 440 size_t user_pattern_size = 0; 441 void *addr = NULL; 442 const struct mlx5_flow_expand_node *next = NULL; 443 struct rte_flow_item missed_item; 444 int missed = 0; 445 int elt = 0; 446 const struct rte_flow_item *last_expand_item = NULL; 447 448 memset(&missed_item, 0, sizeof(missed_item)); 449 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 450 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 451 if (lsize > size) 452 return -EINVAL; 453 buf->entry[0].priority = 0; 454 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 455 buf->entries = 0; 456 addr = buf->entry[0].pattern; 457 for (item = pattern, item_idx = 0; 458 item->type != RTE_FLOW_ITEM_TYPE_END; 459 item++, item_idx++) { 460 if (!mlx5_flow_is_rss_expandable_item(item)) { 461 user_pattern_size += sizeof(*item); 462 continue; 463 } 464 last_expand_item = item; 465 last_expand_item_idx = item_idx; 466 i = 0; 467 while (node->next && node->next[i]) { 468 next = &graph[node->next[i]]; 469 if (next->type == item->type) 470 break; 471 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 472 node = next; 473 i = 0; 474 } else { 475 ++i; 476 } 477 } 478 if (next) 479 node = next; 480 user_pattern_size += sizeof(*item); 481 } 482 user_pattern_size += sizeof(*item); /* Handle END item. */ 483 lsize += user_pattern_size; 484 if (lsize > size) 485 return -EINVAL; 486 /* Copy the user pattern in the first entry of the buffer. */ 487 rte_memcpy(addr, pattern, user_pattern_size); 488 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 489 buf->entries = 1; 490 /* Start expanding. */ 491 memset(flow_items, 0, sizeof(flow_items)); 492 user_pattern_size -= sizeof(*item); 493 /* 494 * Check if the last valid item has spec set, need complete pattern, 495 * and the pattern can be used for expansion. 496 */ 497 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 498 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 499 /* Item type END indicates expansion is not required. */ 500 return lsize; 501 } 502 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 503 next = NULL; 504 missed = 1; 505 i = 0; 506 while (node->next && node->next[i]) { 507 next = &graph[node->next[i]]; 508 if (next->type == missed_item.type) { 509 flow_items[0].type = missed_item.type; 510 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 511 break; 512 } 513 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 514 node = next; 515 i = 0; 516 } else { 517 ++i; 518 } 519 next = NULL; 520 } 521 } 522 if (next && missed) { 523 elt = 2; /* missed item + item end. */ 524 node = next; 525 lsize += elt * sizeof(*item) + user_pattern_size; 526 if (lsize > size) 527 return -EINVAL; 528 if (node->rss_types & types) { 529 buf->entry[buf->entries].priority = 1; 530 buf->entry[buf->entries].pattern = addr; 531 buf->entries++; 532 rte_memcpy(addr, buf->entry[0].pattern, 533 user_pattern_size); 534 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 535 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 536 addr = (void *)(((uintptr_t)addr) + 537 elt * sizeof(*item)); 538 } 539 } else if (last_expand_item != NULL) { 540 node = mlx5_flow_expand_rss_adjust_node(pattern, 541 last_expand_item_idx, graph, node); 542 } 543 memset(flow_items, 0, sizeof(flow_items)); 544 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 545 node->next); 546 stack[stack_pos] = next_node; 547 node = next_node ? &graph[*next_node] : NULL; 548 while (node) { 549 flow_items[stack_pos].type = node->type; 550 if (node->rss_types & types) { 551 size_t n; 552 /* 553 * compute the number of items to copy from the 554 * expansion and copy it. 555 * When the stack_pos is 0, there are 1 element in it, 556 * plus the addition END item. 557 */ 558 elt = stack_pos + 2; 559 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 560 lsize += elt * sizeof(*item) + user_pattern_size; 561 if (lsize > size) 562 return -EINVAL; 563 n = elt * sizeof(*item); 564 MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); 565 buf->entry[buf->entries].priority = 566 stack_pos + 1 + missed; 567 buf->entry[buf->entries].pattern = addr; 568 buf->entries++; 569 rte_memcpy(addr, buf->entry[0].pattern, 570 user_pattern_size); 571 addr = (void *)(((uintptr_t)addr) + 572 user_pattern_size); 573 rte_memcpy(addr, &missed_item, 574 missed * sizeof(*item)); 575 addr = (void *)(((uintptr_t)addr) + 576 missed * sizeof(*item)); 577 rte_memcpy(addr, flow_items, n); 578 addr = (void *)(((uintptr_t)addr) + n); 579 } 580 /* Go deeper. */ 581 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 582 node->next) { 583 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 584 node->next); 585 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 586 rte_errno = E2BIG; 587 return -rte_errno; 588 } 589 stack[stack_pos] = next_node; 590 } else if (*(next_node + 1)) { 591 /* Follow up with the next possibility. */ 592 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 593 ++next_node); 594 } else if (!stack_pos) { 595 /* 596 * Completing the traverse over the different paths. 597 * The next_node is advanced to the terminator. 598 */ 599 ++next_node; 600 } else { 601 /* Move to the next path. */ 602 while (stack_pos) { 603 next_node = stack[--stack_pos]; 604 next_node++; 605 if (*next_node) 606 break; 607 } 608 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 609 next_node); 610 stack[stack_pos] = next_node; 611 } 612 node = next_node && *next_node ? &graph[*next_node] : NULL; 613 }; 614 return lsize; 615 } 616 617 /** 618 * Expand SQN flows into several possible flows according to the Tx queue 619 * number 620 * 621 * @param[in] buf 622 * Buffer to store the result expansion. 623 * @param[in] size 624 * Buffer size in bytes. If 0, @p buf can be NULL. 625 * @param[in] pattern 626 * User flow pattern. 627 * @param[in] sq_specs 628 * Buffer to store sq spec. 629 * 630 * @return 631 * 0 for success and negative value for failure 632 * 633 */ 634 static int 635 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 636 const struct rte_flow_item *pattern, 637 struct mlx5_rte_flow_item_sq *sq_specs) 638 { 639 const struct rte_flow_item *item; 640 bool port_representor = false; 641 size_t user_pattern_size = 0; 642 struct rte_eth_dev *dev; 643 struct mlx5_priv *priv; 644 void *addr = NULL; 645 uint16_t port_id; 646 size_t lsize; 647 int elt = 2; 648 uint16_t i; 649 650 buf->entries = 0; 651 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 652 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 653 const struct rte_flow_item_ethdev *pid_v = item->spec; 654 655 if (!pid_v) 656 return 0; 657 port_id = pid_v->port_id; 658 port_representor = true; 659 } 660 user_pattern_size += sizeof(*item); 661 } 662 if (!port_representor) 663 return 0; 664 dev = &rte_eth_devices[port_id]; 665 priv = dev->data->dev_private; 666 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 667 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 668 sizeof(buf->entry[0]) * priv->txqs_n; 669 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 670 return -EINVAL; 671 addr = buf->entry[0].pattern; 672 for (i = 0; i != priv->txqs_n; ++i) { 673 struct rte_flow_item pattern_add[] = { 674 { 675 .type = (enum rte_flow_item_type) 676 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 677 .spec = &sq_specs[i], 678 }, 679 { 680 .type = RTE_FLOW_ITEM_TYPE_END, 681 }, 682 }; 683 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 684 685 if (txq == NULL) 686 return -EINVAL; 687 buf->entry[i].pattern = addr; 688 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 689 mlx5_txq_release(dev, i); 690 rte_memcpy(addr, pattern, user_pattern_size); 691 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 692 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 693 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 694 buf->entries++; 695 } 696 return 0; 697 } 698 699 enum mlx5_expansion { 700 MLX5_EXPANSION_ROOT, 701 MLX5_EXPANSION_ROOT_OUTER, 702 MLX5_EXPANSION_OUTER_ETH, 703 MLX5_EXPANSION_OUTER_VLAN, 704 MLX5_EXPANSION_OUTER_IPV4, 705 MLX5_EXPANSION_OUTER_IPV4_UDP, 706 MLX5_EXPANSION_OUTER_IPV4_TCP, 707 MLX5_EXPANSION_OUTER_IPV4_ESP, 708 MLX5_EXPANSION_OUTER_IPV4_ICMP, 709 MLX5_EXPANSION_OUTER_IPV6, 710 MLX5_EXPANSION_OUTER_IPV6_UDP, 711 MLX5_EXPANSION_OUTER_IPV6_TCP, 712 MLX5_EXPANSION_OUTER_IPV6_ESP, 713 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 714 MLX5_EXPANSION_VXLAN, 715 MLX5_EXPANSION_STD_VXLAN, 716 MLX5_EXPANSION_L3_VXLAN, 717 MLX5_EXPANSION_VXLAN_GPE, 718 MLX5_EXPANSION_GRE, 719 MLX5_EXPANSION_NVGRE, 720 MLX5_EXPANSION_GRE_KEY, 721 MLX5_EXPANSION_MPLS, 722 MLX5_EXPANSION_ETH, 723 MLX5_EXPANSION_VLAN, 724 MLX5_EXPANSION_IPV4, 725 MLX5_EXPANSION_IPV4_UDP, 726 MLX5_EXPANSION_IPV4_TCP, 727 MLX5_EXPANSION_IPV4_ESP, 728 MLX5_EXPANSION_IPV4_ICMP, 729 MLX5_EXPANSION_IPV6, 730 MLX5_EXPANSION_IPV6_UDP, 731 MLX5_EXPANSION_IPV6_TCP, 732 MLX5_EXPANSION_IPV6_ESP, 733 MLX5_EXPANSION_IPV6_ICMP6, 734 MLX5_EXPANSION_IPV6_FRAG_EXT, 735 MLX5_EXPANSION_GTP, 736 MLX5_EXPANSION_GENEVE, 737 }; 738 739 /** Supported expansion of items. */ 740 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 741 [MLX5_EXPANSION_ROOT] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 743 MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_END, 746 }, 747 [MLX5_EXPANSION_ROOT_OUTER] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 749 MLX5_EXPANSION_OUTER_IPV4, 750 MLX5_EXPANSION_OUTER_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_END, 752 }, 753 [MLX5_EXPANSION_OUTER_ETH] = { 754 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 755 .type = RTE_FLOW_ITEM_TYPE_ETH, 756 .rss_types = 0, 757 }, 758 [MLX5_EXPANSION_OUTER_VLAN] = { 759 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 760 MLX5_EXPANSION_OUTER_IPV6), 761 .type = RTE_FLOW_ITEM_TYPE_VLAN, 762 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 763 }, 764 [MLX5_EXPANSION_OUTER_IPV4] = { 765 .next = MLX5_FLOW_EXPAND_RSS_NEXT 766 (MLX5_EXPANSION_OUTER_IPV4_UDP, 767 MLX5_EXPANSION_OUTER_IPV4_TCP, 768 MLX5_EXPANSION_OUTER_IPV4_ESP, 769 MLX5_EXPANSION_OUTER_IPV4_ICMP, 770 MLX5_EXPANSION_GRE, 771 MLX5_EXPANSION_NVGRE, 772 MLX5_EXPANSION_IPV4, 773 MLX5_EXPANSION_IPV6), 774 .type = RTE_FLOW_ITEM_TYPE_IPV4, 775 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 776 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 777 }, 778 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 779 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 780 MLX5_EXPANSION_VXLAN_GPE, 781 MLX5_EXPANSION_MPLS, 782 MLX5_EXPANSION_GENEVE, 783 MLX5_EXPANSION_GTP), 784 .type = RTE_FLOW_ITEM_TYPE_UDP, 785 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 786 }, 787 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 788 .type = RTE_FLOW_ITEM_TYPE_TCP, 789 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 790 }, 791 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 792 .type = RTE_FLOW_ITEM_TYPE_ESP, 793 .rss_types = RTE_ETH_RSS_ESP, 794 }, 795 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 796 .type = RTE_FLOW_ITEM_TYPE_ICMP, 797 }, 798 [MLX5_EXPANSION_OUTER_IPV6] = { 799 .next = MLX5_FLOW_EXPAND_RSS_NEXT 800 (MLX5_EXPANSION_OUTER_IPV6_UDP, 801 MLX5_EXPANSION_OUTER_IPV6_TCP, 802 MLX5_EXPANSION_OUTER_IPV6_ESP, 803 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 804 MLX5_EXPANSION_IPV4, 805 MLX5_EXPANSION_IPV6, 806 MLX5_EXPANSION_GRE, 807 MLX5_EXPANSION_NVGRE), 808 .type = RTE_FLOW_ITEM_TYPE_IPV6, 809 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 810 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 811 }, 812 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 813 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 814 MLX5_EXPANSION_VXLAN_GPE, 815 MLX5_EXPANSION_MPLS, 816 MLX5_EXPANSION_GENEVE, 817 MLX5_EXPANSION_GTP), 818 .type = RTE_FLOW_ITEM_TYPE_UDP, 819 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 820 }, 821 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 822 .type = RTE_FLOW_ITEM_TYPE_TCP, 823 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 824 }, 825 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 826 .type = RTE_FLOW_ITEM_TYPE_ESP, 827 .rss_types = RTE_ETH_RSS_ESP, 828 }, 829 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 830 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 831 }, 832 [MLX5_EXPANSION_VXLAN] = { 833 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 834 MLX5_EXPANSION_IPV4, 835 MLX5_EXPANSION_IPV6), 836 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 837 }, 838 [MLX5_EXPANSION_STD_VXLAN] = { 839 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 840 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 841 }, 842 [MLX5_EXPANSION_L3_VXLAN] = { 843 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 844 MLX5_EXPANSION_IPV6), 845 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 846 }, 847 [MLX5_EXPANSION_VXLAN_GPE] = { 848 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 849 MLX5_EXPANSION_IPV4, 850 MLX5_EXPANSION_IPV6), 851 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 852 }, 853 [MLX5_EXPANSION_GRE] = { 854 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 855 MLX5_EXPANSION_IPV4, 856 MLX5_EXPANSION_IPV6, 857 MLX5_EXPANSION_GRE_KEY, 858 MLX5_EXPANSION_MPLS), 859 .type = RTE_FLOW_ITEM_TYPE_GRE, 860 }, 861 [MLX5_EXPANSION_GRE_KEY] = { 862 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 863 MLX5_EXPANSION_IPV6, 864 MLX5_EXPANSION_MPLS), 865 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 866 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 867 }, 868 [MLX5_EXPANSION_NVGRE] = { 869 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 870 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 871 }, 872 [MLX5_EXPANSION_MPLS] = { 873 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 874 MLX5_EXPANSION_IPV6, 875 MLX5_EXPANSION_ETH), 876 .type = RTE_FLOW_ITEM_TYPE_MPLS, 877 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 878 }, 879 [MLX5_EXPANSION_ETH] = { 880 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 881 .type = RTE_FLOW_ITEM_TYPE_ETH, 882 }, 883 [MLX5_EXPANSION_VLAN] = { 884 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 885 MLX5_EXPANSION_IPV6), 886 .type = RTE_FLOW_ITEM_TYPE_VLAN, 887 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 888 }, 889 [MLX5_EXPANSION_IPV4] = { 890 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 891 MLX5_EXPANSION_IPV4_TCP, 892 MLX5_EXPANSION_IPV4_ESP, 893 MLX5_EXPANSION_IPV4_ICMP), 894 .type = RTE_FLOW_ITEM_TYPE_IPV4, 895 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 896 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 897 }, 898 [MLX5_EXPANSION_IPV4_UDP] = { 899 .type = RTE_FLOW_ITEM_TYPE_UDP, 900 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 901 }, 902 [MLX5_EXPANSION_IPV4_TCP] = { 903 .type = RTE_FLOW_ITEM_TYPE_TCP, 904 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 905 }, 906 [MLX5_EXPANSION_IPV4_ESP] = { 907 .type = RTE_FLOW_ITEM_TYPE_ESP, 908 .rss_types = RTE_ETH_RSS_ESP, 909 }, 910 [MLX5_EXPANSION_IPV4_ICMP] = { 911 .type = RTE_FLOW_ITEM_TYPE_ICMP, 912 }, 913 [MLX5_EXPANSION_IPV6] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 915 MLX5_EXPANSION_IPV6_TCP, 916 MLX5_EXPANSION_IPV6_ESP, 917 MLX5_EXPANSION_IPV6_ICMP6, 918 MLX5_EXPANSION_IPV6_FRAG_EXT), 919 .type = RTE_FLOW_ITEM_TYPE_IPV6, 920 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 921 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 922 }, 923 [MLX5_EXPANSION_IPV6_UDP] = { 924 .type = RTE_FLOW_ITEM_TYPE_UDP, 925 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 926 }, 927 [MLX5_EXPANSION_IPV6_TCP] = { 928 .type = RTE_FLOW_ITEM_TYPE_TCP, 929 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 930 }, 931 [MLX5_EXPANSION_IPV6_ESP] = { 932 .type = RTE_FLOW_ITEM_TYPE_ESP, 933 .rss_types = RTE_ETH_RSS_ESP, 934 }, 935 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 936 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 937 }, 938 [MLX5_EXPANSION_IPV6_ICMP6] = { 939 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 940 }, 941 [MLX5_EXPANSION_GTP] = { 942 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 943 MLX5_EXPANSION_IPV6), 944 .type = RTE_FLOW_ITEM_TYPE_GTP, 945 }, 946 [MLX5_EXPANSION_GENEVE] = { 947 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 948 MLX5_EXPANSION_IPV4, 949 MLX5_EXPANSION_IPV6), 950 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 951 }, 952 }; 953 954 static struct rte_flow_action_handle * 955 mlx5_action_handle_create(struct rte_eth_dev *dev, 956 const struct rte_flow_indir_action_conf *conf, 957 const struct rte_flow_action *action, 958 struct rte_flow_error *error); 959 static int mlx5_action_handle_destroy 960 (struct rte_eth_dev *dev, 961 struct rte_flow_action_handle *handle, 962 struct rte_flow_error *error); 963 static int mlx5_action_handle_update 964 (struct rte_eth_dev *dev, 965 struct rte_flow_action_handle *handle, 966 const void *update, 967 struct rte_flow_error *error); 968 static int mlx5_action_handle_query 969 (struct rte_eth_dev *dev, 970 const struct rte_flow_action_handle *handle, 971 void *data, 972 struct rte_flow_error *error); 973 static int 974 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 975 struct rte_flow_tunnel *app_tunnel, 976 struct rte_flow_action **actions, 977 uint32_t *num_of_actions, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 981 struct rte_flow_tunnel *app_tunnel, 982 struct rte_flow_item **items, 983 uint32_t *num_of_items, 984 struct rte_flow_error *error); 985 static int 986 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 987 struct rte_flow_item *pmd_items, 988 uint32_t num_items, struct rte_flow_error *err); 989 static int 990 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 991 struct rte_flow_action *pmd_actions, 992 uint32_t num_actions, 993 struct rte_flow_error *err); 994 static int 995 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 996 struct rte_mbuf *m, 997 struct rte_flow_restore_info *info, 998 struct rte_flow_error *err); 999 static struct rte_flow_item_flex_handle * 1000 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 1001 const struct rte_flow_item_flex_conf *conf, 1002 struct rte_flow_error *error); 1003 static int 1004 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 1005 const struct rte_flow_item_flex_handle *handle, 1006 struct rte_flow_error *error); 1007 static int 1008 mlx5_flow_info_get(struct rte_eth_dev *dev, 1009 struct rte_flow_port_info *port_info, 1010 struct rte_flow_queue_info *queue_info, 1011 struct rte_flow_error *error); 1012 static int 1013 mlx5_flow_port_configure(struct rte_eth_dev *dev, 1014 const struct rte_flow_port_attr *port_attr, 1015 uint16_t nb_queue, 1016 const struct rte_flow_queue_attr *queue_attr[], 1017 struct rte_flow_error *err); 1018 1019 static struct rte_flow_pattern_template * 1020 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 1021 const struct rte_flow_pattern_template_attr *attr, 1022 const struct rte_flow_item items[], 1023 struct rte_flow_error *error); 1024 1025 static int 1026 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1027 struct rte_flow_pattern_template *template, 1028 struct rte_flow_error *error); 1029 static struct rte_flow_actions_template * 1030 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1031 const struct rte_flow_actions_template_attr *attr, 1032 const struct rte_flow_action actions[], 1033 const struct rte_flow_action masks[], 1034 struct rte_flow_error *error); 1035 static int 1036 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1037 struct rte_flow_actions_template *template, 1038 struct rte_flow_error *error); 1039 1040 static struct rte_flow_template_table * 1041 mlx5_flow_table_create(struct rte_eth_dev *dev, 1042 const struct rte_flow_template_table_attr *attr, 1043 struct rte_flow_pattern_template *item_templates[], 1044 uint8_t nb_item_templates, 1045 struct rte_flow_actions_template *action_templates[], 1046 uint8_t nb_action_templates, 1047 struct rte_flow_error *error); 1048 static int 1049 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1050 struct rte_flow_template_table *table, 1051 struct rte_flow_error *error); 1052 static int 1053 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 1054 uint32_t group_id, 1055 const struct rte_flow_group_attr *attr, 1056 const struct rte_flow_action actions[], 1057 struct rte_flow_error *error); 1058 1059 static int 1060 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1061 struct rte_flow_action_handle *handle, 1062 const void *update, void *query, 1063 enum rte_flow_query_update_mode qu_mode, 1064 struct rte_flow_error *error); 1065 1066 static struct rte_flow_action_list_handle * 1067 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 1068 const struct rte_flow_indir_action_conf *conf, 1069 const struct rte_flow_action *actions, 1070 struct rte_flow_error *error); 1071 1072 static int 1073 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 1074 struct rte_flow_action_list_handle *handle, 1075 struct rte_flow_error *error); 1076 1077 static int 1078 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 1079 const 1080 struct rte_flow_action_list_handle *handle, 1081 const void **update, void **query, 1082 enum rte_flow_query_update_mode mode, 1083 struct rte_flow_error *error); 1084 1085 static int 1086 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 1087 const struct rte_flow_template_table *table, 1088 const struct rte_flow_item pattern[], 1089 uint8_t pattern_template_index, 1090 uint32_t *hash, struct rte_flow_error *error); 1091 static int 1092 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 1093 const struct rte_flow_item pattern[], 1094 enum rte_flow_encap_hash_field dest_field, 1095 uint8_t *hash, 1096 struct rte_flow_error *error); 1097 1098 static const struct rte_flow_ops mlx5_flow_ops = { 1099 .validate = mlx5_flow_validate, 1100 .create = mlx5_flow_create, 1101 .destroy = mlx5_flow_destroy, 1102 .flush = mlx5_flow_flush, 1103 .isolate = mlx5_flow_isolate, 1104 .query = mlx5_flow_query, 1105 .dev_dump = mlx5_flow_dev_dump, 1106 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1107 .get_aged_flows = mlx5_flow_get_aged_flows, 1108 .action_handle_create = mlx5_action_handle_create, 1109 .action_handle_destroy = mlx5_action_handle_destroy, 1110 .action_handle_update = mlx5_action_handle_update, 1111 .action_handle_query = mlx5_action_handle_query, 1112 .action_handle_query_update = mlx5_action_handle_query_update, 1113 .action_list_handle_create = mlx5_action_list_handle_create, 1114 .action_list_handle_destroy = mlx5_action_list_handle_destroy, 1115 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1116 .tunnel_match = mlx5_flow_tunnel_match, 1117 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1118 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1119 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1120 .flex_item_create = mlx5_flow_flex_item_create, 1121 .flex_item_release = mlx5_flow_flex_item_release, 1122 .info_get = mlx5_flow_info_get, 1123 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1124 .configure = mlx5_flow_port_configure, 1125 .pattern_template_create = mlx5_flow_pattern_template_create, 1126 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1127 .actions_template_create = mlx5_flow_actions_template_create, 1128 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1129 .template_table_create = mlx5_flow_table_create, 1130 .template_table_destroy = mlx5_flow_table_destroy, 1131 .group_set_miss_actions = mlx5_flow_group_set_miss_actions, 1132 .action_list_handle_query_update = 1133 mlx5_flow_action_list_handle_query_update, 1134 .flow_calc_table_hash = mlx5_flow_calc_table_hash, 1135 .flow_calc_encap_hash = mlx5_flow_calc_encap_hash, 1136 }; 1137 1138 /* Tunnel information. */ 1139 struct mlx5_flow_tunnel_info { 1140 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1141 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1142 }; 1143 1144 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1145 { 1146 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1147 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1148 }, 1149 { 1150 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1151 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1152 }, 1153 { 1154 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1155 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1156 }, 1157 { 1158 .tunnel = MLX5_FLOW_LAYER_GRE, 1159 .ptype = RTE_PTYPE_TUNNEL_GRE, 1160 }, 1161 { 1162 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1163 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1164 }, 1165 { 1166 .tunnel = MLX5_FLOW_LAYER_MPLS, 1167 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1168 }, 1169 { 1170 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1171 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1172 }, 1173 { 1174 .tunnel = MLX5_FLOW_LAYER_IPIP, 1175 .ptype = RTE_PTYPE_TUNNEL_IP, 1176 }, 1177 { 1178 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1179 .ptype = RTE_PTYPE_TUNNEL_IP, 1180 }, 1181 { 1182 .tunnel = MLX5_FLOW_LAYER_GTP, 1183 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1184 }, 1185 }; 1186 1187 1188 1189 /** 1190 * Translate tag ID to register. 1191 * 1192 * @param[in] dev 1193 * Pointer to the Ethernet device structure. 1194 * @param[in] feature 1195 * The feature that request the register. 1196 * @param[in] id 1197 * The request register ID. 1198 * @param[out] error 1199 * Error description in case of any. 1200 * 1201 * @return 1202 * The request register on success, a negative errno 1203 * value otherwise and rte_errno is set. 1204 */ 1205 int 1206 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1207 enum mlx5_feature_name feature, 1208 uint32_t id, 1209 struct rte_flow_error *error) 1210 { 1211 struct mlx5_priv *priv = dev->data->dev_private; 1212 struct mlx5_sh_config *config = &priv->sh->config; 1213 struct mlx5_dev_registers *reg = &priv->sh->registers; 1214 enum modify_reg start_reg; 1215 bool skip_mtr_reg = false; 1216 1217 switch (feature) { 1218 case MLX5_HAIRPIN_RX: 1219 return REG_B; 1220 case MLX5_HAIRPIN_TX: 1221 return REG_A; 1222 case MLX5_METADATA_RX: 1223 switch (config->dv_xmeta_en) { 1224 case MLX5_XMETA_MODE_LEGACY: 1225 return REG_B; 1226 case MLX5_XMETA_MODE_META16: 1227 return REG_C_0; 1228 case MLX5_XMETA_MODE_META32: 1229 return REG_C_1; 1230 case MLX5_XMETA_MODE_META32_HWS: 1231 return REG_C_1; 1232 } 1233 break; 1234 case MLX5_METADATA_TX: 1235 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1236 return REG_C_1; 1237 } else { 1238 return REG_A; 1239 } 1240 case MLX5_METADATA_FDB: 1241 switch (config->dv_xmeta_en) { 1242 case MLX5_XMETA_MODE_LEGACY: 1243 return REG_NON; 1244 case MLX5_XMETA_MODE_META16: 1245 return REG_C_0; 1246 case MLX5_XMETA_MODE_META32: 1247 return REG_C_1; 1248 case MLX5_XMETA_MODE_META32_HWS: 1249 return REG_C_1; 1250 } 1251 break; 1252 case MLX5_FLOW_MARK: 1253 switch (config->dv_xmeta_en) { 1254 case MLX5_XMETA_MODE_LEGACY: 1255 case MLX5_XMETA_MODE_META32_HWS: 1256 return REG_NON; 1257 case MLX5_XMETA_MODE_META16: 1258 return REG_C_1; 1259 case MLX5_XMETA_MODE_META32: 1260 return REG_C_0; 1261 } 1262 break; 1263 case MLX5_MTR_ID: 1264 /* 1265 * If meter color and meter id share one register, flow match 1266 * should use the meter color register for match. 1267 */ 1268 if (priv->mtr_reg_share) 1269 return reg->aso_reg; 1270 else 1271 return reg->aso_reg != REG_C_2 ? REG_C_2 : 1272 REG_C_3; 1273 case MLX5_MTR_COLOR: 1274 case MLX5_ASO_FLOW_HIT: 1275 case MLX5_ASO_CONNTRACK: 1276 case MLX5_SAMPLE_ID: 1277 /* All features use the same REG_C. */ 1278 MLX5_ASSERT(reg->aso_reg != REG_NON); 1279 return reg->aso_reg; 1280 case MLX5_COPY_MARK: 1281 /* 1282 * Metadata COPY_MARK register using is in meter suffix sub 1283 * flow while with meter. It's safe to share the same register. 1284 */ 1285 return reg->aso_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1286 case MLX5_APP_TAG: 1287 /* 1288 * If meter is enable, it will engage the register for color 1289 * match and flow match. If meter color match is not using the 1290 * REG_C_2, need to skip the REG_C_x be used by meter color 1291 * match. 1292 * If meter is disable, free to use all available registers. 1293 */ 1294 start_reg = reg->aso_reg != REG_C_2 ? REG_C_2 : 1295 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1296 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1297 if (id > (uint32_t)(REG_C_7 - start_reg)) 1298 return rte_flow_error_set(error, EINVAL, 1299 RTE_FLOW_ERROR_TYPE_ITEM, 1300 NULL, "invalid tag id"); 1301 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1302 return rte_flow_error_set(error, ENOTSUP, 1303 RTE_FLOW_ERROR_TYPE_ITEM, 1304 NULL, "unsupported tag id"); 1305 /* 1306 * This case means meter is using the REG_C_x great than 2. 1307 * Take care not to conflict with meter color REG_C_x. 1308 * If the available index REG_C_y >= REG_C_x, skip the 1309 * color register. 1310 */ 1311 if (skip_mtr_reg && priv->sh->flow_mreg_c 1312 [id + start_reg - REG_C_0] >= reg->aso_reg) { 1313 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1314 return rte_flow_error_set(error, EINVAL, 1315 RTE_FLOW_ERROR_TYPE_ITEM, 1316 NULL, "invalid tag id"); 1317 if (priv->sh->flow_mreg_c 1318 [id + 1 + start_reg - REG_C_0] != REG_NON) 1319 return priv->sh->flow_mreg_c 1320 [id + 1 + start_reg - REG_C_0]; 1321 return rte_flow_error_set(error, ENOTSUP, 1322 RTE_FLOW_ERROR_TYPE_ITEM, 1323 NULL, "unsupported tag id"); 1324 } 1325 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1326 } 1327 MLX5_ASSERT(false); 1328 return rte_flow_error_set(error, EINVAL, 1329 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1330 NULL, "invalid feature name"); 1331 } 1332 1333 /** 1334 * Check extensive flow metadata register support. 1335 * 1336 * @param dev 1337 * Pointer to rte_eth_dev structure. 1338 * 1339 * @return 1340 * True if device supports extensive flow metadata register, otherwise false. 1341 */ 1342 bool 1343 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1344 { 1345 struct mlx5_priv *priv = dev->data->dev_private; 1346 1347 /* 1348 * Having available reg_c can be regarded inclusively as supporting 1349 * extensive flow metadata register, which could mean, 1350 * - metadata register copy action by modify header. 1351 * - 16 modify header actions is supported. 1352 * - reg_c's are preserved across different domain (FDB and NIC) on 1353 * packet loopback by flow lookup miss. 1354 */ 1355 return priv->sh->flow_mreg_c[2] != REG_NON; 1356 } 1357 1358 /** 1359 * Get the lowest priority. 1360 * 1361 * @param[in] dev 1362 * Pointer to the Ethernet device structure. 1363 * @param[in] attributes 1364 * Pointer to device flow rule attributes. 1365 * 1366 * @return 1367 * The value of lowest priority of flow. 1368 */ 1369 uint32_t 1370 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1371 const struct rte_flow_attr *attr) 1372 { 1373 struct mlx5_priv *priv = dev->data->dev_private; 1374 1375 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1376 return priv->sh->flow_max_priority - 2; 1377 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1378 } 1379 1380 /** 1381 * Calculate matcher priority of the flow. 1382 * 1383 * @param[in] dev 1384 * Pointer to the Ethernet device structure. 1385 * @param[in] attr 1386 * Pointer to device flow rule attributes. 1387 * @param[in] subpriority 1388 * The priority based on the items. 1389 * @param[in] external 1390 * Flow is user flow. 1391 * @return 1392 * The matcher priority of the flow. 1393 */ 1394 uint16_t 1395 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1396 const struct rte_flow_attr *attr, 1397 uint32_t subpriority, bool external) 1398 { 1399 uint16_t priority = (uint16_t)attr->priority; 1400 struct mlx5_priv *priv = dev->data->dev_private; 1401 1402 /* NIC root rules */ 1403 if (!attr->group && !attr->transfer) { 1404 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1405 priority = priv->sh->flow_max_priority - 1; 1406 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1407 /* FDB root rules */ 1408 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1409 attr->group == 0 && 1410 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1411 return (priv->sh->flow_max_priority - 1) * 3; 1412 } 1413 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1414 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1415 return priority * 3 + subpriority; 1416 } 1417 1418 /** 1419 * Verify the @p item specifications (spec, last, mask) are compatible with the 1420 * NIC capabilities. 1421 * 1422 * @param[in] item 1423 * Item specification. 1424 * @param[in] mask 1425 * @p item->mask or flow default bit-masks. 1426 * @param[in] nic_mask 1427 * Bit-masks covering supported fields by the NIC to compare with user mask. 1428 * @param[in] size 1429 * Bit-masks size in bytes. 1430 * @param[in] range_accepted 1431 * True if range of values is accepted for specific fields, false otherwise. 1432 * @param[out] error 1433 * Pointer to error structure. 1434 * 1435 * @return 1436 * 0 on success, a negative errno value otherwise and rte_errno is set. 1437 */ 1438 int 1439 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1440 const uint8_t *mask, 1441 const uint8_t *nic_mask, 1442 unsigned int size, 1443 bool range_accepted, 1444 struct rte_flow_error *error) 1445 { 1446 unsigned int i; 1447 1448 MLX5_ASSERT(nic_mask); 1449 for (i = 0; i < size; ++i) 1450 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1451 return rte_flow_error_set(error, ENOTSUP, 1452 RTE_FLOW_ERROR_TYPE_ITEM, 1453 item, 1454 "mask enables non supported" 1455 " bits"); 1456 if (!item->spec && (item->mask || item->last)) 1457 return rte_flow_error_set(error, EINVAL, 1458 RTE_FLOW_ERROR_TYPE_ITEM, item, 1459 "mask/last without a spec is not" 1460 " supported"); 1461 if (item->spec && item->last && !range_accepted) { 1462 uint8_t spec[size]; 1463 uint8_t last[size]; 1464 unsigned int i; 1465 int ret; 1466 1467 for (i = 0; i < size; ++i) { 1468 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1469 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1470 } 1471 ret = memcmp(spec, last, size); 1472 if (ret != 0) 1473 return rte_flow_error_set(error, EINVAL, 1474 RTE_FLOW_ERROR_TYPE_ITEM, 1475 item, 1476 "range is not valid"); 1477 } 1478 return 0; 1479 } 1480 1481 /** 1482 * Adjust the hash fields according to the @p flow information. 1483 * 1484 * @param[in] dev_flow. 1485 * Pointer to the mlx5_flow. 1486 * @param[in] tunnel 1487 * 1 when the hash field is for a tunnel item. 1488 * @param[in] layer_types 1489 * RTE_ETH_RSS_* types. 1490 * @param[in] hash_fields 1491 * Item hash fields. 1492 * 1493 * @return 1494 * The hash fields that should be used. 1495 */ 1496 uint64_t 1497 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1498 int tunnel __rte_unused, uint64_t layer_types, 1499 uint64_t hash_fields) 1500 { 1501 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1502 int rss_request_inner = rss_desc->level >= 2; 1503 1504 /* Check RSS hash level for tunnel. */ 1505 if (tunnel && rss_request_inner) 1506 hash_fields |= IBV_RX_HASH_INNER; 1507 else if (tunnel || rss_request_inner) 1508 return 0; 1509 #endif 1510 /* Check if requested layer matches RSS hash fields. */ 1511 if (!(rss_desc->types & layer_types)) 1512 return 0; 1513 return hash_fields; 1514 } 1515 1516 /** 1517 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1518 * if several tunnel rules are used on this queue, the tunnel ptype will be 1519 * cleared. 1520 * 1521 * @param rxq_ctrl 1522 * Rx queue to update. 1523 */ 1524 static void 1525 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1526 { 1527 unsigned int i; 1528 uint32_t tunnel_ptype = 0; 1529 1530 /* Look up for the ptype to use. */ 1531 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1532 if (!rxq_ctrl->flow_tunnels_n[i]) 1533 continue; 1534 if (!tunnel_ptype) { 1535 tunnel_ptype = tunnels_info[i].ptype; 1536 } else { 1537 tunnel_ptype = 0; 1538 break; 1539 } 1540 } 1541 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1542 } 1543 1544 /** 1545 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1546 * flow. 1547 * 1548 * @param[in] dev 1549 * Pointer to the Ethernet device structure. 1550 * @param[in] dev_handle 1551 * Pointer to device flow handle structure. 1552 */ 1553 void 1554 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1555 struct mlx5_flow_handle *dev_handle) 1556 { 1557 struct mlx5_priv *priv = dev->data->dev_private; 1558 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1559 struct mlx5_ind_table_obj *ind_tbl = NULL; 1560 unsigned int i; 1561 1562 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1563 struct mlx5_hrxq *hrxq; 1564 1565 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1566 dev_handle->rix_hrxq); 1567 if (hrxq) 1568 ind_tbl = hrxq->ind_table; 1569 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1570 struct mlx5_shared_action_rss *shared_rss; 1571 1572 shared_rss = mlx5_ipool_get 1573 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1574 dev_handle->rix_srss); 1575 if (shared_rss) 1576 ind_tbl = shared_rss->ind_tbl; 1577 } 1578 if (!ind_tbl) 1579 return; 1580 for (i = 0; i != ind_tbl->queues_n; ++i) { 1581 int idx = ind_tbl->queues[i]; 1582 struct mlx5_rxq_ctrl *rxq_ctrl; 1583 1584 if (mlx5_is_external_rxq(dev, idx)) 1585 continue; 1586 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1587 MLX5_ASSERT(rxq_ctrl != NULL); 1588 if (rxq_ctrl == NULL) 1589 continue; 1590 /* 1591 * To support metadata register copy on Tx loopback, 1592 * this must be always enabled (metadata may arive 1593 * from other port - not from local flows only. 1594 */ 1595 if (tunnel) { 1596 unsigned int j; 1597 1598 /* Increase the counter matching the flow. */ 1599 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1600 if ((tunnels_info[j].tunnel & 1601 dev_handle->layers) == 1602 tunnels_info[j].tunnel) { 1603 rxq_ctrl->flow_tunnels_n[j]++; 1604 break; 1605 } 1606 } 1607 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1608 } 1609 } 1610 } 1611 1612 static void 1613 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1614 { 1615 struct mlx5_priv *priv = dev->data->dev_private; 1616 struct mlx5_rxq_ctrl *rxq_ctrl; 1617 uint16_t port_id; 1618 1619 if (priv->sh->shared_mark_enabled) 1620 return; 1621 if (priv->master || priv->representor) { 1622 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1623 struct mlx5_priv *opriv = 1624 rte_eth_devices[port_id].data->dev_private; 1625 1626 if (!opriv || 1627 opriv->sh != priv->sh || 1628 opriv->domain_id != priv->domain_id || 1629 opriv->mark_enabled) 1630 continue; 1631 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1632 rxq_ctrl->rxq.mark = 1; 1633 } 1634 opriv->mark_enabled = 1; 1635 } 1636 } else { 1637 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1638 rxq_ctrl->rxq.mark = 1; 1639 } 1640 priv->mark_enabled = 1; 1641 } 1642 priv->sh->shared_mark_enabled = 1; 1643 } 1644 1645 /** 1646 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1647 * 1648 * @param[in] dev 1649 * Pointer to the Ethernet device structure. 1650 * @param[in] flow 1651 * Pointer to flow structure. 1652 */ 1653 static void 1654 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1655 { 1656 struct mlx5_priv *priv = dev->data->dev_private; 1657 uint32_t handle_idx; 1658 struct mlx5_flow_handle *dev_handle; 1659 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1660 1661 MLX5_ASSERT(wks); 1662 if (wks->mark) 1663 flow_rxq_mark_flag_set(dev); 1664 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1665 handle_idx, dev_handle, next) 1666 flow_drv_rxq_flags_set(dev, dev_handle); 1667 } 1668 1669 /** 1670 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1671 * device flow if no other flow uses it with the same kind of request. 1672 * 1673 * @param dev 1674 * Pointer to Ethernet device. 1675 * @param[in] dev_handle 1676 * Pointer to the device flow handle structure. 1677 */ 1678 static void 1679 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1680 struct mlx5_flow_handle *dev_handle) 1681 { 1682 struct mlx5_priv *priv = dev->data->dev_private; 1683 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1684 struct mlx5_ind_table_obj *ind_tbl = NULL; 1685 unsigned int i; 1686 1687 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1688 struct mlx5_hrxq *hrxq; 1689 1690 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1691 dev_handle->rix_hrxq); 1692 if (hrxq) 1693 ind_tbl = hrxq->ind_table; 1694 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1695 struct mlx5_shared_action_rss *shared_rss; 1696 1697 shared_rss = mlx5_ipool_get 1698 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1699 dev_handle->rix_srss); 1700 if (shared_rss) 1701 ind_tbl = shared_rss->ind_tbl; 1702 } 1703 if (!ind_tbl) 1704 return; 1705 MLX5_ASSERT(dev->data->dev_started); 1706 for (i = 0; i != ind_tbl->queues_n; ++i) { 1707 int idx = ind_tbl->queues[i]; 1708 struct mlx5_rxq_ctrl *rxq_ctrl; 1709 1710 if (mlx5_is_external_rxq(dev, idx)) 1711 continue; 1712 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1713 MLX5_ASSERT(rxq_ctrl != NULL); 1714 if (rxq_ctrl == NULL) 1715 continue; 1716 if (tunnel) { 1717 unsigned int j; 1718 1719 /* Decrease the counter matching the flow. */ 1720 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1721 if ((tunnels_info[j].tunnel & 1722 dev_handle->layers) == 1723 tunnels_info[j].tunnel) { 1724 rxq_ctrl->flow_tunnels_n[j]--; 1725 break; 1726 } 1727 } 1728 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1729 } 1730 } 1731 } 1732 1733 /** 1734 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1735 * @p flow if no other flow uses it with the same kind of request. 1736 * 1737 * @param dev 1738 * Pointer to Ethernet device. 1739 * @param[in] flow 1740 * Pointer to the flow. 1741 */ 1742 static void 1743 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1744 { 1745 struct mlx5_priv *priv = dev->data->dev_private; 1746 uint32_t handle_idx; 1747 struct mlx5_flow_handle *dev_handle; 1748 1749 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1750 handle_idx, dev_handle, next) 1751 flow_drv_rxq_flags_trim(dev, dev_handle); 1752 } 1753 1754 /** 1755 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1756 * 1757 * @param dev 1758 * Pointer to Ethernet device. 1759 */ 1760 static void 1761 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1762 { 1763 struct mlx5_priv *priv = dev->data->dev_private; 1764 unsigned int i; 1765 1766 for (i = 0; i != priv->rxqs_n; ++i) { 1767 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1768 unsigned int j; 1769 1770 if (rxq == NULL || rxq->ctrl == NULL) 1771 continue; 1772 rxq->ctrl->rxq.mark = 0; 1773 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1774 rxq->ctrl->flow_tunnels_n[j] = 0; 1775 rxq->ctrl->rxq.tunnel = 0; 1776 } 1777 priv->mark_enabled = 0; 1778 priv->sh->shared_mark_enabled = 0; 1779 } 1780 1781 static uint64_t mlx5_restore_info_dynflag; 1782 1783 int 1784 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 1785 { 1786 struct mlx5_priv *priv = dev->data->dev_private; 1787 uint64_t supported = 0; 1788 1789 if (!is_tunnel_offload_active(dev)) { 1790 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 1791 supported |= RTE_ETH_RX_METADATA_USER_MARK; 1792 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) { 1793 DRV_LOG(DEBUG, 1794 "tunnel offload was not activated, consider setting dv_xmeta_en=%d", 1795 MLX5_XMETA_MODE_MISS_INFO); 1796 } 1797 } else { 1798 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 1799 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 && 1800 mlx5_restore_info_dynflag == 0) 1801 mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag(); 1802 } 1803 1804 if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) 1805 priv->tunnel_enabled = 1; 1806 else 1807 priv->tunnel_enabled = 0; 1808 1809 *features &= supported; 1810 return 0; 1811 } 1812 1813 /** 1814 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1815 * 1816 * @param[in] dev 1817 * Pointer to the Ethernet device structure. 1818 */ 1819 void 1820 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) 1821 { 1822 struct mlx5_priv *priv = dev->data->dev_private; 1823 uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID; 1824 unsigned int i; 1825 1826 if (priv->tunnel_enabled) 1827 mark_flag |= mlx5_restore_info_dynflag; 1828 1829 for (i = 0; i != priv->rxqs_n; ++i) { 1830 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1831 struct mlx5_rxq_data *data; 1832 1833 if (rxq == NULL || rxq->ctrl == NULL) 1834 continue; 1835 data = &rxq->ctrl->rxq; 1836 if (!rte_flow_dynf_metadata_avail()) { 1837 data->dynf_meta = 0; 1838 data->flow_meta_mask = 0; 1839 data->flow_meta_offset = -1; 1840 data->flow_meta_port_mask = 0; 1841 } else { 1842 data->dynf_meta = 1; 1843 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1844 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1845 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1846 } 1847 data->mark_flag = mark_flag; 1848 } 1849 } 1850 1851 /* 1852 * return a pointer to the desired action in the list of actions. 1853 * 1854 * @param[in] actions 1855 * The list of actions to search the action in. 1856 * @param[in] action 1857 * The action to find. 1858 * 1859 * @return 1860 * Pointer to the action in the list, if found. NULL otherwise. 1861 */ 1862 const struct rte_flow_action * 1863 mlx5_flow_find_action(const struct rte_flow_action *actions, 1864 enum rte_flow_action_type action) 1865 { 1866 if (actions == NULL) 1867 return NULL; 1868 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1869 if (actions->type == action) 1870 return actions; 1871 return NULL; 1872 } 1873 1874 /* 1875 * Validate the flag action. 1876 * 1877 * @param[in] action_flags 1878 * Bit-fields that holds the actions detected until now. 1879 * @param[in] attr 1880 * Attributes of flow that includes this action. 1881 * @param[out] error 1882 * Pointer to error structure. 1883 * 1884 * @return 1885 * 0 on success, a negative errno value otherwise and rte_errno is set. 1886 */ 1887 int 1888 mlx5_flow_validate_action_flag(uint64_t action_flags, 1889 const struct rte_flow_attr *attr, 1890 struct rte_flow_error *error) 1891 { 1892 if (action_flags & MLX5_FLOW_ACTION_MARK) 1893 return rte_flow_error_set(error, EINVAL, 1894 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1895 "can't mark and flag in same flow"); 1896 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1897 return rte_flow_error_set(error, EINVAL, 1898 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1899 "can't have 2 flag" 1900 " actions in same flow"); 1901 if (attr->egress) 1902 return rte_flow_error_set(error, ENOTSUP, 1903 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1904 "flag action not supported for " 1905 "egress"); 1906 return 0; 1907 } 1908 1909 /* 1910 * Validate the mark action. 1911 * 1912 * @param[in] action 1913 * Pointer to the queue action. 1914 * @param[in] action_flags 1915 * Bit-fields that holds the actions detected until now. 1916 * @param[in] attr 1917 * Attributes of flow that includes this action. 1918 * @param[out] error 1919 * Pointer to error structure. 1920 * 1921 * @return 1922 * 0 on success, a negative errno value otherwise and rte_errno is set. 1923 */ 1924 int 1925 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1926 uint64_t action_flags, 1927 const struct rte_flow_attr *attr, 1928 struct rte_flow_error *error) 1929 { 1930 const struct rte_flow_action_mark *mark = action->conf; 1931 1932 if (!mark) 1933 return rte_flow_error_set(error, EINVAL, 1934 RTE_FLOW_ERROR_TYPE_ACTION, 1935 action, 1936 "configuration cannot be null"); 1937 if (mark->id >= MLX5_FLOW_MARK_MAX) 1938 return rte_flow_error_set(error, EINVAL, 1939 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1940 &mark->id, 1941 "mark id must in 0 <= id < " 1942 RTE_STR(MLX5_FLOW_MARK_MAX)); 1943 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1944 return rte_flow_error_set(error, EINVAL, 1945 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1946 "can't flag and mark in same flow"); 1947 if (action_flags & MLX5_FLOW_ACTION_MARK) 1948 return rte_flow_error_set(error, EINVAL, 1949 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1950 "can't have 2 mark actions in same" 1951 " flow"); 1952 if (attr->egress) 1953 return rte_flow_error_set(error, ENOTSUP, 1954 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1955 "mark action not supported for " 1956 "egress"); 1957 return 0; 1958 } 1959 1960 /* 1961 * Validate the drop action. 1962 * 1963 * @param[in] dev 1964 * Pointer to the Ethernet device structure. 1965 * @param[in] is_root 1966 * True if flow is validated for root table. False otherwise. 1967 * @param[in] attr 1968 * Attributes of flow that includes this action. 1969 * @param[out] error 1970 * Pointer to error structure. 1971 * 1972 * @return 1973 * 0 on success, a negative errno value otherwise and rte_errno is set. 1974 */ 1975 int 1976 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 1977 bool is_root, 1978 const struct rte_flow_attr *attr, 1979 struct rte_flow_error *error) 1980 { 1981 struct mlx5_priv *priv = dev->data->dev_private; 1982 1983 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 1984 return rte_flow_error_set(error, ENOTSUP, 1985 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1986 "drop action not supported for " 1987 "egress"); 1988 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 1989 !priv->sh->dr_root_drop_action_en) { 1990 return rte_flow_error_set(error, ENOTSUP, 1991 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 1992 "drop action not supported for " 1993 "egress and transfer on group 0"); 1994 } 1995 return 0; 1996 } 1997 1998 /* 1999 * Validate the queue action. 2000 * 2001 * @param[in] action 2002 * Pointer to the queue action. 2003 * @param[in] action_flags 2004 * Bit-fields that holds the actions detected until now. 2005 * @param[in] dev 2006 * Pointer to the Ethernet device structure. 2007 * @param[in] attr 2008 * Attributes of flow that includes this action. 2009 * @param[out] error 2010 * Pointer to error structure. 2011 * 2012 * @return 2013 * 0 on success, a negative errno value otherwise and rte_errno is set. 2014 */ 2015 int 2016 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 2017 uint64_t action_flags, 2018 struct rte_eth_dev *dev, 2019 const struct rte_flow_attr *attr, 2020 struct rte_flow_error *error) 2021 { 2022 struct mlx5_priv *priv = dev->data->dev_private; 2023 const struct rte_flow_action_queue *queue = action->conf; 2024 2025 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2026 return rte_flow_error_set(error, EINVAL, 2027 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2028 "can't have 2 fate actions in" 2029 " same flow"); 2030 if (attr->egress) 2031 return rte_flow_error_set(error, ENOTSUP, 2032 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2033 "queue action not supported for egress."); 2034 if (mlx5_is_external_rxq(dev, queue->index)) 2035 return 0; 2036 if (!priv->rxqs_n) 2037 return rte_flow_error_set(error, EINVAL, 2038 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2039 NULL, "No Rx queues configured"); 2040 if (queue->index >= priv->rxqs_n) 2041 return rte_flow_error_set(error, EINVAL, 2042 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2043 &queue->index, 2044 "queue index out of range"); 2045 if (mlx5_rxq_get(dev, queue->index) == NULL) 2046 return rte_flow_error_set(error, EINVAL, 2047 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2048 &queue->index, 2049 "queue is not configured"); 2050 return 0; 2051 } 2052 2053 /** 2054 * Validate queue numbers for device RSS. 2055 * 2056 * @param[in] dev 2057 * Configured device. 2058 * @param[in] queues 2059 * Array of queue numbers. 2060 * @param[in] queues_n 2061 * Size of the @p queues array. 2062 * @param[out] error 2063 * On error, filled with a textual error description. 2064 * @param[out] queue_idx 2065 * On error, filled with an offending queue index in @p queues array. 2066 * 2067 * @return 2068 * 0 on success, a negative errno code on error. 2069 */ 2070 static int 2071 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2072 const uint16_t *queues, uint32_t queues_n, 2073 const char **error, uint32_t *queue_idx) 2074 { 2075 const struct mlx5_priv *priv = dev->data->dev_private; 2076 bool is_hairpin = false; 2077 bool is_ext_rss = false; 2078 uint32_t i; 2079 2080 for (i = 0; i != queues_n; ++i) { 2081 struct mlx5_rxq_ctrl *rxq_ctrl; 2082 2083 if (mlx5_is_external_rxq(dev, queues[0])) { 2084 is_ext_rss = true; 2085 continue; 2086 } 2087 if (is_ext_rss) { 2088 *error = "Combining external and regular RSS queues is not supported"; 2089 *queue_idx = i; 2090 return -ENOTSUP; 2091 } 2092 if (queues[i] >= priv->rxqs_n) { 2093 *error = "queue index out of range"; 2094 *queue_idx = i; 2095 return -EINVAL; 2096 } 2097 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2098 if (rxq_ctrl == NULL) { 2099 *error = "queue is not configured"; 2100 *queue_idx = i; 2101 return -EINVAL; 2102 } 2103 if (i == 0 && rxq_ctrl->is_hairpin) 2104 is_hairpin = true; 2105 if (is_hairpin != rxq_ctrl->is_hairpin) { 2106 *error = "combining hairpin and regular RSS queues is not supported"; 2107 *queue_idx = i; 2108 return -ENOTSUP; 2109 } 2110 } 2111 return 0; 2112 } 2113 2114 /* 2115 * Validate the rss action. 2116 * 2117 * @param[in] dev 2118 * Pointer to the Ethernet device structure. 2119 * @param[in] action 2120 * Pointer to the queue action. 2121 * @param[out] error 2122 * Pointer to error structure. 2123 * 2124 * @return 2125 * 0 on success, a negative errno value otherwise and rte_errno is set. 2126 */ 2127 int 2128 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2129 const struct rte_flow_action *action, 2130 struct rte_flow_error *error) 2131 { 2132 struct mlx5_priv *priv = dev->data->dev_private; 2133 const struct rte_flow_action_rss *rss = action->conf; 2134 int ret; 2135 const char *message; 2136 uint32_t queue_idx; 2137 2138 if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 2139 DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT", 2140 dev->data->port_id); 2141 } else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2142 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2143 return rte_flow_error_set(error, ENOTSUP, 2144 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2145 &rss->func, 2146 "RSS hash function not supported"); 2147 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2148 if (rss->level > 2) 2149 #else 2150 if (rss->level > 1) 2151 #endif 2152 return rte_flow_error_set(error, ENOTSUP, 2153 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2154 &rss->level, 2155 "tunnel RSS is not supported"); 2156 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2157 if (rss->key_len == 0 && rss->key != NULL) 2158 return rte_flow_error_set(error, ENOTSUP, 2159 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2160 &rss->key_len, 2161 "RSS hash key length 0"); 2162 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2163 return rte_flow_error_set(error, ENOTSUP, 2164 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2165 &rss->key_len, 2166 "RSS hash key too small"); 2167 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2168 return rte_flow_error_set(error, ENOTSUP, 2169 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2170 &rss->key_len, 2171 "RSS hash key too large"); 2172 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2173 return rte_flow_error_set(error, ENOTSUP, 2174 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2175 &rss->queue_num, 2176 "number of queues too large"); 2177 if (rss->types & MLX5_RSS_HF_MASK) 2178 return rte_flow_error_set(error, ENOTSUP, 2179 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2180 &rss->types, 2181 "some RSS protocols are not" 2182 " supported"); 2183 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2184 !(rss->types & RTE_ETH_RSS_IP)) 2185 return rte_flow_error_set(error, EINVAL, 2186 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2187 "L3 partial RSS requested but L3 RSS" 2188 " type not specified"); 2189 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2190 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2191 return rte_flow_error_set(error, EINVAL, 2192 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2193 "L4 partial RSS requested but L4 RSS" 2194 " type not specified"); 2195 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2196 return rte_flow_error_set(error, EINVAL, 2197 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2198 NULL, "No Rx queues configured"); 2199 if (!rss->queue_num) 2200 return rte_flow_error_set(error, EINVAL, 2201 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2202 NULL, "No queues configured"); 2203 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2204 &message, &queue_idx); 2205 if (ret != 0) { 2206 return rte_flow_error_set(error, -ret, 2207 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2208 &rss->queue[queue_idx], message); 2209 } 2210 return 0; 2211 } 2212 2213 /* 2214 * Validate the rss action. 2215 * 2216 * @param[in] action 2217 * Pointer to the queue action. 2218 * @param[in] action_flags 2219 * Bit-fields that holds the actions detected until now. 2220 * @param[in] dev 2221 * Pointer to the Ethernet device structure. 2222 * @param[in] attr 2223 * Attributes of flow that includes this action. 2224 * @param[in] item_flags 2225 * Items that were detected. 2226 * @param[out] error 2227 * Pointer to error structure. 2228 * 2229 * @return 2230 * 0 on success, a negative errno value otherwise and rte_errno is set. 2231 */ 2232 int 2233 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2234 uint64_t action_flags, 2235 struct rte_eth_dev *dev, 2236 const struct rte_flow_attr *attr, 2237 uint64_t item_flags, 2238 struct rte_flow_error *error) 2239 { 2240 const struct rte_flow_action_rss *rss = action->conf; 2241 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2242 int ret; 2243 2244 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2245 return rte_flow_error_set(error, EINVAL, 2246 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2247 "can't have 2 fate actions" 2248 " in same flow"); 2249 ret = mlx5_validate_action_rss(dev, action, error); 2250 if (ret) 2251 return ret; 2252 if (attr->egress) 2253 return rte_flow_error_set(error, ENOTSUP, 2254 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2255 "rss action not supported for " 2256 "egress"); 2257 if (rss->level > 1 && !tunnel) 2258 return rte_flow_error_set(error, EINVAL, 2259 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2260 "inner RSS is not supported for " 2261 "non-tunnel flows"); 2262 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2263 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2264 return rte_flow_error_set(error, EINVAL, 2265 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2266 "RSS on eCPRI is not supported now"); 2267 } 2268 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2269 !(item_flags & 2270 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2271 rss->level > 1) 2272 return rte_flow_error_set(error, EINVAL, 2273 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2274 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2275 return 0; 2276 } 2277 2278 /* 2279 * Validate the default miss action. 2280 * 2281 * @param[in] action_flags 2282 * Bit-fields that holds the actions detected until now. 2283 * @param[out] error 2284 * Pointer to error structure. 2285 * 2286 * @return 2287 * 0 on success, a negative errno value otherwise and rte_errno is set. 2288 */ 2289 int 2290 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2291 const struct rte_flow_attr *attr, 2292 struct rte_flow_error *error) 2293 { 2294 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2295 return rte_flow_error_set(error, EINVAL, 2296 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2297 "can't have 2 fate actions in" 2298 " same flow"); 2299 if (attr->egress) 2300 return rte_flow_error_set(error, ENOTSUP, 2301 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2302 "default miss action not supported " 2303 "for egress"); 2304 if (attr->group) 2305 return rte_flow_error_set(error, ENOTSUP, 2306 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2307 "only group 0 is supported"); 2308 if (attr->transfer) 2309 return rte_flow_error_set(error, ENOTSUP, 2310 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2311 NULL, "transfer is not supported"); 2312 return 0; 2313 } 2314 2315 /* 2316 * Validate the count action. 2317 * 2318 * @param[in] dev 2319 * Pointer to the Ethernet device structure. 2320 * @param[in] attr 2321 * Attributes of flow that includes this action. 2322 * @param[out] error 2323 * Pointer to error structure. 2324 * 2325 * @return 2326 * 0 on success, a negative errno value otherwise and rte_errno is set. 2327 */ 2328 int 2329 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2330 const struct rte_flow_attr *attr, 2331 struct rte_flow_error *error) 2332 { 2333 if (attr->egress) 2334 return rte_flow_error_set(error, ENOTSUP, 2335 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2336 "count action not supported for " 2337 "egress"); 2338 return 0; 2339 } 2340 2341 /* 2342 * Validate the ASO CT action. 2343 * 2344 * @param[in] dev 2345 * Pointer to the Ethernet device structure. 2346 * @param[in] conntrack 2347 * Pointer to the CT action profile. 2348 * @param[out] error 2349 * Pointer to error structure. 2350 * 2351 * @return 2352 * 0 on success, a negative errno value otherwise and rte_errno is set. 2353 */ 2354 int 2355 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2356 const struct rte_flow_action_conntrack *conntrack, 2357 struct rte_flow_error *error) 2358 { 2359 RTE_SET_USED(dev); 2360 2361 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2362 return rte_flow_error_set(error, EINVAL, 2363 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2364 "Invalid CT state"); 2365 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2366 return rte_flow_error_set(error, EINVAL, 2367 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2368 "Invalid last TCP packet flag"); 2369 return 0; 2370 } 2371 2372 /** 2373 * Validate the level value for modify field action. 2374 * 2375 * @param[in] data 2376 * Pointer to the rte_flow_field_data structure either src or dst. 2377 * @param[out] error 2378 * Pointer to error structure. 2379 * 2380 * @return 2381 * 0 on success, a negative errno value otherwise and rte_errno is set. 2382 */ 2383 int 2384 flow_validate_modify_field_level(const struct rte_flow_field_data *data, 2385 struct rte_flow_error *error) 2386 { 2387 if (data->level == 0 || data->field == RTE_FLOW_FIELD_FLEX_ITEM) 2388 return 0; 2389 if (data->field != RTE_FLOW_FIELD_TAG && 2390 data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) { 2391 if (data->level > 1) 2392 return rte_flow_error_set(error, ENOTSUP, 2393 RTE_FLOW_ERROR_TYPE_ACTION, 2394 NULL, 2395 "inner header fields modification is not supported"); 2396 return 0; 2397 } 2398 if (data->tag_index != 0) 2399 return rte_flow_error_set(error, EINVAL, 2400 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2401 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2402 /* 2403 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2404 * 'tag_index' field. In old API, it was provided using 'level' field 2405 * and it is still supported for backwards compatibility. 2406 */ 2407 DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field."); 2408 return 0; 2409 } 2410 2411 /** 2412 * Validate ICMP6 item. 2413 * 2414 * @param[in] item 2415 * Item specification. 2416 * @param[in] item_flags 2417 * Bit-fields that holds the items detected until now. 2418 * @param[in] ext_vlan_sup 2419 * Whether extended VLAN features are supported or not. 2420 * @param[out] error 2421 * Pointer to error structure. 2422 * 2423 * @return 2424 * 0 on success, a negative errno value otherwise and rte_errno is set. 2425 */ 2426 int 2427 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2428 uint64_t item_flags, 2429 uint8_t target_protocol, 2430 struct rte_flow_error *error) 2431 { 2432 const struct rte_flow_item_icmp6 *mask = item->mask; 2433 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2434 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2435 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2436 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2437 MLX5_FLOW_LAYER_OUTER_L4; 2438 int ret; 2439 2440 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2441 return rte_flow_error_set(error, EINVAL, 2442 RTE_FLOW_ERROR_TYPE_ITEM, item, 2443 "protocol filtering not compatible" 2444 " with ICMP6 layer"); 2445 if (!(item_flags & l3m)) 2446 return rte_flow_error_set(error, EINVAL, 2447 RTE_FLOW_ERROR_TYPE_ITEM, item, 2448 "IPv6 is mandatory to filter on" 2449 " ICMP6"); 2450 if (item_flags & l4m) 2451 return rte_flow_error_set(error, EINVAL, 2452 RTE_FLOW_ERROR_TYPE_ITEM, item, 2453 "multiple L4 layers not supported"); 2454 if (!mask) 2455 mask = &rte_flow_item_icmp6_mask; 2456 ret = mlx5_flow_item_acceptable 2457 (item, (const uint8_t *)mask, 2458 (const uint8_t *)&rte_flow_item_icmp6_mask, 2459 sizeof(struct rte_flow_item_icmp6), 2460 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2461 if (ret < 0) 2462 return ret; 2463 return 0; 2464 } 2465 2466 /** 2467 * Validate ICMP6 echo request/reply item. 2468 * 2469 * @param[in] item 2470 * Item specification. 2471 * @param[in] item_flags 2472 * Bit-fields that holds the items detected until now. 2473 * @param[in] ext_vlan_sup 2474 * Whether extended VLAN features are supported or not. 2475 * @param[out] error 2476 * Pointer to error structure. 2477 * 2478 * @return 2479 * 0 on success, a negative errno value otherwise and rte_errno is set. 2480 */ 2481 int 2482 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2483 uint64_t item_flags, 2484 uint8_t target_protocol, 2485 struct rte_flow_error *error) 2486 { 2487 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2488 const struct rte_flow_item_icmp6_echo nic_mask = { 2489 .hdr.base.type = 0xff, 2490 .hdr.base.code = 0xff, 2491 .hdr.identifier = RTE_BE16(0xffff), 2492 .hdr.sequence = RTE_BE16(0xffff), 2493 }; 2494 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2495 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2496 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2497 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2498 MLX5_FLOW_LAYER_OUTER_L4; 2499 int ret; 2500 2501 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2502 return rte_flow_error_set(error, EINVAL, 2503 RTE_FLOW_ERROR_TYPE_ITEM, item, 2504 "protocol filtering not compatible" 2505 " with ICMP6 layer"); 2506 if (!(item_flags & l3m)) 2507 return rte_flow_error_set(error, EINVAL, 2508 RTE_FLOW_ERROR_TYPE_ITEM, item, 2509 "IPv6 is mandatory to filter on" 2510 " ICMP6"); 2511 if (item_flags & l4m) 2512 return rte_flow_error_set(error, EINVAL, 2513 RTE_FLOW_ERROR_TYPE_ITEM, item, 2514 "multiple L4 layers not supported"); 2515 if (!mask) 2516 mask = &nic_mask; 2517 ret = mlx5_flow_item_acceptable 2518 (item, (const uint8_t *)mask, 2519 (const uint8_t *)&nic_mask, 2520 sizeof(struct rte_flow_item_icmp6_echo), 2521 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2522 if (ret < 0) 2523 return ret; 2524 return 0; 2525 } 2526 2527 /** 2528 * Validate ICMP item. 2529 * 2530 * @param[in] item 2531 * Item specification. 2532 * @param[in] item_flags 2533 * Bit-fields that holds the items detected until now. 2534 * @param[out] error 2535 * Pointer to error structure. 2536 * 2537 * @return 2538 * 0 on success, a negative errno value otherwise and rte_errno is set. 2539 */ 2540 int 2541 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2542 uint64_t item_flags, 2543 uint8_t target_protocol, 2544 struct rte_flow_error *error) 2545 { 2546 const struct rte_flow_item_icmp *mask = item->mask; 2547 const struct rte_flow_item_icmp nic_mask = { 2548 .hdr.icmp_type = 0xff, 2549 .hdr.icmp_code = 0xff, 2550 .hdr.icmp_ident = RTE_BE16(0xffff), 2551 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2552 }; 2553 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2554 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2555 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2556 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2557 MLX5_FLOW_LAYER_OUTER_L4; 2558 int ret; 2559 2560 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2561 return rte_flow_error_set(error, EINVAL, 2562 RTE_FLOW_ERROR_TYPE_ITEM, item, 2563 "protocol filtering not compatible" 2564 " with ICMP layer"); 2565 if (!(item_flags & l3m)) 2566 return rte_flow_error_set(error, EINVAL, 2567 RTE_FLOW_ERROR_TYPE_ITEM, item, 2568 "IPv4 is mandatory to filter" 2569 " on ICMP"); 2570 if (item_flags & l4m) 2571 return rte_flow_error_set(error, EINVAL, 2572 RTE_FLOW_ERROR_TYPE_ITEM, item, 2573 "multiple L4 layers not supported"); 2574 if (!mask) 2575 mask = &nic_mask; 2576 ret = mlx5_flow_item_acceptable 2577 (item, (const uint8_t *)mask, 2578 (const uint8_t *)&nic_mask, 2579 sizeof(struct rte_flow_item_icmp), 2580 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2581 if (ret < 0) 2582 return ret; 2583 return 0; 2584 } 2585 2586 /** 2587 * Validate Ethernet item. 2588 * 2589 * @param[in] item 2590 * Item specification. 2591 * @param[in] item_flags 2592 * Bit-fields that holds the items detected until now. 2593 * @param[out] error 2594 * Pointer to error structure. 2595 * 2596 * @return 2597 * 0 on success, a negative errno value otherwise and rte_errno is set. 2598 */ 2599 int 2600 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2601 uint64_t item_flags, bool ext_vlan_sup, 2602 struct rte_flow_error *error) 2603 { 2604 const struct rte_flow_item_eth *mask = item->mask; 2605 const struct rte_flow_item_eth nic_mask = { 2606 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2607 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2608 .hdr.ether_type = RTE_BE16(0xffff), 2609 .has_vlan = ext_vlan_sup ? 1 : 0, 2610 }; 2611 int ret; 2612 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2613 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2614 MLX5_FLOW_LAYER_OUTER_L2; 2615 2616 if (item_flags & ethm) 2617 return rte_flow_error_set(error, ENOTSUP, 2618 RTE_FLOW_ERROR_TYPE_ITEM, item, 2619 "multiple L2 layers not supported"); 2620 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2621 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2622 return rte_flow_error_set(error, EINVAL, 2623 RTE_FLOW_ERROR_TYPE_ITEM, item, 2624 "L2 layer should not follow " 2625 "L3 layers"); 2626 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2627 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2628 return rte_flow_error_set(error, EINVAL, 2629 RTE_FLOW_ERROR_TYPE_ITEM, item, 2630 "L2 layer should not follow VLAN"); 2631 if (item_flags & MLX5_FLOW_LAYER_GTP) 2632 return rte_flow_error_set(error, EINVAL, 2633 RTE_FLOW_ERROR_TYPE_ITEM, item, 2634 "L2 layer should not follow GTP"); 2635 if (!mask) 2636 mask = &rte_flow_item_eth_mask; 2637 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2638 (const uint8_t *)&nic_mask, 2639 sizeof(struct rte_flow_item_eth), 2640 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2641 return ret; 2642 } 2643 2644 /** 2645 * Validate VLAN item. 2646 * 2647 * @param[in] item 2648 * Item specification. 2649 * @param[in] item_flags 2650 * Bit-fields that holds the items detected until now. 2651 * @param[in] dev 2652 * Ethernet device flow is being created on. 2653 * @param[out] error 2654 * Pointer to error structure. 2655 * 2656 * @return 2657 * 0 on success, a negative errno value otherwise and rte_errno is set. 2658 */ 2659 int 2660 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2661 uint64_t item_flags, 2662 struct rte_eth_dev *dev, 2663 struct rte_flow_error *error) 2664 { 2665 const struct rte_flow_item_vlan *spec = item->spec; 2666 const struct rte_flow_item_vlan *mask = item->mask; 2667 const struct rte_flow_item_vlan nic_mask = { 2668 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2669 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2670 }; 2671 uint16_t vlan_tag = 0; 2672 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2673 int ret; 2674 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2675 MLX5_FLOW_LAYER_INNER_L4) : 2676 (MLX5_FLOW_LAYER_OUTER_L3 | 2677 MLX5_FLOW_LAYER_OUTER_L4); 2678 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2679 MLX5_FLOW_LAYER_OUTER_VLAN; 2680 2681 if (item_flags & vlanm) 2682 return rte_flow_error_set(error, EINVAL, 2683 RTE_FLOW_ERROR_TYPE_ITEM, item, 2684 "multiple VLAN layers not supported"); 2685 else if ((item_flags & l34m) != 0) 2686 return rte_flow_error_set(error, EINVAL, 2687 RTE_FLOW_ERROR_TYPE_ITEM, item, 2688 "VLAN cannot follow L3/L4 layer"); 2689 if (!mask) 2690 mask = &rte_flow_item_vlan_mask; 2691 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2692 (const uint8_t *)&nic_mask, 2693 sizeof(struct rte_flow_item_vlan), 2694 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2695 if (ret) 2696 return ret; 2697 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2698 struct mlx5_priv *priv = dev->data->dev_private; 2699 2700 if (priv->vmwa_context) { 2701 /* 2702 * Non-NULL context means we have a virtual machine 2703 * and SR-IOV enabled, we have to create VLAN interface 2704 * to make hypervisor to setup E-Switch vport 2705 * context correctly. We avoid creating the multiple 2706 * VLAN interfaces, so we cannot support VLAN tag mask. 2707 */ 2708 return rte_flow_error_set(error, EINVAL, 2709 RTE_FLOW_ERROR_TYPE_ITEM, 2710 item, 2711 "VLAN tag mask is not" 2712 " supported in virtual" 2713 " environment"); 2714 } 2715 } 2716 if (spec) { 2717 vlan_tag = spec->hdr.vlan_tci; 2718 vlan_tag &= mask->hdr.vlan_tci; 2719 } 2720 /* 2721 * From verbs perspective an empty VLAN is equivalent 2722 * to a packet without VLAN layer. 2723 */ 2724 if (!vlan_tag) 2725 return rte_flow_error_set(error, EINVAL, 2726 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2727 item->spec, 2728 "VLAN cannot be empty"); 2729 return 0; 2730 } 2731 2732 /** 2733 * Validate IPV4 item. 2734 * 2735 * @param[in] item 2736 * Item specification. 2737 * @param[in] item_flags 2738 * Bit-fields that holds the items detected until now. 2739 * @param[in] last_item 2740 * Previous validated item in the pattern items. 2741 * @param[in] ether_type 2742 * Type in the ethernet layer header (including dot1q). 2743 * @param[in] acc_mask 2744 * Acceptable mask, if NULL default internal default mask 2745 * will be used to check whether item fields are supported. 2746 * @param[in] range_accepted 2747 * True if range of values is accepted for specific fields, false otherwise. 2748 * @param[out] error 2749 * Pointer to error structure. 2750 * 2751 * @return 2752 * 0 on success, a negative errno value otherwise and rte_errno is set. 2753 */ 2754 int 2755 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2756 uint64_t item_flags, 2757 uint64_t last_item, 2758 uint16_t ether_type, 2759 const struct rte_flow_item_ipv4 *acc_mask, 2760 bool range_accepted, 2761 struct rte_flow_error *error) 2762 { 2763 const struct rte_flow_item_ipv4 *mask = item->mask; 2764 const struct rte_flow_item_ipv4 *spec = item->spec; 2765 const struct rte_flow_item_ipv4 nic_mask = { 2766 .hdr = { 2767 .src_addr = RTE_BE32(0xffffffff), 2768 .dst_addr = RTE_BE32(0xffffffff), 2769 .type_of_service = 0xff, 2770 .next_proto_id = 0xff, 2771 }, 2772 }; 2773 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2774 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2775 MLX5_FLOW_LAYER_OUTER_L3; 2776 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2777 MLX5_FLOW_LAYER_OUTER_L4; 2778 int ret; 2779 uint8_t next_proto = 0xFF; 2780 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2781 MLX5_FLOW_LAYER_OUTER_VLAN | 2782 MLX5_FLOW_LAYER_INNER_VLAN); 2783 2784 if ((last_item & l2_vlan) && ether_type && 2785 ether_type != RTE_ETHER_TYPE_IPV4) 2786 return rte_flow_error_set(error, EINVAL, 2787 RTE_FLOW_ERROR_TYPE_ITEM, item, 2788 "IPv4 cannot follow L2/VLAN layer " 2789 "which ether type is not IPv4"); 2790 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2791 if (mask && spec) 2792 next_proto = mask->hdr.next_proto_id & 2793 spec->hdr.next_proto_id; 2794 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2795 return rte_flow_error_set(error, EINVAL, 2796 RTE_FLOW_ERROR_TYPE_ITEM, 2797 item, 2798 "multiple tunnel " 2799 "not supported"); 2800 } 2801 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2802 return rte_flow_error_set(error, EINVAL, 2803 RTE_FLOW_ERROR_TYPE_ITEM, item, 2804 "wrong tunnel type - IPv6 specified " 2805 "but IPv4 item provided"); 2806 if (item_flags & l3m) 2807 return rte_flow_error_set(error, ENOTSUP, 2808 RTE_FLOW_ERROR_TYPE_ITEM, item, 2809 "multiple L3 layers not supported"); 2810 else if (item_flags & l4m) 2811 return rte_flow_error_set(error, EINVAL, 2812 RTE_FLOW_ERROR_TYPE_ITEM, item, 2813 "L3 cannot follow an L4 layer."); 2814 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2815 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2816 return rte_flow_error_set(error, EINVAL, 2817 RTE_FLOW_ERROR_TYPE_ITEM, item, 2818 "L3 cannot follow an NVGRE layer."); 2819 if (!mask) 2820 mask = &rte_flow_item_ipv4_mask; 2821 else if (mask->hdr.next_proto_id != 0 && 2822 mask->hdr.next_proto_id != 0xff) 2823 return rte_flow_error_set(error, EINVAL, 2824 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2825 "partial mask is not supported" 2826 " for protocol"); 2827 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2828 acc_mask ? (const uint8_t *)acc_mask 2829 : (const uint8_t *)&nic_mask, 2830 sizeof(struct rte_flow_item_ipv4), 2831 range_accepted, error); 2832 if (ret < 0) 2833 return ret; 2834 return 0; 2835 } 2836 2837 /** 2838 * Validate IPV6 item. 2839 * 2840 * @param[in] item 2841 * Item specification. 2842 * @param[in] item_flags 2843 * Bit-fields that holds the items detected until now. 2844 * @param[in] last_item 2845 * Previous validated item in the pattern items. 2846 * @param[in] ether_type 2847 * Type in the ethernet layer header (including dot1q). 2848 * @param[in] acc_mask 2849 * Acceptable mask, if NULL default internal default mask 2850 * will be used to check whether item fields are supported. 2851 * @param[out] error 2852 * Pointer to error structure. 2853 * 2854 * @return 2855 * 0 on success, a negative errno value otherwise and rte_errno is set. 2856 */ 2857 int 2858 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2859 uint64_t item_flags, 2860 uint64_t last_item, 2861 uint16_t ether_type, 2862 const struct rte_flow_item_ipv6 *acc_mask, 2863 struct rte_flow_error *error) 2864 { 2865 const struct rte_flow_item_ipv6 *mask = item->mask; 2866 const struct rte_flow_item_ipv6 *spec = item->spec; 2867 const struct rte_flow_item_ipv6 nic_mask = { 2868 .hdr = { 2869 .src_addr = 2870 "\xff\xff\xff\xff\xff\xff\xff\xff" 2871 "\xff\xff\xff\xff\xff\xff\xff\xff", 2872 .dst_addr = 2873 "\xff\xff\xff\xff\xff\xff\xff\xff" 2874 "\xff\xff\xff\xff\xff\xff\xff\xff", 2875 .vtc_flow = RTE_BE32(0xffffffff), 2876 .proto = 0xff, 2877 }, 2878 }; 2879 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2880 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2881 MLX5_FLOW_LAYER_OUTER_L3; 2882 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2883 MLX5_FLOW_LAYER_OUTER_L4; 2884 int ret; 2885 uint8_t next_proto = 0xFF; 2886 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2887 MLX5_FLOW_LAYER_OUTER_VLAN | 2888 MLX5_FLOW_LAYER_INNER_VLAN); 2889 2890 if ((last_item & l2_vlan) && ether_type && 2891 ether_type != RTE_ETHER_TYPE_IPV6) 2892 return rte_flow_error_set(error, EINVAL, 2893 RTE_FLOW_ERROR_TYPE_ITEM, item, 2894 "IPv6 cannot follow L2/VLAN layer " 2895 "which ether type is not IPv6"); 2896 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2897 next_proto = spec->hdr.proto; 2898 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2899 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2900 return rte_flow_error_set(error, EINVAL, 2901 RTE_FLOW_ERROR_TYPE_ITEM, 2902 item, 2903 "multiple tunnel " 2904 "not supported"); 2905 } 2906 if (next_proto == IPPROTO_HOPOPTS || 2907 next_proto == IPPROTO_ROUTING || 2908 next_proto == IPPROTO_FRAGMENT || 2909 next_proto == IPPROTO_ESP || 2910 next_proto == IPPROTO_AH || 2911 next_proto == IPPROTO_DSTOPTS) 2912 return rte_flow_error_set(error, EINVAL, 2913 RTE_FLOW_ERROR_TYPE_ITEM, item, 2914 "IPv6 proto (next header) should " 2915 "not be set as extension header"); 2916 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2917 return rte_flow_error_set(error, EINVAL, 2918 RTE_FLOW_ERROR_TYPE_ITEM, item, 2919 "wrong tunnel type - IPv4 specified " 2920 "but IPv6 item provided"); 2921 if (item_flags & l3m) 2922 return rte_flow_error_set(error, ENOTSUP, 2923 RTE_FLOW_ERROR_TYPE_ITEM, item, 2924 "multiple L3 layers not supported"); 2925 else if (item_flags & l4m) 2926 return rte_flow_error_set(error, EINVAL, 2927 RTE_FLOW_ERROR_TYPE_ITEM, item, 2928 "L3 cannot follow an L4 layer."); 2929 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2930 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2931 return rte_flow_error_set(error, EINVAL, 2932 RTE_FLOW_ERROR_TYPE_ITEM, item, 2933 "L3 cannot follow an NVGRE layer."); 2934 if (!mask) 2935 mask = &rte_flow_item_ipv6_mask; 2936 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2937 acc_mask ? (const uint8_t *)acc_mask 2938 : (const uint8_t *)&nic_mask, 2939 sizeof(struct rte_flow_item_ipv6), 2940 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2941 if (ret < 0) 2942 return ret; 2943 return 0; 2944 } 2945 2946 /** 2947 * Validate UDP item. 2948 * 2949 * @param[in] item 2950 * Item specification. 2951 * @param[in] item_flags 2952 * Bit-fields that holds the items detected until now. 2953 * @param[in] target_protocol 2954 * The next protocol in the previous item. 2955 * @param[in] flow_mask 2956 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2957 * @param[out] error 2958 * Pointer to error structure. 2959 * 2960 * @return 2961 * 0 on success, a negative errno value otherwise and rte_errno is set. 2962 */ 2963 int 2964 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2965 uint64_t item_flags, 2966 uint8_t target_protocol, 2967 struct rte_flow_error *error) 2968 { 2969 const struct rte_flow_item_udp *mask = item->mask; 2970 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2971 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2972 MLX5_FLOW_LAYER_OUTER_L3; 2973 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2974 MLX5_FLOW_LAYER_OUTER_L4; 2975 int ret; 2976 2977 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2978 return rte_flow_error_set(error, EINVAL, 2979 RTE_FLOW_ERROR_TYPE_ITEM, item, 2980 "protocol filtering not compatible" 2981 " with UDP layer"); 2982 if (!(item_flags & l3m)) 2983 return rte_flow_error_set(error, EINVAL, 2984 RTE_FLOW_ERROR_TYPE_ITEM, item, 2985 "L3 is mandatory to filter on L4"); 2986 if (item_flags & l4m) 2987 return rte_flow_error_set(error, EINVAL, 2988 RTE_FLOW_ERROR_TYPE_ITEM, item, 2989 "multiple L4 layers not supported"); 2990 if (!mask) 2991 mask = &rte_flow_item_udp_mask; 2992 ret = mlx5_flow_item_acceptable 2993 (item, (const uint8_t *)mask, 2994 (const uint8_t *)&rte_flow_item_udp_mask, 2995 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2996 error); 2997 if (ret < 0) 2998 return ret; 2999 return 0; 3000 } 3001 3002 /** 3003 * Validate TCP item. 3004 * 3005 * @param[in] item 3006 * Item specification. 3007 * @param[in] item_flags 3008 * Bit-fields that holds the items detected until now. 3009 * @param[in] target_protocol 3010 * The next protocol in the previous item. 3011 * @param[out] error 3012 * Pointer to error structure. 3013 * 3014 * @return 3015 * 0 on success, a negative errno value otherwise and rte_errno is set. 3016 */ 3017 int 3018 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 3019 uint64_t item_flags, 3020 uint8_t target_protocol, 3021 const struct rte_flow_item_tcp *flow_mask, 3022 struct rte_flow_error *error) 3023 { 3024 const struct rte_flow_item_tcp *mask = item->mask; 3025 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3026 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3027 MLX5_FLOW_LAYER_OUTER_L3; 3028 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3029 MLX5_FLOW_LAYER_OUTER_L4; 3030 int ret; 3031 3032 MLX5_ASSERT(flow_mask); 3033 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3034 return rte_flow_error_set(error, EINVAL, 3035 RTE_FLOW_ERROR_TYPE_ITEM, item, 3036 "protocol filtering not compatible" 3037 " with TCP layer"); 3038 if (!(item_flags & l3m)) 3039 return rte_flow_error_set(error, EINVAL, 3040 RTE_FLOW_ERROR_TYPE_ITEM, item, 3041 "L3 is mandatory to filter on L4"); 3042 if (item_flags & l4m) 3043 return rte_flow_error_set(error, EINVAL, 3044 RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "multiple L4 layers not supported"); 3046 if (!mask) 3047 mask = &rte_flow_item_tcp_mask; 3048 ret = mlx5_flow_item_acceptable 3049 (item, (const uint8_t *)mask, 3050 (const uint8_t *)flow_mask, 3051 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3052 error); 3053 if (ret < 0) 3054 return ret; 3055 return 0; 3056 } 3057 3058 /** 3059 * Validate VXLAN item. 3060 * 3061 * @param[in] dev 3062 * Pointer to the Ethernet device structure. 3063 * @param[in] udp_dport 3064 * UDP destination port 3065 * @param[in] item 3066 * Item specification. 3067 * @param[in] item_flags 3068 * Bit-fields that holds the items detected until now. 3069 * @param root 3070 * Whether action is on root table. 3071 * @param[out] error 3072 * Pointer to error structure. 3073 * 3074 * @return 3075 * 0 on success, a negative errno value otherwise and rte_errno is set. 3076 */ 3077 int 3078 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3079 uint16_t udp_dport, 3080 const struct rte_flow_item *item, 3081 uint64_t item_flags, 3082 bool root, 3083 struct rte_flow_error *error) 3084 { 3085 const struct rte_flow_item_vxlan *spec = item->spec; 3086 const struct rte_flow_item_vxlan *mask = item->mask; 3087 int ret; 3088 struct mlx5_priv *priv = dev->data->dev_private; 3089 union vni { 3090 uint32_t vlan_id; 3091 uint8_t vni[4]; 3092 } id = { .vlan_id = 0, }; 3093 const struct rte_flow_item_vxlan nic_mask = { 3094 .hdr.vni = "\xff\xff\xff", 3095 .hdr.rsvd1 = 0xff, 3096 }; 3097 const struct rte_flow_item_vxlan *valid_mask; 3098 3099 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3100 return rte_flow_error_set(error, ENOTSUP, 3101 RTE_FLOW_ERROR_TYPE_ITEM, item, 3102 "multiple tunnel layers not" 3103 " supported"); 3104 valid_mask = &rte_flow_item_vxlan_mask; 3105 /* 3106 * Verify only UDPv4 is present as defined in 3107 * https://tools.ietf.org/html/rfc7348 3108 */ 3109 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3110 return rte_flow_error_set(error, EINVAL, 3111 RTE_FLOW_ERROR_TYPE_ITEM, item, 3112 "no outer UDP layer found"); 3113 if (!mask) 3114 mask = &rte_flow_item_vxlan_mask; 3115 3116 if (priv->sh->steering_format_version != 3117 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3118 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3119 /* non-root table */ 3120 if (!root && priv->sh->misc5_cap) 3121 valid_mask = &nic_mask; 3122 /* Group zero in NIC domain */ 3123 if (!root && priv->sh->tunnel_header_0_1) 3124 valid_mask = &nic_mask; 3125 } 3126 ret = mlx5_flow_item_acceptable 3127 (item, (const uint8_t *)mask, 3128 (const uint8_t *)valid_mask, 3129 sizeof(struct rte_flow_item_vxlan), 3130 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3131 if (ret < 0) 3132 return ret; 3133 if (spec) { 3134 memcpy(&id.vni[1], spec->hdr.vni, 3); 3135 memcpy(&id.vni[1], mask->hdr.vni, 3); 3136 } 3137 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3138 return rte_flow_error_set(error, ENOTSUP, 3139 RTE_FLOW_ERROR_TYPE_ITEM, item, 3140 "VXLAN tunnel must be fully defined"); 3141 return 0; 3142 } 3143 3144 /** 3145 * Validate VXLAN_GPE item. 3146 * 3147 * @param[in] item 3148 * Item specification. 3149 * @param[in] item_flags 3150 * Bit-fields that holds the items detected until now. 3151 * @param[in] priv 3152 * Pointer to the private data structure. 3153 * @param[in] target_protocol 3154 * The next protocol in the previous item. 3155 * @param[out] error 3156 * Pointer to error structure. 3157 * 3158 * @return 3159 * 0 on success, a negative errno value otherwise and rte_errno is set. 3160 */ 3161 int 3162 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3163 uint64_t item_flags, 3164 struct rte_eth_dev *dev, 3165 struct rte_flow_error *error) 3166 { 3167 struct mlx5_priv *priv = dev->data->dev_private; 3168 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3169 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3170 int ret; 3171 union vni { 3172 uint32_t vlan_id; 3173 uint8_t vni[4]; 3174 } id = { .vlan_id = 0, }; 3175 3176 struct rte_flow_item_vxlan_gpe nic_mask = { 3177 .vni = "\xff\xff\xff", 3178 .protocol = 0xff, 3179 .flags = 0xff, 3180 }; 3181 3182 if (!priv->sh->config.l3_vxlan_en) 3183 return rte_flow_error_set(error, ENOTSUP, 3184 RTE_FLOW_ERROR_TYPE_ITEM, item, 3185 "L3 VXLAN is not enabled by device" 3186 " parameter and/or not configured in" 3187 " firmware"); 3188 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3189 return rte_flow_error_set(error, ENOTSUP, 3190 RTE_FLOW_ERROR_TYPE_ITEM, item, 3191 "multiple tunnel layers not" 3192 " supported"); 3193 /* 3194 * Verify only UDPv4 is present as defined in 3195 * https://tools.ietf.org/html/rfc7348 3196 */ 3197 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3198 return rte_flow_error_set(error, EINVAL, 3199 RTE_FLOW_ERROR_TYPE_ITEM, item, 3200 "no outer UDP layer found"); 3201 if (!mask) 3202 mask = &rte_flow_item_vxlan_gpe_mask; 3203 if (priv->sh->misc5_cap && priv->sh->tunnel_header_0_1) { 3204 nic_mask.rsvd0[0] = 0xff; 3205 nic_mask.rsvd0[1] = 0xff; 3206 nic_mask.rsvd1 = 0xff; 3207 } 3208 ret = mlx5_flow_item_acceptable 3209 (item, (const uint8_t *)mask, 3210 (const uint8_t *)&nic_mask, 3211 sizeof(struct rte_flow_item_vxlan_gpe), 3212 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3213 if (ret < 0) 3214 return ret; 3215 if (spec) { 3216 memcpy(&id.vni[1], spec->hdr.vni, 3); 3217 memcpy(&id.vni[1], mask->hdr.vni, 3); 3218 } 3219 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3220 return rte_flow_error_set(error, ENOTSUP, 3221 RTE_FLOW_ERROR_TYPE_ITEM, item, 3222 "VXLAN-GPE tunnel must be fully" 3223 " defined"); 3224 return 0; 3225 } 3226 /** 3227 * Validate GRE Key item. 3228 * 3229 * @param[in] item 3230 * Item specification. 3231 * @param[in] item_flags 3232 * Bit flags to mark detected items. 3233 * @param[in] gre_item 3234 * Pointer to gre_item 3235 * @param[out] error 3236 * Pointer to error structure. 3237 * 3238 * @return 3239 * 0 on success, a negative errno value otherwise and rte_errno is set. 3240 */ 3241 int 3242 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3243 uint64_t item_flags, 3244 const struct rte_flow_item *gre_item, 3245 struct rte_flow_error *error) 3246 { 3247 const rte_be32_t *mask = item->mask; 3248 int ret = 0; 3249 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3250 const struct rte_flow_item_gre *gre_spec; 3251 const struct rte_flow_item_gre *gre_mask; 3252 3253 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3254 return rte_flow_error_set(error, ENOTSUP, 3255 RTE_FLOW_ERROR_TYPE_ITEM, item, 3256 "Multiple GRE key not support"); 3257 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3258 return rte_flow_error_set(error, ENOTSUP, 3259 RTE_FLOW_ERROR_TYPE_ITEM, item, 3260 "No preceding GRE header"); 3261 if (item_flags & MLX5_FLOW_LAYER_INNER) 3262 return rte_flow_error_set(error, ENOTSUP, 3263 RTE_FLOW_ERROR_TYPE_ITEM, item, 3264 "GRE key following a wrong item"); 3265 gre_mask = gre_item->mask; 3266 if (!gre_mask) 3267 gre_mask = &rte_flow_item_gre_mask; 3268 gre_spec = gre_item->spec; 3269 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3270 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3271 return rte_flow_error_set(error, EINVAL, 3272 RTE_FLOW_ERROR_TYPE_ITEM, item, 3273 "Key bit must be on"); 3274 3275 if (!mask) 3276 mask = &gre_key_default_mask; 3277 ret = mlx5_flow_item_acceptable 3278 (item, (const uint8_t *)mask, 3279 (const uint8_t *)&gre_key_default_mask, 3280 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3281 return ret; 3282 } 3283 3284 /** 3285 * Validate GRE optional item. 3286 * 3287 * @param[in] dev 3288 * Pointer to the Ethernet device structure. 3289 * @param[in] item 3290 * Item specification. 3291 * @param[in] item_flags 3292 * Bit flags to mark detected items. 3293 * @param[in] attr 3294 * Flow rule attributes. 3295 * @param[in] gre_item 3296 * Pointer to gre_item 3297 * @param[out] error 3298 * Pointer to error structure. 3299 * 3300 * @return 3301 * 0 on success, a negative errno value otherwise and rte_errno is set. 3302 */ 3303 int 3304 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3305 const struct rte_flow_item *item, 3306 uint64_t item_flags, 3307 const struct rte_flow_attr *attr, 3308 const struct rte_flow_item *gre_item, 3309 struct rte_flow_error *error) 3310 { 3311 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3312 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3313 const struct rte_flow_item_gre_opt *spec = item->spec; 3314 const struct rte_flow_item_gre_opt *mask = item->mask; 3315 struct mlx5_priv *priv = dev->data->dev_private; 3316 int ret = 0; 3317 struct rte_flow_item_gre_opt nic_mask = { 3318 .checksum_rsvd = { 3319 .checksum = RTE_BE16(UINT16_MAX), 3320 .reserved1 = 0x0, 3321 }, 3322 .key = { 3323 .key = RTE_BE32(UINT32_MAX), 3324 }, 3325 .sequence = { 3326 .sequence = RTE_BE32(UINT32_MAX), 3327 }, 3328 }; 3329 3330 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3331 return rte_flow_error_set(error, ENOTSUP, 3332 RTE_FLOW_ERROR_TYPE_ITEM, item, 3333 "No preceding GRE header"); 3334 if (item_flags & MLX5_FLOW_LAYER_INNER) 3335 return rte_flow_error_set(error, ENOTSUP, 3336 RTE_FLOW_ERROR_TYPE_ITEM, item, 3337 "GRE option following a wrong item"); 3338 if (!spec || !mask) 3339 return rte_flow_error_set(error, EINVAL, 3340 RTE_FLOW_ERROR_TYPE_ITEM, item, 3341 "At least one field gre_option(checksum/key/sequence) must be specified"); 3342 if (!gre_mask) 3343 gre_mask = &rte_flow_item_gre_mask; 3344 if (mask->checksum_rsvd.checksum) 3345 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3346 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3347 return rte_flow_error_set(error, EINVAL, 3348 RTE_FLOW_ERROR_TYPE_ITEM, 3349 item, 3350 "Checksum bit must be on"); 3351 if (mask->key.key) 3352 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3353 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3354 return rte_flow_error_set(error, EINVAL, 3355 RTE_FLOW_ERROR_TYPE_ITEM, 3356 item, "Key bit must be on"); 3357 if (mask->sequence.sequence) 3358 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3359 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3360 return rte_flow_error_set(error, EINVAL, 3361 RTE_FLOW_ERROR_TYPE_ITEM, 3362 item, 3363 "Sequence bit must be on"); 3364 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3365 if (priv->sh->steering_format_version == 3366 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3367 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3368 !priv->sh->misc5_cap) || 3369 (!(priv->sh->tunnel_header_0_1 && 3370 priv->sh->tunnel_header_2_3) && 3371 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3372 return rte_flow_error_set(error, EINVAL, 3373 RTE_FLOW_ERROR_TYPE_ITEM, 3374 item, 3375 "Checksum/Sequence not supported"); 3376 } 3377 ret = mlx5_flow_item_acceptable 3378 (item, (const uint8_t *)mask, 3379 (const uint8_t *)&nic_mask, 3380 sizeof(struct rte_flow_item_gre_opt), 3381 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3382 return ret; 3383 } 3384 3385 /** 3386 * Validate GRE item. 3387 * 3388 * @param[in] item 3389 * Item specification. 3390 * @param[in] item_flags 3391 * Bit flags to mark detected items. 3392 * @param[in] target_protocol 3393 * The next protocol in the previous item. 3394 * @param[out] error 3395 * Pointer to error structure. 3396 * 3397 * @return 3398 * 0 on success, a negative errno value otherwise and rte_errno is set. 3399 */ 3400 int 3401 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3402 uint64_t item_flags, 3403 uint8_t target_protocol, 3404 struct rte_flow_error *error) 3405 { 3406 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3407 const struct rte_flow_item_gre *mask = item->mask; 3408 int ret; 3409 const struct rte_flow_item_gre nic_mask = { 3410 .c_rsvd0_ver = RTE_BE16(0xB000), 3411 .protocol = RTE_BE16(UINT16_MAX), 3412 }; 3413 3414 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3415 return rte_flow_error_set(error, EINVAL, 3416 RTE_FLOW_ERROR_TYPE_ITEM, item, 3417 "protocol filtering not compatible" 3418 " with this GRE layer"); 3419 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3420 return rte_flow_error_set(error, ENOTSUP, 3421 RTE_FLOW_ERROR_TYPE_ITEM, item, 3422 "multiple tunnel layers not" 3423 " supported"); 3424 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3425 return rte_flow_error_set(error, ENOTSUP, 3426 RTE_FLOW_ERROR_TYPE_ITEM, item, 3427 "L3 Layer is missing"); 3428 if (!mask) 3429 mask = &rte_flow_item_gre_mask; 3430 ret = mlx5_flow_item_acceptable 3431 (item, (const uint8_t *)mask, 3432 (const uint8_t *)&nic_mask, 3433 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3434 error); 3435 if (ret < 0) 3436 return ret; 3437 #ifndef HAVE_MLX5DV_DR 3438 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3439 if (spec && (spec->protocol & mask->protocol)) 3440 return rte_flow_error_set(error, ENOTSUP, 3441 RTE_FLOW_ERROR_TYPE_ITEM, item, 3442 "without MPLS support the" 3443 " specification cannot be used for" 3444 " filtering"); 3445 #endif 3446 #endif 3447 return 0; 3448 } 3449 3450 /** 3451 * Validate Geneve item. 3452 * 3453 * @param[in] item 3454 * Item specification. 3455 * @param[in] itemFlags 3456 * Bit-fields that holds the items detected until now. 3457 * @param[in] enPriv 3458 * Pointer to the private data structure. 3459 * @param[out] error 3460 * Pointer to error structure. 3461 * 3462 * @return 3463 * 0 on success, a negative errno value otherwise and rte_errno is set. 3464 */ 3465 3466 int 3467 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3468 uint64_t item_flags, 3469 struct rte_eth_dev *dev, 3470 struct rte_flow_error *error) 3471 { 3472 struct mlx5_priv *priv = dev->data->dev_private; 3473 const struct rte_flow_item_geneve *spec = item->spec; 3474 const struct rte_flow_item_geneve *mask = item->mask; 3475 int ret; 3476 uint16_t gbhdr; 3477 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3478 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3479 const struct rte_flow_item_geneve nic_mask = { 3480 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3481 .vni = "\xff\xff\xff", 3482 .protocol = RTE_BE16(UINT16_MAX), 3483 }; 3484 3485 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3486 return rte_flow_error_set(error, ENOTSUP, 3487 RTE_FLOW_ERROR_TYPE_ITEM, item, 3488 "L3 Geneve is not enabled by device" 3489 " parameter and/or not configured in" 3490 " firmware"); 3491 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3492 return rte_flow_error_set(error, ENOTSUP, 3493 RTE_FLOW_ERROR_TYPE_ITEM, item, 3494 "multiple tunnel layers not" 3495 " supported"); 3496 /* 3497 * Verify only UDPv4 is present as defined in 3498 * https://tools.ietf.org/html/rfc7348 3499 */ 3500 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3501 return rte_flow_error_set(error, EINVAL, 3502 RTE_FLOW_ERROR_TYPE_ITEM, item, 3503 "no outer UDP layer found"); 3504 if (!mask) 3505 mask = &rte_flow_item_geneve_mask; 3506 ret = mlx5_flow_item_acceptable 3507 (item, (const uint8_t *)mask, 3508 (const uint8_t *)&nic_mask, 3509 sizeof(struct rte_flow_item_geneve), 3510 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3511 if (ret) 3512 return ret; 3513 if (spec) { 3514 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3515 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3516 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3517 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3518 return rte_flow_error_set(error, ENOTSUP, 3519 RTE_FLOW_ERROR_TYPE_ITEM, 3520 item, 3521 "Geneve protocol unsupported" 3522 " fields are being used"); 3523 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3524 return rte_flow_error_set 3525 (error, ENOTSUP, 3526 RTE_FLOW_ERROR_TYPE_ITEM, 3527 item, 3528 "Unsupported Geneve options length"); 3529 } 3530 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3531 return rte_flow_error_set 3532 (error, ENOTSUP, 3533 RTE_FLOW_ERROR_TYPE_ITEM, item, 3534 "Geneve tunnel must be fully defined"); 3535 return 0; 3536 } 3537 3538 /** 3539 * Validate Geneve TLV option item. 3540 * 3541 * @param[in] item 3542 * Item specification. 3543 * @param[in] last_item 3544 * Previous validated item in the pattern items. 3545 * @param[in] geneve_item 3546 * Previous GENEVE item specification. 3547 * @param[in] dev 3548 * Pointer to the rte_eth_dev structure. 3549 * @param[out] error 3550 * Pointer to error structure. 3551 * 3552 * @return 3553 * 0 on success, a negative errno value otherwise and rte_errno is set. 3554 */ 3555 int 3556 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3557 uint64_t last_item, 3558 const struct rte_flow_item *geneve_item, 3559 struct rte_eth_dev *dev, 3560 struct rte_flow_error *error) 3561 { 3562 struct mlx5_priv *priv = dev->data->dev_private; 3563 struct mlx5_dev_ctx_shared *sh = priv->sh; 3564 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3565 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3566 uint8_t data_max_supported = 3567 hca_attr->max_geneve_tlv_option_data_len * 4; 3568 const struct rte_flow_item_geneve *geneve_spec; 3569 const struct rte_flow_item_geneve *geneve_mask; 3570 const struct rte_flow_item_geneve_opt *spec = item->spec; 3571 const struct rte_flow_item_geneve_opt *mask = item->mask; 3572 unsigned int i; 3573 unsigned int data_len; 3574 uint8_t tlv_option_len; 3575 uint16_t optlen_m, optlen_v; 3576 const struct rte_flow_item_geneve_opt full_mask = { 3577 .option_class = RTE_BE16(0xffff), 3578 .option_type = 0xff, 3579 .option_len = 0x1f, 3580 }; 3581 3582 if (!mask) 3583 mask = &rte_flow_item_geneve_opt_mask; 3584 if (!spec) 3585 return rte_flow_error_set 3586 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3587 "Geneve TLV opt class/type/length must be specified"); 3588 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3589 return rte_flow_error_set 3590 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3591 "Geneve TLV opt length exceeds the limit (31)"); 3592 /* Check if class type and length masks are full. */ 3593 if (full_mask.option_class != mask->option_class || 3594 full_mask.option_type != mask->option_type || 3595 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3596 return rte_flow_error_set 3597 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3598 "Geneve TLV opt class/type/length masks must be full"); 3599 /* Check if length is supported */ 3600 if ((uint32_t)spec->option_len > 3601 hca_attr->max_geneve_tlv_option_data_len) 3602 return rte_flow_error_set 3603 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3604 "Geneve TLV opt length not supported"); 3605 if (hca_attr->max_geneve_tlv_options > 1) 3606 DRV_LOG(DEBUG, 3607 "max_geneve_tlv_options supports more than 1 option"); 3608 /* Check GENEVE item preceding. */ 3609 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3610 return rte_flow_error_set 3611 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3612 "Geneve opt item must be preceded with Geneve item"); 3613 geneve_spec = geneve_item->spec; 3614 geneve_mask = geneve_item->mask ? geneve_item->mask : 3615 &rte_flow_item_geneve_mask; 3616 /* Check if GENEVE TLV option size doesn't exceed option length */ 3617 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3618 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3619 tlv_option_len = spec->option_len & mask->option_len; 3620 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3621 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3622 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3623 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3624 if ((optlen_v & optlen_m) <= tlv_option_len) 3625 return rte_flow_error_set 3626 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3627 "GENEVE TLV option length exceeds optlen"); 3628 } 3629 /* Check if length is 0 or data is 0. */ 3630 if (spec->data == NULL || spec->option_len == 0) 3631 return rte_flow_error_set 3632 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3633 "Geneve TLV opt with zero data/length not supported"); 3634 /* Check not all data & mask are 0. */ 3635 data_len = spec->option_len * 4; 3636 if (mask->data == NULL) { 3637 for (i = 0; i < data_len; i++) 3638 if (spec->data[i]) 3639 break; 3640 if (i == data_len) 3641 return rte_flow_error_set(error, ENOTSUP, 3642 RTE_FLOW_ERROR_TYPE_ITEM, item, 3643 "Can't match on Geneve option data 0"); 3644 } else { 3645 for (i = 0; i < data_len; i++) 3646 if (spec->data[i] & mask->data[i]) 3647 break; 3648 if (i == data_len) 3649 return rte_flow_error_set(error, ENOTSUP, 3650 RTE_FLOW_ERROR_TYPE_ITEM, item, 3651 "Can't match on Geneve option data and mask 0"); 3652 /* Check data mask supported. */ 3653 for (i = data_max_supported; i < data_len ; i++) 3654 if (mask->data[i]) 3655 return rte_flow_error_set(error, ENOTSUP, 3656 RTE_FLOW_ERROR_TYPE_ITEM, item, 3657 "Data mask is of unsupported size"); 3658 } 3659 /* Check GENEVE option is supported in NIC. */ 3660 if (!hca_attr->geneve_tlv_opt) 3661 return rte_flow_error_set 3662 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3663 "Geneve TLV opt not supported"); 3664 /* Check if we already have geneve option with different type/class. */ 3665 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3666 geneve_opt_resource = sh->geneve_tlv_option_resource; 3667 if (geneve_opt_resource != NULL) 3668 if (geneve_opt_resource->option_class != spec->option_class || 3669 geneve_opt_resource->option_type != spec->option_type || 3670 geneve_opt_resource->length != spec->option_len) { 3671 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3672 return rte_flow_error_set(error, ENOTSUP, 3673 RTE_FLOW_ERROR_TYPE_ITEM, item, 3674 "Only one Geneve TLV option supported"); 3675 } 3676 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3677 return 0; 3678 } 3679 3680 /** 3681 * Validate MPLS item. 3682 * 3683 * @param[in] dev 3684 * Pointer to the rte_eth_dev structure. 3685 * @param[in] item 3686 * Item specification. 3687 * @param[in] item_flags 3688 * Bit-fields that holds the items detected until now. 3689 * @param[in] prev_layer 3690 * The protocol layer indicated in previous item. 3691 * @param[out] error 3692 * Pointer to error structure. 3693 * 3694 * @return 3695 * 0 on success, a negative errno value otherwise and rte_errno is set. 3696 */ 3697 int 3698 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3699 const struct rte_flow_item *item __rte_unused, 3700 uint64_t item_flags __rte_unused, 3701 uint64_t prev_layer __rte_unused, 3702 struct rte_flow_error *error) 3703 { 3704 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3705 const struct rte_flow_item_mpls *mask = item->mask; 3706 struct mlx5_priv *priv = dev->data->dev_private; 3707 int ret; 3708 3709 if (!priv->sh->dev_cap.mpls_en) 3710 return rte_flow_error_set(error, ENOTSUP, 3711 RTE_FLOW_ERROR_TYPE_ITEM, item, 3712 "MPLS not supported or" 3713 " disabled in firmware" 3714 " configuration."); 3715 /* MPLS over UDP, GRE is allowed */ 3716 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3717 MLX5_FLOW_LAYER_GRE | 3718 MLX5_FLOW_LAYER_GRE_KEY))) 3719 return rte_flow_error_set(error, EINVAL, 3720 RTE_FLOW_ERROR_TYPE_ITEM, item, 3721 "protocol filtering not compatible" 3722 " with MPLS layer"); 3723 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3724 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3725 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3726 return rte_flow_error_set(error, ENOTSUP, 3727 RTE_FLOW_ERROR_TYPE_ITEM, item, 3728 "multiple tunnel layers not" 3729 " supported"); 3730 if (!mask) 3731 mask = &rte_flow_item_mpls_mask; 3732 ret = mlx5_flow_item_acceptable 3733 (item, (const uint8_t *)mask, 3734 (const uint8_t *)&rte_flow_item_mpls_mask, 3735 sizeof(struct rte_flow_item_mpls), 3736 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3737 if (ret < 0) 3738 return ret; 3739 return 0; 3740 #else 3741 return rte_flow_error_set(error, ENOTSUP, 3742 RTE_FLOW_ERROR_TYPE_ITEM, item, 3743 "MPLS is not supported by Verbs, please" 3744 " update."); 3745 #endif 3746 } 3747 3748 /** 3749 * Validate NVGRE item. 3750 * 3751 * @param[in] item 3752 * Item specification. 3753 * @param[in] item_flags 3754 * Bit flags to mark detected items. 3755 * @param[in] target_protocol 3756 * The next protocol in the previous item. 3757 * @param[out] error 3758 * Pointer to error structure. 3759 * 3760 * @return 3761 * 0 on success, a negative errno value otherwise and rte_errno is set. 3762 */ 3763 int 3764 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3765 uint64_t item_flags, 3766 uint8_t target_protocol, 3767 struct rte_flow_error *error) 3768 { 3769 const struct rte_flow_item_nvgre *mask = item->mask; 3770 int ret; 3771 3772 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3773 return rte_flow_error_set(error, EINVAL, 3774 RTE_FLOW_ERROR_TYPE_ITEM, item, 3775 "protocol filtering not compatible" 3776 " with this GRE layer"); 3777 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3778 return rte_flow_error_set(error, ENOTSUP, 3779 RTE_FLOW_ERROR_TYPE_ITEM, item, 3780 "multiple tunnel layers not" 3781 " supported"); 3782 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3783 return rte_flow_error_set(error, ENOTSUP, 3784 RTE_FLOW_ERROR_TYPE_ITEM, item, 3785 "L3 Layer is missing"); 3786 if (!mask) 3787 mask = &rte_flow_item_nvgre_mask; 3788 ret = mlx5_flow_item_acceptable 3789 (item, (const uint8_t *)mask, 3790 (const uint8_t *)&rte_flow_item_nvgre_mask, 3791 sizeof(struct rte_flow_item_nvgre), 3792 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3793 if (ret < 0) 3794 return ret; 3795 return 0; 3796 } 3797 3798 /** 3799 * Validate eCPRI item. 3800 * 3801 * @param[in] item 3802 * Item specification. 3803 * @param[in] item_flags 3804 * Bit-fields that holds the items detected until now. 3805 * @param[in] last_item 3806 * Previous validated item in the pattern items. 3807 * @param[in] ether_type 3808 * Type in the ethernet layer header (including dot1q). 3809 * @param[in] acc_mask 3810 * Acceptable mask, if NULL default internal default mask 3811 * will be used to check whether item fields are supported. 3812 * @param[out] error 3813 * Pointer to error structure. 3814 * 3815 * @return 3816 * 0 on success, a negative errno value otherwise and rte_errno is set. 3817 */ 3818 int 3819 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3820 uint64_t item_flags, 3821 uint64_t last_item, 3822 uint16_t ether_type, 3823 const struct rte_flow_item_ecpri *acc_mask, 3824 struct rte_flow_error *error) 3825 { 3826 const struct rte_flow_item_ecpri *mask = item->mask; 3827 const struct rte_flow_item_ecpri nic_mask = { 3828 .hdr = { 3829 .common = { 3830 .u32 = 3831 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3832 .type = 0xFF, 3833 }).u32), 3834 }, 3835 .dummy[0] = 0xFFFFFFFF, 3836 }, 3837 }; 3838 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3839 MLX5_FLOW_LAYER_OUTER_VLAN); 3840 struct rte_flow_item_ecpri mask_lo; 3841 3842 if (!(last_item & outer_l2_vlan) && 3843 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3844 return rte_flow_error_set(error, EINVAL, 3845 RTE_FLOW_ERROR_TYPE_ITEM, item, 3846 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3847 if ((last_item & outer_l2_vlan) && ether_type && 3848 ether_type != RTE_ETHER_TYPE_ECPRI) 3849 return rte_flow_error_set(error, EINVAL, 3850 RTE_FLOW_ERROR_TYPE_ITEM, item, 3851 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3852 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3853 return rte_flow_error_set(error, EINVAL, 3854 RTE_FLOW_ERROR_TYPE_ITEM, item, 3855 "eCPRI with tunnel is not supported right now"); 3856 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3857 return rte_flow_error_set(error, ENOTSUP, 3858 RTE_FLOW_ERROR_TYPE_ITEM, item, 3859 "multiple L3 layers not supported"); 3860 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3861 return rte_flow_error_set(error, EINVAL, 3862 RTE_FLOW_ERROR_TYPE_ITEM, item, 3863 "eCPRI cannot coexist with a TCP layer"); 3864 /* In specification, eCPRI could be over UDP layer. */ 3865 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3866 return rte_flow_error_set(error, EINVAL, 3867 RTE_FLOW_ERROR_TYPE_ITEM, item, 3868 "eCPRI over UDP layer is not yet supported right now"); 3869 /* Mask for type field in common header could be zero. */ 3870 if (!mask) 3871 mask = &rte_flow_item_ecpri_mask; 3872 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3873 /* Input mask is in big-endian format. */ 3874 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3875 return rte_flow_error_set(error, EINVAL, 3876 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3877 "partial mask is not supported for protocol"); 3878 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3879 return rte_flow_error_set(error, EINVAL, 3880 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3881 "message header mask must be after a type mask"); 3882 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3883 acc_mask ? (const uint8_t *)acc_mask 3884 : (const uint8_t *)&nic_mask, 3885 sizeof(struct rte_flow_item_ecpri), 3886 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3887 } 3888 3889 /** 3890 * Validate the NSH item. 3891 * 3892 * @param[in] dev 3893 * Pointer to Ethernet device on which flow rule is being created on. 3894 * @param[out] error 3895 * Pointer to error structure. 3896 * 3897 * @return 3898 * 0 on success, a negative errno value otherwise and rte_errno is set. 3899 */ 3900 int 3901 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev, 3902 const struct rte_flow_item *item, 3903 struct rte_flow_error *error) 3904 { 3905 struct mlx5_priv *priv = dev->data->dev_private; 3906 3907 if (item->mask) { 3908 return rte_flow_error_set(error, ENOTSUP, 3909 RTE_FLOW_ERROR_TYPE_ITEM, item, 3910 "NSH fields matching is not supported"); 3911 } 3912 3913 if (!priv->sh->config.dv_flow_en) { 3914 return rte_flow_error_set(error, ENOTSUP, 3915 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3916 NULL, "NSH support requires DV flow interface"); 3917 } 3918 3919 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) { 3920 return rte_flow_error_set(error, ENOTSUP, 3921 RTE_FLOW_ERROR_TYPE_ITEM, item, 3922 "Current FW does not support matching on NSH"); 3923 } 3924 3925 return 0; 3926 } 3927 3928 static int 3929 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3930 const struct rte_flow_attr *attr __rte_unused, 3931 const struct rte_flow_item items[] __rte_unused, 3932 const struct rte_flow_action actions[] __rte_unused, 3933 bool external __rte_unused, 3934 int hairpin __rte_unused, 3935 struct rte_flow_error *error) 3936 { 3937 return rte_flow_error_set(error, ENOTSUP, 3938 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3939 } 3940 3941 static struct mlx5_flow * 3942 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3943 const struct rte_flow_attr *attr __rte_unused, 3944 const struct rte_flow_item items[] __rte_unused, 3945 const struct rte_flow_action actions[] __rte_unused, 3946 struct rte_flow_error *error) 3947 { 3948 rte_flow_error_set(error, ENOTSUP, 3949 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3950 return NULL; 3951 } 3952 3953 static int 3954 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3955 struct mlx5_flow *dev_flow __rte_unused, 3956 const struct rte_flow_attr *attr __rte_unused, 3957 const struct rte_flow_item items[] __rte_unused, 3958 const struct rte_flow_action actions[] __rte_unused, 3959 struct rte_flow_error *error) 3960 { 3961 return rte_flow_error_set(error, ENOTSUP, 3962 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3963 } 3964 3965 static int 3966 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3967 struct rte_flow *flow __rte_unused, 3968 struct rte_flow_error *error) 3969 { 3970 return rte_flow_error_set(error, ENOTSUP, 3971 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3972 } 3973 3974 static void 3975 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3976 struct rte_flow *flow __rte_unused) 3977 { 3978 } 3979 3980 static void 3981 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3982 struct rte_flow *flow __rte_unused) 3983 { 3984 } 3985 3986 static int 3987 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3988 struct rte_flow *flow __rte_unused, 3989 const struct rte_flow_action *actions __rte_unused, 3990 void *data __rte_unused, 3991 struct rte_flow_error *error) 3992 { 3993 return rte_flow_error_set(error, ENOTSUP, 3994 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3995 } 3996 3997 static int 3998 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3999 uint32_t domains __rte_unused, 4000 uint32_t flags __rte_unused) 4001 { 4002 return 0; 4003 } 4004 4005 int 4006 flow_null_get_aged_flows(struct rte_eth_dev *dev, 4007 void **context __rte_unused, 4008 uint32_t nb_contexts __rte_unused, 4009 struct rte_flow_error *error __rte_unused) 4010 { 4011 DRV_LOG(ERR, "port %u get aged flows is not supported.", 4012 dev->data->port_id); 4013 return -ENOTSUP; 4014 } 4015 4016 uint32_t 4017 flow_null_counter_allocate(struct rte_eth_dev *dev) 4018 { 4019 DRV_LOG(ERR, "port %u counter allocate is not supported.", 4020 dev->data->port_id); 4021 return 0; 4022 } 4023 4024 void 4025 flow_null_counter_free(struct rte_eth_dev *dev, 4026 uint32_t counter __rte_unused) 4027 { 4028 DRV_LOG(ERR, "port %u counter free is not supported.", 4029 dev->data->port_id); 4030 } 4031 4032 int 4033 flow_null_counter_query(struct rte_eth_dev *dev, 4034 uint32_t counter __rte_unused, 4035 bool clear __rte_unused, 4036 uint64_t *pkts __rte_unused, 4037 uint64_t *bytes __rte_unused, 4038 void **action __rte_unused) 4039 { 4040 DRV_LOG(ERR, "port %u counter query is not supported.", 4041 dev->data->port_id); 4042 return -ENOTSUP; 4043 } 4044 4045 /* Void driver to protect from null pointer reference. */ 4046 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 4047 .validate = flow_null_validate, 4048 .prepare = flow_null_prepare, 4049 .translate = flow_null_translate, 4050 .apply = flow_null_apply, 4051 .remove = flow_null_remove, 4052 .destroy = flow_null_destroy, 4053 .query = flow_null_query, 4054 .sync_domain = flow_null_sync_domain, 4055 .get_aged_flows = flow_null_get_aged_flows, 4056 .counter_alloc = flow_null_counter_allocate, 4057 .counter_free = flow_null_counter_free, 4058 .counter_query = flow_null_counter_query 4059 }; 4060 4061 /** 4062 * Select flow driver type according to flow attributes and device 4063 * configuration. 4064 * 4065 * @param[in] dev 4066 * Pointer to the dev structure. 4067 * @param[in] attr 4068 * Pointer to the flow attributes. 4069 * 4070 * @return 4071 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4072 */ 4073 static enum mlx5_flow_drv_type 4074 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4075 { 4076 struct mlx5_priv *priv = dev->data->dev_private; 4077 /* The OS can determine first a specific flow type (DV, VERBS) */ 4078 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4079 4080 if (type != MLX5_FLOW_TYPE_MAX) 4081 return type; 4082 /* 4083 * Currently when dv_flow_en == 2, only HW steering engine is 4084 * supported. New engines can also be chosen here if ready. 4085 */ 4086 if (priv->sh->config.dv_flow_en == 2) 4087 return MLX5_FLOW_TYPE_HW; 4088 if (!attr) 4089 return MLX5_FLOW_TYPE_MIN; 4090 /* If no OS specific type - continue with DV/VERBS selection */ 4091 if (attr->transfer && priv->sh->config.dv_esw_en) 4092 type = MLX5_FLOW_TYPE_DV; 4093 if (!attr->transfer) 4094 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4095 MLX5_FLOW_TYPE_VERBS; 4096 return type; 4097 } 4098 4099 #define flow_get_drv_ops(type) flow_drv_ops[type] 4100 4101 /** 4102 * Flow driver validation API. This abstracts calling driver specific functions. 4103 * The type of flow driver is determined according to flow attributes. 4104 * 4105 * @param[in] dev 4106 * Pointer to the dev structure. 4107 * @param[in] attr 4108 * Pointer to the flow attributes. 4109 * @param[in] items 4110 * Pointer to the list of items. 4111 * @param[in] actions 4112 * Pointer to the list of actions. 4113 * @param[in] external 4114 * This flow rule is created by request external to PMD. 4115 * @param[in] hairpin 4116 * Number of hairpin TX actions, 0 means classic flow. 4117 * @param[out] error 4118 * Pointer to the error structure. 4119 * 4120 * @return 4121 * 0 on success, a negative errno value otherwise and rte_errno is set. 4122 */ 4123 static inline int 4124 flow_drv_validate(struct rte_eth_dev *dev, 4125 const struct rte_flow_attr *attr, 4126 const struct rte_flow_item items[], 4127 const struct rte_flow_action actions[], 4128 bool external, int hairpin, struct rte_flow_error *error) 4129 { 4130 const struct mlx5_flow_driver_ops *fops; 4131 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4132 4133 fops = flow_get_drv_ops(type); 4134 return fops->validate(dev, attr, items, actions, external, 4135 hairpin, error); 4136 } 4137 4138 /** 4139 * Flow driver preparation API. This abstracts calling driver specific 4140 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4141 * calculates the size of memory required for device flow, allocates the memory, 4142 * initializes the device flow and returns the pointer. 4143 * 4144 * @note 4145 * This function initializes device flow structure such as dv or verbs in 4146 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4147 * rest. For example, adding returning device flow to flow->dev_flow list and 4148 * setting backward reference to the flow should be done out of this function. 4149 * layers field is not filled either. 4150 * 4151 * @param[in] dev 4152 * Pointer to the dev structure. 4153 * @param[in] attr 4154 * Pointer to the flow attributes. 4155 * @param[in] items 4156 * Pointer to the list of items. 4157 * @param[in] actions 4158 * Pointer to the list of actions. 4159 * @param[in] flow_idx 4160 * This memory pool index to the flow. 4161 * @param[out] error 4162 * Pointer to the error structure. 4163 * 4164 * @return 4165 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4166 */ 4167 static inline struct mlx5_flow * 4168 flow_drv_prepare(struct rte_eth_dev *dev, 4169 const struct rte_flow *flow, 4170 const struct rte_flow_attr *attr, 4171 const struct rte_flow_item items[], 4172 const struct rte_flow_action actions[], 4173 uint32_t flow_idx, 4174 struct rte_flow_error *error) 4175 { 4176 const struct mlx5_flow_driver_ops *fops; 4177 enum mlx5_flow_drv_type type = flow->drv_type; 4178 struct mlx5_flow *mlx5_flow = NULL; 4179 4180 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4181 fops = flow_get_drv_ops(type); 4182 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4183 if (mlx5_flow) 4184 mlx5_flow->flow_idx = flow_idx; 4185 return mlx5_flow; 4186 } 4187 4188 /** 4189 * Flow driver translation API. This abstracts calling driver specific 4190 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4191 * translates a generic flow into a driver flow. flow_drv_prepare() must 4192 * precede. 4193 * 4194 * @note 4195 * dev_flow->layers could be filled as a result of parsing during translation 4196 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4197 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4198 * flow->actions could be overwritten even though all the expanded dev_flows 4199 * have the same actions. 4200 * 4201 * @param[in] dev 4202 * Pointer to the rte dev structure. 4203 * @param[in, out] dev_flow 4204 * Pointer to the mlx5 flow. 4205 * @param[in] attr 4206 * Pointer to the flow attributes. 4207 * @param[in] items 4208 * Pointer to the list of items. 4209 * @param[in] actions 4210 * Pointer to the list of actions. 4211 * @param[out] error 4212 * Pointer to the error structure. 4213 * 4214 * @return 4215 * 0 on success, a negative errno value otherwise and rte_errno is set. 4216 */ 4217 static inline int 4218 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4219 const struct rte_flow_attr *attr, 4220 const struct rte_flow_item items[], 4221 const struct rte_flow_action actions[], 4222 struct rte_flow_error *error) 4223 { 4224 const struct mlx5_flow_driver_ops *fops; 4225 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4226 4227 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4228 fops = flow_get_drv_ops(type); 4229 return fops->translate(dev, dev_flow, attr, items, actions, error); 4230 } 4231 4232 /** 4233 * Flow driver apply API. This abstracts calling driver specific functions. 4234 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4235 * translated driver flows on to device. flow_drv_translate() must precede. 4236 * 4237 * @param[in] dev 4238 * Pointer to Ethernet device structure. 4239 * @param[in, out] flow 4240 * Pointer to flow structure. 4241 * @param[out] error 4242 * Pointer to error structure. 4243 * 4244 * @return 4245 * 0 on success, a negative errno value otherwise and rte_errno is set. 4246 */ 4247 static inline int 4248 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4249 struct rte_flow_error *error) 4250 { 4251 const struct mlx5_flow_driver_ops *fops; 4252 enum mlx5_flow_drv_type type = flow->drv_type; 4253 4254 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4255 fops = flow_get_drv_ops(type); 4256 return fops->apply(dev, flow, error); 4257 } 4258 4259 /** 4260 * Flow driver destroy API. This abstracts calling driver specific functions. 4261 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4262 * on device and releases resources of the flow. 4263 * 4264 * @param[in] dev 4265 * Pointer to Ethernet device. 4266 * @param[in, out] flow 4267 * Pointer to flow structure. 4268 */ 4269 static inline void 4270 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4271 { 4272 const struct mlx5_flow_driver_ops *fops; 4273 enum mlx5_flow_drv_type type = flow->drv_type; 4274 4275 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4276 fops = flow_get_drv_ops(type); 4277 fops->destroy(dev, flow); 4278 } 4279 4280 /** 4281 * Flow driver find RSS policy tbl API. This abstracts calling driver 4282 * specific functions. Parent flow (rte_flow) should have driver 4283 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4284 * 4285 * @param[in] dev 4286 * Pointer to Ethernet device. 4287 * @param[in, out] flow 4288 * Pointer to flow structure. 4289 * @param[in] policy 4290 * Pointer to meter policy table. 4291 * @param[in] rss_desc 4292 * Pointer to rss_desc 4293 */ 4294 static struct mlx5_flow_meter_sub_policy * 4295 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4296 struct rte_flow *flow, 4297 struct mlx5_flow_meter_policy *policy, 4298 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4299 { 4300 const struct mlx5_flow_driver_ops *fops; 4301 enum mlx5_flow_drv_type type = flow->drv_type; 4302 4303 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4304 fops = flow_get_drv_ops(type); 4305 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4306 } 4307 4308 /** 4309 * Flow driver color tag rule API. This abstracts calling driver 4310 * specific functions. Parent flow (rte_flow) should have driver 4311 * type (drv_type). It will create the color tag rules in hierarchy meter. 4312 * 4313 * @param[in] dev 4314 * Pointer to Ethernet device. 4315 * @param[in, out] flow 4316 * Pointer to flow structure. 4317 * @param[in] fm 4318 * Pointer to flow meter structure. 4319 * @param[in] src_port 4320 * The src port this extra rule should use. 4321 * @param[in] item 4322 * The src port id match item. 4323 * @param[out] error 4324 * Pointer to error structure. 4325 */ 4326 static int 4327 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4328 struct rte_flow *flow, 4329 struct mlx5_flow_meter_info *fm, 4330 int32_t src_port, 4331 const struct rte_flow_item *item, 4332 struct rte_flow_error *error) 4333 { 4334 const struct mlx5_flow_driver_ops *fops; 4335 enum mlx5_flow_drv_type type = flow->drv_type; 4336 4337 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4338 fops = flow_get_drv_ops(type); 4339 return fops->meter_hierarchy_rule_create(dev, fm, 4340 src_port, item, error); 4341 } 4342 4343 /** 4344 * Get RSS action from the action list. 4345 * 4346 * @param[in] dev 4347 * Pointer to Ethernet device. 4348 * @param[in] actions 4349 * Pointer to the list of actions. 4350 * @param[in] flow 4351 * Parent flow structure pointer. 4352 * 4353 * @return 4354 * Pointer to the RSS action if exist, else return NULL. 4355 */ 4356 static const struct rte_flow_action_rss* 4357 flow_get_rss_action(struct rte_eth_dev *dev, 4358 const struct rte_flow_action actions[]) 4359 { 4360 struct mlx5_priv *priv = dev->data->dev_private; 4361 const struct rte_flow_action_rss *rss = NULL; 4362 struct mlx5_meter_policy_action_container *acg; 4363 struct mlx5_meter_policy_action_container *acy; 4364 4365 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4366 switch (actions->type) { 4367 case RTE_FLOW_ACTION_TYPE_RSS: 4368 rss = actions->conf; 4369 break; 4370 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4371 { 4372 const struct rte_flow_action_sample *sample = 4373 actions->conf; 4374 const struct rte_flow_action *act = sample->actions; 4375 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4376 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4377 rss = act->conf; 4378 break; 4379 } 4380 case RTE_FLOW_ACTION_TYPE_METER: 4381 { 4382 uint32_t mtr_idx; 4383 struct mlx5_flow_meter_info *fm; 4384 struct mlx5_flow_meter_policy *policy; 4385 const struct rte_flow_action_meter *mtr = actions->conf; 4386 4387 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4388 if (fm && !fm->def_policy) { 4389 policy = mlx5_flow_meter_policy_find(dev, 4390 fm->policy_id, NULL); 4391 MLX5_ASSERT(policy); 4392 if (policy->is_hierarchy) { 4393 policy = 4394 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4395 policy); 4396 if (!policy) 4397 return NULL; 4398 } 4399 if (policy->is_rss) { 4400 acg = 4401 &policy->act_cnt[RTE_COLOR_GREEN]; 4402 acy = 4403 &policy->act_cnt[RTE_COLOR_YELLOW]; 4404 if (acg->fate_action == 4405 MLX5_FLOW_FATE_SHARED_RSS) 4406 rss = acg->rss->conf; 4407 else if (acy->fate_action == 4408 MLX5_FLOW_FATE_SHARED_RSS) 4409 rss = acy->rss->conf; 4410 } 4411 } 4412 break; 4413 } 4414 default: 4415 break; 4416 } 4417 } 4418 return rss; 4419 } 4420 4421 /** 4422 * Get ASO age action by index. 4423 * 4424 * @param[in] dev 4425 * Pointer to the Ethernet device structure. 4426 * @param[in] age_idx 4427 * Index to the ASO age action. 4428 * 4429 * @return 4430 * The specified ASO age action. 4431 */ 4432 struct mlx5_aso_age_action* 4433 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4434 { 4435 uint16_t pool_idx = age_idx & UINT16_MAX; 4436 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4437 struct mlx5_priv *priv = dev->data->dev_private; 4438 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4439 struct mlx5_aso_age_pool *pool; 4440 4441 rte_rwlock_read_lock(&mng->resize_rwl); 4442 pool = mng->pools[pool_idx]; 4443 rte_rwlock_read_unlock(&mng->resize_rwl); 4444 return &pool->actions[offset - 1]; 4445 } 4446 4447 /* maps indirect action to translated direct in some actions array */ 4448 struct mlx5_translated_action_handle { 4449 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4450 int index; /**< Index in related array of rte_flow_action. */ 4451 }; 4452 4453 /** 4454 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4455 * direct action if translation possible. 4456 * This functionality used to run same execution path for both direct and 4457 * indirect actions on flow create. All necessary preparations for indirect 4458 * action handling should be performed on *handle* actions list returned 4459 * from this call. 4460 * 4461 * @param[in] dev 4462 * Pointer to Ethernet device. 4463 * @param[in] actions 4464 * List of actions to translate. 4465 * @param[out] handle 4466 * List to store translated indirect action object handles. 4467 * @param[in, out] indir_n 4468 * Size of *handle* array. On return should be updated with number of 4469 * indirect actions retrieved from the *actions* list. 4470 * @param[out] translated_actions 4471 * List of actions where all indirect actions were translated to direct 4472 * if possible. NULL if no translation took place. 4473 * @param[out] error 4474 * Pointer to the error structure. 4475 * 4476 * @return 4477 * 0 on success, a negative errno value otherwise and rte_errno is set. 4478 */ 4479 static int 4480 flow_action_handles_translate(struct rte_eth_dev *dev, 4481 const struct rte_flow_action actions[], 4482 struct mlx5_translated_action_handle *handle, 4483 int *indir_n, 4484 struct rte_flow_action **translated_actions, 4485 struct rte_flow_error *error) 4486 { 4487 struct mlx5_priv *priv = dev->data->dev_private; 4488 struct rte_flow_action *translated = NULL; 4489 size_t actions_size; 4490 int n; 4491 int copied_n = 0; 4492 struct mlx5_translated_action_handle *handle_end = NULL; 4493 4494 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4495 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4496 continue; 4497 if (copied_n == *indir_n) { 4498 return rte_flow_error_set 4499 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4500 NULL, "too many shared actions"); 4501 } 4502 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4503 sizeof(actions[n].conf)); 4504 handle[copied_n].index = n; 4505 copied_n++; 4506 } 4507 n++; 4508 *indir_n = copied_n; 4509 if (!copied_n) 4510 return 0; 4511 actions_size = sizeof(struct rte_flow_action) * n; 4512 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4513 if (!translated) { 4514 rte_errno = ENOMEM; 4515 return -ENOMEM; 4516 } 4517 memcpy(translated, actions, actions_size); 4518 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4519 struct mlx5_shared_action_rss *shared_rss; 4520 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4521 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4522 uint32_t idx = act_idx & 4523 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4524 4525 switch (type) { 4526 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4527 shared_rss = mlx5_ipool_get 4528 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4529 translated[handle->index].type = 4530 RTE_FLOW_ACTION_TYPE_RSS; 4531 translated[handle->index].conf = 4532 &shared_rss->origin; 4533 break; 4534 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4535 translated[handle->index].type = 4536 (enum rte_flow_action_type) 4537 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4538 translated[handle->index].conf = (void *)(uintptr_t)idx; 4539 break; 4540 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4541 translated[handle->index].type = 4542 (enum rte_flow_action_type) 4543 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4544 translated[handle->index].conf = (void *)(uintptr_t)idx; 4545 break; 4546 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4547 if (priv->sh->flow_hit_aso_en) { 4548 translated[handle->index].type = 4549 (enum rte_flow_action_type) 4550 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4551 translated[handle->index].conf = 4552 (void *)(uintptr_t)idx; 4553 break; 4554 } 4555 /* Fall-through */ 4556 case MLX5_INDIRECT_ACTION_TYPE_CT: 4557 if (priv->sh->ct_aso_en) { 4558 translated[handle->index].type = 4559 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4560 translated[handle->index].conf = 4561 (void *)(uintptr_t)idx; 4562 break; 4563 } 4564 /* Fall-through */ 4565 default: 4566 mlx5_free(translated); 4567 return rte_flow_error_set 4568 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4569 NULL, "invalid indirect action type"); 4570 } 4571 } 4572 *translated_actions = translated; 4573 return 0; 4574 } 4575 4576 /** 4577 * Get Shared RSS action from the action list. 4578 * 4579 * @param[in] dev 4580 * Pointer to Ethernet device. 4581 * @param[in] shared 4582 * Pointer to the list of actions. 4583 * @param[in] shared_n 4584 * Actions list length. 4585 * 4586 * @return 4587 * The MLX5 RSS action ID if exists, otherwise return 0. 4588 */ 4589 static uint32_t 4590 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4591 struct mlx5_translated_action_handle *handle, 4592 int shared_n) 4593 { 4594 struct mlx5_translated_action_handle *handle_end; 4595 struct mlx5_priv *priv = dev->data->dev_private; 4596 struct mlx5_shared_action_rss *shared_rss; 4597 4598 4599 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4600 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4601 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4602 uint32_t idx = act_idx & 4603 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4604 switch (type) { 4605 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4606 shared_rss = mlx5_ipool_get 4607 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4608 idx); 4609 __atomic_fetch_add(&shared_rss->refcnt, 1, 4610 __ATOMIC_RELAXED); 4611 return idx; 4612 default: 4613 break; 4614 } 4615 } 4616 return 0; 4617 } 4618 4619 static unsigned int 4620 find_graph_root(uint32_t rss_level) 4621 { 4622 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4623 MLX5_EXPANSION_ROOT_OUTER; 4624 } 4625 4626 /** 4627 * Get layer flags from the prefix flow. 4628 * 4629 * Some flows may be split to several subflows, the prefix subflow gets the 4630 * match items and the suffix sub flow gets the actions. 4631 * Some actions need the user defined match item flags to get the detail for 4632 * the action. 4633 * This function helps the suffix flow to get the item layer flags from prefix 4634 * subflow. 4635 * 4636 * @param[in] dev_flow 4637 * Pointer the created prefix subflow. 4638 * 4639 * @return 4640 * The layers get from prefix subflow. 4641 */ 4642 static inline uint64_t 4643 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4644 { 4645 uint64_t layers = 0; 4646 4647 /* 4648 * Layers bits could be localization, but usually the compiler will 4649 * help to do the optimization work for source code. 4650 * If no decap actions, use the layers directly. 4651 */ 4652 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4653 return dev_flow->handle->layers; 4654 /* Convert L3 layers with decap action. */ 4655 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4656 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4657 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4658 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4659 /* Convert L4 layers with decap action. */ 4660 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4661 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4662 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4663 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4664 return layers; 4665 } 4666 4667 /** 4668 * Get metadata split action information. 4669 * 4670 * @param[in] actions 4671 * Pointer to the list of actions. 4672 * @param[out] qrss 4673 * Pointer to the return pointer. 4674 * @param[out] qrss_type 4675 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4676 * if no QUEUE/RSS is found. 4677 * @param[out] encap_idx 4678 * Pointer to the index of the encap action if exists, otherwise the last 4679 * action index. 4680 * 4681 * @return 4682 * Total number of actions. 4683 */ 4684 static int 4685 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4686 const struct rte_flow_action **qrss, 4687 int *encap_idx) 4688 { 4689 const struct rte_flow_action_raw_encap *raw_encap; 4690 int actions_n = 0; 4691 int raw_decap_idx = -1; 4692 4693 *encap_idx = -1; 4694 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4695 switch (actions->type) { 4696 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4697 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4698 *encap_idx = actions_n; 4699 break; 4700 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4701 raw_decap_idx = actions_n; 4702 break; 4703 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4704 raw_encap = actions->conf; 4705 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4706 *encap_idx = raw_decap_idx != -1 ? 4707 raw_decap_idx : actions_n; 4708 break; 4709 case RTE_FLOW_ACTION_TYPE_QUEUE: 4710 case RTE_FLOW_ACTION_TYPE_RSS: 4711 *qrss = actions; 4712 break; 4713 default: 4714 break; 4715 } 4716 actions_n++; 4717 } 4718 if (*encap_idx == -1) 4719 *encap_idx = actions_n; 4720 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4721 return actions_n + 1; 4722 } 4723 4724 /** 4725 * Check if the action will change packet. 4726 * 4727 * @param dev 4728 * Pointer to Ethernet device. 4729 * @param[in] type 4730 * action type. 4731 * 4732 * @return 4733 * true if action will change packet, false otherwise. 4734 */ 4735 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4736 enum rte_flow_action_type type) 4737 { 4738 struct mlx5_priv *priv = dev->data->dev_private; 4739 4740 switch (type) { 4741 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4742 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4743 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4744 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4745 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4746 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4747 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4748 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4749 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4750 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4751 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4752 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4753 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4754 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4755 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4756 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4757 case RTE_FLOW_ACTION_TYPE_SET_META: 4758 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4759 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4760 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4761 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4762 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4763 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4764 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4765 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4766 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4767 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4768 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4769 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4770 return true; 4771 case RTE_FLOW_ACTION_TYPE_FLAG: 4772 case RTE_FLOW_ACTION_TYPE_MARK: 4773 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4774 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4775 return true; 4776 else 4777 return false; 4778 default: 4779 return false; 4780 } 4781 } 4782 4783 /** 4784 * Check meter action from the action list. 4785 * 4786 * @param dev 4787 * Pointer to Ethernet device. 4788 * @param[in] actions 4789 * Pointer to the list of actions. 4790 * @param[out] has_mtr 4791 * Pointer to the meter exist flag. 4792 * @param[out] has_modify 4793 * Pointer to the flag showing there's packet change action. 4794 * @param[out] meter_id 4795 * Pointer to the meter id. 4796 * 4797 * @return 4798 * Total number of actions. 4799 */ 4800 static int 4801 flow_check_meter_action(struct rte_eth_dev *dev, 4802 const struct rte_flow_action actions[], 4803 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4804 { 4805 const struct rte_flow_action_meter *mtr = NULL; 4806 int actions_n = 0; 4807 4808 MLX5_ASSERT(has_mtr); 4809 *has_mtr = false; 4810 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4811 switch (actions->type) { 4812 case RTE_FLOW_ACTION_TYPE_METER: 4813 mtr = actions->conf; 4814 *meter_id = mtr->mtr_id; 4815 *has_mtr = true; 4816 break; 4817 default: 4818 break; 4819 } 4820 if (!*has_mtr) 4821 *has_modify |= flow_check_modify_action_type(dev, 4822 actions->type); 4823 actions_n++; 4824 } 4825 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4826 return actions_n + 1; 4827 } 4828 4829 /** 4830 * Check if the flow should be split due to hairpin. 4831 * The reason for the split is that in current HW we can't 4832 * support encap and push-vlan on Rx, so if a flow contains 4833 * these actions we move it to Tx. 4834 * 4835 * @param dev 4836 * Pointer to Ethernet device. 4837 * @param[in] attr 4838 * Flow rule attributes. 4839 * @param[in] actions 4840 * Associated actions (list terminated by the END action). 4841 * 4842 * @return 4843 * > 0 the number of actions and the flow should be split, 4844 * 0 when no split required. 4845 */ 4846 static int 4847 flow_check_hairpin_split(struct rte_eth_dev *dev, 4848 const struct rte_flow_attr *attr, 4849 const struct rte_flow_action actions[]) 4850 { 4851 int queue_action = 0; 4852 int action_n = 0; 4853 int split = 0; 4854 int push_vlan = 0; 4855 const struct rte_flow_action_queue *queue; 4856 const struct rte_flow_action_rss *rss; 4857 const struct rte_flow_action_raw_encap *raw_encap; 4858 const struct rte_eth_hairpin_conf *conf; 4859 4860 if (!attr->ingress) 4861 return 0; 4862 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4863 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4864 push_vlan = 1; 4865 switch (actions->type) { 4866 case RTE_FLOW_ACTION_TYPE_QUEUE: 4867 queue = actions->conf; 4868 if (queue == NULL) 4869 return 0; 4870 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4871 if (conf == NULL || conf->tx_explicit != 0) 4872 return 0; 4873 queue_action = 1; 4874 action_n++; 4875 break; 4876 case RTE_FLOW_ACTION_TYPE_RSS: 4877 rss = actions->conf; 4878 if (rss == NULL || rss->queue_num == 0) 4879 return 0; 4880 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4881 if (conf == NULL || conf->tx_explicit != 0) 4882 return 0; 4883 queue_action = 1; 4884 action_n++; 4885 break; 4886 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4887 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4888 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4889 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4890 split++; 4891 action_n++; 4892 break; 4893 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4894 if (push_vlan) 4895 split++; 4896 action_n++; 4897 break; 4898 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4899 raw_encap = actions->conf; 4900 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4901 split++; 4902 action_n++; 4903 break; 4904 default: 4905 action_n++; 4906 break; 4907 } 4908 } 4909 if (split && queue_action) 4910 return action_n; 4911 return 0; 4912 } 4913 4914 /* Declare flow create/destroy prototype in advance. */ 4915 static uint32_t 4916 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4917 const struct rte_flow_attr *attr, 4918 const struct rte_flow_item items[], 4919 const struct rte_flow_action actions[], 4920 bool external, struct rte_flow_error *error); 4921 4922 static void 4923 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4924 uint32_t flow_idx); 4925 4926 int 4927 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4928 struct mlx5_list_entry *entry, void *cb_ctx) 4929 { 4930 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4931 struct mlx5_flow_mreg_copy_resource *mcp_res = 4932 container_of(entry, typeof(*mcp_res), hlist_ent); 4933 4934 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4935 } 4936 4937 struct mlx5_list_entry * 4938 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4939 { 4940 struct rte_eth_dev *dev = tool_ctx; 4941 struct mlx5_priv *priv = dev->data->dev_private; 4942 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4943 struct mlx5_flow_mreg_copy_resource *mcp_res; 4944 struct rte_flow_error *error = ctx->error; 4945 uint32_t idx = 0; 4946 int ret; 4947 uint32_t mark_id = *(uint32_t *)(ctx->data); 4948 struct rte_flow_attr attr = { 4949 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4950 .ingress = 1, 4951 }; 4952 struct mlx5_rte_flow_item_tag tag_spec = { 4953 .data = mark_id, 4954 }; 4955 struct rte_flow_item items[] = { 4956 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4957 }; 4958 struct rte_flow_action_mark ftag = { 4959 .id = mark_id, 4960 }; 4961 struct mlx5_flow_action_copy_mreg cp_mreg = { 4962 .dst = REG_B, 4963 .src = REG_NON, 4964 }; 4965 struct rte_flow_action_jump jump = { 4966 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4967 }; 4968 struct rte_flow_action actions[] = { 4969 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4970 }; 4971 4972 /* Fill the register fields in the flow. */ 4973 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4974 if (ret < 0) 4975 return NULL; 4976 tag_spec.id = ret; 4977 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4978 if (ret < 0) 4979 return NULL; 4980 cp_mreg.src = ret; 4981 /* Provide the full width of FLAG specific value. */ 4982 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4983 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4984 /* Build a new flow. */ 4985 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4986 items[0] = (struct rte_flow_item){ 4987 .type = (enum rte_flow_item_type) 4988 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4989 .spec = &tag_spec, 4990 }; 4991 items[1] = (struct rte_flow_item){ 4992 .type = RTE_FLOW_ITEM_TYPE_END, 4993 }; 4994 actions[0] = (struct rte_flow_action){ 4995 .type = (enum rte_flow_action_type) 4996 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4997 .conf = &ftag, 4998 }; 4999 actions[1] = (struct rte_flow_action){ 5000 .type = (enum rte_flow_action_type) 5001 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5002 .conf = &cp_mreg, 5003 }; 5004 actions[2] = (struct rte_flow_action){ 5005 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5006 .conf = &jump, 5007 }; 5008 actions[3] = (struct rte_flow_action){ 5009 .type = RTE_FLOW_ACTION_TYPE_END, 5010 }; 5011 } else { 5012 /* Default rule, wildcard match. */ 5013 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 5014 items[0] = (struct rte_flow_item){ 5015 .type = RTE_FLOW_ITEM_TYPE_END, 5016 }; 5017 actions[0] = (struct rte_flow_action){ 5018 .type = (enum rte_flow_action_type) 5019 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5020 .conf = &cp_mreg, 5021 }; 5022 actions[1] = (struct rte_flow_action){ 5023 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5024 .conf = &jump, 5025 }; 5026 actions[2] = (struct rte_flow_action){ 5027 .type = RTE_FLOW_ACTION_TYPE_END, 5028 }; 5029 } 5030 /* Build a new entry. */ 5031 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5032 if (!mcp_res) { 5033 rte_errno = ENOMEM; 5034 return NULL; 5035 } 5036 mcp_res->idx = idx; 5037 mcp_res->mark_id = mark_id; 5038 /* 5039 * The copy Flows are not included in any list. There 5040 * ones are referenced from other Flows and can not 5041 * be applied, removed, deleted in arbitrary order 5042 * by list traversing. 5043 */ 5044 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 5045 &attr, items, actions, false, error); 5046 if (!mcp_res->rix_flow) { 5047 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 5048 return NULL; 5049 } 5050 return &mcp_res->hlist_ent; 5051 } 5052 5053 struct mlx5_list_entry * 5054 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5055 void *cb_ctx __rte_unused) 5056 { 5057 struct rte_eth_dev *dev = tool_ctx; 5058 struct mlx5_priv *priv = dev->data->dev_private; 5059 struct mlx5_flow_mreg_copy_resource *mcp_res; 5060 uint32_t idx = 0; 5061 5062 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5063 if (!mcp_res) { 5064 rte_errno = ENOMEM; 5065 return NULL; 5066 } 5067 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 5068 mcp_res->idx = idx; 5069 return &mcp_res->hlist_ent; 5070 } 5071 5072 void 5073 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5074 { 5075 struct mlx5_flow_mreg_copy_resource *mcp_res = 5076 container_of(entry, typeof(*mcp_res), hlist_ent); 5077 struct rte_eth_dev *dev = tool_ctx; 5078 struct mlx5_priv *priv = dev->data->dev_private; 5079 5080 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5081 } 5082 5083 /** 5084 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5085 * 5086 * As mark_id is unique, if there's already a registered flow for the mark_id, 5087 * return by increasing the reference counter of the resource. Otherwise, create 5088 * the resource (mcp_res) and flow. 5089 * 5090 * Flow looks like, 5091 * - If ingress port is ANY and reg_c[1] is mark_id, 5092 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5093 * 5094 * For default flow (zero mark_id), flow is like, 5095 * - If ingress port is ANY, 5096 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5097 * 5098 * @param dev 5099 * Pointer to Ethernet device. 5100 * @param mark_id 5101 * ID of MARK action, zero means default flow for META. 5102 * @param[out] error 5103 * Perform verbose error reporting if not NULL. 5104 * 5105 * @return 5106 * Associated resource on success, NULL otherwise and rte_errno is set. 5107 */ 5108 static struct mlx5_flow_mreg_copy_resource * 5109 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5110 struct rte_flow_error *error) 5111 { 5112 struct mlx5_priv *priv = dev->data->dev_private; 5113 struct mlx5_list_entry *entry; 5114 struct mlx5_flow_cb_ctx ctx = { 5115 .dev = dev, 5116 .error = error, 5117 .data = &mark_id, 5118 }; 5119 5120 /* Check if already registered. */ 5121 MLX5_ASSERT(priv->mreg_cp_tbl); 5122 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5123 if (!entry) 5124 return NULL; 5125 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5126 hlist_ent); 5127 } 5128 5129 void 5130 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5131 { 5132 struct mlx5_flow_mreg_copy_resource *mcp_res = 5133 container_of(entry, typeof(*mcp_res), hlist_ent); 5134 struct rte_eth_dev *dev = tool_ctx; 5135 struct mlx5_priv *priv = dev->data->dev_private; 5136 5137 MLX5_ASSERT(mcp_res->rix_flow); 5138 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5139 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5140 } 5141 5142 /** 5143 * Release flow in RX_CP_TBL. 5144 * 5145 * @param dev 5146 * Pointer to Ethernet device. 5147 * @flow 5148 * Parent flow for wich copying is provided. 5149 */ 5150 static void 5151 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5152 struct rte_flow *flow) 5153 { 5154 struct mlx5_flow_mreg_copy_resource *mcp_res; 5155 struct mlx5_priv *priv = dev->data->dev_private; 5156 5157 if (!flow->rix_mreg_copy) 5158 return; 5159 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5160 flow->rix_mreg_copy); 5161 if (!mcp_res || !priv->mreg_cp_tbl) 5162 return; 5163 MLX5_ASSERT(mcp_res->rix_flow); 5164 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5165 flow->rix_mreg_copy = 0; 5166 } 5167 5168 /** 5169 * Remove the default copy action from RX_CP_TBL. 5170 * 5171 * This functions is called in the mlx5_dev_start(). No thread safe 5172 * is guaranteed. 5173 * 5174 * @param dev 5175 * Pointer to Ethernet device. 5176 */ 5177 static void 5178 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5179 { 5180 struct mlx5_list_entry *entry; 5181 struct mlx5_priv *priv = dev->data->dev_private; 5182 struct mlx5_flow_cb_ctx ctx; 5183 uint32_t mark_id; 5184 5185 /* Check if default flow is registered. */ 5186 if (!priv->mreg_cp_tbl) 5187 return; 5188 mark_id = MLX5_DEFAULT_COPY_ID; 5189 ctx.data = &mark_id; 5190 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5191 if (!entry) 5192 return; 5193 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5194 } 5195 5196 /** 5197 * Add the default copy action in RX_CP_TBL. 5198 * 5199 * This functions is called in the mlx5_dev_start(). No thread safe 5200 * is guaranteed. 5201 * 5202 * @param dev 5203 * Pointer to Ethernet device. 5204 * @param[out] error 5205 * Perform verbose error reporting if not NULL. 5206 * 5207 * @return 5208 * 0 for success, negative value otherwise and rte_errno is set. 5209 */ 5210 static int 5211 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5212 struct rte_flow_error *error) 5213 { 5214 struct mlx5_priv *priv = dev->data->dev_private; 5215 struct mlx5_flow_mreg_copy_resource *mcp_res; 5216 struct mlx5_flow_cb_ctx ctx; 5217 uint32_t mark_id; 5218 5219 /* Check whether extensive metadata feature is engaged. */ 5220 if (!priv->sh->config.dv_flow_en || 5221 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5222 !mlx5_flow_ext_mreg_supported(dev) || 5223 !priv->sh->dv_regc0_mask) 5224 return 0; 5225 /* 5226 * Add default mreg copy flow may be called multiple time, but 5227 * only be called once in stop. Avoid register it twice. 5228 */ 5229 mark_id = MLX5_DEFAULT_COPY_ID; 5230 ctx.data = &mark_id; 5231 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5232 return 0; 5233 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5234 if (!mcp_res) 5235 return -rte_errno; 5236 return 0; 5237 } 5238 5239 /** 5240 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5241 * 5242 * All the flow having Q/RSS action should be split by 5243 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5244 * performs the following, 5245 * - CQE->flow_tag := reg_c[1] (MARK) 5246 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5247 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5248 * but there should be a flow per each MARK ID set by MARK action. 5249 * 5250 * For the aforementioned reason, if there's a MARK action in flow's action 5251 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5252 * the MARK ID to CQE's flow_tag like, 5253 * - If reg_c[1] is mark_id, 5254 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5255 * 5256 * For SET_META action which stores value in reg_c[0], as the destination is 5257 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5258 * MARK ID means the default flow. The default flow looks like, 5259 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5260 * 5261 * @param dev 5262 * Pointer to Ethernet device. 5263 * @param flow 5264 * Pointer to flow structure. 5265 * @param[in] actions 5266 * Pointer to the list of actions. 5267 * @param[out] error 5268 * Perform verbose error reporting if not NULL. 5269 * 5270 * @return 5271 * 0 on success, negative value otherwise and rte_errno is set. 5272 */ 5273 static int 5274 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5275 struct rte_flow *flow, 5276 const struct rte_flow_action *actions, 5277 struct rte_flow_error *error) 5278 { 5279 struct mlx5_priv *priv = dev->data->dev_private; 5280 struct mlx5_sh_config *config = &priv->sh->config; 5281 struct mlx5_flow_mreg_copy_resource *mcp_res; 5282 const struct rte_flow_action_mark *mark; 5283 5284 /* Check whether extensive metadata feature is engaged. */ 5285 if (!config->dv_flow_en || 5286 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5287 !mlx5_flow_ext_mreg_supported(dev) || 5288 !priv->sh->dv_regc0_mask) 5289 return 0; 5290 /* Find MARK action. */ 5291 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5292 switch (actions->type) { 5293 case RTE_FLOW_ACTION_TYPE_FLAG: 5294 mcp_res = flow_mreg_add_copy_action 5295 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5296 if (!mcp_res) 5297 return -rte_errno; 5298 flow->rix_mreg_copy = mcp_res->idx; 5299 return 0; 5300 case RTE_FLOW_ACTION_TYPE_MARK: 5301 mark = (const struct rte_flow_action_mark *) 5302 actions->conf; 5303 mcp_res = 5304 flow_mreg_add_copy_action(dev, mark->id, error); 5305 if (!mcp_res) 5306 return -rte_errno; 5307 flow->rix_mreg_copy = mcp_res->idx; 5308 return 0; 5309 default: 5310 break; 5311 } 5312 } 5313 return 0; 5314 } 5315 5316 #define MLX5_MAX_SPLIT_ACTIONS 24 5317 #define MLX5_MAX_SPLIT_ITEMS 24 5318 5319 /** 5320 * Split the hairpin flow. 5321 * Since HW can't support encap and push-vlan on Rx, we move these 5322 * actions to Tx. 5323 * If the count action is after the encap then we also 5324 * move the count action. in this case the count will also measure 5325 * the outer bytes. 5326 * 5327 * @param dev 5328 * Pointer to Ethernet device. 5329 * @param[in] actions 5330 * Associated actions (list terminated by the END action). 5331 * @param[out] actions_rx 5332 * Rx flow actions. 5333 * @param[out] actions_tx 5334 * Tx flow actions.. 5335 * @param[out] pattern_tx 5336 * The pattern items for the Tx flow. 5337 * @param[out] flow_id 5338 * The flow ID connected to this flow. 5339 * 5340 * @return 5341 * 0 on success. 5342 */ 5343 static int 5344 flow_hairpin_split(struct rte_eth_dev *dev, 5345 const struct rte_flow_action actions[], 5346 struct rte_flow_action actions_rx[], 5347 struct rte_flow_action actions_tx[], 5348 struct rte_flow_item pattern_tx[], 5349 uint32_t flow_id) 5350 { 5351 const struct rte_flow_action_raw_encap *raw_encap; 5352 const struct rte_flow_action_raw_decap *raw_decap; 5353 struct mlx5_rte_flow_action_set_tag *set_tag; 5354 struct rte_flow_action *tag_action; 5355 struct mlx5_rte_flow_item_tag *tag_item; 5356 struct rte_flow_item *item; 5357 char *addr; 5358 int push_vlan = 0; 5359 int encap = 0; 5360 5361 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5362 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5363 push_vlan = 1; 5364 switch (actions->type) { 5365 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5366 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5367 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5368 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5369 rte_memcpy(actions_tx, actions, 5370 sizeof(struct rte_flow_action)); 5371 actions_tx++; 5372 break; 5373 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5374 if (push_vlan) { 5375 rte_memcpy(actions_tx, actions, 5376 sizeof(struct rte_flow_action)); 5377 actions_tx++; 5378 } else { 5379 rte_memcpy(actions_rx, actions, 5380 sizeof(struct rte_flow_action)); 5381 actions_rx++; 5382 } 5383 break; 5384 case RTE_FLOW_ACTION_TYPE_COUNT: 5385 if (encap) { 5386 rte_memcpy(actions_tx, actions, 5387 sizeof(struct rte_flow_action)); 5388 actions_tx++; 5389 } else { 5390 rte_memcpy(actions_rx, actions, 5391 sizeof(struct rte_flow_action)); 5392 actions_rx++; 5393 } 5394 break; 5395 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5396 raw_encap = actions->conf; 5397 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5398 memcpy(actions_tx, actions, 5399 sizeof(struct rte_flow_action)); 5400 actions_tx++; 5401 encap = 1; 5402 } else { 5403 rte_memcpy(actions_rx, actions, 5404 sizeof(struct rte_flow_action)); 5405 actions_rx++; 5406 } 5407 break; 5408 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5409 raw_decap = actions->conf; 5410 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5411 memcpy(actions_tx, actions, 5412 sizeof(struct rte_flow_action)); 5413 actions_tx++; 5414 } else { 5415 rte_memcpy(actions_rx, actions, 5416 sizeof(struct rte_flow_action)); 5417 actions_rx++; 5418 } 5419 break; 5420 default: 5421 rte_memcpy(actions_rx, actions, 5422 sizeof(struct rte_flow_action)); 5423 actions_rx++; 5424 break; 5425 } 5426 } 5427 /* Add set meta action and end action for the Rx flow. */ 5428 tag_action = actions_rx; 5429 tag_action->type = (enum rte_flow_action_type) 5430 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5431 actions_rx++; 5432 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5433 actions_rx++; 5434 set_tag = (void *)actions_rx; 5435 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5436 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5437 .data = flow_id, 5438 }; 5439 MLX5_ASSERT(set_tag->id > REG_NON); 5440 tag_action->conf = set_tag; 5441 /* Create Tx item list. */ 5442 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5443 addr = (void *)&pattern_tx[2]; 5444 item = pattern_tx; 5445 item->type = (enum rte_flow_item_type) 5446 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5447 tag_item = (void *)addr; 5448 tag_item->data = flow_id; 5449 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5450 MLX5_ASSERT(set_tag->id > REG_NON); 5451 item->spec = tag_item; 5452 addr += sizeof(struct mlx5_rte_flow_item_tag); 5453 tag_item = (void *)addr; 5454 tag_item->data = UINT32_MAX; 5455 tag_item->id = UINT16_MAX; 5456 item->mask = tag_item; 5457 item->last = NULL; 5458 item++; 5459 item->type = RTE_FLOW_ITEM_TYPE_END; 5460 return 0; 5461 } 5462 5463 /** 5464 * The last stage of splitting chain, just creates the subflow 5465 * without any modification. 5466 * 5467 * @param[in] dev 5468 * Pointer to Ethernet device. 5469 * @param[in] flow 5470 * Parent flow structure pointer. 5471 * @param[in, out] sub_flow 5472 * Pointer to return the created subflow, may be NULL. 5473 * @param[in] attr 5474 * Flow rule attributes. 5475 * @param[in] items 5476 * Pattern specification (list terminated by the END pattern item). 5477 * @param[in] actions 5478 * Associated actions (list terminated by the END action). 5479 * @param[in] flow_split_info 5480 * Pointer to flow split info structure. 5481 * @param[out] error 5482 * Perform verbose error reporting if not NULL. 5483 * @return 5484 * 0 on success, negative value otherwise 5485 */ 5486 static int 5487 flow_create_split_inner(struct rte_eth_dev *dev, 5488 struct rte_flow *flow, 5489 struct mlx5_flow **sub_flow, 5490 const struct rte_flow_attr *attr, 5491 const struct rte_flow_item items[], 5492 const struct rte_flow_action actions[], 5493 struct mlx5_flow_split_info *flow_split_info, 5494 struct rte_flow_error *error) 5495 { 5496 struct mlx5_flow *dev_flow; 5497 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5498 5499 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5500 flow_split_info->flow_idx, error); 5501 if (!dev_flow) 5502 return -rte_errno; 5503 dev_flow->flow = flow; 5504 dev_flow->external = flow_split_info->external; 5505 dev_flow->skip_scale = flow_split_info->skip_scale; 5506 /* Subflow object was created, we must include one in the list. */ 5507 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5508 dev_flow->handle, next); 5509 /* 5510 * If dev_flow is as one of the suffix flow, some actions in suffix 5511 * flow may need some user defined item layer flags, and pass the 5512 * Metadata rxq mark flag to suffix flow as well. 5513 */ 5514 if (flow_split_info->prefix_layers) 5515 dev_flow->handle->layers = flow_split_info->prefix_layers; 5516 if (flow_split_info->prefix_mark) { 5517 MLX5_ASSERT(wks); 5518 wks->mark = 1; 5519 } 5520 if (sub_flow) 5521 *sub_flow = dev_flow; 5522 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5523 dev_flow->dv.table_id = flow_split_info->table_id; 5524 #endif 5525 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5526 } 5527 5528 /** 5529 * Get the sub policy of a meter. 5530 * 5531 * @param[in] dev 5532 * Pointer to Ethernet device. 5533 * @param[in] flow 5534 * Parent flow structure pointer. 5535 * @param wks 5536 * Pointer to thread flow work space. 5537 * @param[in] attr 5538 * Flow rule attributes. 5539 * @param[in] items 5540 * Pattern specification (list terminated by the END pattern item). 5541 * @param[out] error 5542 * Perform verbose error reporting if not NULL. 5543 * 5544 * @return 5545 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5546 */ 5547 static struct mlx5_flow_meter_sub_policy * 5548 get_meter_sub_policy(struct rte_eth_dev *dev, 5549 struct rte_flow *flow, 5550 struct mlx5_flow_workspace *wks, 5551 const struct rte_flow_attr *attr, 5552 const struct rte_flow_item items[], 5553 struct rte_flow_error *error) 5554 { 5555 struct mlx5_flow_meter_policy *policy; 5556 struct mlx5_flow_meter_policy *final_policy; 5557 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5558 5559 policy = wks->policy; 5560 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5561 if (final_policy->is_rss || final_policy->is_queue) { 5562 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5563 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5564 uint32_t i; 5565 5566 /* 5567 * This is a tmp dev_flow, 5568 * no need to register any matcher for it in translate. 5569 */ 5570 wks->skip_matcher_reg = 1; 5571 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5572 struct mlx5_flow dev_flow = {0}; 5573 struct mlx5_flow_handle dev_handle = { {0} }; 5574 uint8_t fate = final_policy->act_cnt[i].fate_action; 5575 5576 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5577 const struct rte_flow_action_rss *rss_act = 5578 final_policy->act_cnt[i].rss->conf; 5579 struct rte_flow_action rss_actions[2] = { 5580 [0] = { 5581 .type = RTE_FLOW_ACTION_TYPE_RSS, 5582 .conf = rss_act, 5583 }, 5584 [1] = { 5585 .type = RTE_FLOW_ACTION_TYPE_END, 5586 .conf = NULL, 5587 } 5588 }; 5589 5590 dev_flow.handle = &dev_handle; 5591 dev_flow.ingress = attr->ingress; 5592 dev_flow.flow = flow; 5593 dev_flow.external = 0; 5594 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5595 dev_flow.dv.transfer = attr->transfer; 5596 #endif 5597 /** 5598 * Translate RSS action to get rss hash fields. 5599 */ 5600 if (flow_drv_translate(dev, &dev_flow, attr, 5601 items, rss_actions, error)) 5602 goto exit; 5603 rss_desc_v[i] = wks->rss_desc; 5604 rss_desc_v[i].symmetric_hash_function = 5605 dev_flow.symmetric_hash_function; 5606 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5607 rss_desc_v[i].hash_fields = 5608 dev_flow.hash_fields; 5609 rss_desc_v[i].queue_num = 5610 rss_desc_v[i].hash_fields ? 5611 rss_desc_v[i].queue_num : 1; 5612 rss_desc_v[i].tunnel = 5613 !!(dev_flow.handle->layers & 5614 MLX5_FLOW_LAYER_TUNNEL); 5615 /* Use the RSS queues in the containers. */ 5616 rss_desc_v[i].queue = 5617 (uint16_t *)(uintptr_t)rss_act->queue; 5618 rss_desc[i] = &rss_desc_v[i]; 5619 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5620 /* This is queue action. */ 5621 rss_desc_v[i] = wks->rss_desc; 5622 rss_desc_v[i].key_len = 0; 5623 rss_desc_v[i].hash_fields = 0; 5624 rss_desc_v[i].queue = 5625 &final_policy->act_cnt[i].queue; 5626 rss_desc_v[i].queue_num = 1; 5627 rss_desc[i] = &rss_desc_v[i]; 5628 } else { 5629 rss_desc[i] = NULL; 5630 } 5631 } 5632 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5633 flow, policy, rss_desc); 5634 } else { 5635 enum mlx5_meter_domain mtr_domain = 5636 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5637 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5638 MLX5_MTR_DOMAIN_INGRESS); 5639 sub_policy = policy->sub_policys[mtr_domain][0]; 5640 } 5641 if (!sub_policy) 5642 rte_flow_error_set(error, EINVAL, 5643 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5644 "Failed to get meter sub-policy."); 5645 exit: 5646 return sub_policy; 5647 } 5648 5649 /** 5650 * Split the meter flow. 5651 * 5652 * As meter flow will split to three sub flow, other than meter 5653 * action, the other actions make sense to only meter accepts 5654 * the packet. If it need to be dropped, no other additional 5655 * actions should be take. 5656 * 5657 * One kind of special action which decapsulates the L3 tunnel 5658 * header will be in the prefix sub flow, as not to take the 5659 * L3 tunnel header into account. 5660 * 5661 * @param[in] dev 5662 * Pointer to Ethernet device. 5663 * @param[in] flow 5664 * Parent flow structure pointer. 5665 * @param wks 5666 * Pointer to thread flow work space. 5667 * @param[in] attr 5668 * Flow rule attributes. 5669 * @param[in] items 5670 * Pattern specification (list terminated by the END pattern item). 5671 * @param[out] sfx_items 5672 * Suffix flow match items (list terminated by the END pattern item). 5673 * @param[in] actions 5674 * Associated actions (list terminated by the END action). 5675 * @param[out] actions_sfx 5676 * Suffix flow actions. 5677 * @param[out] actions_pre 5678 * Prefix flow actions. 5679 * @param[out] mtr_flow_id 5680 * Pointer to meter flow id. 5681 * @param[out] error 5682 * Perform verbose error reporting if not NULL. 5683 * 5684 * @return 5685 * 0 on success, a negative errno value otherwise and rte_errno is set. 5686 */ 5687 static int 5688 flow_meter_split_prep(struct rte_eth_dev *dev, 5689 struct rte_flow *flow, 5690 struct mlx5_flow_workspace *wks, 5691 const struct rte_flow_attr *attr, 5692 const struct rte_flow_item items[], 5693 struct rte_flow_item sfx_items[], 5694 const struct rte_flow_action actions[], 5695 struct rte_flow_action actions_sfx[], 5696 struct rte_flow_action actions_pre[], 5697 uint32_t *mtr_flow_id, 5698 struct rte_flow_error *error) 5699 { 5700 struct mlx5_priv *priv = dev->data->dev_private; 5701 struct mlx5_flow_meter_info *fm = wks->fm; 5702 struct rte_flow_action *tag_action = NULL; 5703 struct rte_flow_item *tag_item; 5704 struct mlx5_rte_flow_action_set_tag *set_tag; 5705 const struct rte_flow_action_raw_encap *raw_encap; 5706 const struct rte_flow_action_raw_decap *raw_decap; 5707 struct mlx5_rte_flow_item_tag *tag_item_spec; 5708 struct mlx5_rte_flow_item_tag *tag_item_mask; 5709 uint32_t tag_id = 0; 5710 struct rte_flow_item *vlan_item_dst = NULL; 5711 const struct rte_flow_item *vlan_item_src = NULL; 5712 const struct rte_flow_item *orig_items = items; 5713 struct rte_flow_action *hw_mtr_action; 5714 struct rte_flow_action *action_pre_head = NULL; 5715 uint16_t flow_src_port = priv->representor_id; 5716 bool mtr_first; 5717 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5718 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5719 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5720 uint32_t flow_id = 0; 5721 uint32_t flow_id_reversed = 0; 5722 uint8_t flow_id_bits = 0; 5723 bool after_meter = false; 5724 int shift; 5725 5726 /* Prepare the suffix subflow items. */ 5727 tag_item = sfx_items++; 5728 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5729 int item_type = items->type; 5730 5731 switch (item_type) { 5732 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5733 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5734 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5735 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5736 return -rte_errno; 5737 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5738 flow_src_port != priv->representor_id) { 5739 if (flow_drv_mtr_hierarchy_rule_create(dev, 5740 flow, fm, 5741 flow_src_port, 5742 items, 5743 error)) 5744 return -rte_errno; 5745 } 5746 memcpy(sfx_items, items, sizeof(*sfx_items)); 5747 sfx_items++; 5748 break; 5749 case RTE_FLOW_ITEM_TYPE_VLAN: 5750 /* Determine if copy vlan item below. */ 5751 vlan_item_src = items; 5752 vlan_item_dst = sfx_items++; 5753 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5754 break; 5755 default: 5756 break; 5757 } 5758 } 5759 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5760 sfx_items++; 5761 mtr_first = priv->sh->meter_aso_en && 5762 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5763 /* For ASO meter, meter must be before tag in TX direction. */ 5764 if (mtr_first) { 5765 action_pre_head = actions_pre++; 5766 /* Leave space for tag action. */ 5767 tag_action = actions_pre++; 5768 } 5769 /* Prepare the actions for prefix and suffix flow. */ 5770 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5771 struct rte_flow_action *action_cur = NULL; 5772 5773 switch (actions->type) { 5774 case RTE_FLOW_ACTION_TYPE_METER: 5775 if (mtr_first) { 5776 action_cur = action_pre_head; 5777 } else { 5778 /* Leave space for tag action. */ 5779 tag_action = actions_pre++; 5780 action_cur = actions_pre++; 5781 } 5782 after_meter = true; 5783 break; 5784 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5785 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5786 action_cur = actions_pre++; 5787 break; 5788 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5789 raw_encap = actions->conf; 5790 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5791 action_cur = actions_pre++; 5792 break; 5793 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5794 raw_decap = actions->conf; 5795 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5796 action_cur = actions_pre++; 5797 break; 5798 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5799 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5800 if (vlan_item_dst && vlan_item_src) { 5801 memcpy(vlan_item_dst, vlan_item_src, 5802 sizeof(*vlan_item_dst)); 5803 /* 5804 * Convert to internal match item, it is used 5805 * for vlan push and set vid. 5806 */ 5807 vlan_item_dst->type = (enum rte_flow_item_type) 5808 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5809 } 5810 break; 5811 case RTE_FLOW_ACTION_TYPE_COUNT: 5812 if (fm->def_policy) 5813 action_cur = after_meter ? 5814 actions_sfx++ : actions_pre++; 5815 break; 5816 default: 5817 break; 5818 } 5819 if (!action_cur) 5820 action_cur = (fm->def_policy) ? 5821 actions_sfx++ : actions_pre++; 5822 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5823 } 5824 /* Add end action to the actions. */ 5825 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5826 if (priv->sh->meter_aso_en) { 5827 /** 5828 * For ASO meter, need to add an extra jump action explicitly, 5829 * to jump from meter to policer table. 5830 */ 5831 struct mlx5_flow_meter_sub_policy *sub_policy; 5832 struct mlx5_flow_tbl_data_entry *tbl_data; 5833 5834 if (!fm->def_policy) { 5835 sub_policy = get_meter_sub_policy(dev, flow, wks, 5836 attr, orig_items, 5837 error); 5838 if (!sub_policy) 5839 return -rte_errno; 5840 } else { 5841 enum mlx5_meter_domain mtr_domain = 5842 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5843 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5844 MLX5_MTR_DOMAIN_INGRESS); 5845 5846 sub_policy = 5847 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5848 } 5849 tbl_data = container_of(sub_policy->tbl_rsc, 5850 struct mlx5_flow_tbl_data_entry, tbl); 5851 hw_mtr_action = actions_pre++; 5852 hw_mtr_action->type = (enum rte_flow_action_type) 5853 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5854 hw_mtr_action->conf = tbl_data->jump.action; 5855 } 5856 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5857 actions_pre++; 5858 if (!tag_action) 5859 return rte_flow_error_set(error, ENOMEM, 5860 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5861 NULL, "No tag action space."); 5862 if (!mtr_flow_id) { 5863 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5864 goto exit; 5865 } 5866 /* Only default-policy Meter creates mtr flow id. */ 5867 if (fm->def_policy) { 5868 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5869 if (!tag_id) 5870 return rte_flow_error_set(error, ENOMEM, 5871 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5872 "Failed to allocate meter flow id."); 5873 flow_id = tag_id - 1; 5874 flow_id_bits = (!flow_id) ? 1 : 5875 (MLX5_REG_BITS - rte_clz32(flow_id)); 5876 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5877 mtr_reg_bits) { 5878 mlx5_ipool_free(fm->flow_ipool, tag_id); 5879 return rte_flow_error_set(error, EINVAL, 5880 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5881 "Meter flow id exceeds max limit."); 5882 } 5883 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5884 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5885 } 5886 /* Build tag actions and items for meter_id/meter flow_id. */ 5887 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5888 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5889 tag_item_mask = tag_item_spec + 1; 5890 /* Both flow_id and meter_id share the same register. */ 5891 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5892 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5893 0, error), 5894 .offset = mtr_id_offset, 5895 .length = mtr_reg_bits, 5896 .data = flow->meter, 5897 }; 5898 /* 5899 * The color Reg bits used by flow_id are growing from 5900 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5901 */ 5902 for (shift = 0; shift < flow_id_bits; shift++) 5903 flow_id_reversed = (flow_id_reversed << 1) | 5904 ((flow_id >> shift) & 0x1); 5905 set_tag->data |= 5906 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5907 tag_item_spec->id = set_tag->id; 5908 tag_item_spec->data = set_tag->data << mtr_id_offset; 5909 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5910 tag_action->type = (enum rte_flow_action_type) 5911 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5912 tag_action->conf = set_tag; 5913 tag_item->type = (enum rte_flow_item_type) 5914 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5915 tag_item->spec = tag_item_spec; 5916 tag_item->last = NULL; 5917 tag_item->mask = tag_item_mask; 5918 exit: 5919 if (mtr_flow_id) 5920 *mtr_flow_id = tag_id; 5921 return 0; 5922 } 5923 5924 /** 5925 * Split action list having QUEUE/RSS for metadata register copy. 5926 * 5927 * Once Q/RSS action is detected in user's action list, the flow action 5928 * should be split in order to copy metadata registers, which will happen in 5929 * RX_CP_TBL like, 5930 * - CQE->flow_tag := reg_c[1] (MARK) 5931 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5932 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5933 * This is because the last action of each flow must be a terminal action 5934 * (QUEUE, RSS or DROP). 5935 * 5936 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5937 * stored and kept in the mlx5_flow structure per each sub_flow. 5938 * 5939 * The Q/RSS action is replaced with, 5940 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5941 * And the following JUMP action is added at the end, 5942 * - JUMP, to RX_CP_TBL. 5943 * 5944 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5945 * flow_create_split_metadata() routine. The flow will look like, 5946 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5947 * 5948 * @param dev 5949 * Pointer to Ethernet device. 5950 * @param[out] split_actions 5951 * Pointer to store split actions to jump to CP_TBL. 5952 * @param[in] actions 5953 * Pointer to the list of original flow actions. 5954 * @param[in] qrss 5955 * Pointer to the Q/RSS action. 5956 * @param[in] actions_n 5957 * Number of original actions. 5958 * @param[in] mtr_sfx 5959 * Check if it is in meter suffix table. 5960 * @param[out] error 5961 * Perform verbose error reporting if not NULL. 5962 * 5963 * @return 5964 * non-zero unique flow_id on success, otherwise 0 and 5965 * error/rte_error are set. 5966 */ 5967 static uint32_t 5968 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5969 struct rte_flow_action *split_actions, 5970 const struct rte_flow_action *actions, 5971 const struct rte_flow_action *qrss, 5972 int actions_n, int mtr_sfx, 5973 struct rte_flow_error *error) 5974 { 5975 struct mlx5_priv *priv = dev->data->dev_private; 5976 struct mlx5_rte_flow_action_set_tag *set_tag; 5977 struct rte_flow_action_jump *jump; 5978 const int qrss_idx = qrss - actions; 5979 uint32_t flow_id = 0; 5980 int ret = 0; 5981 5982 /* 5983 * Given actions will be split 5984 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5985 * - Add jump to mreg CP_TBL. 5986 * As a result, there will be one more action. 5987 */ 5988 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5989 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5990 ++actions_n; 5991 set_tag = (void *)(split_actions + actions_n); 5992 /* 5993 * If we are not the meter suffix flow, add the tag action. 5994 * Since meter suffix flow already has the tag added. 5995 */ 5996 if (!mtr_sfx) { 5997 /* 5998 * Allocate the new subflow ID. This one is unique within 5999 * device and not shared with representors. Otherwise, 6000 * we would have to resolve multi-thread access synch 6001 * issue. Each flow on the shared device is appended 6002 * with source vport identifier, so the resulting 6003 * flows will be unique in the shared (by master and 6004 * representors) domain even if they have coinciding 6005 * IDs. 6006 */ 6007 mlx5_ipool_malloc(priv->sh->ipool 6008 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 6009 if (!flow_id) 6010 return rte_flow_error_set(error, ENOMEM, 6011 RTE_FLOW_ERROR_TYPE_ACTION, 6012 NULL, "can't allocate id " 6013 "for split Q/RSS subflow"); 6014 /* Internal SET_TAG action to set flow ID. */ 6015 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 6016 .data = flow_id, 6017 }; 6018 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 6019 if (ret < 0) 6020 return ret; 6021 set_tag->id = ret; 6022 /* Construct new actions array. */ 6023 /* Replace QUEUE/RSS action. */ 6024 split_actions[qrss_idx] = (struct rte_flow_action){ 6025 .type = (enum rte_flow_action_type) 6026 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6027 .conf = set_tag, 6028 }; 6029 } else { 6030 /* 6031 * If we are the suffix flow of meter, tag already exist. 6032 * Set the QUEUE/RSS action to void. 6033 */ 6034 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 6035 } 6036 /* JUMP action to jump to mreg copy table (CP_TBL). */ 6037 jump = (void *)(set_tag + 1); 6038 *jump = (struct rte_flow_action_jump){ 6039 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6040 }; 6041 split_actions[actions_n - 2] = (struct rte_flow_action){ 6042 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6043 .conf = jump, 6044 }; 6045 split_actions[actions_n - 1] = (struct rte_flow_action){ 6046 .type = RTE_FLOW_ACTION_TYPE_END, 6047 }; 6048 return flow_id; 6049 } 6050 6051 /** 6052 * Extend the given action list for Tx metadata copy. 6053 * 6054 * Copy the given action list to the ext_actions and add flow metadata register 6055 * copy action in order to copy reg_a set by WQE to reg_c[0]. 6056 * 6057 * @param[out] ext_actions 6058 * Pointer to the extended action list. 6059 * @param[in] actions 6060 * Pointer to the list of actions. 6061 * @param[in] actions_n 6062 * Number of actions in the list. 6063 * @param[out] error 6064 * Perform verbose error reporting if not NULL. 6065 * @param[in] encap_idx 6066 * The encap action index. 6067 * 6068 * @return 6069 * 0 on success, negative value otherwise 6070 */ 6071 static int 6072 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6073 struct rte_flow_action *ext_actions, 6074 const struct rte_flow_action *actions, 6075 int actions_n, struct rte_flow_error *error, 6076 int encap_idx) 6077 { 6078 struct mlx5_flow_action_copy_mreg *cp_mreg = 6079 (struct mlx5_flow_action_copy_mreg *) 6080 (ext_actions + actions_n + 1); 6081 int ret; 6082 6083 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6084 if (ret < 0) 6085 return ret; 6086 cp_mreg->dst = ret; 6087 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6088 if (ret < 0) 6089 return ret; 6090 cp_mreg->src = ret; 6091 if (encap_idx != 0) 6092 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6093 if (encap_idx == actions_n - 1) { 6094 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6095 .type = (enum rte_flow_action_type) 6096 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6097 .conf = cp_mreg, 6098 }; 6099 ext_actions[actions_n] = (struct rte_flow_action){ 6100 .type = RTE_FLOW_ACTION_TYPE_END, 6101 }; 6102 } else { 6103 ext_actions[encap_idx] = (struct rte_flow_action){ 6104 .type = (enum rte_flow_action_type) 6105 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6106 .conf = cp_mreg, 6107 }; 6108 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6109 sizeof(*ext_actions) * (actions_n - encap_idx)); 6110 } 6111 return 0; 6112 } 6113 6114 /** 6115 * Check the match action from the action list. 6116 * 6117 * @param[in] actions 6118 * Pointer to the list of actions. 6119 * @param[in] attr 6120 * Flow rule attributes. 6121 * @param[in] action 6122 * The action to be check if exist. 6123 * @param[out] match_action_pos 6124 * Pointer to the position of the matched action if exists, otherwise is -1. 6125 * @param[out] qrss_action_pos 6126 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6127 * @param[out] modify_after_mirror 6128 * Pointer to the flag of modify action after FDB mirroring. 6129 * 6130 * @return 6131 * > 0 the total number of actions. 6132 * 0 if not found match action in action list. 6133 */ 6134 static int 6135 flow_check_match_action(const struct rte_flow_action actions[], 6136 const struct rte_flow_attr *attr, 6137 enum rte_flow_action_type action, 6138 int *match_action_pos, int *qrss_action_pos, 6139 int *modify_after_mirror) 6140 { 6141 const struct rte_flow_action_sample *sample; 6142 const struct rte_flow_action_raw_decap *decap; 6143 const struct rte_flow_action *action_cur = NULL; 6144 int actions_n = 0; 6145 uint32_t ratio = 0; 6146 int sub_type = 0; 6147 int flag = 0; 6148 int fdb_mirror = 0; 6149 6150 *match_action_pos = -1; 6151 *qrss_action_pos = -1; 6152 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6153 if (actions->type == action) { 6154 flag = 1; 6155 *match_action_pos = actions_n; 6156 } 6157 switch (actions->type) { 6158 case RTE_FLOW_ACTION_TYPE_QUEUE: 6159 case RTE_FLOW_ACTION_TYPE_RSS: 6160 *qrss_action_pos = actions_n; 6161 break; 6162 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6163 sample = actions->conf; 6164 ratio = sample->ratio; 6165 sub_type = ((const struct rte_flow_action *) 6166 (sample->actions))->type; 6167 if (ratio == 1 && attr->transfer && 6168 sub_type != RTE_FLOW_ACTION_TYPE_END) 6169 fdb_mirror = 1; 6170 break; 6171 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6172 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6173 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6174 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6175 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6176 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6177 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6178 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6179 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6180 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6181 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6182 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6183 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6184 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6185 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6186 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6187 case RTE_FLOW_ACTION_TYPE_FLAG: 6188 case RTE_FLOW_ACTION_TYPE_MARK: 6189 case RTE_FLOW_ACTION_TYPE_SET_META: 6190 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6191 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6192 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6193 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6194 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6195 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6196 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6197 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6198 case RTE_FLOW_ACTION_TYPE_METER: 6199 if (fdb_mirror) 6200 *modify_after_mirror = 1; 6201 break; 6202 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6203 decap = actions->conf; 6204 action_cur = actions; 6205 while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID) 6206 ; 6207 if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6208 const struct rte_flow_action_raw_encap *encap = 6209 action_cur->conf; 6210 if (decap->size <= 6211 MLX5_ENCAPSULATION_DECISION_SIZE && 6212 encap->size > 6213 MLX5_ENCAPSULATION_DECISION_SIZE) 6214 /* L3 encap. */ 6215 break; 6216 } 6217 if (fdb_mirror) 6218 *modify_after_mirror = 1; 6219 break; 6220 default: 6221 break; 6222 } 6223 actions_n++; 6224 } 6225 if (flag && fdb_mirror && !*modify_after_mirror) { 6226 /* FDB mirroring uses the destination array to implement 6227 * instead of FLOW_SAMPLER object. 6228 */ 6229 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6230 flag = 0; 6231 } 6232 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6233 return flag ? actions_n + 1 : 0; 6234 } 6235 6236 #define SAMPLE_SUFFIX_ITEM 3 6237 6238 /** 6239 * Split the sample flow. 6240 * 6241 * As sample flow will split to two sub flow, sample flow with 6242 * sample action, the other actions will move to new suffix flow. 6243 * 6244 * Also add unique tag id with tag action in the sample flow, 6245 * the same tag id will be as match in the suffix flow. 6246 * 6247 * @param dev 6248 * Pointer to Ethernet device. 6249 * @param[in] add_tag 6250 * Add extra tag action flag. 6251 * @param[out] sfx_items 6252 * Suffix flow match items (list terminated by the END pattern item). 6253 * @param[in] actions 6254 * Associated actions (list terminated by the END action). 6255 * @param[out] actions_sfx 6256 * Suffix flow actions. 6257 * @param[out] actions_pre 6258 * Prefix flow actions. 6259 * @param[in] actions_n 6260 * The total number of actions. 6261 * @param[in] sample_action_pos 6262 * The sample action position. 6263 * @param[in] qrss_action_pos 6264 * The Queue/RSS action position. 6265 * @param[in] jump_table 6266 * Add extra jump action flag. 6267 * @param[out] error 6268 * Perform verbose error reporting if not NULL. 6269 * 6270 * @return 6271 * 0 on success, or unique flow_id, a negative errno value 6272 * otherwise and rte_errno is set. 6273 */ 6274 static int 6275 flow_sample_split_prep(struct rte_eth_dev *dev, 6276 int add_tag, 6277 const struct rte_flow_item items[], 6278 struct rte_flow_item sfx_items[], 6279 const struct rte_flow_action actions[], 6280 struct rte_flow_action actions_sfx[], 6281 struct rte_flow_action actions_pre[], 6282 int actions_n, 6283 int sample_action_pos, 6284 int qrss_action_pos, 6285 int jump_table, 6286 struct rte_flow_error *error) 6287 { 6288 struct mlx5_priv *priv = dev->data->dev_private; 6289 struct mlx5_rte_flow_action_set_tag *set_tag; 6290 struct mlx5_rte_flow_item_tag *tag_spec; 6291 struct mlx5_rte_flow_item_tag *tag_mask; 6292 struct rte_flow_action_jump *jump_action; 6293 uint32_t tag_id = 0; 6294 int append_index = 0; 6295 int set_tag_idx = -1; 6296 int index; 6297 int ret; 6298 6299 if (sample_action_pos < 0) 6300 return rte_flow_error_set(error, EINVAL, 6301 RTE_FLOW_ERROR_TYPE_ACTION, 6302 NULL, "invalid position of sample " 6303 "action in list"); 6304 /* Prepare the actions for prefix and suffix flow. */ 6305 if (add_tag) { 6306 /* Update the new added tag action index preceding 6307 * the PUSH_VLAN or ENCAP action. 6308 */ 6309 const struct rte_flow_action_raw_encap *raw_encap; 6310 const struct rte_flow_action *action = actions; 6311 int encap_idx; 6312 int action_idx = 0; 6313 int raw_decap_idx = -1; 6314 int push_vlan_idx = -1; 6315 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6316 switch (action->type) { 6317 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6318 raw_decap_idx = action_idx; 6319 break; 6320 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6321 raw_encap = action->conf; 6322 if (raw_encap->size > 6323 MLX5_ENCAPSULATION_DECISION_SIZE) { 6324 encap_idx = raw_decap_idx != -1 ? 6325 raw_decap_idx : action_idx; 6326 if (encap_idx < sample_action_pos && 6327 push_vlan_idx == -1) 6328 set_tag_idx = encap_idx; 6329 } 6330 break; 6331 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6332 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6333 encap_idx = action_idx; 6334 if (encap_idx < sample_action_pos && 6335 push_vlan_idx == -1) 6336 set_tag_idx = encap_idx; 6337 break; 6338 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6339 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6340 if (action_idx < sample_action_pos && 6341 push_vlan_idx == -1) { 6342 set_tag_idx = action_idx; 6343 push_vlan_idx = action_idx; 6344 } 6345 break; 6346 default: 6347 break; 6348 } 6349 action_idx++; 6350 } 6351 } 6352 /* Prepare the actions for prefix and suffix flow. */ 6353 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6354 index = qrss_action_pos; 6355 /* Put the preceding the Queue/RSS action into prefix flow. */ 6356 if (index != 0) 6357 memcpy(actions_pre, actions, 6358 sizeof(struct rte_flow_action) * index); 6359 /* Put others preceding the sample action into prefix flow. */ 6360 if (sample_action_pos > index + 1) 6361 memcpy(actions_pre + index, actions + index + 1, 6362 sizeof(struct rte_flow_action) * 6363 (sample_action_pos - index - 1)); 6364 index = sample_action_pos - 1; 6365 /* Put Queue/RSS action into Suffix flow. */ 6366 memcpy(actions_sfx, actions + qrss_action_pos, 6367 sizeof(struct rte_flow_action)); 6368 actions_sfx++; 6369 } else if (add_tag && set_tag_idx >= 0) { 6370 if (set_tag_idx > 0) 6371 memcpy(actions_pre, actions, 6372 sizeof(struct rte_flow_action) * set_tag_idx); 6373 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6374 sizeof(struct rte_flow_action) * 6375 (sample_action_pos - set_tag_idx)); 6376 index = sample_action_pos; 6377 } else { 6378 index = sample_action_pos; 6379 if (index != 0) 6380 memcpy(actions_pre, actions, 6381 sizeof(struct rte_flow_action) * index); 6382 } 6383 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6384 * For CX6DX and above, metadata registers Cx preserve their value, 6385 * add an extra tag action for NIC-RX and E-Switch Domain. 6386 */ 6387 if (add_tag) { 6388 /* Prepare the prefix tag action. */ 6389 append_index++; 6390 set_tag = (void *)(actions_pre + actions_n + append_index); 6391 /* Trust VF/SF on CX5 not supported meter so that the reserved 6392 * metadata regC is REG_NON, back to use application tag 6393 * index 0. 6394 */ 6395 if (unlikely(priv->sh->registers.aso_reg == REG_NON)) 6396 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6397 else 6398 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6399 if (ret < 0) 6400 return ret; 6401 mlx5_ipool_malloc(priv->sh->ipool 6402 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6403 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6404 .id = ret, 6405 .data = tag_id, 6406 }; 6407 /* Prepare the suffix subflow items. */ 6408 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6409 tag_spec->data = tag_id; 6410 tag_spec->id = set_tag->id; 6411 tag_mask = tag_spec + 1; 6412 tag_mask->data = UINT32_MAX; 6413 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6414 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6415 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6416 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6417 memcpy(sfx_items, items, sizeof(*sfx_items)); 6418 sfx_items++; 6419 break; 6420 } 6421 } 6422 sfx_items[0] = (struct rte_flow_item){ 6423 .type = (enum rte_flow_item_type) 6424 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6425 .spec = tag_spec, 6426 .last = NULL, 6427 .mask = tag_mask, 6428 }; 6429 sfx_items[1] = (struct rte_flow_item){ 6430 .type = (enum rte_flow_item_type) 6431 RTE_FLOW_ITEM_TYPE_END, 6432 }; 6433 /* Prepare the tag action in prefix subflow. */ 6434 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6435 actions_pre[set_tag_idx] = 6436 (struct rte_flow_action){ 6437 .type = (enum rte_flow_action_type) 6438 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6439 .conf = set_tag, 6440 }; 6441 /* Update next sample position due to add one tag action */ 6442 index += 1; 6443 } 6444 /* Copy the sample action into prefix flow. */ 6445 memcpy(actions_pre + index, actions + sample_action_pos, 6446 sizeof(struct rte_flow_action)); 6447 index += 1; 6448 /* For the modify action after the sample action in E-Switch mirroring, 6449 * Add the extra jump action in prefix subflow and jump into the next 6450 * table, then do the modify action in the new table. 6451 */ 6452 if (jump_table) { 6453 /* Prepare the prefix jump action. */ 6454 append_index++; 6455 jump_action = (void *)(actions_pre + actions_n + append_index); 6456 jump_action->group = jump_table; 6457 actions_pre[index++] = 6458 (struct rte_flow_action){ 6459 .type = (enum rte_flow_action_type) 6460 RTE_FLOW_ACTION_TYPE_JUMP, 6461 .conf = jump_action, 6462 }; 6463 } 6464 actions_pre[index] = (struct rte_flow_action){ 6465 .type = (enum rte_flow_action_type) 6466 RTE_FLOW_ACTION_TYPE_END, 6467 }; 6468 /* Put the actions after sample into Suffix flow. */ 6469 memcpy(actions_sfx, actions + sample_action_pos + 1, 6470 sizeof(struct rte_flow_action) * 6471 (actions_n - sample_action_pos - 1)); 6472 return tag_id; 6473 } 6474 6475 /** 6476 * The splitting for metadata feature. 6477 * 6478 * - Q/RSS action on NIC Rx should be split in order to pass by 6479 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6480 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6481 * 6482 * - All the actions on NIC Tx should have a mreg copy action to 6483 * copy reg_a from WQE to reg_c[0]. 6484 * 6485 * @param dev 6486 * Pointer to Ethernet device. 6487 * @param[in] flow 6488 * Parent flow structure pointer. 6489 * @param[in] attr 6490 * Flow rule attributes. 6491 * @param[in] items 6492 * Pattern specification (list terminated by the END pattern item). 6493 * @param[in] actions 6494 * Associated actions (list terminated by the END action). 6495 * @param[in] flow_split_info 6496 * Pointer to flow split info structure. 6497 * @param[out] error 6498 * Perform verbose error reporting if not NULL. 6499 * @return 6500 * 0 on success, negative value otherwise 6501 */ 6502 static int 6503 flow_create_split_metadata(struct rte_eth_dev *dev, 6504 struct rte_flow *flow, 6505 const struct rte_flow_attr *attr, 6506 const struct rte_flow_item items[], 6507 const struct rte_flow_action actions[], 6508 struct mlx5_flow_split_info *flow_split_info, 6509 struct rte_flow_error *error) 6510 { 6511 struct mlx5_priv *priv = dev->data->dev_private; 6512 struct mlx5_sh_config *config = &priv->sh->config; 6513 const struct rte_flow_action *qrss = NULL; 6514 struct rte_flow_action *ext_actions = NULL; 6515 struct mlx5_flow *dev_flow = NULL; 6516 uint32_t qrss_id = 0; 6517 int mtr_sfx = 0; 6518 size_t act_size; 6519 int actions_n; 6520 int encap_idx; 6521 int ret; 6522 6523 /* Check whether extensive metadata feature is engaged. */ 6524 if (!config->dv_flow_en || 6525 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6526 !mlx5_flow_ext_mreg_supported(dev)) 6527 return flow_create_split_inner(dev, flow, NULL, attr, items, 6528 actions, flow_split_info, error); 6529 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6530 &encap_idx); 6531 if (qrss) { 6532 /* Exclude hairpin flows from splitting. */ 6533 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6534 const struct rte_flow_action_queue *queue; 6535 6536 queue = qrss->conf; 6537 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6538 qrss = NULL; 6539 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6540 const struct rte_flow_action_rss *rss; 6541 6542 rss = qrss->conf; 6543 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6544 qrss = NULL; 6545 } 6546 } 6547 if (qrss) { 6548 /* Check if it is in meter suffix table. */ 6549 mtr_sfx = attr->group == 6550 ((attr->transfer && priv->fdb_def_rule) ? 6551 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6552 MLX5_FLOW_TABLE_LEVEL_METER); 6553 /* 6554 * Q/RSS action on NIC Rx should be split in order to pass by 6555 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6556 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6557 */ 6558 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6559 sizeof(struct rte_flow_action_set_tag) + 6560 sizeof(struct rte_flow_action_jump); 6561 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6562 SOCKET_ID_ANY); 6563 if (!ext_actions) 6564 return rte_flow_error_set(error, ENOMEM, 6565 RTE_FLOW_ERROR_TYPE_ACTION, 6566 NULL, "no memory to split " 6567 "metadata flow"); 6568 /* 6569 * Create the new actions list with removed Q/RSS action 6570 * and appended set tag and jump to register copy table 6571 * (RX_CP_TBL). We should preallocate unique tag ID here 6572 * in advance, because it is needed for set tag action. 6573 */ 6574 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6575 qrss, actions_n, 6576 mtr_sfx, error); 6577 if (!mtr_sfx && !qrss_id) { 6578 ret = -rte_errno; 6579 goto exit; 6580 } 6581 } else if (attr->egress) { 6582 /* 6583 * All the actions on NIC Tx should have a metadata register 6584 * copy action to copy reg_a from WQE to reg_c[meta] 6585 */ 6586 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6587 sizeof(struct mlx5_flow_action_copy_mreg); 6588 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6589 SOCKET_ID_ANY); 6590 if (!ext_actions) 6591 return rte_flow_error_set(error, ENOMEM, 6592 RTE_FLOW_ERROR_TYPE_ACTION, 6593 NULL, "no memory to split " 6594 "metadata flow"); 6595 /* Create the action list appended with copy register. */ 6596 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6597 actions_n, error, encap_idx); 6598 if (ret < 0) 6599 goto exit; 6600 } 6601 /* Add the unmodified original or prefix subflow. */ 6602 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6603 items, ext_actions ? ext_actions : 6604 actions, flow_split_info, error); 6605 if (ret < 0) 6606 goto exit; 6607 MLX5_ASSERT(dev_flow); 6608 if (qrss) { 6609 const struct rte_flow_attr q_attr = { 6610 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6611 .ingress = 1, 6612 }; 6613 /* Internal PMD action to set register. */ 6614 struct mlx5_rte_flow_item_tag q_tag_spec = { 6615 .data = qrss_id, 6616 .id = REG_NON, 6617 }; 6618 struct rte_flow_item q_items[] = { 6619 { 6620 .type = (enum rte_flow_item_type) 6621 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6622 .spec = &q_tag_spec, 6623 .last = NULL, 6624 .mask = NULL, 6625 }, 6626 { 6627 .type = RTE_FLOW_ITEM_TYPE_END, 6628 }, 6629 }; 6630 struct rte_flow_action q_actions[] = { 6631 { 6632 .type = qrss->type, 6633 .conf = qrss->conf, 6634 }, 6635 { 6636 .type = RTE_FLOW_ACTION_TYPE_END, 6637 }, 6638 }; 6639 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6640 6641 /* 6642 * Configure the tag item only if there is no meter subflow. 6643 * Since tag is already marked in the meter suffix subflow 6644 * we can just use the meter suffix items as is. 6645 */ 6646 if (qrss_id) { 6647 /* Not meter subflow. */ 6648 MLX5_ASSERT(!mtr_sfx); 6649 /* 6650 * Put unique id in prefix flow due to it is destroyed 6651 * after suffix flow and id will be freed after there 6652 * is no actual flows with this id and identifier 6653 * reallocation becomes possible (for example, for 6654 * other flows in other threads). 6655 */ 6656 dev_flow->handle->split_flow_id = qrss_id; 6657 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6658 error); 6659 if (ret < 0) 6660 goto exit; 6661 q_tag_spec.id = ret; 6662 } 6663 dev_flow = NULL; 6664 /* Add suffix subflow to execute Q/RSS. */ 6665 flow_split_info->prefix_layers = layers; 6666 flow_split_info->prefix_mark = 0; 6667 flow_split_info->table_id = 0; 6668 ret = flow_create_split_inner(dev, flow, &dev_flow, 6669 &q_attr, mtr_sfx ? items : 6670 q_items, q_actions, 6671 flow_split_info, error); 6672 if (ret < 0) 6673 goto exit; 6674 /* qrss ID should be freed if failed. */ 6675 qrss_id = 0; 6676 MLX5_ASSERT(dev_flow); 6677 } 6678 6679 exit: 6680 /* 6681 * We do not destroy the partially created sub_flows in case of error. 6682 * These ones are included into parent flow list and will be destroyed 6683 * by flow_drv_destroy. 6684 */ 6685 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6686 qrss_id); 6687 mlx5_free(ext_actions); 6688 return ret; 6689 } 6690 6691 /** 6692 * Create meter internal drop flow with the original pattern. 6693 * 6694 * @param dev 6695 * Pointer to Ethernet device. 6696 * @param[in] flow 6697 * Parent flow structure pointer. 6698 * @param[in] attr 6699 * Flow rule attributes. 6700 * @param[in] items 6701 * Pattern specification (list terminated by the END pattern item). 6702 * @param[in] flow_split_info 6703 * Pointer to flow split info structure. 6704 * @param[in] fm 6705 * Pointer to flow meter structure. 6706 * @param[out] error 6707 * Perform verbose error reporting if not NULL. 6708 * @return 6709 * 0 on success, negative value otherwise 6710 */ 6711 static uint32_t 6712 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6713 struct rte_flow *flow, 6714 const struct rte_flow_attr *attr, 6715 const struct rte_flow_item items[], 6716 struct mlx5_flow_split_info *flow_split_info, 6717 struct mlx5_flow_meter_info *fm, 6718 struct rte_flow_error *error) 6719 { 6720 struct mlx5_flow *dev_flow = NULL; 6721 struct rte_flow_attr drop_attr = *attr; 6722 struct rte_flow_action drop_actions[3]; 6723 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6724 6725 MLX5_ASSERT(fm->drop_cnt); 6726 drop_actions[0].type = 6727 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6728 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6729 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6730 drop_actions[1].conf = NULL; 6731 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6732 drop_actions[2].conf = NULL; 6733 drop_split_info.external = false; 6734 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6735 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6736 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6737 return flow_create_split_inner(dev, flow, &dev_flow, 6738 &drop_attr, items, drop_actions, 6739 &drop_split_info, error); 6740 } 6741 6742 /** 6743 * The splitting for meter feature. 6744 * 6745 * - The meter flow will be split to two flows as prefix and 6746 * suffix flow. The packets make sense only it pass the prefix 6747 * meter action. 6748 * 6749 * - Reg_C_5 is used for the packet to match betweend prefix and 6750 * suffix flow. 6751 * 6752 * @param dev 6753 * Pointer to Ethernet device. 6754 * @param[in] flow 6755 * Parent flow structure pointer. 6756 * @param[in] attr 6757 * Flow rule attributes. 6758 * @param[in] items 6759 * Pattern specification (list terminated by the END pattern item). 6760 * @param[in] actions 6761 * Associated actions (list terminated by the END action). 6762 * @param[in] flow_split_info 6763 * Pointer to flow split info structure. 6764 * @param[out] error 6765 * Perform verbose error reporting if not NULL. 6766 * @return 6767 * 0 on success, negative value otherwise 6768 */ 6769 static int 6770 flow_create_split_meter(struct rte_eth_dev *dev, 6771 struct rte_flow *flow, 6772 const struct rte_flow_attr *attr, 6773 const struct rte_flow_item items[], 6774 const struct rte_flow_action actions[], 6775 struct mlx5_flow_split_info *flow_split_info, 6776 struct rte_flow_error *error) 6777 { 6778 struct mlx5_priv *priv = dev->data->dev_private; 6779 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6780 struct rte_flow_action *sfx_actions = NULL; 6781 struct rte_flow_action *pre_actions = NULL; 6782 struct rte_flow_item *sfx_items = NULL; 6783 struct mlx5_flow *dev_flow = NULL; 6784 struct rte_flow_attr sfx_attr = *attr; 6785 struct mlx5_flow_meter_info *fm = NULL; 6786 uint8_t skip_scale_restore; 6787 bool has_mtr = false; 6788 bool has_modify = false; 6789 bool set_mtr_reg = true; 6790 bool is_mtr_hierarchy = false; 6791 uint32_t meter_id = 0; 6792 uint32_t mtr_idx = 0; 6793 uint32_t mtr_flow_id = 0; 6794 size_t act_size; 6795 size_t item_size; 6796 int actions_n = 0; 6797 int ret = 0; 6798 6799 if (priv->mtr_en) 6800 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6801 &has_modify, &meter_id); 6802 if (has_mtr) { 6803 if (flow->meter) { 6804 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6805 if (!fm) 6806 return rte_flow_error_set(error, EINVAL, 6807 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6808 NULL, "Meter not found."); 6809 } else { 6810 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6811 if (!fm) 6812 return rte_flow_error_set(error, EINVAL, 6813 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6814 NULL, "Meter not found."); 6815 ret = mlx5_flow_meter_attach(priv, fm, 6816 &sfx_attr, error); 6817 if (ret) 6818 return -rte_errno; 6819 flow->meter = mtr_idx; 6820 } 6821 MLX5_ASSERT(wks); 6822 wks->fm = fm; 6823 if (!fm->def_policy) { 6824 wks->policy = mlx5_flow_meter_policy_find(dev, 6825 fm->policy_id, 6826 NULL); 6827 MLX5_ASSERT(wks->policy); 6828 if (wks->policy->mark) 6829 wks->mark = 1; 6830 if (wks->policy->is_hierarchy) { 6831 wks->final_policy = 6832 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6833 wks->policy); 6834 if (!wks->final_policy) 6835 return rte_flow_error_set(error, 6836 EINVAL, 6837 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6838 "Failed to find terminal policy of hierarchy."); 6839 is_mtr_hierarchy = true; 6840 } 6841 } 6842 /* 6843 * If it isn't default-policy Meter, and 6844 * 1. Not meter hierarchy and there's no action in flow to change 6845 * packet (modify/encap/decap etc.), OR 6846 * 2. No drop count needed for this meter. 6847 * Then no need to use regC to save meter id anymore. 6848 */ 6849 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6850 set_mtr_reg = false; 6851 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6852 #define METER_PREFIX_ACTION 7 6853 act_size = (sizeof(struct rte_flow_action) * 6854 (actions_n + METER_PREFIX_ACTION)) + 6855 sizeof(struct mlx5_rte_flow_action_set_tag); 6856 /* Suffix items: tag, vlan, port id, end. */ 6857 #define METER_SUFFIX_ITEM 4 6858 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6859 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6860 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6861 0, SOCKET_ID_ANY); 6862 if (!sfx_actions) 6863 return rte_flow_error_set(error, ENOMEM, 6864 RTE_FLOW_ERROR_TYPE_ACTION, 6865 NULL, "no memory to split " 6866 "meter flow"); 6867 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6868 act_size); 6869 /* There's no suffix flow for meter of non-default policy. */ 6870 if (!fm->def_policy) 6871 pre_actions = sfx_actions + 1; 6872 else 6873 pre_actions = sfx_actions + actions_n; 6874 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6875 items, sfx_items, actions, 6876 sfx_actions, pre_actions, 6877 (set_mtr_reg ? &mtr_flow_id : NULL), 6878 error); 6879 if (ret) { 6880 ret = -rte_errno; 6881 goto exit; 6882 } 6883 /* Add the prefix subflow. */ 6884 skip_scale_restore = flow_split_info->skip_scale; 6885 flow_split_info->skip_scale |= 6886 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6887 ret = flow_create_split_inner(dev, flow, &dev_flow, 6888 attr, items, pre_actions, 6889 flow_split_info, error); 6890 flow_split_info->skip_scale = skip_scale_restore; 6891 if (ret) { 6892 if (mtr_flow_id) 6893 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6894 ret = -rte_errno; 6895 goto exit; 6896 } 6897 if (mtr_flow_id) { 6898 dev_flow->handle->split_flow_id = mtr_flow_id; 6899 dev_flow->handle->is_meter_flow_id = 1; 6900 } 6901 if (!fm->def_policy) { 6902 if (!set_mtr_reg && fm->drop_cnt) 6903 ret = 6904 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6905 &sfx_attr, items, 6906 flow_split_info, 6907 fm, error); 6908 goto exit; 6909 } 6910 /* Setting the sfx group atrr. */ 6911 sfx_attr.group = sfx_attr.transfer ? 6912 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6913 MLX5_FLOW_TABLE_LEVEL_METER; 6914 flow_split_info->prefix_layers = 6915 flow_get_prefix_layer_flags(dev_flow); 6916 flow_split_info->prefix_mark |= wks->mark; 6917 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6918 } 6919 /* Add the prefix subflow. */ 6920 ret = flow_create_split_metadata(dev, flow, 6921 &sfx_attr, sfx_items ? 6922 sfx_items : items, 6923 sfx_actions ? sfx_actions : actions, 6924 flow_split_info, error); 6925 exit: 6926 if (sfx_actions) 6927 mlx5_free(sfx_actions); 6928 return ret; 6929 } 6930 6931 /** 6932 * The splitting for sample feature. 6933 * 6934 * Once Sample action is detected in the action list, the flow actions should 6935 * be split into prefix sub flow and suffix sub flow. 6936 * 6937 * The original items remain in the prefix sub flow, all actions preceding the 6938 * sample action and the sample action itself will be copied to the prefix 6939 * sub flow, the actions following the sample action will be copied to the 6940 * suffix sub flow, Queue action always be located in the suffix sub flow. 6941 * 6942 * In order to make the packet from prefix sub flow matches with suffix sub 6943 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6944 * flow uses tag item with the unique flow id. 6945 * 6946 * @param dev 6947 * Pointer to Ethernet device. 6948 * @param[in] flow 6949 * Parent flow structure pointer. 6950 * @param[in] attr 6951 * Flow rule attributes. 6952 * @param[in] items 6953 * Pattern specification (list terminated by the END pattern item). 6954 * @param[in] actions 6955 * Associated actions (list terminated by the END action). 6956 * @param[in] flow_split_info 6957 * Pointer to flow split info structure. 6958 * @param[out] error 6959 * Perform verbose error reporting if not NULL. 6960 * @return 6961 * 0 on success, negative value otherwise 6962 */ 6963 static int 6964 flow_create_split_sample(struct rte_eth_dev *dev, 6965 struct rte_flow *flow, 6966 const struct rte_flow_attr *attr, 6967 const struct rte_flow_item items[], 6968 const struct rte_flow_action actions[], 6969 struct mlx5_flow_split_info *flow_split_info, 6970 struct rte_flow_error *error) 6971 { 6972 struct mlx5_priv *priv = dev->data->dev_private; 6973 struct rte_flow_action *sfx_actions = NULL; 6974 struct rte_flow_action *pre_actions = NULL; 6975 struct rte_flow_item *sfx_items = NULL; 6976 struct mlx5_flow *dev_flow = NULL; 6977 struct rte_flow_attr sfx_attr = *attr; 6978 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6979 struct mlx5_flow_dv_sample_resource *sample_res; 6980 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6981 struct mlx5_flow_tbl_resource *sfx_tbl; 6982 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6983 #endif 6984 size_t act_size; 6985 size_t item_size; 6986 uint32_t fdb_tx = 0; 6987 int32_t tag_id = 0; 6988 int actions_n = 0; 6989 int sample_action_pos; 6990 int qrss_action_pos; 6991 int add_tag = 0; 6992 int modify_after_mirror = 0; 6993 uint16_t jump_table = 0; 6994 const uint32_t next_ft_step = 1; 6995 int ret = 0; 6996 struct mlx5_priv *item_port_priv = NULL; 6997 const struct rte_flow_item *item; 6998 6999 if (priv->sampler_en) 7000 actions_n = flow_check_match_action(actions, attr, 7001 RTE_FLOW_ACTION_TYPE_SAMPLE, 7002 &sample_action_pos, &qrss_action_pos, 7003 &modify_after_mirror); 7004 if (actions_n) { 7005 /* The prefix actions must includes sample, tag, end. */ 7006 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 7007 + sizeof(struct mlx5_rte_flow_action_set_tag); 7008 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 7009 sizeof(struct mlx5_rte_flow_item_tag) * 2; 7010 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 7011 item_size), 0, SOCKET_ID_ANY); 7012 if (!sfx_actions) 7013 return rte_flow_error_set(error, ENOMEM, 7014 RTE_FLOW_ERROR_TYPE_ACTION, 7015 NULL, "no memory to split " 7016 "sample flow"); 7017 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 7018 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 7019 const struct rte_flow_item_port_id *spec; 7020 7021 spec = (const struct rte_flow_item_port_id *)item->spec; 7022 if (spec) 7023 item_port_priv = 7024 mlx5_port_to_eswitch_info(spec->id, true); 7025 break; 7026 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 7027 const struct rte_flow_item_ethdev *spec; 7028 7029 spec = (const struct rte_flow_item_ethdev *)item->spec; 7030 if (spec) 7031 item_port_priv = 7032 mlx5_port_to_eswitch_info(spec->port_id, true); 7033 break; 7034 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 7035 const struct rte_flow_item_ethdev *spec; 7036 7037 spec = (const struct rte_flow_item_ethdev *)item->spec; 7038 if (spec) 7039 item_port_priv = 7040 mlx5_port_to_eswitch_info(spec->port_id, true); 7041 break; 7042 } 7043 } 7044 /* The representor_id is UINT16_MAX for uplink. */ 7045 fdb_tx = (attr->transfer && 7046 flow_source_vport_representor(priv, item_port_priv)); 7047 /* 7048 * When reg_c_preserve is set, metadata registers Cx preserve 7049 * their value even through packet duplication. 7050 */ 7051 add_tag = (!fdb_tx || 7052 priv->sh->cdev->config.hca_attr.reg_c_preserve); 7053 if (add_tag) 7054 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 7055 + act_size); 7056 if (modify_after_mirror) 7057 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 7058 next_ft_step; 7059 pre_actions = sfx_actions + actions_n; 7060 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 7061 actions, sfx_actions, 7062 pre_actions, actions_n, 7063 sample_action_pos, 7064 qrss_action_pos, jump_table, 7065 error); 7066 if (tag_id < 0 || (add_tag && !tag_id)) { 7067 ret = -rte_errno; 7068 goto exit; 7069 } 7070 if (modify_after_mirror) 7071 flow_split_info->skip_scale = 7072 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 7073 /* Add the prefix subflow. */ 7074 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7075 items, pre_actions, 7076 flow_split_info, error); 7077 if (ret) { 7078 ret = -rte_errno; 7079 goto exit; 7080 } 7081 dev_flow->handle->split_flow_id = tag_id; 7082 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7083 if (!modify_after_mirror) { 7084 /* Set the sfx group attr. */ 7085 sample_res = (struct mlx5_flow_dv_sample_resource *) 7086 dev_flow->dv.sample_res; 7087 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7088 sample_res->normal_path_tbl; 7089 sfx_tbl_data = container_of(sfx_tbl, 7090 struct mlx5_flow_tbl_data_entry, 7091 tbl); 7092 sfx_attr.group = sfx_attr.transfer ? 7093 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7094 } else { 7095 MLX5_ASSERT(attr->transfer); 7096 sfx_attr.group = jump_table; 7097 } 7098 flow_split_info->prefix_layers = 7099 flow_get_prefix_layer_flags(dev_flow); 7100 MLX5_ASSERT(wks); 7101 flow_split_info->prefix_mark |= wks->mark; 7102 /* Suffix group level already be scaled with factor, set 7103 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7104 * again in translation. 7105 */ 7106 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7107 #endif 7108 } 7109 /* Add the suffix subflow. */ 7110 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7111 sfx_items ? sfx_items : items, 7112 sfx_actions ? sfx_actions : actions, 7113 flow_split_info, error); 7114 exit: 7115 if (sfx_actions) 7116 mlx5_free(sfx_actions); 7117 return ret; 7118 } 7119 7120 /** 7121 * Split the flow to subflow set. The splitters might be linked 7122 * in the chain, like this: 7123 * flow_create_split_outer() calls: 7124 * flow_create_split_meter() calls: 7125 * flow_create_split_metadata(meter_subflow_0) calls: 7126 * flow_create_split_inner(metadata_subflow_0) 7127 * flow_create_split_inner(metadata_subflow_1) 7128 * flow_create_split_inner(metadata_subflow_2) 7129 * flow_create_split_metadata(meter_subflow_1) calls: 7130 * flow_create_split_inner(metadata_subflow_0) 7131 * flow_create_split_inner(metadata_subflow_1) 7132 * flow_create_split_inner(metadata_subflow_2) 7133 * 7134 * This provide flexible way to add new levels of flow splitting. 7135 * The all of successfully created subflows are included to the 7136 * parent flow dev_flow list. 7137 * 7138 * @param dev 7139 * Pointer to Ethernet device. 7140 * @param[in] flow 7141 * Parent flow structure pointer. 7142 * @param[in] attr 7143 * Flow rule attributes. 7144 * @param[in] items 7145 * Pattern specification (list terminated by the END pattern item). 7146 * @param[in] actions 7147 * Associated actions (list terminated by the END action). 7148 * @param[in] flow_split_info 7149 * Pointer to flow split info structure. 7150 * @param[out] error 7151 * Perform verbose error reporting if not NULL. 7152 * @return 7153 * 0 on success, negative value otherwise 7154 */ 7155 static int 7156 flow_create_split_outer(struct rte_eth_dev *dev, 7157 struct rte_flow *flow, 7158 const struct rte_flow_attr *attr, 7159 const struct rte_flow_item items[], 7160 const struct rte_flow_action actions[], 7161 struct mlx5_flow_split_info *flow_split_info, 7162 struct rte_flow_error *error) 7163 { 7164 int ret; 7165 7166 ret = flow_create_split_sample(dev, flow, attr, items, 7167 actions, flow_split_info, error); 7168 MLX5_ASSERT(ret <= 0); 7169 return ret; 7170 } 7171 7172 static inline struct mlx5_flow_tunnel * 7173 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7174 { 7175 struct mlx5_flow_tunnel *tunnel; 7176 7177 #pragma GCC diagnostic push 7178 #pragma GCC diagnostic ignored "-Wcast-qual" 7179 tunnel = (typeof(tunnel))flow->tunnel; 7180 #pragma GCC diagnostic pop 7181 7182 return tunnel; 7183 } 7184 7185 /** 7186 * Create a flow and add it to @p list. 7187 * 7188 * @param dev 7189 * Pointer to Ethernet device. 7190 * @param list 7191 * Pointer to a TAILQ flow list. If this parameter NULL, 7192 * no list insertion occurred, flow is just created, 7193 * this is caller's responsibility to track the 7194 * created flow. 7195 * @param[in] attr 7196 * Flow rule attributes. 7197 * @param[in] items 7198 * Pattern specification (list terminated by the END pattern item). 7199 * @param[in] actions 7200 * Associated actions (list terminated by the END action). 7201 * @param[in] external 7202 * This flow rule is created by request external to PMD. 7203 * @param[out] error 7204 * Perform verbose error reporting if not NULL. 7205 * 7206 * @return 7207 * A flow index on success, 0 otherwise and rte_errno is set. 7208 */ 7209 static uint32_t 7210 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7211 const struct rte_flow_attr *attr, 7212 const struct rte_flow_item items[], 7213 const struct rte_flow_action original_actions[], 7214 bool external, struct rte_flow_error *error) 7215 { 7216 struct mlx5_priv *priv = dev->data->dev_private; 7217 struct rte_flow *flow = NULL; 7218 struct mlx5_flow *dev_flow; 7219 const struct rte_flow_action_rss *rss = NULL; 7220 struct mlx5_translated_action_handle 7221 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7222 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7223 union { 7224 struct mlx5_flow_expand_rss buf; 7225 uint8_t buffer[8192]; 7226 } expand_buffer; 7227 union { 7228 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7229 uint8_t buffer[2048]; 7230 } actions_rx; 7231 union { 7232 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7233 uint8_t buffer[2048]; 7234 } actions_hairpin_tx; 7235 union { 7236 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7237 uint8_t buffer[2048]; 7238 } items_tx; 7239 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7240 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7241 struct mlx5_flow_rss_desc *rss_desc; 7242 const struct rte_flow_action *p_actions_rx; 7243 uint32_t i; 7244 uint32_t idx = 0; 7245 int hairpin_flow; 7246 struct rte_flow_attr attr_tx = { .priority = 0 }; 7247 const struct rte_flow_action *actions; 7248 struct rte_flow_action *translated_actions = NULL; 7249 struct mlx5_flow_tunnel *tunnel; 7250 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7251 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7252 struct mlx5_flow_split_info flow_split_info = { 7253 .external = !!external, 7254 .skip_scale = 0, 7255 .flow_idx = 0, 7256 .prefix_mark = 0, 7257 .prefix_layers = 0, 7258 .table_id = 0 7259 }; 7260 int ret; 7261 7262 MLX5_ASSERT(wks); 7263 rss_desc = &wks->rss_desc; 7264 ret = flow_action_handles_translate(dev, original_actions, 7265 indir_actions, 7266 &indir_actions_n, 7267 &translated_actions, error); 7268 if (ret < 0) { 7269 MLX5_ASSERT(translated_actions == NULL); 7270 return 0; 7271 } 7272 actions = translated_actions ? translated_actions : original_actions; 7273 p_actions_rx = actions; 7274 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7275 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7276 external, hairpin_flow, error); 7277 if (ret < 0) 7278 goto error_before_hairpin_split; 7279 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7280 if (!flow) { 7281 rte_errno = ENOMEM; 7282 goto error_before_hairpin_split; 7283 } 7284 if (hairpin_flow > 0) { 7285 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7286 rte_errno = EINVAL; 7287 goto error_before_hairpin_split; 7288 } 7289 flow_hairpin_split(dev, actions, actions_rx.actions, 7290 actions_hairpin_tx.actions, items_tx.items, 7291 idx); 7292 p_actions_rx = actions_rx.actions; 7293 } 7294 flow_split_info.flow_idx = idx; 7295 flow->drv_type = flow_get_drv_type(dev, attr); 7296 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7297 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7298 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7299 /* RSS Action only works on NIC RX domain */ 7300 if (attr->ingress) 7301 rss = flow_get_rss_action(dev, p_actions_rx); 7302 if (rss) { 7303 MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); 7304 rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func); 7305 /* 7306 * The following information is required by 7307 * mlx5_flow_hashfields_adjust() in advance. 7308 */ 7309 rss_desc->level = rss->level; 7310 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7311 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7312 } 7313 flow->dev_handles = 0; 7314 if (rss && rss->types) { 7315 unsigned int graph_root; 7316 7317 graph_root = find_graph_root(rss->level); 7318 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7319 items, rss->types, 7320 mlx5_support_expansion, graph_root); 7321 MLX5_ASSERT(ret > 0 && 7322 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7323 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7324 for (i = 0; i < buf->entries; ++i) 7325 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7326 } 7327 } else { 7328 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7329 sizeof(expand_buffer.buffer), 7330 items, sq_specs); 7331 if (ret) { 7332 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7333 NULL, "not enough memory for rte_flow"); 7334 goto error; 7335 } 7336 if (buf->entries == 0) { 7337 buf->entries = 1; 7338 buf->entry[0].pattern = (void *)(uintptr_t)items; 7339 } 7340 } 7341 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7342 indir_actions_n); 7343 for (i = 0; i < buf->entries; ++i) { 7344 /* Initialize flow split data. */ 7345 flow_split_info.prefix_layers = 0; 7346 flow_split_info.prefix_mark = 0; 7347 flow_split_info.skip_scale = 0; 7348 /* 7349 * The splitter may create multiple dev_flows, 7350 * depending on configuration. In the simplest 7351 * case it just creates unmodified original flow. 7352 */ 7353 ret = flow_create_split_outer(dev, flow, attr, 7354 buf->entry[i].pattern, 7355 p_actions_rx, &flow_split_info, 7356 error); 7357 if (ret < 0) 7358 goto error; 7359 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7360 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7361 p_actions_rx, 7362 idx, 7363 wks->flows[0].tunnel, 7364 &default_miss_ctx, 7365 error); 7366 if (ret < 0) { 7367 mlx5_free(default_miss_ctx.queue); 7368 goto error; 7369 } 7370 } 7371 } 7372 /* Create the tx flow. */ 7373 if (hairpin_flow) { 7374 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7375 attr_tx.ingress = 0; 7376 attr_tx.egress = 1; 7377 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7378 actions_hairpin_tx.actions, 7379 idx, error); 7380 if (!dev_flow) 7381 goto error; 7382 dev_flow->flow = flow; 7383 dev_flow->external = 0; 7384 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7385 dev_flow->handle, next); 7386 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7387 items_tx.items, 7388 actions_hairpin_tx.actions, error); 7389 if (ret < 0) 7390 goto error; 7391 } 7392 /* 7393 * Update the metadata register copy table. If extensive 7394 * metadata feature is enabled and registers are supported 7395 * we might create the extra rte_flow for each unique 7396 * MARK/FLAG action ID. 7397 * 7398 * The table is updated for ingress and transfer flows only, because 7399 * the egress Flows belong to the different device and 7400 * copy table should be updated in peer NIC Rx domain. 7401 */ 7402 if ((attr->ingress || attr->transfer) && 7403 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7404 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7405 if (ret) 7406 goto error; 7407 } 7408 /* 7409 * If the flow is external (from application) OR device is started, 7410 * OR mreg discover, then apply immediately. 7411 */ 7412 if (external || dev->data->dev_started || 7413 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7414 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7415 ret = flow_drv_apply(dev, flow, error); 7416 if (ret < 0) 7417 goto error; 7418 } 7419 flow->type = type; 7420 flow_rxq_flags_set(dev, flow); 7421 rte_free(translated_actions); 7422 tunnel = flow_tunnel_from_rule(wks->flows); 7423 if (tunnel) { 7424 flow->tunnel = 1; 7425 flow->tunnel_id = tunnel->tunnel_id; 7426 __atomic_fetch_add(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7427 mlx5_free(default_miss_ctx.queue); 7428 } 7429 mlx5_flow_pop_thread_workspace(); 7430 return idx; 7431 error: 7432 MLX5_ASSERT(flow); 7433 ret = rte_errno; /* Save rte_errno before cleanup. */ 7434 flow_mreg_del_copy_action(dev, flow); 7435 flow_drv_destroy(dev, flow); 7436 if (rss_desc->shared_rss) 7437 __atomic_fetch_sub(&((struct mlx5_shared_action_rss *) 7438 mlx5_ipool_get 7439 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7440 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7441 mlx5_ipool_free(priv->flows[type], idx); 7442 rte_errno = ret; /* Restore rte_errno. */ 7443 ret = rte_errno; 7444 rte_errno = ret; 7445 error_before_hairpin_split: 7446 mlx5_flow_pop_thread_workspace(); 7447 rte_free(translated_actions); 7448 return 0; 7449 } 7450 7451 /** 7452 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7453 * incoming packets to table 1. 7454 * 7455 * Other flow rules, requested for group n, will be created in 7456 * e-switch table n+1. 7457 * Jump action to e-switch group n will be created to group n+1. 7458 * 7459 * Used when working in switchdev mode, to utilise advantages of table 1 7460 * and above. 7461 * 7462 * @param dev 7463 * Pointer to Ethernet device. 7464 * 7465 * @return 7466 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7467 */ 7468 struct rte_flow * 7469 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7470 { 7471 const struct rte_flow_attr attr = { 7472 .group = 0, 7473 .priority = 0, 7474 .ingress = 0, 7475 .egress = 0, 7476 .transfer = 1, 7477 }; 7478 const struct rte_flow_item pattern = { 7479 .type = RTE_FLOW_ITEM_TYPE_END, 7480 }; 7481 struct rte_flow_action_jump jump = { 7482 .group = 1, 7483 }; 7484 const struct rte_flow_action actions[] = { 7485 { 7486 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7487 .conf = &jump, 7488 }, 7489 { 7490 .type = RTE_FLOW_ACTION_TYPE_END, 7491 }, 7492 }; 7493 struct rte_flow_error error; 7494 7495 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7496 &attr, &pattern, 7497 actions, false, &error); 7498 } 7499 7500 /** 7501 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7502 * and sq number, directs all packets to peer vport. 7503 * 7504 * @param dev 7505 * Pointer to Ethernet device. 7506 * @param sq_num 7507 * SQ number. 7508 * 7509 * @return 7510 * Flow ID on success, 0 otherwise and rte_errno is set. 7511 */ 7512 uint32_t 7513 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7514 { 7515 struct rte_flow_attr attr = { 7516 .group = 0, 7517 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7518 .ingress = 0, 7519 .egress = 0, 7520 .transfer = 1, 7521 }; 7522 struct rte_flow_item_port_id port_spec = { 7523 .id = MLX5_PORT_ESW_MGR, 7524 }; 7525 struct mlx5_rte_flow_item_sq sq_spec = { 7526 .queue = sq_num, 7527 }; 7528 struct rte_flow_item pattern[] = { 7529 { 7530 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7531 .spec = &port_spec, 7532 }, 7533 { 7534 .type = (enum rte_flow_item_type) 7535 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7536 .spec = &sq_spec, 7537 }, 7538 { 7539 .type = RTE_FLOW_ITEM_TYPE_END, 7540 }, 7541 }; 7542 struct rte_flow_action_jump jump = { 7543 .group = 1, 7544 }; 7545 struct rte_flow_action_port_id port = { 7546 .id = dev->data->port_id, 7547 }; 7548 struct rte_flow_action actions[] = { 7549 { 7550 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7551 .conf = &jump, 7552 }, 7553 { 7554 .type = RTE_FLOW_ACTION_TYPE_END, 7555 }, 7556 }; 7557 struct rte_flow_error error; 7558 7559 /* 7560 * Creates group 0, highest priority jump flow. 7561 * Matches txq to bypass kernel packets. 7562 */ 7563 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7564 false, &error) == 0) 7565 return 0; 7566 /* Create group 1, lowest priority redirect flow for txq. */ 7567 attr.group = 1; 7568 actions[0].conf = &port; 7569 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7570 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7571 actions, false, &error); 7572 } 7573 7574 /** 7575 * Validate a flow supported by the NIC. 7576 * 7577 * @see rte_flow_validate() 7578 * @see rte_flow_ops 7579 */ 7580 int 7581 mlx5_flow_validate(struct rte_eth_dev *dev, 7582 const struct rte_flow_attr *attr, 7583 const struct rte_flow_item items[], 7584 const struct rte_flow_action original_actions[], 7585 struct rte_flow_error *error) 7586 { 7587 int hairpin_flow; 7588 struct mlx5_translated_action_handle 7589 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7590 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7591 const struct rte_flow_action *actions; 7592 struct rte_flow_action *translated_actions = NULL; 7593 int ret = flow_action_handles_translate(dev, original_actions, 7594 indir_actions, 7595 &indir_actions_n, 7596 &translated_actions, error); 7597 7598 if (ret) 7599 return ret; 7600 actions = translated_actions ? translated_actions : original_actions; 7601 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7602 ret = flow_drv_validate(dev, attr, items, actions, 7603 true, hairpin_flow, error); 7604 rte_free(translated_actions); 7605 return ret; 7606 } 7607 7608 static int 7609 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7610 const struct rte_flow_attr *attr, 7611 const uint32_t orig_prio, 7612 const struct rte_flow_item *items, 7613 const struct rte_flow_action *actions, 7614 uint32_t flow_idx) 7615 { 7616 struct mlx5_priv *priv = dev->data->dev_private; 7617 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7618 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7619 struct rte_flow_error error; 7620 int len, ret; 7621 7622 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7623 if (!flow_info) { 7624 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7625 return -1; 7626 } 7627 flow_info->orig_prio = orig_prio; 7628 flow_info->attr = *attr; 7629 /* Standby mode rule awlays saves it in low priority entry. */ 7630 flow_info->flow_idx_low_prio = flow_idx; 7631 7632 /* Store matching items. */ 7633 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7634 if (ret <= 0) { 7635 DRV_LOG(ERR, "Can't get items length."); 7636 goto end; 7637 } 7638 len = RTE_ALIGN(ret, 16); 7639 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7640 if (!flow_info->items) { 7641 DRV_LOG(ERR, "No enough memory for items caching."); 7642 goto end; 7643 } 7644 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7645 if (ret <= 0) { 7646 DRV_LOG(ERR, "Can't duplicate items."); 7647 goto end; 7648 } 7649 7650 /* Store flow actions. */ 7651 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7652 if (ret <= 0) { 7653 DRV_LOG(ERR, "Can't get actions length."); 7654 goto end; 7655 } 7656 len = RTE_ALIGN(ret, 16); 7657 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7658 if (!flow_info->actions) { 7659 DRV_LOG(ERR, "No enough memory for actions caching."); 7660 goto end; 7661 } 7662 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7663 if (ret <= 0) { 7664 DRV_LOG(ERR, "Can't duplicate actions."); 7665 goto end; 7666 } 7667 7668 /* Insert to the list end. */ 7669 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7670 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7671 } else { 7672 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7673 while (LIST_NEXT(tmp_info, next)) 7674 tmp_info = LIST_NEXT(tmp_info, next); 7675 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7676 } 7677 return 0; 7678 end: 7679 if (flow_info->items) 7680 mlx5_free(flow_info->items); 7681 if (flow_info->actions) 7682 mlx5_free(flow_info->actions); 7683 mlx5_free(flow_info); 7684 return -1; 7685 } 7686 7687 static int 7688 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7689 { 7690 struct mlx5_priv *priv = dev->data->dev_private; 7691 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7692 struct mlx5_dv_flow_info *flow_info; 7693 struct rte_flow_attr attr; 7694 struct rte_flow_error error; 7695 struct rte_flow *high, *low; 7696 7697 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7698 while (flow_info) { 7699 /* DUP flow may have the same priority. */ 7700 if (flow_info->orig_prio != flow_info->attr.priority) { 7701 attr = flow_info->attr; 7702 if (orig_prio) 7703 attr.priority = flow_info->orig_prio; 7704 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7705 &attr, flow_info->items, flow_info->actions, 7706 true, &error); 7707 if (!flow_info->flow_idx_high_prio) { 7708 DRV_LOG(ERR, "Priority toggle failed internally."); 7709 goto err; 7710 } 7711 } 7712 flow_info = LIST_NEXT(flow_info, next); 7713 } 7714 /* Delete the low priority rules and swap the flow handle. */ 7715 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7716 while (flow_info) { 7717 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7718 if (flow_info->orig_prio != flow_info->attr.priority) { 7719 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7720 flow_info->flow_idx_high_prio); 7721 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7722 flow_info->flow_idx_low_prio); 7723 if (high && low) { 7724 RTE_SWAP(*low, *high); 7725 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7726 flow_info->flow_idx_low_prio); 7727 flow_info->flow_idx_high_prio = 0; 7728 } 7729 } 7730 flow_info = LIST_NEXT(flow_info, next); 7731 } 7732 return 0; 7733 err: 7734 /* Destroy preceding successful high priority rules. */ 7735 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7736 while (flow_info) { 7737 if (flow_info->orig_prio != flow_info->attr.priority) { 7738 if (flow_info->flow_idx_high_prio) 7739 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7740 flow_info->flow_idx_high_prio); 7741 else 7742 break; 7743 flow_info->flow_idx_high_prio = 0; 7744 } 7745 flow_info = LIST_NEXT(flow_info, next); 7746 } 7747 return -1; 7748 } 7749 7750 /** 7751 * Set the mode of the flow engine of a process to active or standby during live migration. 7752 * 7753 * @param[in] mode 7754 * MLX5 flow engine mode, @see `enum rte_pmd_mlx5_flow_engine_mode`. 7755 * @param[in] flags 7756 * Flow engine mode specific flags. 7757 * 7758 * @return 7759 * Negative value on error, positive on success. 7760 */ 7761 int 7762 rte_pmd_mlx5_flow_engine_set_mode(enum rte_pmd_mlx5_flow_engine_mode mode, uint32_t flags) 7763 { 7764 struct mlx5_priv *priv; 7765 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info; 7766 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7767 uint16_t port, port_id; 7768 uint16_t toggle_num = 0; 7769 struct rte_eth_dev *dev; 7770 enum rte_pmd_mlx5_flow_engine_mode orig_mode; 7771 uint32_t orig_flags; 7772 bool need_toggle = false; 7773 7774 /* Check if flags combinations are supported. */ 7775 if (flags && flags != RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7776 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7777 return -1; 7778 } 7779 MLX5_ETH_FOREACH_DEV(port, NULL) { 7780 dev = &rte_eth_devices[port]; 7781 priv = dev->data->dev_private; 7782 mode_info = &priv->mode_info; 7783 /* No mode change. Assume all devices hold the same mode. */ 7784 if (mode_info->mode == mode) { 7785 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7786 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7787 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7788 port, mode_info->mode_flag); 7789 orig_mode = mode_info->mode; 7790 orig_flags = mode_info->mode_flag; 7791 goto err; 7792 } 7793 mode_info->mode_flag = flags; 7794 toggle_num++; 7795 continue; 7796 } 7797 /* Active -> standby. */ 7798 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY) { 7799 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7800 DRV_LOG(ERR, "Cached rule existed"); 7801 orig_mode = mode_info->mode; 7802 orig_flags = mode_info->mode_flag; 7803 goto err; 7804 } 7805 mode_info->mode_flag = flags; 7806 mode_info->mode = mode; 7807 toggle_num++; 7808 /* Standby -> active. */ 7809 } else if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7810 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7811 DRV_LOG(INFO, "No cached rule existed"); 7812 } else { 7813 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7814 orig_mode = mode_info->mode; 7815 orig_flags = mode_info->mode_flag; 7816 need_toggle = true; 7817 goto err; 7818 } 7819 } 7820 toggle_num++; 7821 } 7822 } 7823 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7824 /* Clear cache flow rules. */ 7825 MLX5_ETH_FOREACH_DEV(port, NULL) { 7826 priv = rte_eth_devices[port].data->dev_private; 7827 mode_info = &priv->mode_info; 7828 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7829 while (flow_info) { 7830 tmp_info = LIST_NEXT(flow_info, next); 7831 LIST_REMOVE(flow_info, next); 7832 mlx5_free(flow_info->actions); 7833 mlx5_free(flow_info->items); 7834 mlx5_free(flow_info); 7835 flow_info = tmp_info; 7836 } 7837 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7838 } 7839 } 7840 return toggle_num; 7841 err: 7842 /* Rollback all preceding successful ports. */ 7843 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7844 if (port_id == port) 7845 break; 7846 priv = rte_eth_devices[port_id].data->dev_private; 7847 mode_info = &priv->mode_info; 7848 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7849 mlx5_flow_cache_flow_toggle(dev, false)) 7850 return -EPERM; 7851 mode_info->mode = orig_mode; 7852 mode_info->mode_flag = orig_flags; 7853 } 7854 return -EINVAL; 7855 } 7856 /** 7857 * Create a flow. 7858 * 7859 * @see rte_flow_create() 7860 * @see rte_flow_ops 7861 */ 7862 struct rte_flow * 7863 mlx5_flow_create(struct rte_eth_dev *dev, 7864 const struct rte_flow_attr *attr, 7865 const struct rte_flow_item items[], 7866 const struct rte_flow_action actions[], 7867 struct rte_flow_error *error) 7868 { 7869 struct mlx5_priv *priv = dev->data->dev_private; 7870 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7871 uint32_t prio = attr->priority; 7872 uint32_t flow_idx; 7873 7874 if (priv->sh->config.dv_flow_en == 2) { 7875 rte_flow_error_set(error, ENOTSUP, 7876 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7877 NULL, 7878 "Flow non-Q creation not supported"); 7879 return NULL; 7880 } 7881 /* 7882 * If the device is not started yet, it is not allowed to created a 7883 * flow from application. PMD default flows and traffic control flows 7884 * are not affected. 7885 */ 7886 if (unlikely(!dev->data->dev_started)) { 7887 DRV_LOG(DEBUG, "port %u is not started when " 7888 "inserting a flow", dev->data->port_id); 7889 rte_flow_error_set(error, ENODEV, 7890 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7891 NULL, 7892 "port not started"); 7893 return NULL; 7894 } 7895 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7896 if (attr->transfer || 7897 (attr->ingress && !(priv->mode_info.mode_flag & 7898 RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 7899 new_attr->priority += 1; 7900 } 7901 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 7902 if (!flow_idx) 7903 return NULL; 7904 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7905 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 7906 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7907 flow_idx = 0; 7908 } 7909 } 7910 return (void *)(uintptr_t)flow_idx; 7911 } 7912 7913 /** 7914 * Destroy a flow in a list. 7915 * 7916 * @param dev 7917 * Pointer to Ethernet device. 7918 * @param[in] flow_idx 7919 * Index of flow to destroy. 7920 */ 7921 static void 7922 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7923 uint32_t flow_idx) 7924 { 7925 struct mlx5_priv *priv = dev->data->dev_private; 7926 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7927 7928 if (!flow) 7929 return; 7930 MLX5_ASSERT(flow->type == type); 7931 /* 7932 * Update RX queue flags only if port is started, otherwise it is 7933 * already clean. 7934 */ 7935 if (dev->data->dev_started) 7936 flow_rxq_flags_trim(dev, flow); 7937 flow_drv_destroy(dev, flow); 7938 if (flow->tunnel) { 7939 struct mlx5_flow_tunnel *tunnel; 7940 7941 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7942 RTE_VERIFY(tunnel); 7943 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 7944 mlx5_flow_tunnel_free(dev, tunnel); 7945 } 7946 flow_mreg_del_copy_action(dev, flow); 7947 mlx5_ipool_free(priv->flows[type], flow_idx); 7948 } 7949 7950 /** 7951 * Destroy all flows. 7952 * 7953 * @param dev 7954 * Pointer to Ethernet device. 7955 * @param type 7956 * Flow type to be flushed. 7957 * @param active 7958 * If flushing is called actively. 7959 */ 7960 void 7961 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7962 bool active) 7963 { 7964 struct mlx5_priv *priv = dev->data->dev_private; 7965 uint32_t num_flushed = 0, fidx = 1; 7966 struct rte_flow *flow; 7967 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7968 struct mlx5_dv_flow_info *flow_info; 7969 7970 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7971 if (priv->sh->config.dv_flow_en == 2 && 7972 type == MLX5_FLOW_TYPE_GEN) { 7973 flow_hw_q_flow_flush(dev, NULL); 7974 return; 7975 } 7976 #endif 7977 7978 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7979 flow_list_destroy(dev, type, fidx); 7980 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 7981 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7982 while (flow_info) { 7983 /* Romove the cache flow info. */ 7984 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 7985 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 7986 LIST_REMOVE(flow_info, next); 7987 mlx5_free(flow_info->items); 7988 mlx5_free(flow_info->actions); 7989 mlx5_free(flow_info); 7990 break; 7991 } 7992 flow_info = LIST_NEXT(flow_info, next); 7993 } 7994 } 7995 num_flushed++; 7996 } 7997 if (active) { 7998 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7999 dev->data->port_id, num_flushed); 8000 } 8001 } 8002 8003 /** 8004 * Stop all default actions for flows. 8005 * 8006 * @param dev 8007 * Pointer to Ethernet device. 8008 */ 8009 void 8010 mlx5_flow_stop_default(struct rte_eth_dev *dev) 8011 { 8012 flow_mreg_del_default_copy_action(dev); 8013 flow_rxq_flags_clear(dev); 8014 } 8015 8016 /** 8017 * Set rxq flag. 8018 * 8019 * @param[in] dev 8020 * Pointer to the rte_eth_dev structure. 8021 * @param[in] enable 8022 * Flag to enable or not. 8023 */ 8024 void 8025 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 8026 { 8027 struct mlx5_priv *priv = dev->data->dev_private; 8028 unsigned int i; 8029 8030 if ((!priv->mark_enabled && !enable) || 8031 (priv->mark_enabled && enable)) 8032 return; 8033 for (i = 0; i < priv->rxqs_n; ++i) { 8034 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 8035 8036 /* With RXQ start/stop feature, RXQ might be stopped. */ 8037 if (!rxq_ctrl) 8038 continue; 8039 rxq_ctrl->rxq.mark = enable; 8040 } 8041 priv->mark_enabled = enable; 8042 } 8043 8044 /** 8045 * Start all default actions for flows. 8046 * 8047 * @param dev 8048 * Pointer to Ethernet device. 8049 * @return 8050 * 0 on success, a negative errno value otherwise and rte_errno is set. 8051 */ 8052 int 8053 mlx5_flow_start_default(struct rte_eth_dev *dev) 8054 { 8055 struct rte_flow_error error; 8056 8057 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8058 return flow_mreg_add_default_copy_action(dev, &error); 8059 } 8060 8061 /** 8062 * Release key of thread specific flow workspace data. 8063 */ 8064 void 8065 flow_release_workspace(void *data) 8066 { 8067 struct mlx5_flow_workspace *wks = data; 8068 struct mlx5_flow_workspace *next; 8069 8070 while (wks) { 8071 next = wks->next; 8072 free(wks); 8073 wks = next; 8074 } 8075 } 8076 8077 /** 8078 * Get thread specific current flow workspace. 8079 * 8080 * @return pointer to thread specific flow workspace data, NULL on error. 8081 */ 8082 struct mlx5_flow_workspace* 8083 mlx5_flow_get_thread_workspace(void) 8084 { 8085 struct mlx5_flow_workspace *data; 8086 8087 data = mlx5_flow_os_get_specific_workspace(); 8088 MLX5_ASSERT(data && data->inuse); 8089 if (!data || !data->inuse) 8090 DRV_LOG(ERR, "flow workspace not initialized."); 8091 return data; 8092 } 8093 8094 /** 8095 * Allocate and init new flow workspace. 8096 * 8097 * @return pointer to flow workspace data, NULL on error. 8098 */ 8099 static struct mlx5_flow_workspace* 8100 flow_alloc_thread_workspace(void) 8101 { 8102 size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); 8103 size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; 8104 struct mlx5_flow_workspace *data = calloc(1, data_size + 8105 rss_queue_array_size); 8106 8107 if (!data) { 8108 DRV_LOG(ERR, "Failed to allocate flow workspace memory."); 8109 return NULL; 8110 } 8111 data->rss_desc.queue = RTE_PTR_ADD(data, data_size); 8112 return data; 8113 } 8114 8115 /** 8116 * Get new thread specific flow workspace. 8117 * 8118 * If current workspace inuse, create new one and set as current. 8119 * 8120 * @return pointer to thread specific flow workspace data, NULL on error. 8121 */ 8122 struct mlx5_flow_workspace* 8123 mlx5_flow_push_thread_workspace(void) 8124 { 8125 struct mlx5_flow_workspace *curr; 8126 struct mlx5_flow_workspace *data; 8127 8128 curr = mlx5_flow_os_get_specific_workspace(); 8129 if (!curr) { 8130 data = flow_alloc_thread_workspace(); 8131 if (!data) 8132 return NULL; 8133 mlx5_flow_os_workspace_gc_add(data); 8134 } else if (!curr->inuse) { 8135 data = curr; 8136 } else if (curr->next) { 8137 data = curr->next; 8138 } else { 8139 data = flow_alloc_thread_workspace(); 8140 if (!data) 8141 return NULL; 8142 curr->next = data; 8143 data->prev = curr; 8144 } 8145 data->inuse = 1; 8146 data->flow_idx = 0; 8147 /* Set as current workspace */ 8148 if (mlx5_flow_os_set_specific_workspace(data)) 8149 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8150 return data; 8151 } 8152 8153 /** 8154 * Close current thread specific flow workspace. 8155 * 8156 * If previous workspace available, set it as current. 8157 * 8158 * @return pointer to thread specific flow workspace data, NULL on error. 8159 */ 8160 void 8161 mlx5_flow_pop_thread_workspace(void) 8162 { 8163 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8164 8165 if (!data) 8166 return; 8167 if (!data->inuse) { 8168 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8169 return; 8170 } 8171 data->inuse = 0; 8172 if (!data->prev) 8173 return; 8174 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8175 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8176 } 8177 8178 /** 8179 * Verify the flow list is empty 8180 * 8181 * @param dev 8182 * Pointer to Ethernet device. 8183 * 8184 * @return the number of flows not released. 8185 */ 8186 int 8187 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8188 { 8189 struct mlx5_priv *priv = dev->data->dev_private; 8190 struct rte_flow *flow; 8191 uint32_t idx = 0; 8192 int ret = 0, i; 8193 8194 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8195 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8196 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8197 dev->data->port_id, (void *)flow); 8198 ret++; 8199 } 8200 } 8201 return ret; 8202 } 8203 8204 /** 8205 * Enable default hairpin egress flow. 8206 * 8207 * @param dev 8208 * Pointer to Ethernet device. 8209 * @param sq_num 8210 * The SQ hw number. 8211 * 8212 * @return 8213 * 0 on success, a negative errno value otherwise and rte_errno is set. 8214 */ 8215 int 8216 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8217 uint32_t sq_num) 8218 { 8219 const struct rte_flow_attr attr = { 8220 .egress = 1, 8221 .priority = 0, 8222 }; 8223 struct mlx5_rte_flow_item_sq queue_spec = { 8224 .queue = sq_num, 8225 }; 8226 struct mlx5_rte_flow_item_sq queue_mask = { 8227 .queue = UINT32_MAX, 8228 }; 8229 struct rte_flow_item items[] = { 8230 { 8231 .type = (enum rte_flow_item_type) 8232 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8233 .spec = &queue_spec, 8234 .last = NULL, 8235 .mask = &queue_mask, 8236 }, 8237 { 8238 .type = RTE_FLOW_ITEM_TYPE_END, 8239 }, 8240 }; 8241 struct rte_flow_action_jump jump = { 8242 .group = MLX5_HAIRPIN_TX_TABLE, 8243 }; 8244 struct rte_flow_action actions[2]; 8245 uint32_t flow_idx; 8246 struct rte_flow_error error; 8247 8248 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8249 actions[0].conf = &jump; 8250 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8251 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8252 &attr, items, actions, false, &error); 8253 if (!flow_idx) { 8254 DRV_LOG(DEBUG, 8255 "Failed to create ctrl flow: rte_errno(%d)," 8256 " type(%d), message(%s)", 8257 rte_errno, error.type, 8258 error.message ? error.message : " (no stated reason)"); 8259 return -rte_errno; 8260 } 8261 return 0; 8262 } 8263 8264 /** 8265 * Enable a control flow configured from the control plane. 8266 * 8267 * @param dev 8268 * Pointer to Ethernet device. 8269 * @param eth_spec 8270 * An Ethernet flow spec to apply. 8271 * @param eth_mask 8272 * An Ethernet flow mask to apply. 8273 * @param vlan_spec 8274 * A VLAN flow spec to apply. 8275 * @param vlan_mask 8276 * A VLAN flow mask to apply. 8277 * 8278 * @return 8279 * 0 on success, a negative errno value otherwise and rte_errno is set. 8280 */ 8281 int 8282 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8283 struct rte_flow_item_eth *eth_spec, 8284 struct rte_flow_item_eth *eth_mask, 8285 struct rte_flow_item_vlan *vlan_spec, 8286 struct rte_flow_item_vlan *vlan_mask) 8287 { 8288 struct mlx5_priv *priv = dev->data->dev_private; 8289 const struct rte_flow_attr attr = { 8290 .ingress = 1, 8291 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8292 }; 8293 struct rte_flow_item items[] = { 8294 { 8295 .type = RTE_FLOW_ITEM_TYPE_ETH, 8296 .spec = eth_spec, 8297 .last = NULL, 8298 .mask = eth_mask, 8299 }, 8300 { 8301 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8302 RTE_FLOW_ITEM_TYPE_END, 8303 .spec = vlan_spec, 8304 .last = NULL, 8305 .mask = vlan_mask, 8306 }, 8307 { 8308 .type = RTE_FLOW_ITEM_TYPE_END, 8309 }, 8310 }; 8311 uint16_t queue[priv->reta_idx_n]; 8312 struct rte_flow_action_rss action_rss = { 8313 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8314 .level = 0, 8315 .types = priv->rss_conf.rss_hf, 8316 .key_len = priv->rss_conf.rss_key_len, 8317 .queue_num = priv->reta_idx_n, 8318 .key = priv->rss_conf.rss_key, 8319 .queue = queue, 8320 }; 8321 struct rte_flow_action actions[] = { 8322 { 8323 .type = RTE_FLOW_ACTION_TYPE_RSS, 8324 .conf = &action_rss, 8325 }, 8326 { 8327 .type = RTE_FLOW_ACTION_TYPE_END, 8328 }, 8329 }; 8330 uint32_t flow_idx; 8331 struct rte_flow_error error; 8332 unsigned int i; 8333 8334 if (!priv->reta_idx_n || !priv->rxqs_n) { 8335 return 0; 8336 } 8337 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8338 action_rss.types = 0; 8339 for (i = 0; i != priv->reta_idx_n; ++i) 8340 queue[i] = (*priv->reta_idx)[i]; 8341 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8342 &attr, items, actions, false, &error); 8343 if (!flow_idx) 8344 return -rte_errno; 8345 return 0; 8346 } 8347 8348 /** 8349 * Enable a flow control configured from the control plane. 8350 * 8351 * @param dev 8352 * Pointer to Ethernet device. 8353 * @param eth_spec 8354 * An Ethernet flow spec to apply. 8355 * @param eth_mask 8356 * An Ethernet flow mask to apply. 8357 * 8358 * @return 8359 * 0 on success, a negative errno value otherwise and rte_errno is set. 8360 */ 8361 int 8362 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8363 struct rte_flow_item_eth *eth_spec, 8364 struct rte_flow_item_eth *eth_mask) 8365 { 8366 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8367 } 8368 8369 /** 8370 * Create default miss flow rule matching lacp traffic 8371 * 8372 * @param dev 8373 * Pointer to Ethernet device. 8374 * @param eth_spec 8375 * An Ethernet flow spec to apply. 8376 * 8377 * @return 8378 * 0 on success, a negative errno value otherwise and rte_errno is set. 8379 */ 8380 int 8381 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8382 { 8383 /* 8384 * The LACP matching is done by only using ether type since using 8385 * a multicast dst mac causes kernel to give low priority to this flow. 8386 */ 8387 static const struct rte_flow_item_eth lacp_spec = { 8388 .hdr.ether_type = RTE_BE16(0x8809), 8389 }; 8390 static const struct rte_flow_item_eth lacp_mask = { 8391 .hdr.ether_type = 0xffff, 8392 }; 8393 const struct rte_flow_attr attr = { 8394 .ingress = 1, 8395 }; 8396 struct rte_flow_item items[] = { 8397 { 8398 .type = RTE_FLOW_ITEM_TYPE_ETH, 8399 .spec = &lacp_spec, 8400 .mask = &lacp_mask, 8401 }, 8402 { 8403 .type = RTE_FLOW_ITEM_TYPE_END, 8404 }, 8405 }; 8406 struct rte_flow_action actions[] = { 8407 { 8408 .type = (enum rte_flow_action_type) 8409 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8410 }, 8411 { 8412 .type = RTE_FLOW_ACTION_TYPE_END, 8413 }, 8414 }; 8415 struct rte_flow_error error; 8416 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8417 &attr, items, actions, 8418 false, &error); 8419 8420 if (!flow_idx) 8421 return -rte_errno; 8422 return 0; 8423 } 8424 8425 /** 8426 * Destroy a flow. 8427 * 8428 * @see rte_flow_destroy() 8429 * @see rte_flow_ops 8430 */ 8431 int 8432 mlx5_flow_destroy(struct rte_eth_dev *dev, 8433 struct rte_flow *flow, 8434 struct rte_flow_error *error __rte_unused) 8435 { 8436 struct mlx5_priv *priv = dev->data->dev_private; 8437 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8438 struct mlx5_dv_flow_info *flow_info; 8439 8440 if (priv->sh->config.dv_flow_en == 2) 8441 return rte_flow_error_set(error, ENOTSUP, 8442 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8443 NULL, 8444 "Flow non-Q destruction not supported"); 8445 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8446 (uintptr_t)(void *)flow); 8447 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8448 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8449 while (flow_info) { 8450 /* Romove the cache flow info. */ 8451 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8452 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8453 LIST_REMOVE(flow_info, next); 8454 mlx5_free(flow_info->items); 8455 mlx5_free(flow_info->actions); 8456 mlx5_free(flow_info); 8457 break; 8458 } 8459 flow_info = LIST_NEXT(flow_info, next); 8460 } 8461 } 8462 return 0; 8463 } 8464 8465 /** 8466 * Destroy all flows. 8467 * 8468 * @see rte_flow_flush() 8469 * @see rte_flow_ops 8470 */ 8471 int 8472 mlx5_flow_flush(struct rte_eth_dev *dev, 8473 struct rte_flow_error *error __rte_unused) 8474 { 8475 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8476 return 0; 8477 } 8478 8479 /** 8480 * Isolated mode. 8481 * 8482 * @see rte_flow_isolate() 8483 * @see rte_flow_ops 8484 */ 8485 int 8486 mlx5_flow_isolate(struct rte_eth_dev *dev, 8487 int enable, 8488 struct rte_flow_error *error) 8489 { 8490 struct mlx5_priv *priv = dev->data->dev_private; 8491 8492 if (dev->data->dev_started) { 8493 rte_flow_error_set(error, EBUSY, 8494 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8495 NULL, 8496 "port must be stopped first"); 8497 return -rte_errno; 8498 } 8499 if (!enable && !priv->sh->config.repr_matching) 8500 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8501 "isolated mode cannot be disabled when " 8502 "representor matching is disabled"); 8503 priv->isolated = !!enable; 8504 if (enable) 8505 dev->dev_ops = &mlx5_dev_ops_isolate; 8506 else 8507 dev->dev_ops = &mlx5_dev_ops; 8508 8509 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8510 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8511 8512 return 0; 8513 } 8514 8515 /** 8516 * Query a flow. 8517 * 8518 * @see rte_flow_query() 8519 * @see rte_flow_ops 8520 */ 8521 static int 8522 flow_drv_query(struct rte_eth_dev *dev, 8523 struct rte_flow *eflow, 8524 const struct rte_flow_action *actions, 8525 void *data, 8526 struct rte_flow_error *error) 8527 { 8528 struct mlx5_priv *priv = dev->data->dev_private; 8529 const struct mlx5_flow_driver_ops *fops; 8530 struct rte_flow *flow = NULL; 8531 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8532 8533 if (priv->sh->config.dv_flow_en == 2) { 8534 #ifdef HAVE_MLX5_HWS_SUPPORT 8535 flow = eflow; 8536 ftype = MLX5_FLOW_TYPE_HW; 8537 #endif 8538 } else { 8539 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8540 (uintptr_t)(void *)eflow); 8541 } 8542 if (!flow) { 8543 return rte_flow_error_set(error, ENOENT, 8544 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8545 NULL, 8546 "invalid flow handle"); 8547 } 8548 if (ftype == MLX5_FLOW_TYPE_MIN) 8549 ftype = flow->drv_type; 8550 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8551 fops = flow_get_drv_ops(ftype); 8552 8553 return fops->query(dev, flow, actions, data, error); 8554 } 8555 8556 /** 8557 * Query a flow. 8558 * 8559 * @see rte_flow_query() 8560 * @see rte_flow_ops 8561 */ 8562 int 8563 mlx5_flow_query(struct rte_eth_dev *dev, 8564 struct rte_flow *flow, 8565 const struct rte_flow_action *actions, 8566 void *data, 8567 struct rte_flow_error *error) 8568 { 8569 int ret; 8570 8571 ret = flow_drv_query(dev, flow, actions, data, 8572 error); 8573 if (ret < 0) 8574 return ret; 8575 return 0; 8576 } 8577 8578 /** 8579 * Get rte_flow callbacks. 8580 * 8581 * @param dev 8582 * Pointer to Ethernet device structure. 8583 * @param ops 8584 * Pointer to operation-specific structure. 8585 * 8586 * @return 0 8587 */ 8588 int 8589 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8590 const struct rte_flow_ops **ops) 8591 { 8592 *ops = &mlx5_flow_ops; 8593 return 0; 8594 } 8595 8596 /** 8597 * Validate meter policy actions. 8598 * Dispatcher for action type specific validation. 8599 * 8600 * @param[in] dev 8601 * Pointer to the Ethernet device structure. 8602 * @param[in] action 8603 * The meter policy action object to validate. 8604 * @param[in] attr 8605 * Attributes of flow to determine steering domain. 8606 * @param[out] is_rss 8607 * Is RSS or not. 8608 * @param[out] domain_bitmap 8609 * Domain bitmap. 8610 * @param[out] is_def_policy 8611 * Is default policy or not. 8612 * @param[out] error 8613 * Perform verbose error reporting if not NULL. Initialized in case of 8614 * error only. 8615 * 8616 * @return 8617 * 0 on success, otherwise negative errno value. 8618 */ 8619 int 8620 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8621 const struct rte_flow_action *actions[RTE_COLORS], 8622 struct rte_flow_attr *attr, 8623 bool *is_rss, 8624 uint8_t *domain_bitmap, 8625 uint8_t *policy_mode, 8626 struct rte_mtr_error *error) 8627 { 8628 const struct mlx5_flow_driver_ops *fops; 8629 8630 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8631 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8632 domain_bitmap, policy_mode, error); 8633 } 8634 8635 /** 8636 * Destroy the meter table set. 8637 * 8638 * @param[in] dev 8639 * Pointer to Ethernet device. 8640 * @param[in] mtr_policy 8641 * Meter policy struct. 8642 */ 8643 void 8644 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8645 struct mlx5_flow_meter_policy *mtr_policy) 8646 { 8647 const struct mlx5_flow_driver_ops *fops; 8648 8649 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8650 fops->destroy_mtr_acts(dev, mtr_policy); 8651 } 8652 8653 /** 8654 * Create policy action, lock free, 8655 * (mutex should be acquired by caller). 8656 * Dispatcher for action type specific call. 8657 * 8658 * @param[in] dev 8659 * Pointer to the Ethernet device structure. 8660 * @param[in] mtr_policy 8661 * Meter policy struct. 8662 * @param[in] action 8663 * Action specification used to create meter actions. 8664 * @param[in] attr 8665 * Flow rule attributes. 8666 * @param[out] error 8667 * Perform verbose error reporting if not NULL. Initialized in case of 8668 * error only. 8669 * 8670 * @return 8671 * 0 on success, otherwise negative errno value. 8672 */ 8673 int 8674 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8675 struct mlx5_flow_meter_policy *mtr_policy, 8676 const struct rte_flow_action *actions[RTE_COLORS], 8677 struct rte_flow_attr *attr, 8678 struct rte_mtr_error *error) 8679 { 8680 const struct mlx5_flow_driver_ops *fops; 8681 8682 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8683 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8684 } 8685 8686 /** 8687 * Create policy rules, lock free, 8688 * (mutex should be acquired by caller). 8689 * Dispatcher for action type specific call. 8690 * 8691 * @param[in] dev 8692 * Pointer to the Ethernet device structure. 8693 * @param[in] mtr_policy 8694 * Meter policy struct. 8695 * 8696 * @return 8697 * 0 on success, -1 otherwise. 8698 */ 8699 int 8700 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8701 struct mlx5_flow_meter_policy *mtr_policy) 8702 { 8703 const struct mlx5_flow_driver_ops *fops; 8704 8705 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8706 return fops->create_policy_rules(dev, mtr_policy); 8707 } 8708 8709 /** 8710 * Destroy policy rules, lock free, 8711 * (mutex should be acquired by caller). 8712 * Dispatcher for action type specific call. 8713 * 8714 * @param[in] dev 8715 * Pointer to the Ethernet device structure. 8716 * @param[in] mtr_policy 8717 * Meter policy struct. 8718 */ 8719 void 8720 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8721 struct mlx5_flow_meter_policy *mtr_policy) 8722 { 8723 const struct mlx5_flow_driver_ops *fops; 8724 8725 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8726 fops->destroy_policy_rules(dev, mtr_policy); 8727 } 8728 8729 /** 8730 * Destroy the default policy table set. 8731 * 8732 * @param[in] dev 8733 * Pointer to Ethernet device. 8734 */ 8735 void 8736 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8737 { 8738 const struct mlx5_flow_driver_ops *fops; 8739 8740 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8741 fops->destroy_def_policy(dev); 8742 } 8743 8744 /** 8745 * Destroy the default policy table set. 8746 * 8747 * @param[in] dev 8748 * Pointer to Ethernet device. 8749 * 8750 * @return 8751 * 0 on success, -1 otherwise. 8752 */ 8753 int 8754 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8755 { 8756 const struct mlx5_flow_driver_ops *fops; 8757 8758 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8759 return fops->create_def_policy(dev); 8760 } 8761 8762 /** 8763 * Create the needed meter and suffix tables. 8764 * 8765 * @param[in] dev 8766 * Pointer to Ethernet device. 8767 * 8768 * @return 8769 * 0 on success, -1 otherwise. 8770 */ 8771 int 8772 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8773 struct mlx5_flow_meter_info *fm, 8774 uint32_t mtr_idx, 8775 uint8_t domain_bitmap) 8776 { 8777 const struct mlx5_flow_driver_ops *fops; 8778 8779 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8780 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8781 } 8782 8783 /** 8784 * Destroy the meter table set. 8785 * 8786 * @param[in] dev 8787 * Pointer to Ethernet device. 8788 * @param[in] tbl 8789 * Pointer to the meter table set. 8790 */ 8791 void 8792 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8793 struct mlx5_flow_meter_info *fm) 8794 { 8795 const struct mlx5_flow_driver_ops *fops; 8796 8797 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8798 fops->destroy_mtr_tbls(dev, fm); 8799 } 8800 8801 /** 8802 * Destroy the global meter drop table. 8803 * 8804 * @param[in] dev 8805 * Pointer to Ethernet device. 8806 */ 8807 void 8808 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8809 { 8810 const struct mlx5_flow_driver_ops *fops; 8811 8812 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8813 fops->destroy_mtr_drop_tbls(dev); 8814 } 8815 8816 /** 8817 * Destroy the sub policy table with RX queue. 8818 * 8819 * @param[in] dev 8820 * Pointer to Ethernet device. 8821 * @param[in] mtr_policy 8822 * Pointer to meter policy table. 8823 */ 8824 void 8825 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8826 struct mlx5_flow_meter_policy *mtr_policy) 8827 { 8828 const struct mlx5_flow_driver_ops *fops; 8829 8830 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8831 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8832 } 8833 8834 /** 8835 * Allocate the needed aso flow meter id. 8836 * 8837 * @param[in] dev 8838 * Pointer to Ethernet device. 8839 * 8840 * @return 8841 * Index to aso flow meter on success, NULL otherwise. 8842 */ 8843 uint32_t 8844 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8845 { 8846 const struct mlx5_flow_driver_ops *fops; 8847 8848 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8849 return fops->create_meter(dev); 8850 } 8851 8852 /** 8853 * Free the aso flow meter id. 8854 * 8855 * @param[in] dev 8856 * Pointer to Ethernet device. 8857 * @param[in] mtr_idx 8858 * Index to aso flow meter to be free. 8859 * 8860 * @return 8861 * 0 on success. 8862 */ 8863 void 8864 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8865 { 8866 const struct mlx5_flow_driver_ops *fops; 8867 8868 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8869 fops->free_meter(dev, mtr_idx); 8870 } 8871 8872 /** 8873 * Allocate a counter. 8874 * 8875 * @param[in] dev 8876 * Pointer to Ethernet device structure. 8877 * 8878 * @return 8879 * Index to allocated counter on success, 0 otherwise. 8880 */ 8881 uint32_t 8882 mlx5_counter_alloc(struct rte_eth_dev *dev) 8883 { 8884 struct rte_flow_attr attr = { .transfer = 0 }; 8885 8886 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8887 (dev); 8888 } 8889 8890 /** 8891 * Free a counter. 8892 * 8893 * @param[in] dev 8894 * Pointer to Ethernet device structure. 8895 * @param[in] cnt 8896 * Index to counter to be free. 8897 */ 8898 void 8899 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8900 { 8901 struct rte_flow_attr attr = { .transfer = 0 }; 8902 8903 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8904 } 8905 8906 /** 8907 * Query counter statistics. 8908 * 8909 * @param[in] dev 8910 * Pointer to Ethernet device structure. 8911 * @param[in] cnt 8912 * Index to counter to query. 8913 * @param[in] clear 8914 * Set to clear counter statistics. 8915 * @param[out] pkts 8916 * The counter hits packets number to save. 8917 * @param[out] bytes 8918 * The counter hits bytes number to save. 8919 * 8920 * @return 8921 * 0 on success, a negative errno value otherwise. 8922 */ 8923 int 8924 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8925 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8926 { 8927 struct rte_flow_attr attr = { .transfer = 0 }; 8928 8929 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8930 (dev, cnt, clear, pkts, bytes, action); 8931 } 8932 8933 /** 8934 * Get information about HWS pre-configurable resources. 8935 * 8936 * @param[in] dev 8937 * Pointer to the rte_eth_dev structure. 8938 * @param[out] port_info 8939 * Pointer to port information. 8940 * @param[out] queue_info 8941 * Pointer to queue information. 8942 * @param[out] error 8943 * Pointer to error structure. 8944 * 8945 * @return 8946 * 0 on success, a negative errno value otherwise and rte_errno is set. 8947 */ 8948 static int 8949 mlx5_flow_info_get(struct rte_eth_dev *dev, 8950 struct rte_flow_port_info *port_info, 8951 struct rte_flow_queue_info *queue_info, 8952 struct rte_flow_error *error) 8953 { 8954 const struct mlx5_flow_driver_ops *fops; 8955 struct rte_flow_attr attr = {0}; 8956 8957 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8958 return rte_flow_error_set(error, ENOTSUP, 8959 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8960 NULL, 8961 "info get with incorrect steering mode"); 8962 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8963 return fops->info_get(dev, port_info, queue_info, error); 8964 } 8965 8966 /** 8967 * Configure port HWS resources. 8968 * 8969 * @param[in] dev 8970 * Pointer to the rte_eth_dev structure. 8971 * @param[in] port_attr 8972 * Port configuration attributes. 8973 * @param[in] nb_queue 8974 * Number of queue. 8975 * @param[in] queue_attr 8976 * Array that holds attributes for each flow queue. 8977 * @param[out] error 8978 * Pointer to error structure. 8979 * 8980 * @return 8981 * 0 on success, a negative errno value otherwise and rte_errno is set. 8982 */ 8983 static int 8984 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8985 const struct rte_flow_port_attr *port_attr, 8986 uint16_t nb_queue, 8987 const struct rte_flow_queue_attr *queue_attr[], 8988 struct rte_flow_error *error) 8989 { 8990 const struct mlx5_flow_driver_ops *fops; 8991 struct rte_flow_attr attr = {0}; 8992 8993 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8994 return rte_flow_error_set(error, ENOTSUP, 8995 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8996 NULL, 8997 "port configure with incorrect steering mode"); 8998 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8999 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 9000 } 9001 9002 /** 9003 * Validate item template. 9004 * 9005 * @param[in] dev 9006 * Pointer to the rte_eth_dev structure. 9007 * @param[in] attr 9008 * Pointer to the item template attributes. 9009 * @param[in] items 9010 * The template item pattern. 9011 * @param[out] error 9012 * Pointer to error structure. 9013 * 9014 * @return 9015 * 0 on success, a negative errno value otherwise and rte_errno is set. 9016 */ 9017 int 9018 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 9019 const struct rte_flow_pattern_template_attr *attr, 9020 const struct rte_flow_item items[], 9021 struct rte_flow_error *error) 9022 { 9023 const struct mlx5_flow_driver_ops *fops; 9024 struct rte_flow_attr fattr = {0}; 9025 9026 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9027 rte_flow_error_set(error, ENOTSUP, 9028 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9029 "pattern validate with incorrect steering mode"); 9030 return -ENOTSUP; 9031 } 9032 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9033 return fops->pattern_validate(dev, attr, items, error); 9034 } 9035 9036 /** 9037 * Create flow item template. 9038 * 9039 * @param[in] dev 9040 * Pointer to the rte_eth_dev structure. 9041 * @param[in] attr 9042 * Pointer to the item template attributes. 9043 * @param[in] items 9044 * The template item pattern. 9045 * @param[out] error 9046 * Pointer to error structure. 9047 * 9048 * @return 9049 * 0 on success, a negative errno value otherwise and rte_errno is set. 9050 */ 9051 static struct rte_flow_pattern_template * 9052 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9053 const struct rte_flow_pattern_template_attr *attr, 9054 const struct rte_flow_item items[], 9055 struct rte_flow_error *error) 9056 { 9057 const struct mlx5_flow_driver_ops *fops; 9058 struct rte_flow_attr fattr = {0}; 9059 9060 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9061 rte_flow_error_set(error, ENOTSUP, 9062 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9063 NULL, 9064 "pattern create with incorrect steering mode"); 9065 return NULL; 9066 } 9067 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9068 return fops->pattern_template_create(dev, attr, items, error); 9069 } 9070 9071 /** 9072 * Destroy flow item template. 9073 * 9074 * @param[in] dev 9075 * Pointer to the rte_eth_dev structure. 9076 * @param[in] template 9077 * Pointer to the item template to be destroyed. 9078 * @param[out] error 9079 * Pointer to error structure. 9080 * 9081 * @return 9082 * 0 on success, a negative errno value otherwise and rte_errno is set. 9083 */ 9084 static int 9085 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9086 struct rte_flow_pattern_template *template, 9087 struct rte_flow_error *error) 9088 { 9089 const struct mlx5_flow_driver_ops *fops; 9090 struct rte_flow_attr attr = {0}; 9091 9092 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9093 return rte_flow_error_set(error, ENOTSUP, 9094 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9095 NULL, 9096 "pattern destroy with incorrect steering mode"); 9097 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9098 return fops->pattern_template_destroy(dev, template, error); 9099 } 9100 9101 /** 9102 * Validate flow actions template. 9103 * 9104 * @param[in] dev 9105 * Pointer to the rte_eth_dev structure. 9106 * @param[in] attr 9107 * Pointer to the action template attributes. 9108 * @param[in] actions 9109 * Associated actions (list terminated by the END action). 9110 * @param[in] masks 9111 * List of actions that marks which of the action's member is constant. 9112 * @param[out] error 9113 * Pointer to error structure. 9114 * 9115 * @return 9116 * 0 on success, a negative errno value otherwise and rte_errno is set. 9117 */ 9118 int 9119 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9120 const struct rte_flow_actions_template_attr *attr, 9121 const struct rte_flow_action actions[], 9122 const struct rte_flow_action masks[], 9123 struct rte_flow_error *error) 9124 { 9125 const struct mlx5_flow_driver_ops *fops; 9126 struct rte_flow_attr fattr = {0}; 9127 9128 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9129 rte_flow_error_set(error, ENOTSUP, 9130 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9131 "actions validate with incorrect steering mode"); 9132 return -ENOTSUP; 9133 } 9134 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9135 return fops->actions_validate(dev, attr, actions, masks, error); 9136 } 9137 9138 /** 9139 * Create flow item template. 9140 * 9141 * @param[in] dev 9142 * Pointer to the rte_eth_dev structure. 9143 * @param[in] attr 9144 * Pointer to the action template attributes. 9145 * @param[in] actions 9146 * Associated actions (list terminated by the END action). 9147 * @param[in] masks 9148 * List of actions that marks which of the action's member is constant. 9149 * @param[out] error 9150 * Pointer to error structure. 9151 * 9152 * @return 9153 * 0 on success, a negative errno value otherwise and rte_errno is set. 9154 */ 9155 static struct rte_flow_actions_template * 9156 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9157 const struct rte_flow_actions_template_attr *attr, 9158 const struct rte_flow_action actions[], 9159 const struct rte_flow_action masks[], 9160 struct rte_flow_error *error) 9161 { 9162 const struct mlx5_flow_driver_ops *fops; 9163 struct rte_flow_attr fattr = {0}; 9164 9165 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9166 rte_flow_error_set(error, ENOTSUP, 9167 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9168 NULL, 9169 "action create with incorrect steering mode"); 9170 return NULL; 9171 } 9172 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9173 return fops->actions_template_create(dev, attr, actions, masks, error); 9174 } 9175 9176 /** 9177 * Destroy flow action template. 9178 * 9179 * @param[in] dev 9180 * Pointer to the rte_eth_dev structure. 9181 * @param[in] template 9182 * Pointer to the action template to be destroyed. 9183 * @param[out] error 9184 * Pointer to error structure. 9185 * 9186 * @return 9187 * 0 on success, a negative errno value otherwise and rte_errno is set. 9188 */ 9189 static int 9190 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9191 struct rte_flow_actions_template *template, 9192 struct rte_flow_error *error) 9193 { 9194 const struct mlx5_flow_driver_ops *fops; 9195 struct rte_flow_attr attr = {0}; 9196 9197 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9198 return rte_flow_error_set(error, ENOTSUP, 9199 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9200 NULL, 9201 "action destroy with incorrect steering mode"); 9202 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9203 return fops->actions_template_destroy(dev, template, error); 9204 } 9205 9206 /** 9207 * Create flow table. 9208 * 9209 * @param[in] dev 9210 * Pointer to the rte_eth_dev structure. 9211 * @param[in] attr 9212 * Pointer to the table attributes. 9213 * @param[in] item_templates 9214 * Item template array to be binded to the table. 9215 * @param[in] nb_item_templates 9216 * Number of item template. 9217 * @param[in] action_templates 9218 * Action template array to be binded to the table. 9219 * @param[in] nb_action_templates 9220 * Number of action template. 9221 * @param[out] error 9222 * Pointer to error structure. 9223 * 9224 * @return 9225 * Table on success, NULL otherwise and rte_errno is set. 9226 */ 9227 static struct rte_flow_template_table * 9228 mlx5_flow_table_create(struct rte_eth_dev *dev, 9229 const struct rte_flow_template_table_attr *attr, 9230 struct rte_flow_pattern_template *item_templates[], 9231 uint8_t nb_item_templates, 9232 struct rte_flow_actions_template *action_templates[], 9233 uint8_t nb_action_templates, 9234 struct rte_flow_error *error) 9235 { 9236 const struct mlx5_flow_driver_ops *fops; 9237 struct rte_flow_attr fattr = {0}; 9238 9239 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9240 rte_flow_error_set(error, ENOTSUP, 9241 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9242 NULL, 9243 "table create with incorrect steering mode"); 9244 return NULL; 9245 } 9246 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9247 return fops->template_table_create(dev, 9248 attr, 9249 item_templates, 9250 nb_item_templates, 9251 action_templates, 9252 nb_action_templates, 9253 error); 9254 } 9255 9256 /** 9257 * PMD destroy flow table. 9258 * 9259 * @param[in] dev 9260 * Pointer to the rte_eth_dev structure. 9261 * @param[in] table 9262 * Pointer to the table to be destroyed. 9263 * @param[out] error 9264 * Pointer to error structure. 9265 * 9266 * @return 9267 * 0 on success, a negative errno value otherwise and rte_errno is set. 9268 */ 9269 static int 9270 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9271 struct rte_flow_template_table *table, 9272 struct rte_flow_error *error) 9273 { 9274 const struct mlx5_flow_driver_ops *fops; 9275 struct rte_flow_attr attr = {0}; 9276 9277 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9278 return rte_flow_error_set(error, ENOTSUP, 9279 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9280 NULL, 9281 "table destroy with incorrect steering mode"); 9282 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9283 return fops->template_table_destroy(dev, table, error); 9284 } 9285 9286 /** 9287 * PMD group set miss actions. 9288 * 9289 * @param[in] dev 9290 * Pointer to the rte_eth_dev structure. 9291 * @param[in] attr 9292 * Pointer to group attributes 9293 * @param[in] actions 9294 * Array of actions 9295 * @param[out] error 9296 * Pointer to error structure. 9297 * 9298 * @return 9299 * 0 on success, a negative errno value otherwise and rte_errno is set. 9300 */ 9301 static int 9302 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 9303 uint32_t group_id, 9304 const struct rte_flow_group_attr *attr, 9305 const struct rte_flow_action actions[], 9306 struct rte_flow_error *error) 9307 { 9308 const struct mlx5_flow_driver_ops *fops; 9309 struct rte_flow_attr fattr = {0}; 9310 9311 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9312 return rte_flow_error_set(error, ENOTSUP, 9313 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9314 NULL, 9315 "group set miss actions with incorrect steering mode"); 9316 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9317 return fops->group_set_miss_actions(dev, group_id, attr, actions, error); 9318 } 9319 9320 /** 9321 * Allocate a new memory for the counter values wrapped by all the needed 9322 * management. 9323 * 9324 * @param[in] sh 9325 * Pointer to mlx5_dev_ctx_shared object. 9326 * 9327 * @return 9328 * 0 on success, a negative errno value otherwise. 9329 */ 9330 static int 9331 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9332 { 9333 struct mlx5_counter_stats_mem_mng *mem_mng; 9334 volatile struct flow_counter_stats *raw_data; 9335 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9336 int size = (sizeof(struct flow_counter_stats) * 9337 MLX5_COUNTERS_PER_POOL + 9338 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9339 sizeof(struct mlx5_counter_stats_mem_mng); 9340 size_t pgsize = rte_mem_page_size(); 9341 uint8_t *mem; 9342 int ret; 9343 int i; 9344 9345 if (pgsize == (size_t)-1) { 9346 DRV_LOG(ERR, "Failed to get mem page size"); 9347 rte_errno = ENOMEM; 9348 return -ENOMEM; 9349 } 9350 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9351 if (!mem) { 9352 rte_errno = ENOMEM; 9353 return -ENOMEM; 9354 } 9355 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9356 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9357 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9358 sh->cdev->pdn, mem, size, 9359 &mem_mng->wm); 9360 if (ret) { 9361 rte_errno = errno; 9362 mlx5_free(mem); 9363 return -rte_errno; 9364 } 9365 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9366 raw_data = (volatile struct flow_counter_stats *)mem; 9367 for (i = 0; i < raws_n; ++i) { 9368 mem_mng->raws[i].mem_mng = mem_mng; 9369 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9370 } 9371 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9372 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9373 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9374 next); 9375 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9376 sh->sws_cmng.mem_mng = mem_mng; 9377 return 0; 9378 } 9379 9380 /** 9381 * Set the statistic memory to the new counter pool. 9382 * 9383 * @param[in] sh 9384 * Pointer to mlx5_dev_ctx_shared object. 9385 * @param[in] pool 9386 * Pointer to the pool to set the statistic memory. 9387 * 9388 * @return 9389 * 0 on success, a negative errno value otherwise. 9390 */ 9391 static int 9392 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9393 struct mlx5_flow_counter_pool *pool) 9394 { 9395 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9396 /* Resize statistic memory once used out. */ 9397 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9398 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9399 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9400 return -1; 9401 } 9402 rte_spinlock_lock(&pool->sl); 9403 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9404 rte_spinlock_unlock(&pool->sl); 9405 pool->raw_hw = NULL; 9406 return 0; 9407 } 9408 9409 #define MLX5_POOL_QUERY_FREQ_US 1000000 9410 9411 /** 9412 * Set the periodic procedure for triggering asynchronous batch queries for all 9413 * the counter pools. 9414 * 9415 * @param[in] sh 9416 * Pointer to mlx5_dev_ctx_shared object. 9417 */ 9418 void 9419 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9420 { 9421 uint32_t pools_n, us; 9422 9423 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9424 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9425 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9426 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9427 sh->sws_cmng.query_thread_on = 0; 9428 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9429 } else { 9430 sh->sws_cmng.query_thread_on = 1; 9431 } 9432 } 9433 9434 /** 9435 * The periodic procedure for triggering asynchronous batch queries for all the 9436 * counter pools. This function is probably called by the host thread. 9437 * 9438 * @param[in] arg 9439 * The parameter for the alarm process. 9440 */ 9441 void 9442 mlx5_flow_query_alarm(void *arg) 9443 { 9444 struct mlx5_dev_ctx_shared *sh = arg; 9445 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9446 uint16_t pool_index = cmng->pool_index; 9447 struct mlx5_flow_counter_pool *pool; 9448 uint16_t n_valid; 9449 int ret; 9450 9451 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9452 goto set_alarm; 9453 rte_spinlock_lock(&cmng->pool_update_sl); 9454 pool = cmng->pools[pool_index]; 9455 n_valid = cmng->n_valid; 9456 rte_spinlock_unlock(&cmng->pool_update_sl); 9457 /* Set the statistic memory to the new created pool. */ 9458 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9459 goto set_alarm; 9460 if (pool->raw_hw) 9461 /* There is a pool query in progress. */ 9462 goto set_alarm; 9463 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9464 if (!pool->raw_hw) 9465 /* No free counter statistics raw memory. */ 9466 goto set_alarm; 9467 /* 9468 * Identify the counters released between query trigger and query 9469 * handle more efficiently. The counter released in this gap period 9470 * should wait for a new round of query as the new arrived packets 9471 * will not be taken into account. 9472 */ 9473 pool->query_gen++; 9474 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9475 MLX5_COUNTERS_PER_POOL, 9476 NULL, NULL, 9477 pool->raw_hw->mem_mng->wm.lkey, 9478 (void *)(uintptr_t) 9479 pool->raw_hw->data, 9480 sh->devx_comp, 9481 (uint64_t)(uintptr_t)pool); 9482 if (ret) { 9483 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9484 " %d", pool->min_dcs->id); 9485 pool->raw_hw = NULL; 9486 goto set_alarm; 9487 } 9488 LIST_REMOVE(pool->raw_hw, next); 9489 cmng->pending_queries++; 9490 pool_index++; 9491 if (pool_index >= n_valid) 9492 pool_index = 0; 9493 set_alarm: 9494 cmng->pool_index = pool_index; 9495 mlx5_set_query_alarm(sh); 9496 } 9497 9498 /** 9499 * Check and callback event for new aged flow in the counter pool 9500 * 9501 * @param[in] sh 9502 * Pointer to mlx5_dev_ctx_shared object. 9503 * @param[in] pool 9504 * Pointer to Current counter pool. 9505 */ 9506 static void 9507 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9508 struct mlx5_flow_counter_pool *pool) 9509 { 9510 struct mlx5_priv *priv; 9511 struct mlx5_flow_counter *cnt; 9512 struct mlx5_age_info *age_info; 9513 struct mlx5_age_param *age_param; 9514 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9515 struct mlx5_counter_stats_raw *prev = pool->raw; 9516 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9517 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9518 uint16_t expected = AGE_CANDIDATE; 9519 uint32_t i; 9520 9521 pool->time_of_last_age_check = curr_time; 9522 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9523 cnt = MLX5_POOL_GET_CNT(pool, i); 9524 age_param = MLX5_CNT_TO_AGE(cnt); 9525 if (__atomic_load_n(&age_param->state, 9526 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9527 continue; 9528 if (cur->data[i].hits != prev->data[i].hits) { 9529 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9530 __ATOMIC_RELAXED); 9531 continue; 9532 } 9533 if (__atomic_fetch_add(&age_param->sec_since_last_hit, 9534 time_delta, 9535 __ATOMIC_RELAXED) + time_delta <= age_param->timeout) 9536 continue; 9537 /** 9538 * Hold the lock first, or if between the 9539 * state AGE_TMOUT and tailq operation the 9540 * release happened, the release procedure 9541 * may delete a non-existent tailq node. 9542 */ 9543 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9544 age_info = GET_PORT_AGE_INFO(priv); 9545 rte_spinlock_lock(&age_info->aged_sl); 9546 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9547 AGE_TMOUT, false, 9548 __ATOMIC_RELAXED, 9549 __ATOMIC_RELAXED)) { 9550 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9551 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9552 } 9553 rte_spinlock_unlock(&age_info->aged_sl); 9554 } 9555 mlx5_age_event_prepare(sh); 9556 } 9557 9558 /** 9559 * Handler for the HW respond about ready values from an asynchronous batch 9560 * query. This function is probably called by the host thread. 9561 * 9562 * @param[in] sh 9563 * The pointer to the shared device context. 9564 * @param[in] async_id 9565 * The Devx async ID. 9566 * @param[in] status 9567 * The status of the completion. 9568 */ 9569 void 9570 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9571 uint64_t async_id, int status) 9572 { 9573 struct mlx5_flow_counter_pool *pool = 9574 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9575 struct mlx5_counter_stats_raw *raw_to_free; 9576 uint8_t query_gen = pool->query_gen ^ 1; 9577 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9578 enum mlx5_counter_type cnt_type = 9579 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9580 MLX5_COUNTER_TYPE_ORIGIN; 9581 9582 if (unlikely(status)) { 9583 raw_to_free = pool->raw_hw; 9584 } else { 9585 raw_to_free = pool->raw; 9586 if (pool->is_aged) 9587 mlx5_flow_aging_check(sh, pool); 9588 rte_spinlock_lock(&pool->sl); 9589 pool->raw = pool->raw_hw; 9590 rte_spinlock_unlock(&pool->sl); 9591 /* Be sure the new raw counters data is updated in memory. */ 9592 rte_io_wmb(); 9593 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9594 rte_spinlock_lock(&cmng->csl[cnt_type]); 9595 TAILQ_CONCAT(&cmng->counters[cnt_type], 9596 &pool->counters[query_gen], next); 9597 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9598 } 9599 } 9600 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9601 pool->raw_hw = NULL; 9602 sh->sws_cmng.pending_queries--; 9603 } 9604 9605 static int 9606 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9607 const struct flow_grp_info *grp_info, 9608 struct rte_flow_error *error) 9609 { 9610 if (grp_info->transfer && grp_info->external && 9611 grp_info->fdb_def_rule) { 9612 if (group == UINT32_MAX) 9613 return rte_flow_error_set 9614 (error, EINVAL, 9615 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9616 NULL, 9617 "group index not supported"); 9618 *table = group + 1; 9619 } else { 9620 *table = group; 9621 } 9622 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9623 return 0; 9624 } 9625 9626 /** 9627 * Translate the rte_flow group index to HW table value. 9628 * 9629 * If tunnel offload is disabled, all group ids converted to flow table 9630 * id using the standard method. 9631 * If tunnel offload is enabled, group id can be converted using the 9632 * standard or tunnel conversion method. Group conversion method 9633 * selection depends on flags in `grp_info` parameter: 9634 * - Internal (grp_info.external == 0) groups conversion uses the 9635 * standard method. 9636 * - Group ids in JUMP action converted with the tunnel conversion. 9637 * - Group id in rule attribute conversion depends on a rule type and 9638 * group id value: 9639 * ** non zero group attributes converted with the tunnel method 9640 * ** zero group attribute in non-tunnel rule is converted using the 9641 * standard method - there's only one root table 9642 * ** zero group attribute in steer tunnel rule is converted with the 9643 * standard method - single root table 9644 * ** zero group attribute in match tunnel rule is a special OvS 9645 * case: that value is used for portability reasons. That group 9646 * id is converted with the tunnel conversion method. 9647 * 9648 * @param[in] dev 9649 * Port device 9650 * @param[in] tunnel 9651 * PMD tunnel offload object 9652 * @param[in] group 9653 * rte_flow group index value. 9654 * @param[out] table 9655 * HW table value. 9656 * @param[in] grp_info 9657 * flags used for conversion 9658 * @param[out] error 9659 * Pointer to error structure. 9660 * 9661 * @return 9662 * 0 on success, a negative errno value otherwise and rte_errno is set. 9663 */ 9664 int 9665 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9666 const struct mlx5_flow_tunnel *tunnel, 9667 uint32_t group, uint32_t *table, 9668 const struct flow_grp_info *grp_info, 9669 struct rte_flow_error *error) 9670 { 9671 int ret; 9672 bool standard_translation; 9673 9674 if (!grp_info->skip_scale && grp_info->external && 9675 group < MLX5_MAX_TABLES_EXTERNAL) 9676 group *= MLX5_FLOW_TABLE_FACTOR; 9677 if (is_tunnel_offload_active(dev)) { 9678 standard_translation = !grp_info->external || 9679 grp_info->std_tbl_fix; 9680 } else { 9681 standard_translation = true; 9682 } 9683 DRV_LOG(DEBUG, 9684 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9685 dev->data->port_id, group, grp_info->transfer, 9686 grp_info->external, grp_info->fdb_def_rule, 9687 standard_translation ? "STANDARD" : "TUNNEL"); 9688 if (standard_translation) 9689 ret = flow_group_to_table(dev->data->port_id, group, table, 9690 grp_info, error); 9691 else 9692 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9693 table, error); 9694 9695 return ret; 9696 } 9697 9698 /** 9699 * Discover availability of metadata reg_c's. 9700 * 9701 * Iteratively use test flows to check availability. 9702 * 9703 * @param[in] dev 9704 * Pointer to the Ethernet device structure. 9705 * 9706 * @return 9707 * 0 on success, a negative errno value otherwise and rte_errno is set. 9708 */ 9709 int 9710 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9711 { 9712 struct mlx5_priv *priv = dev->data->dev_private; 9713 enum modify_reg idx; 9714 int n = 0; 9715 9716 /* reg_c[0] and reg_c[1] are reserved. */ 9717 priv->sh->flow_mreg_c[n++] = REG_C_0; 9718 priv->sh->flow_mreg_c[n++] = REG_C_1; 9719 /* Discover availability of other reg_c's. */ 9720 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9721 struct rte_flow_attr attr = { 9722 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9723 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9724 .ingress = 1, 9725 }; 9726 struct rte_flow_item items[] = { 9727 [0] = { 9728 .type = RTE_FLOW_ITEM_TYPE_END, 9729 }, 9730 }; 9731 struct rte_flow_action actions[] = { 9732 [0] = { 9733 .type = (enum rte_flow_action_type) 9734 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9735 .conf = &(struct mlx5_flow_action_copy_mreg){ 9736 .src = REG_C_1, 9737 .dst = idx, 9738 }, 9739 }, 9740 [1] = { 9741 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9742 .conf = &(struct rte_flow_action_jump){ 9743 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9744 }, 9745 }, 9746 [2] = { 9747 .type = RTE_FLOW_ACTION_TYPE_END, 9748 }, 9749 }; 9750 uint32_t flow_idx; 9751 struct rte_flow *flow; 9752 struct rte_flow_error error; 9753 9754 if (!priv->sh->config.dv_flow_en) 9755 break; 9756 /* Create internal flow, validation skips copy action. */ 9757 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9758 items, actions, false, &error); 9759 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9760 flow_idx); 9761 if (!flow) 9762 continue; 9763 priv->sh->flow_mreg_c[n++] = idx; 9764 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9765 } 9766 for (; n < MLX5_MREG_C_NUM; ++n) 9767 priv->sh->flow_mreg_c[n] = REG_NON; 9768 priv->sh->metadata_regc_check_flag = 1; 9769 return 0; 9770 } 9771 9772 int 9773 save_dump_file(const uint8_t *data, uint32_t size, 9774 uint32_t type, uint64_t id, void *arg, FILE *file) 9775 { 9776 char line[BUF_SIZE]; 9777 uint32_t out = 0; 9778 uint32_t k; 9779 uint32_t actions_num; 9780 struct rte_flow_query_count *count; 9781 9782 memset(line, 0, BUF_SIZE); 9783 switch (type) { 9784 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9785 actions_num = *(uint32_t *)(arg); 9786 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9787 type, id, actions_num); 9788 break; 9789 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9790 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9791 type, id); 9792 break; 9793 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9794 count = (struct rte_flow_query_count *)arg; 9795 fprintf(file, 9796 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9797 type, id, count->hits, count->bytes); 9798 return 0; 9799 default: 9800 return -1; 9801 } 9802 9803 for (k = 0; k < size; k++) { 9804 /* Make sure we do not overrun the line buffer length. */ 9805 if (out >= BUF_SIZE - 4) { 9806 line[out] = '\0'; 9807 break; 9808 } 9809 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9810 (data[k]) & 0xff); 9811 } 9812 fprintf(file, "%s\n", line); 9813 return 0; 9814 } 9815 9816 int 9817 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9818 struct rte_flow_query_count *count, struct rte_flow_error *error) 9819 { 9820 struct rte_flow_action action[2]; 9821 enum mlx5_flow_drv_type ftype; 9822 const struct mlx5_flow_driver_ops *fops; 9823 9824 if (!flow) { 9825 return rte_flow_error_set(error, ENOENT, 9826 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9827 NULL, 9828 "invalid flow handle"); 9829 } 9830 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9831 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9832 if (flow->counter) { 9833 memset(count, 0, sizeof(struct rte_flow_query_count)); 9834 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9835 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9836 ftype < MLX5_FLOW_TYPE_MAX); 9837 fops = flow_get_drv_ops(ftype); 9838 return fops->query(dev, flow, action, count, error); 9839 } 9840 return -1; 9841 } 9842 9843 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9844 /** 9845 * Dump flow ipool data to file 9846 * 9847 * @param[in] dev 9848 * The pointer to Ethernet device. 9849 * @param[in] file 9850 * A pointer to a file for output. 9851 * @param[out] error 9852 * Perform verbose error reporting if not NULL. PMDs initialize this 9853 * structure in case of error only. 9854 * @return 9855 * 0 on success, a negative value otherwise. 9856 */ 9857 int 9858 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9859 struct rte_flow *flow, FILE *file, 9860 struct rte_flow_error *error) 9861 { 9862 struct mlx5_priv *priv = dev->data->dev_private; 9863 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9864 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9865 uint32_t handle_idx; 9866 struct mlx5_flow_handle *dh; 9867 struct rte_flow_query_count count; 9868 uint32_t actions_num; 9869 const uint8_t *data; 9870 size_t size; 9871 uint64_t id; 9872 uint32_t type; 9873 void *action = NULL; 9874 9875 if (!flow) { 9876 return rte_flow_error_set(error, ENOENT, 9877 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9878 NULL, 9879 "invalid flow handle"); 9880 } 9881 handle_idx = flow->dev_handles; 9882 /* query counter */ 9883 if (flow->counter && 9884 (!mlx5_counter_query(dev, flow->counter, false, 9885 &count.hits, &count.bytes, &action)) && action) { 9886 id = (uint64_t)(uintptr_t)action; 9887 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9888 save_dump_file(NULL, 0, type, 9889 id, (void *)&count, file); 9890 } 9891 9892 while (handle_idx) { 9893 dh = mlx5_ipool_get(priv->sh->ipool 9894 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9895 if (!dh) 9896 continue; 9897 handle_idx = dh->next.next; 9898 9899 /* Get modify_hdr and encap_decap buf from ipools. */ 9900 encap_decap = NULL; 9901 modify_hdr = dh->dvh.modify_hdr; 9902 9903 if (dh->dvh.rix_encap_decap) { 9904 encap_decap = mlx5_ipool_get(priv->sh->ipool 9905 [MLX5_IPOOL_DECAP_ENCAP], 9906 dh->dvh.rix_encap_decap); 9907 } 9908 if (modify_hdr) { 9909 data = (const uint8_t *)modify_hdr->actions; 9910 size = (size_t)(modify_hdr->actions_num) * 8; 9911 id = (uint64_t)(uintptr_t)modify_hdr->action; 9912 actions_num = modify_hdr->actions_num; 9913 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9914 save_dump_file(data, size, type, id, 9915 (void *)(&actions_num), file); 9916 } 9917 if (encap_decap) { 9918 data = encap_decap->buf; 9919 size = encap_decap->size; 9920 id = (uint64_t)(uintptr_t)encap_decap->action; 9921 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9922 save_dump_file(data, size, type, 9923 id, NULL, file); 9924 } 9925 } 9926 return 0; 9927 } 9928 9929 /** 9930 * Dump all flow's encap_decap/modify_hdr/counter data to file 9931 * 9932 * @param[in] dev 9933 * The pointer to Ethernet device. 9934 * @param[in] file 9935 * A pointer to a file for output. 9936 * @param[out] error 9937 * Perform verbose error reporting if not NULL. PMDs initialize this 9938 * structure in case of error only. 9939 * @return 9940 * 0 on success, a negative value otherwise. 9941 */ 9942 static int 9943 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9944 FILE *file, struct rte_flow_error *error __rte_unused) 9945 { 9946 struct mlx5_priv *priv = dev->data->dev_private; 9947 struct mlx5_dev_ctx_shared *sh = priv->sh; 9948 struct mlx5_hlist *h; 9949 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9950 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9951 struct rte_flow_query_count count; 9952 uint32_t actions_num; 9953 const uint8_t *data; 9954 size_t size; 9955 uint64_t id; 9956 uint32_t type; 9957 uint32_t i; 9958 uint32_t j; 9959 struct mlx5_list_inconst *l_inconst; 9960 struct mlx5_list_entry *e; 9961 int lcore_index; 9962 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9963 uint32_t max; 9964 void *action; 9965 9966 /* encap_decap hlist is lcore_share, get global core cache. */ 9967 i = MLX5_LIST_GLOBAL; 9968 h = sh->encaps_decaps; 9969 if (h) { 9970 for (j = 0; j <= h->mask; j++) { 9971 l_inconst = &h->buckets[j].l; 9972 if (!l_inconst || !l_inconst->cache[i]) 9973 continue; 9974 9975 e = LIST_FIRST(&l_inconst->cache[i]->h); 9976 while (e) { 9977 encap_decap = 9978 (struct mlx5_flow_dv_encap_decap_resource *)e; 9979 data = encap_decap->buf; 9980 size = encap_decap->size; 9981 id = (uint64_t)(uintptr_t)encap_decap->action; 9982 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9983 save_dump_file(data, size, type, 9984 id, NULL, file); 9985 e = LIST_NEXT(e, next); 9986 } 9987 } 9988 } 9989 9990 /* get modify_hdr */ 9991 h = sh->modify_cmds; 9992 if (h) { 9993 lcore_index = rte_lcore_index(rte_lcore_id()); 9994 if (unlikely(lcore_index == -1)) { 9995 lcore_index = MLX5_LIST_NLCORE; 9996 rte_spinlock_lock(&h->l_const.lcore_lock); 9997 } 9998 i = lcore_index; 9999 10000 if (lcore_index == MLX5_LIST_NLCORE) { 10001 for (i = 0; i <= (uint32_t)lcore_index; i++) { 10002 for (j = 0; j <= h->mask; j++) { 10003 l_inconst = &h->buckets[j].l; 10004 if (!l_inconst || !l_inconst->cache[i]) 10005 continue; 10006 10007 e = LIST_FIRST(&l_inconst->cache[i]->h); 10008 while (e) { 10009 modify_hdr = 10010 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10011 data = (const uint8_t *)modify_hdr->actions; 10012 size = (size_t)(modify_hdr->actions_num) * 8; 10013 actions_num = modify_hdr->actions_num; 10014 id = (uint64_t)(uintptr_t)modify_hdr->action; 10015 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10016 save_dump_file(data, size, type, id, 10017 (void *)(&actions_num), file); 10018 e = LIST_NEXT(e, next); 10019 } 10020 } 10021 } 10022 } else { 10023 for (j = 0; j <= h->mask; j++) { 10024 l_inconst = &h->buckets[j].l; 10025 if (!l_inconst || !l_inconst->cache[i]) 10026 continue; 10027 10028 e = LIST_FIRST(&l_inconst->cache[i]->h); 10029 while (e) { 10030 modify_hdr = 10031 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10032 data = (const uint8_t *)modify_hdr->actions; 10033 size = (size_t)(modify_hdr->actions_num) * 8; 10034 actions_num = modify_hdr->actions_num; 10035 id = (uint64_t)(uintptr_t)modify_hdr->action; 10036 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10037 save_dump_file(data, size, type, id, 10038 (void *)(&actions_num), file); 10039 e = LIST_NEXT(e, next); 10040 } 10041 } 10042 } 10043 10044 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10045 rte_spinlock_unlock(&h->l_const.lcore_lock); 10046 } 10047 10048 /* get counter */ 10049 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10050 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10051 for (j = 1; j <= max; j++) { 10052 action = NULL; 10053 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10054 &count.bytes, &action)) && action) { 10055 id = (uint64_t)(uintptr_t)action; 10056 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10057 save_dump_file(NULL, 0, type, 10058 id, (void *)&count, file); 10059 } 10060 } 10061 return 0; 10062 } 10063 #endif 10064 10065 /** 10066 * Dump flow raw hw data to file 10067 * 10068 * @param[in] dev 10069 * The pointer to Ethernet device. 10070 * @param[in] file 10071 * A pointer to a file for output. 10072 * @param[out] error 10073 * Perform verbose error reporting if not NULL. PMDs initialize this 10074 * structure in case of error only. 10075 * @return 10076 * 0 on success, a negative value otherwise. 10077 */ 10078 int 10079 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10080 FILE *file, 10081 struct rte_flow_error *error __rte_unused) 10082 { 10083 struct mlx5_priv *priv = dev->data->dev_private; 10084 struct mlx5_dev_ctx_shared *sh = priv->sh; 10085 uint32_t handle_idx; 10086 int ret; 10087 struct mlx5_flow_handle *dh; 10088 struct rte_flow *flow; 10089 10090 if (!sh->config.dv_flow_en) { 10091 if (fputs("device dv flow disabled\n", file) <= 0) 10092 return -errno; 10093 return -ENOTSUP; 10094 } 10095 10096 /* dump all */ 10097 if (!flow_idx) { 10098 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10099 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10100 return -EINVAL; 10101 10102 if (sh->config.dv_flow_en == 2) 10103 return mlx5dr_debug_dump(priv->dr_ctx, file); 10104 #endif 10105 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10106 sh->rx_domain, 10107 sh->tx_domain, file); 10108 } 10109 /* dump one */ 10110 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10111 (uintptr_t)(void *)flow_idx); 10112 if (!flow) 10113 return -EINVAL; 10114 10115 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10116 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10117 #endif 10118 handle_idx = flow->dev_handles; 10119 while (handle_idx) { 10120 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10121 handle_idx); 10122 if (!dh) 10123 return -ENOENT; 10124 if (dh->drv_flow) { 10125 if (sh->config.dv_flow_en == 2) 10126 return -ENOTSUP; 10127 10128 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10129 file); 10130 if (ret) 10131 return -ENOENT; 10132 } 10133 handle_idx = dh->next.next; 10134 } 10135 return 0; 10136 } 10137 10138 /** 10139 * Get aged-out flows. 10140 * 10141 * @param[in] dev 10142 * Pointer to the Ethernet device structure. 10143 * @param[in] context 10144 * The address of an array of pointers to the aged-out flows contexts. 10145 * @param[in] nb_countexts 10146 * The length of context array pointers. 10147 * @param[out] error 10148 * Perform verbose error reporting if not NULL. Initialized in case of 10149 * error only. 10150 * 10151 * @return 10152 * how many contexts get in success, otherwise negative errno value. 10153 * if nb_contexts is 0, return the amount of all aged contexts. 10154 * if nb_contexts is not 0 , return the amount of aged flows reported 10155 * in the context array. 10156 */ 10157 int 10158 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10159 uint32_t nb_contexts, struct rte_flow_error *error) 10160 { 10161 struct rte_flow_attr attr = { .transfer = 0 }; 10162 10163 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10164 (dev, contexts, nb_contexts, error); 10165 } 10166 10167 /** 10168 * Get aged-out flows per HWS queue. 10169 * 10170 * @param[in] dev 10171 * Pointer to the Ethernet device structure. 10172 * @param[in] queue_id 10173 * Flow queue to query. 10174 * @param[in] context 10175 * The address of an array of pointers to the aged-out flows contexts. 10176 * @param[in] nb_countexts 10177 * The length of context array pointers. 10178 * @param[out] error 10179 * Perform verbose error reporting if not NULL. Initialized in case of 10180 * error only. 10181 * 10182 * @return 10183 * how many contexts get in success, otherwise negative errno value. 10184 * if nb_contexts is 0, return the amount of all aged contexts. 10185 * if nb_contexts is not 0 , return the amount of aged flows reported 10186 * in the context array. 10187 */ 10188 int 10189 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10190 void **contexts, uint32_t nb_contexts, 10191 struct rte_flow_error *error) 10192 { 10193 const struct mlx5_flow_driver_ops *fops; 10194 struct rte_flow_attr attr = { 0 }; 10195 10196 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10197 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10198 return fops->get_q_aged_flows(dev, queue_id, contexts, 10199 nb_contexts, error); 10200 } 10201 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10202 dev->data->port_id, queue_id); 10203 return rte_flow_error_set(error, ENOTSUP, 10204 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10205 "get Q aged flows with incorrect steering mode"); 10206 } 10207 10208 /* Wrapper for driver action_validate op callback */ 10209 static int 10210 flow_drv_action_validate(struct rte_eth_dev *dev, 10211 const struct rte_flow_indir_action_conf *conf, 10212 const struct rte_flow_action *action, 10213 const struct mlx5_flow_driver_ops *fops, 10214 struct rte_flow_error *error) 10215 { 10216 static const char err_msg[] = "indirect action validation unsupported"; 10217 10218 if (!fops->action_validate) { 10219 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10220 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10221 NULL, err_msg); 10222 return -rte_errno; 10223 } 10224 return fops->action_validate(dev, conf, action, error); 10225 } 10226 10227 /** 10228 * Destroys the shared action by handle. 10229 * 10230 * @param dev 10231 * Pointer to Ethernet device structure. 10232 * @param[in] handle 10233 * Handle for the indirect action object to be destroyed. 10234 * @param[out] error 10235 * Perform verbose error reporting if not NULL. PMDs initialize this 10236 * structure in case of error only. 10237 * 10238 * @return 10239 * 0 on success, a negative errno value otherwise and rte_errno is set. 10240 * 10241 * @note: wrapper for driver action_create op callback. 10242 */ 10243 static int 10244 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10245 struct rte_flow_action_handle *handle, 10246 struct rte_flow_error *error) 10247 { 10248 static const char err_msg[] = "indirect action destruction unsupported"; 10249 struct rte_flow_attr attr = { .transfer = 0 }; 10250 const struct mlx5_flow_driver_ops *fops = 10251 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10252 10253 if (!fops->action_destroy) { 10254 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10255 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10256 NULL, err_msg); 10257 return -rte_errno; 10258 } 10259 return fops->action_destroy(dev, handle, error); 10260 } 10261 10262 /* Wrapper for driver action_destroy op callback */ 10263 static int 10264 flow_drv_action_update(struct rte_eth_dev *dev, 10265 struct rte_flow_action_handle *handle, 10266 const void *update, 10267 const struct mlx5_flow_driver_ops *fops, 10268 struct rte_flow_error *error) 10269 { 10270 static const char err_msg[] = "indirect action update unsupported"; 10271 10272 if (!fops->action_update) { 10273 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10274 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10275 NULL, err_msg); 10276 return -rte_errno; 10277 } 10278 return fops->action_update(dev, handle, update, error); 10279 } 10280 10281 /* Wrapper for driver action_destroy op callback */ 10282 static int 10283 flow_drv_action_query(struct rte_eth_dev *dev, 10284 const struct rte_flow_action_handle *handle, 10285 void *data, 10286 const struct mlx5_flow_driver_ops *fops, 10287 struct rte_flow_error *error) 10288 { 10289 static const char err_msg[] = "indirect action query unsupported"; 10290 10291 if (!fops->action_query) { 10292 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10293 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10294 NULL, err_msg); 10295 return -rte_errno; 10296 } 10297 return fops->action_query(dev, handle, data, error); 10298 } 10299 10300 /** 10301 * Create indirect action for reuse in multiple flow rules. 10302 * 10303 * @param dev 10304 * Pointer to Ethernet device structure. 10305 * @param conf 10306 * Pointer to indirect action object configuration. 10307 * @param[in] action 10308 * Action configuration for indirect action object creation. 10309 * @param[out] error 10310 * Perform verbose error reporting if not NULL. PMDs initialize this 10311 * structure in case of error only. 10312 * @return 10313 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10314 */ 10315 static struct rte_flow_action_handle * 10316 mlx5_action_handle_create(struct rte_eth_dev *dev, 10317 const struct rte_flow_indir_action_conf *conf, 10318 const struct rte_flow_action *action, 10319 struct rte_flow_error *error) 10320 { 10321 static const char err_msg[] = "indirect action creation unsupported"; 10322 struct rte_flow_attr attr = { .transfer = 0 }; 10323 const struct mlx5_flow_driver_ops *fops = 10324 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10325 10326 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10327 return NULL; 10328 if (!fops->action_create) { 10329 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10330 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10331 NULL, err_msg); 10332 return NULL; 10333 } 10334 return fops->action_create(dev, conf, action, error); 10335 } 10336 10337 /** 10338 * Updates inplace the indirect action configuration pointed by *handle* 10339 * with the configuration provided as *update* argument. 10340 * The update of the indirect action configuration effects all flow rules 10341 * reusing the action via handle. 10342 * 10343 * @param dev 10344 * Pointer to Ethernet device structure. 10345 * @param[in] handle 10346 * Handle for the indirect action to be updated. 10347 * @param[in] update 10348 * Action specification used to modify the action pointed by handle. 10349 * *update* could be of same type with the action pointed by the *handle* 10350 * handle argument, or some other structures like a wrapper, depending on 10351 * the indirect action type. 10352 * @param[out] error 10353 * Perform verbose error reporting if not NULL. PMDs initialize this 10354 * structure in case of error only. 10355 * 10356 * @return 10357 * 0 on success, a negative errno value otherwise and rte_errno is set. 10358 */ 10359 static int 10360 mlx5_action_handle_update(struct rte_eth_dev *dev, 10361 struct rte_flow_action_handle *handle, 10362 const void *update, 10363 struct rte_flow_error *error) 10364 { 10365 struct rte_flow_attr attr = { .transfer = 0 }; 10366 const struct mlx5_flow_driver_ops *fops = 10367 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10368 int ret; 10369 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 10370 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 10371 10372 switch (type) { 10373 case MLX5_INDIRECT_ACTION_TYPE_CT: 10374 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 10375 ret = 0; 10376 break; 10377 default: 10378 ret = flow_drv_action_validate(dev, NULL, 10379 (const struct rte_flow_action *)update, 10380 fops, error); 10381 } 10382 if (ret) 10383 return ret; 10384 return flow_drv_action_update(dev, handle, update, fops, 10385 error); 10386 } 10387 10388 /** 10389 * Query the indirect action by handle. 10390 * 10391 * This function allows retrieving action-specific data such as counters. 10392 * Data is gathered by special action which may be present/referenced in 10393 * more than one flow rule definition. 10394 * 10395 * see @RTE_FLOW_ACTION_TYPE_COUNT 10396 * 10397 * @param dev 10398 * Pointer to Ethernet device structure. 10399 * @param[in] handle 10400 * Handle for the indirect action to query. 10401 * @param[in, out] data 10402 * Pointer to storage for the associated query data type. 10403 * @param[out] error 10404 * Perform verbose error reporting if not NULL. PMDs initialize this 10405 * structure in case of error only. 10406 * 10407 * @return 10408 * 0 on success, a negative errno value otherwise and rte_errno is set. 10409 */ 10410 static int 10411 mlx5_action_handle_query(struct rte_eth_dev *dev, 10412 const struct rte_flow_action_handle *handle, 10413 void *data, 10414 struct rte_flow_error *error) 10415 { 10416 struct rte_flow_attr attr = { .transfer = 0 }; 10417 const struct mlx5_flow_driver_ops *fops = 10418 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10419 10420 return flow_drv_action_query(dev, handle, data, fops, error); 10421 } 10422 10423 static int 10424 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10425 struct rte_flow_action_handle *handle, 10426 const void *update, void *query, 10427 enum rte_flow_query_update_mode qu_mode, 10428 struct rte_flow_error *error) 10429 { 10430 struct rte_flow_attr attr = { .transfer = 0 }; 10431 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10432 const struct mlx5_flow_driver_ops *fops; 10433 10434 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10435 return rte_flow_error_set(error, ENOTSUP, 10436 RTE_FLOW_ERROR_TYPE_ACTION, 10437 NULL, "invalid driver type"); 10438 fops = flow_get_drv_ops(drv_type); 10439 if (!fops || !fops->action_query_update) 10440 return rte_flow_error_set(error, ENOTSUP, 10441 RTE_FLOW_ERROR_TYPE_ACTION, 10442 NULL, "no query_update handler"); 10443 return fops->action_query_update(dev, handle, update, 10444 query, qu_mode, error); 10445 } 10446 10447 10448 #define MLX5_DRV_FOPS_OR_ERR(dev, fops, drv_cb, ret) \ 10449 { \ 10450 struct rte_flow_attr attr = { .transfer = 0 }; \ 10451 enum mlx5_flow_drv_type drv_type = flow_get_drv_type((dev), &attr); \ 10452 if (drv_type == MLX5_FLOW_TYPE_MIN || \ 10453 drv_type == MLX5_FLOW_TYPE_MAX) { \ 10454 rte_flow_error_set(error, ENOTSUP, \ 10455 RTE_FLOW_ERROR_TYPE_ACTION, \ 10456 NULL, "invalid driver type"); \ 10457 return ret; \ 10458 } \ 10459 (fops) = flow_get_drv_ops(drv_type); \ 10460 if (!(fops) || !(fops)->drv_cb) { \ 10461 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, \ 10462 NULL, "no action_list handler"); \ 10463 return ret; \ 10464 } \ 10465 } 10466 10467 static struct rte_flow_action_list_handle * 10468 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 10469 const struct rte_flow_indir_action_conf *conf, 10470 const struct rte_flow_action *actions, 10471 struct rte_flow_error *error) 10472 { 10473 const struct mlx5_flow_driver_ops *fops; 10474 10475 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_create, NULL); 10476 return fops->action_list_handle_create(dev, conf, actions, error); 10477 } 10478 10479 static int 10480 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 10481 struct rte_flow_action_list_handle *handle, 10482 struct rte_flow_error *error) 10483 { 10484 const struct mlx5_flow_driver_ops *fops; 10485 10486 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_destroy, ENOTSUP); 10487 return fops->action_list_handle_destroy(dev, handle, error); 10488 } 10489 10490 static int 10491 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 10492 const 10493 struct rte_flow_action_list_handle *handle, 10494 const void **update, void **query, 10495 enum rte_flow_query_update_mode mode, 10496 struct rte_flow_error *error) 10497 { 10498 const struct mlx5_flow_driver_ops *fops; 10499 10500 MLX5_DRV_FOPS_OR_ERR(dev, fops, 10501 action_list_handle_query_update, ENOTSUP); 10502 return fops->action_list_handle_query_update(dev, handle, update, query, 10503 mode, error); 10504 } 10505 static int 10506 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 10507 const struct rte_flow_template_table *table, 10508 const struct rte_flow_item pattern[], 10509 uint8_t pattern_template_index, 10510 uint32_t *hash, struct rte_flow_error *error) 10511 { 10512 struct rte_flow_attr attr = { .transfer = 0 }; 10513 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10514 const struct mlx5_flow_driver_ops *fops; 10515 10516 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10517 return rte_flow_error_set(error, ENOTSUP, 10518 RTE_FLOW_ERROR_TYPE_ACTION, 10519 NULL, "invalid driver type"); 10520 fops = flow_get_drv_ops(drv_type); 10521 if (!fops || !fops->action_query_update) 10522 return rte_flow_error_set(error, ENOTSUP, 10523 RTE_FLOW_ERROR_TYPE_ACTION, 10524 NULL, "no query_update handler"); 10525 return fops->flow_calc_table_hash(dev, table, pattern, pattern_template_index, 10526 hash, error); 10527 } 10528 10529 static int 10530 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 10531 const struct rte_flow_item pattern[], 10532 enum rte_flow_encap_hash_field dest_field, 10533 uint8_t *hash, 10534 struct rte_flow_error *error) 10535 { 10536 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, NULL); 10537 const struct mlx5_flow_driver_ops *fops; 10538 10539 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10540 return rte_flow_error_set(error, ENOTSUP, 10541 RTE_FLOW_ERROR_TYPE_ACTION, 10542 NULL, "invalid driver type"); 10543 fops = flow_get_drv_ops(drv_type); 10544 if (!fops || !fops->flow_calc_encap_hash) 10545 return rte_flow_error_set(error, ENOTSUP, 10546 RTE_FLOW_ERROR_TYPE_ACTION, 10547 NULL, "no calc encap hash handler"); 10548 return fops->flow_calc_encap_hash(dev, pattern, dest_field, hash, error); 10549 } 10550 10551 /** 10552 * Destroy all indirect actions (shared RSS). 10553 * 10554 * @param dev 10555 * Pointer to Ethernet device. 10556 * 10557 * @return 10558 * 0 on success, a negative errno value otherwise and rte_errno is set. 10559 */ 10560 int 10561 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10562 { 10563 struct rte_flow_error error; 10564 struct mlx5_priv *priv = dev->data->dev_private; 10565 struct mlx5_shared_action_rss *shared_rss; 10566 int ret = 0; 10567 uint32_t idx; 10568 10569 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10570 priv->rss_shared_actions, idx, shared_rss, next) { 10571 ret |= mlx5_action_handle_destroy(dev, 10572 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10573 } 10574 return ret; 10575 } 10576 10577 /** 10578 * Validate existing indirect actions against current device configuration 10579 * and attach them to device resources. 10580 * 10581 * @param dev 10582 * Pointer to Ethernet device. 10583 * 10584 * @return 10585 * 0 on success, a negative errno value otherwise and rte_errno is set. 10586 */ 10587 int 10588 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10589 { 10590 struct mlx5_priv *priv = dev->data->dev_private; 10591 int ret = 0; 10592 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10593 10594 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10595 const char *message; 10596 uint32_t queue_idx; 10597 10598 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10599 ind_tbl->queues_n, 10600 &message, &queue_idx); 10601 if (ret != 0) { 10602 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10603 dev->data->port_id, ind_tbl->queues[queue_idx], 10604 message); 10605 break; 10606 } 10607 } 10608 if (ret != 0) 10609 return ret; 10610 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10611 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10612 if (ret != 0) { 10613 DRV_LOG(ERR, "Port %u could not attach " 10614 "indirection table obj %p", 10615 dev->data->port_id, (void *)ind_tbl); 10616 goto error; 10617 } 10618 } 10619 10620 return 0; 10621 error: 10622 ind_tbl_last = ind_tbl; 10623 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10624 if (ind_tbl == ind_tbl_last) 10625 break; 10626 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10627 DRV_LOG(CRIT, "Port %u could not detach " 10628 "indirection table obj %p on rollback", 10629 dev->data->port_id, (void *)ind_tbl); 10630 } 10631 return ret; 10632 } 10633 10634 /** 10635 * Detach indirect actions of the device from its resources. 10636 * 10637 * @param dev 10638 * Pointer to Ethernet device. 10639 * 10640 * @return 10641 * 0 on success, a negative errno value otherwise and rte_errno is set. 10642 */ 10643 int 10644 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10645 { 10646 struct mlx5_priv *priv = dev->data->dev_private; 10647 int ret = 0; 10648 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10649 10650 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10651 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10652 if (ret != 0) { 10653 DRV_LOG(ERR, "Port %u could not detach " 10654 "indirection table obj %p", 10655 dev->data->port_id, (void *)ind_tbl); 10656 goto error; 10657 } 10658 } 10659 return 0; 10660 error: 10661 ind_tbl_last = ind_tbl; 10662 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10663 if (ind_tbl == ind_tbl_last) 10664 break; 10665 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10666 DRV_LOG(CRIT, "Port %u could not attach " 10667 "indirection table obj %p on rollback", 10668 dev->data->port_id, (void *)ind_tbl); 10669 } 10670 return ret; 10671 } 10672 10673 #ifndef HAVE_MLX5DV_DR 10674 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10675 #else 10676 #define MLX5_DOMAIN_SYNC_FLOW \ 10677 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10678 #endif 10679 10680 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10681 { 10682 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10683 const struct mlx5_flow_driver_ops *fops; 10684 int ret; 10685 struct rte_flow_attr attr = { .transfer = 0 }; 10686 10687 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10688 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10689 if (ret > 0) 10690 ret = -ret; 10691 return ret; 10692 } 10693 10694 const struct mlx5_flow_tunnel * 10695 mlx5_get_tof(const struct rte_flow_item *item, 10696 const struct rte_flow_action *action, 10697 enum mlx5_tof_rule_type *rule_type) 10698 { 10699 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10700 if (item->type == (typeof(item->type)) 10701 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10702 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10703 return flow_items_to_tunnel(item); 10704 } 10705 } 10706 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10707 if (action->type == (typeof(action->type)) 10708 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10709 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10710 return flow_actions_to_tunnel(action); 10711 } 10712 } 10713 return NULL; 10714 } 10715 10716 /** 10717 * tunnel offload functionality is defined for DV environment only 10718 */ 10719 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10720 __extension__ 10721 union tunnel_offload_mark { 10722 uint32_t val; 10723 struct { 10724 uint32_t app_reserve:8; 10725 uint32_t table_id:15; 10726 uint32_t transfer:1; 10727 uint32_t _unused_:8; 10728 }; 10729 }; 10730 10731 static bool 10732 mlx5_access_tunnel_offload_db 10733 (struct rte_eth_dev *dev, 10734 bool (*match)(struct rte_eth_dev *, 10735 struct mlx5_flow_tunnel *, const void *), 10736 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10737 void (*miss)(struct rte_eth_dev *, void *), 10738 void *ctx, bool lock_op); 10739 10740 static int 10741 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10742 struct rte_flow *flow, 10743 const struct rte_flow_attr *attr, 10744 const struct rte_flow_action *app_actions, 10745 uint32_t flow_idx, 10746 const struct mlx5_flow_tunnel *tunnel, 10747 struct tunnel_default_miss_ctx *ctx, 10748 struct rte_flow_error *error) 10749 { 10750 struct mlx5_priv *priv = dev->data->dev_private; 10751 struct mlx5_flow *dev_flow; 10752 struct rte_flow_attr miss_attr = *attr; 10753 const struct rte_flow_item miss_items[2] = { 10754 { 10755 .type = RTE_FLOW_ITEM_TYPE_ETH, 10756 .spec = NULL, 10757 .last = NULL, 10758 .mask = NULL 10759 }, 10760 { 10761 .type = RTE_FLOW_ITEM_TYPE_END, 10762 .spec = NULL, 10763 .last = NULL, 10764 .mask = NULL 10765 } 10766 }; 10767 union tunnel_offload_mark mark_id; 10768 struct rte_flow_action_mark miss_mark; 10769 struct rte_flow_action miss_actions[3] = { 10770 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10771 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10772 }; 10773 const struct rte_flow_action_jump *jump_data; 10774 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10775 struct flow_grp_info grp_info = { 10776 .external = 1, 10777 .transfer = attr->transfer, 10778 .fdb_def_rule = !!priv->fdb_def_rule, 10779 .std_tbl_fix = 0, 10780 }; 10781 int ret; 10782 10783 if (!attr->transfer) { 10784 uint32_t q_size; 10785 10786 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10787 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10788 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10789 0, SOCKET_ID_ANY); 10790 if (!ctx->queue) 10791 return rte_flow_error_set 10792 (error, ENOMEM, 10793 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10794 NULL, "invalid default miss RSS"); 10795 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10796 ctx->action_rss.level = 0, 10797 ctx->action_rss.types = priv->rss_conf.rss_hf, 10798 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10799 ctx->action_rss.queue_num = priv->reta_idx_n, 10800 ctx->action_rss.key = priv->rss_conf.rss_key, 10801 ctx->action_rss.queue = ctx->queue; 10802 if (!priv->reta_idx_n || !priv->rxqs_n) 10803 return rte_flow_error_set 10804 (error, EINVAL, 10805 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10806 NULL, "invalid port configuration"); 10807 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10808 ctx->action_rss.types = 0; 10809 for (i = 0; i != priv->reta_idx_n; ++i) 10810 ctx->queue[i] = (*priv->reta_idx)[i]; 10811 } else { 10812 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10813 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10814 } 10815 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10816 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10817 jump_data = app_actions->conf; 10818 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10819 miss_attr.group = jump_data->group; 10820 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10821 &flow_table, &grp_info, error); 10822 if (ret) 10823 return rte_flow_error_set(error, EINVAL, 10824 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10825 NULL, "invalid tunnel id"); 10826 mark_id.app_reserve = 0; 10827 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10828 mark_id.transfer = !!attr->transfer; 10829 mark_id._unused_ = 0; 10830 miss_mark.id = mark_id.val; 10831 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10832 miss_items, miss_actions, flow_idx, error); 10833 if (!dev_flow) 10834 return -rte_errno; 10835 dev_flow->flow = flow; 10836 dev_flow->external = true; 10837 dev_flow->tunnel = tunnel; 10838 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10839 /* Subflow object was created, we must include one in the list. */ 10840 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10841 dev_flow->handle, next); 10842 DRV_LOG(DEBUG, 10843 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10844 dev->data->port_id, tunnel->app_tunnel.type, 10845 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10846 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10847 miss_actions, error); 10848 if (!ret) 10849 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10850 error); 10851 10852 return ret; 10853 } 10854 10855 static const struct mlx5_flow_tbl_data_entry * 10856 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10857 { 10858 struct mlx5_priv *priv = dev->data->dev_private; 10859 struct mlx5_dev_ctx_shared *sh = priv->sh; 10860 struct mlx5_list_entry *he; 10861 union tunnel_offload_mark mbits = { .val = mark }; 10862 union mlx5_flow_tbl_key table_key = { 10863 { 10864 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10865 .id = 0, 10866 .reserved = 0, 10867 .dummy = 0, 10868 .is_fdb = !!mbits.transfer, 10869 .is_egress = 0, 10870 } 10871 }; 10872 struct mlx5_flow_cb_ctx ctx = { 10873 .data = &table_key.v64, 10874 }; 10875 10876 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10877 return he ? 10878 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10879 } 10880 10881 static void 10882 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10883 struct mlx5_list_entry *entry) 10884 { 10885 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10886 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10887 10888 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10889 tunnel_flow_tbl_to_id(tte->flow_table)); 10890 mlx5_free(tte); 10891 } 10892 10893 static int 10894 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10895 struct mlx5_list_entry *entry, void *cb_ctx) 10896 { 10897 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10898 union tunnel_tbl_key tbl = { 10899 .val = *(uint64_t *)(ctx->data), 10900 }; 10901 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10902 10903 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10904 } 10905 10906 static struct mlx5_list_entry * 10907 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10908 { 10909 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10910 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10911 struct tunnel_tbl_entry *tte; 10912 union tunnel_tbl_key tbl = { 10913 .val = *(uint64_t *)(ctx->data), 10914 }; 10915 10916 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10917 sizeof(*tte), 0, 10918 SOCKET_ID_ANY); 10919 if (!tte) 10920 goto err; 10921 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10922 &tte->flow_table); 10923 if (tte->flow_table >= MLX5_MAX_TABLES) { 10924 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10925 tte->flow_table); 10926 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10927 tte->flow_table); 10928 goto err; 10929 } else if (!tte->flow_table) { 10930 goto err; 10931 } 10932 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10933 tte->tunnel_id = tbl.tunnel_id; 10934 tte->group = tbl.group; 10935 return &tte->hash; 10936 err: 10937 if (tte) 10938 mlx5_free(tte); 10939 return NULL; 10940 } 10941 10942 static struct mlx5_list_entry * 10943 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10944 struct mlx5_list_entry *oentry, 10945 void *cb_ctx __rte_unused) 10946 { 10947 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10948 0, SOCKET_ID_ANY); 10949 10950 if (!tte) 10951 return NULL; 10952 memcpy(tte, oentry, sizeof(*tte)); 10953 return &tte->hash; 10954 } 10955 10956 static void 10957 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 10958 struct mlx5_list_entry *entry) 10959 { 10960 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10961 10962 mlx5_free(tte); 10963 } 10964 10965 static uint32_t 10966 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 10967 const struct mlx5_flow_tunnel *tunnel, 10968 uint32_t group, uint32_t *table, 10969 struct rte_flow_error *error) 10970 { 10971 struct mlx5_list_entry *he; 10972 struct tunnel_tbl_entry *tte; 10973 union tunnel_tbl_key key = { 10974 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 10975 .group = group 10976 }; 10977 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 10978 struct mlx5_hlist *group_hash; 10979 struct mlx5_flow_cb_ctx ctx = { 10980 .data = &key.val, 10981 }; 10982 10983 group_hash = tunnel ? tunnel->groups : thub->groups; 10984 he = mlx5_hlist_register(group_hash, key.val, &ctx); 10985 if (!he) 10986 return rte_flow_error_set(error, EINVAL, 10987 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10988 NULL, 10989 "tunnel group index not supported"); 10990 tte = container_of(he, typeof(*tte), hash); 10991 *table = tte->flow_table; 10992 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 10993 dev->data->port_id, key.tunnel_id, group, *table); 10994 return 0; 10995 } 10996 10997 static void 10998 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 10999 struct mlx5_flow_tunnel *tunnel) 11000 { 11001 struct mlx5_priv *priv = dev->data->dev_private; 11002 struct mlx5_indexed_pool *ipool; 11003 11004 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11005 dev->data->port_id, tunnel->tunnel_id); 11006 LIST_REMOVE(tunnel, chain); 11007 mlx5_hlist_destroy(tunnel->groups); 11008 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11009 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11010 } 11011 11012 static bool 11013 mlx5_access_tunnel_offload_db 11014 (struct rte_eth_dev *dev, 11015 bool (*match)(struct rte_eth_dev *, 11016 struct mlx5_flow_tunnel *, const void *), 11017 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11018 void (*miss)(struct rte_eth_dev *, void *), 11019 void *ctx, bool lock_op) 11020 { 11021 bool verdict = false; 11022 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11023 struct mlx5_flow_tunnel *tunnel; 11024 11025 rte_spinlock_lock(&thub->sl); 11026 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11027 verdict = match(dev, tunnel, (const void *)ctx); 11028 if (verdict) 11029 break; 11030 } 11031 if (!lock_op) 11032 rte_spinlock_unlock(&thub->sl); 11033 if (verdict && hit) 11034 hit(dev, tunnel, ctx); 11035 if (!verdict && miss) 11036 miss(dev, ctx); 11037 if (lock_op) 11038 rte_spinlock_unlock(&thub->sl); 11039 11040 return verdict; 11041 } 11042 11043 struct tunnel_db_find_tunnel_id_ctx { 11044 uint32_t tunnel_id; 11045 struct mlx5_flow_tunnel *tunnel; 11046 }; 11047 11048 static bool 11049 find_tunnel_id_match(struct rte_eth_dev *dev, 11050 struct mlx5_flow_tunnel *tunnel, const void *x) 11051 { 11052 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11053 11054 RTE_SET_USED(dev); 11055 return tunnel->tunnel_id == ctx->tunnel_id; 11056 } 11057 11058 static void 11059 find_tunnel_id_hit(struct rte_eth_dev *dev, 11060 struct mlx5_flow_tunnel *tunnel, void *x) 11061 { 11062 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11063 RTE_SET_USED(dev); 11064 ctx->tunnel = tunnel; 11065 } 11066 11067 static struct mlx5_flow_tunnel * 11068 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11069 { 11070 struct tunnel_db_find_tunnel_id_ctx ctx = { 11071 .tunnel_id = id, 11072 }; 11073 11074 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11075 find_tunnel_id_hit, NULL, &ctx, true); 11076 11077 return ctx.tunnel; 11078 } 11079 11080 static struct mlx5_flow_tunnel * 11081 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11082 const struct rte_flow_tunnel *app_tunnel) 11083 { 11084 struct mlx5_priv *priv = dev->data->dev_private; 11085 struct mlx5_indexed_pool *ipool; 11086 struct mlx5_flow_tunnel *tunnel; 11087 uint32_t id; 11088 11089 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11090 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11091 if (!tunnel) 11092 return NULL; 11093 if (id >= MLX5_MAX_TUNNELS) { 11094 mlx5_ipool_free(ipool, id); 11095 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11096 return NULL; 11097 } 11098 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11099 priv->sh, 11100 mlx5_flow_tunnel_grp2tbl_create_cb, 11101 mlx5_flow_tunnel_grp2tbl_match_cb, 11102 mlx5_flow_tunnel_grp2tbl_remove_cb, 11103 mlx5_flow_tunnel_grp2tbl_clone_cb, 11104 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11105 if (!tunnel->groups) { 11106 mlx5_ipool_free(ipool, id); 11107 return NULL; 11108 } 11109 /* initiate new PMD tunnel */ 11110 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11111 tunnel->tunnel_id = id; 11112 tunnel->action.type = (typeof(tunnel->action.type)) 11113 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11114 tunnel->action.conf = tunnel; 11115 tunnel->item.type = (typeof(tunnel->item.type)) 11116 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11117 tunnel->item.spec = tunnel; 11118 tunnel->item.last = NULL; 11119 tunnel->item.mask = NULL; 11120 11121 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11122 dev->data->port_id, tunnel->tunnel_id); 11123 11124 return tunnel; 11125 } 11126 11127 struct tunnel_db_get_tunnel_ctx { 11128 const struct rte_flow_tunnel *app_tunnel; 11129 struct mlx5_flow_tunnel *tunnel; 11130 }; 11131 11132 static bool get_tunnel_match(struct rte_eth_dev *dev, 11133 struct mlx5_flow_tunnel *tunnel, const void *x) 11134 { 11135 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11136 11137 RTE_SET_USED(dev); 11138 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11139 sizeof(*ctx->app_tunnel)); 11140 } 11141 11142 static void get_tunnel_hit(struct rte_eth_dev *dev, 11143 struct mlx5_flow_tunnel *tunnel, void *x) 11144 { 11145 /* called under tunnel spinlock protection */ 11146 struct tunnel_db_get_tunnel_ctx *ctx = x; 11147 11148 RTE_SET_USED(dev); 11149 tunnel->refctn++; 11150 ctx->tunnel = tunnel; 11151 } 11152 11153 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11154 { 11155 /* called under tunnel spinlock protection */ 11156 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11157 struct tunnel_db_get_tunnel_ctx *ctx = x; 11158 11159 rte_spinlock_unlock(&thub->sl); 11160 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11161 rte_spinlock_lock(&thub->sl); 11162 if (ctx->tunnel) { 11163 ctx->tunnel->refctn = 1; 11164 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11165 } 11166 } 11167 11168 11169 static int 11170 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11171 const struct rte_flow_tunnel *app_tunnel, 11172 struct mlx5_flow_tunnel **tunnel) 11173 { 11174 struct tunnel_db_get_tunnel_ctx ctx = { 11175 .app_tunnel = app_tunnel, 11176 }; 11177 11178 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11179 get_tunnel_miss, &ctx, true); 11180 *tunnel = ctx.tunnel; 11181 return ctx.tunnel ? 0 : -ENOMEM; 11182 } 11183 11184 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11185 { 11186 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11187 11188 if (!thub) 11189 return; 11190 if (!LIST_EMPTY(&thub->tunnels)) 11191 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11192 mlx5_hlist_destroy(thub->groups); 11193 mlx5_free(thub); 11194 } 11195 11196 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11197 { 11198 int err; 11199 struct mlx5_flow_tunnel_hub *thub; 11200 11201 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11202 0, SOCKET_ID_ANY); 11203 if (!thub) 11204 return -ENOMEM; 11205 LIST_INIT(&thub->tunnels); 11206 rte_spinlock_init(&thub->sl); 11207 thub->groups = mlx5_hlist_create("flow groups", 64, 11208 false, true, sh, 11209 mlx5_flow_tunnel_grp2tbl_create_cb, 11210 mlx5_flow_tunnel_grp2tbl_match_cb, 11211 mlx5_flow_tunnel_grp2tbl_remove_cb, 11212 mlx5_flow_tunnel_grp2tbl_clone_cb, 11213 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11214 if (!thub->groups) { 11215 err = -rte_errno; 11216 goto err; 11217 } 11218 sh->tunnel_hub = thub; 11219 11220 return 0; 11221 11222 err: 11223 if (thub->groups) 11224 mlx5_hlist_destroy(thub->groups); 11225 if (thub) 11226 mlx5_free(thub); 11227 return err; 11228 } 11229 11230 static inline int 11231 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11232 struct rte_flow_tunnel *tunnel, 11233 struct rte_flow_error *error) 11234 { 11235 struct mlx5_priv *priv = dev->data->dev_private; 11236 11237 if (!priv->sh->config.dv_flow_en) 11238 return rte_flow_error_set(error, ENOTSUP, 11239 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11240 "flow DV interface is off"); 11241 if (!is_tunnel_offload_active(dev)) 11242 return rte_flow_error_set(error, ENOTSUP, 11243 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11244 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11245 if (!tunnel) 11246 return rte_flow_error_set(error, EINVAL, 11247 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11248 "no application tunnel"); 11249 switch (tunnel->type) { 11250 default: 11251 return rte_flow_error_set(error, EINVAL, 11252 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11253 "unsupported tunnel type"); 11254 case RTE_FLOW_ITEM_TYPE_VXLAN: 11255 case RTE_FLOW_ITEM_TYPE_GRE: 11256 case RTE_FLOW_ITEM_TYPE_NVGRE: 11257 case RTE_FLOW_ITEM_TYPE_GENEVE: 11258 break; 11259 } 11260 return 0; 11261 } 11262 11263 static int 11264 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11265 struct rte_flow_tunnel *app_tunnel, 11266 struct rte_flow_action **actions, 11267 uint32_t *num_of_actions, 11268 struct rte_flow_error *error) 11269 { 11270 struct mlx5_flow_tunnel *tunnel; 11271 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11272 11273 if (ret) 11274 return ret; 11275 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11276 if (ret < 0) { 11277 return rte_flow_error_set(error, ret, 11278 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11279 "failed to initialize pmd tunnel"); 11280 } 11281 *actions = &tunnel->action; 11282 *num_of_actions = 1; 11283 return 0; 11284 } 11285 11286 static int 11287 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11288 struct rte_flow_tunnel *app_tunnel, 11289 struct rte_flow_item **items, 11290 uint32_t *num_of_items, 11291 struct rte_flow_error *error) 11292 { 11293 struct mlx5_flow_tunnel *tunnel; 11294 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11295 11296 if (ret) 11297 return ret; 11298 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11299 if (ret < 0) { 11300 return rte_flow_error_set(error, ret, 11301 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11302 "failed to initialize pmd tunnel"); 11303 } 11304 *items = &tunnel->item; 11305 *num_of_items = 1; 11306 return 0; 11307 } 11308 11309 struct tunnel_db_element_release_ctx { 11310 struct rte_flow_item *items; 11311 struct rte_flow_action *actions; 11312 uint32_t num_elements; 11313 struct rte_flow_error *error; 11314 int ret; 11315 }; 11316 11317 static bool 11318 tunnel_element_release_match(struct rte_eth_dev *dev, 11319 struct mlx5_flow_tunnel *tunnel, const void *x) 11320 { 11321 const struct tunnel_db_element_release_ctx *ctx = x; 11322 11323 RTE_SET_USED(dev); 11324 if (ctx->num_elements != 1) 11325 return false; 11326 else if (ctx->items) 11327 return ctx->items == &tunnel->item; 11328 else if (ctx->actions) 11329 return ctx->actions == &tunnel->action; 11330 11331 return false; 11332 } 11333 11334 static void 11335 tunnel_element_release_hit(struct rte_eth_dev *dev, 11336 struct mlx5_flow_tunnel *tunnel, void *x) 11337 { 11338 struct tunnel_db_element_release_ctx *ctx = x; 11339 ctx->ret = 0; 11340 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 11341 mlx5_flow_tunnel_free(dev, tunnel); 11342 } 11343 11344 static void 11345 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11346 { 11347 struct tunnel_db_element_release_ctx *ctx = x; 11348 RTE_SET_USED(dev); 11349 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11350 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11351 "invalid argument"); 11352 } 11353 11354 static int 11355 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11356 struct rte_flow_item *pmd_items, 11357 uint32_t num_items, struct rte_flow_error *err) 11358 { 11359 struct tunnel_db_element_release_ctx ctx = { 11360 .items = pmd_items, 11361 .actions = NULL, 11362 .num_elements = num_items, 11363 .error = err, 11364 }; 11365 11366 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11367 tunnel_element_release_hit, 11368 tunnel_element_release_miss, &ctx, false); 11369 11370 return ctx.ret; 11371 } 11372 11373 static int 11374 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11375 struct rte_flow_action *pmd_actions, 11376 uint32_t num_actions, struct rte_flow_error *err) 11377 { 11378 struct tunnel_db_element_release_ctx ctx = { 11379 .items = NULL, 11380 .actions = pmd_actions, 11381 .num_elements = num_actions, 11382 .error = err, 11383 }; 11384 11385 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11386 tunnel_element_release_hit, 11387 tunnel_element_release_miss, &ctx, false); 11388 11389 return ctx.ret; 11390 } 11391 11392 static int 11393 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11394 struct rte_mbuf *m, 11395 struct rte_flow_restore_info *info, 11396 struct rte_flow_error *err) 11397 { 11398 uint64_t ol_flags = m->ol_flags; 11399 const struct mlx5_flow_tbl_data_entry *tble; 11400 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11401 struct mlx5_priv *priv = dev->data->dev_private; 11402 11403 if (priv->tunnel_enabled == 0) 11404 goto err; 11405 if ((ol_flags & mask) != mask) 11406 goto err; 11407 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11408 if (!tble) { 11409 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11410 dev->data->port_id, m->hash.fdir.hi); 11411 goto err; 11412 } 11413 MLX5_ASSERT(tble->tunnel); 11414 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11415 info->group_id = tble->group_id; 11416 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11417 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11418 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11419 11420 return 0; 11421 11422 err: 11423 return rte_flow_error_set(err, EINVAL, 11424 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11425 "failed to get restore info"); 11426 } 11427 11428 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11429 static int 11430 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11431 __rte_unused struct rte_flow_tunnel *app_tunnel, 11432 __rte_unused struct rte_flow_action **actions, 11433 __rte_unused uint32_t *num_of_actions, 11434 __rte_unused struct rte_flow_error *error) 11435 { 11436 return -ENOTSUP; 11437 } 11438 11439 static int 11440 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11441 __rte_unused struct rte_flow_tunnel *app_tunnel, 11442 __rte_unused struct rte_flow_item **items, 11443 __rte_unused uint32_t *num_of_items, 11444 __rte_unused struct rte_flow_error *error) 11445 { 11446 return -ENOTSUP; 11447 } 11448 11449 static int 11450 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11451 __rte_unused struct rte_flow_item *pmd_items, 11452 __rte_unused uint32_t num_items, 11453 __rte_unused struct rte_flow_error *err) 11454 { 11455 return -ENOTSUP; 11456 } 11457 11458 static int 11459 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11460 __rte_unused struct rte_flow_action *pmd_action, 11461 __rte_unused uint32_t num_actions, 11462 __rte_unused struct rte_flow_error *err) 11463 { 11464 return -ENOTSUP; 11465 } 11466 11467 static int 11468 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11469 __rte_unused struct rte_mbuf *m, 11470 __rte_unused struct rte_flow_restore_info *i, 11471 __rte_unused struct rte_flow_error *err) 11472 { 11473 return -ENOTSUP; 11474 } 11475 11476 static int 11477 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11478 __rte_unused struct rte_flow *flow, 11479 __rte_unused const struct rte_flow_attr *attr, 11480 __rte_unused const struct rte_flow_action *actions, 11481 __rte_unused uint32_t flow_idx, 11482 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11483 __rte_unused struct tunnel_default_miss_ctx *ctx, 11484 __rte_unused struct rte_flow_error *error) 11485 { 11486 return -ENOTSUP; 11487 } 11488 11489 static struct mlx5_flow_tunnel * 11490 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11491 __rte_unused uint32_t id) 11492 { 11493 return NULL; 11494 } 11495 11496 static void 11497 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11498 __rte_unused struct mlx5_flow_tunnel *tunnel) 11499 { 11500 } 11501 11502 static uint32_t 11503 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11504 __rte_unused const struct mlx5_flow_tunnel *t, 11505 __rte_unused uint32_t group, 11506 __rte_unused uint32_t *table, 11507 struct rte_flow_error *error) 11508 { 11509 return rte_flow_error_set(error, ENOTSUP, 11510 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11511 "tunnel offload requires DV support"); 11512 } 11513 11514 void 11515 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11516 __rte_unused uint16_t port_id) 11517 { 11518 } 11519 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11520 11521 /* Flex flow item API */ 11522 static struct rte_flow_item_flex_handle * 11523 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11524 const struct rte_flow_item_flex_conf *conf, 11525 struct rte_flow_error *error) 11526 { 11527 static const char err_msg[] = "flex item creation unsupported"; 11528 struct mlx5_priv *priv = dev->data->dev_private; 11529 struct rte_flow_attr attr = { .transfer = 0 }; 11530 const struct mlx5_flow_driver_ops *fops = 11531 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11532 11533 if (!priv->pci_dev) { 11534 rte_flow_error_set(error, ENOTSUP, 11535 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11536 "create flex item on PF only"); 11537 return NULL; 11538 } 11539 switch (priv->pci_dev->id.device_id) { 11540 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11541 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11542 break; 11543 default: 11544 rte_flow_error_set(error, ENOTSUP, 11545 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11546 "flex item available on BlueField ports only"); 11547 return NULL; 11548 } 11549 if (!fops->item_create) { 11550 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11551 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11552 NULL, err_msg); 11553 return NULL; 11554 } 11555 return fops->item_create(dev, conf, error); 11556 } 11557 11558 static int 11559 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11560 const struct rte_flow_item_flex_handle *handle, 11561 struct rte_flow_error *error) 11562 { 11563 static const char err_msg[] = "flex item release unsupported"; 11564 struct rte_flow_attr attr = { .transfer = 0 }; 11565 const struct mlx5_flow_driver_ops *fops = 11566 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11567 11568 if (!fops->item_release) { 11569 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11570 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11571 NULL, err_msg); 11572 return -rte_errno; 11573 } 11574 return fops->item_release(dev, handle, error); 11575 } 11576 11577 static void 11578 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11579 { 11580 int ret; 11581 struct rte_flow_error error; 11582 11583 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11584 char *item_name; 11585 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11586 sizeof(item_name), 11587 (void *)(uintptr_t)item->type, &error); 11588 if (ret > 0) 11589 printf("%s ", item_name); 11590 else 11591 printf("%d\n", (int)item->type); 11592 } 11593 printf("END\n"); 11594 } 11595 11596 static int 11597 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11598 { 11599 const struct rte_flow_item_udp *spec = udp_item->spec; 11600 const struct rte_flow_item_udp *mask = udp_item->mask; 11601 uint16_t udp_dport = 0; 11602 11603 if (spec != NULL) { 11604 if (!mask) 11605 mask = &rte_flow_item_udp_mask; 11606 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11607 mask->hdr.dst_port); 11608 } 11609 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11610 } 11611 11612 static const struct mlx5_flow_expand_node * 11613 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11614 unsigned int item_idx, 11615 const struct mlx5_flow_expand_node graph[], 11616 const struct mlx5_flow_expand_node *node) 11617 { 11618 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11619 11620 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11621 node != NULL && 11622 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11623 /* 11624 * The expansion node is VXLAN and it is also the last 11625 * expandable item in the pattern, so need to continue 11626 * expansion of the inner tunnel. 11627 */ 11628 MLX5_ASSERT(item_idx > 0); 11629 prev_item = pattern + item_idx - 1; 11630 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11631 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11632 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11633 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11634 } 11635 return node; 11636 } 11637 11638 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11639 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11640 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11641 }; 11642 11643 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11644 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11645 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11646 { 9, 10, 11 }, { 12, 13, 14 }, 11647 }; 11648 11649 /** 11650 * Discover the number of available flow priorities. 11651 * 11652 * @param dev 11653 * Ethernet device. 11654 * 11655 * @return 11656 * On success, number of available flow priorities. 11657 * On failure, a negative errno-style code and rte_errno is set. 11658 */ 11659 int 11660 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11661 { 11662 static const uint16_t vprio[] = {8, 16}; 11663 const struct mlx5_priv *priv = dev->data->dev_private; 11664 const struct mlx5_flow_driver_ops *fops; 11665 enum mlx5_flow_drv_type type; 11666 int ret; 11667 11668 type = mlx5_flow_os_get_type(); 11669 if (type == MLX5_FLOW_TYPE_MAX) { 11670 type = MLX5_FLOW_TYPE_VERBS; 11671 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11672 type = MLX5_FLOW_TYPE_DV; 11673 } 11674 fops = flow_get_drv_ops(type); 11675 if (fops->discover_priorities == NULL) { 11676 DRV_LOG(ERR, "Priority discovery not supported"); 11677 rte_errno = ENOTSUP; 11678 return -rte_errno; 11679 } 11680 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11681 if (ret < 0) 11682 return ret; 11683 switch (ret) { 11684 case 8: 11685 ret = RTE_DIM(priority_map_3); 11686 break; 11687 case 16: 11688 ret = RTE_DIM(priority_map_5); 11689 break; 11690 default: 11691 rte_errno = ENOTSUP; 11692 DRV_LOG(ERR, 11693 "port %u maximum priority: %d expected 8/16", 11694 dev->data->port_id, ret); 11695 return -rte_errno; 11696 } 11697 DRV_LOG(INFO, "port %u supported flow priorities:" 11698 " 0-%d for ingress or egress root table," 11699 " 0-%d for non-root table or transfer root table.", 11700 dev->data->port_id, ret - 2, 11701 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11702 return ret; 11703 } 11704 11705 /** 11706 * Adjust flow priority based on the highest layer and the request priority. 11707 * 11708 * @param[in] dev 11709 * Pointer to the Ethernet device structure. 11710 * @param[in] priority 11711 * The rule base priority. 11712 * @param[in] subpriority 11713 * The priority based on the items. 11714 * 11715 * @return 11716 * The new priority. 11717 */ 11718 uint32_t 11719 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11720 uint32_t subpriority) 11721 { 11722 uint32_t res = 0; 11723 struct mlx5_priv *priv = dev->data->dev_private; 11724 11725 switch (priv->sh->flow_max_priority) { 11726 case RTE_DIM(priority_map_3): 11727 res = priority_map_3[priority][subpriority]; 11728 break; 11729 case RTE_DIM(priority_map_5): 11730 res = priority_map_5[priority][subpriority]; 11731 break; 11732 } 11733 return res; 11734 } 11735 11736 /** 11737 * Get the priority for sending traffic to kernel table. 11738 * 11739 * @param[in] dev 11740 * Pointer to the Ethernet device structure. 11741 * 11742 * @return 11743 * On success: the value of priority for sending traffic to kernel table 11744 * On failure: -1 11745 */ 11746 uint32_t 11747 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11748 { 11749 struct mlx5_priv *priv = dev->data->dev_private; 11750 uint32_t res; 11751 11752 switch (priv->sh->flow_max_priority) { 11753 case RTE_DIM(priority_map_5): 11754 res = 15; 11755 break; 11756 case RTE_DIM(priority_map_3): 11757 res = 7; 11758 break; 11759 default: 11760 DRV_LOG(ERR, 11761 "port %u maximum priority: %d expected 8/16", 11762 dev->data->port_id, priv->sh->flow_max_priority); 11763 res = (uint32_t)-1; 11764 } 11765 return res; 11766 } 11767 11768 /** 11769 * Get the E-Switch Manager vport id. 11770 * 11771 * @param[in] dev 11772 * Pointer to the Ethernet device structure. 11773 * 11774 * @return 11775 * The vport id. 11776 */ 11777 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11778 { 11779 struct mlx5_priv *priv = dev->data->dev_private; 11780 struct mlx5_common_device *cdev = priv->sh->cdev; 11781 11782 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11783 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11784 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11785 11786 if (priv->pci_dev == NULL) 11787 return 0; 11788 switch (priv->pci_dev->id.device_id) { 11789 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 11790 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11791 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11792 /* 11793 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11794 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11795 * ECPF vport ID is selected and not the host port (0) in any BF case. 11796 */ 11797 return (int16_t)MLX5_ECPF_VPORT_ID; 11798 default: 11799 return MLX5_PF_VPORT_ID; 11800 } 11801 } 11802 11803 /** 11804 * Parse item to get the vport id. 11805 * 11806 * @param[in] dev 11807 * Pointer to the Ethernet device structure. 11808 * @param[in] item 11809 * The src port id match item. 11810 * @param[out] vport_id 11811 * Pointer to put the vport id. 11812 * @param[out] all_ports 11813 * Indicate if the item matches all ports. 11814 * @param[out] error 11815 * Pointer to error structure. 11816 * 11817 * @return 11818 * 0 on success, a negative errno value otherwise and rte_errno is set. 11819 */ 11820 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11821 const struct rte_flow_item *item, 11822 uint16_t *vport_id, 11823 bool *all_ports, 11824 struct rte_flow_error *error) 11825 { 11826 struct mlx5_priv *port_priv; 11827 const struct rte_flow_item_port_id *pid_v = NULL; 11828 const struct rte_flow_item_ethdev *dev_v = NULL; 11829 uint32_t esw_mgr_port; 11830 uint32_t src_port; 11831 11832 if (all_ports) 11833 *all_ports = false; 11834 switch (item->type) { 11835 case RTE_FLOW_ITEM_TYPE_PORT_ID: 11836 pid_v = item->spec; 11837 if (!pid_v) 11838 return 0; 11839 src_port = pid_v->id; 11840 esw_mgr_port = MLX5_PORT_ESW_MGR; 11841 break; 11842 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 11843 dev_v = item->spec; 11844 if (!dev_v) { 11845 if (all_ports) 11846 *all_ports = true; 11847 return 0; 11848 } 11849 src_port = dev_v->port_id; 11850 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11851 break; 11852 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 11853 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11854 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11855 break; 11856 default: 11857 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11858 NULL, "Incorrect item type."); 11859 } 11860 if (src_port == esw_mgr_port) { 11861 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11862 } else { 11863 port_priv = mlx5_port_to_eswitch_info(src_port, false); 11864 if (!port_priv) 11865 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11866 NULL, "Failed to get port info."); 11867 *vport_id = port_priv->representor_id; 11868 } 11869 11870 return 0; 11871 } 11872 11873 int 11874 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11875 uint16_t *proxy_port_id, 11876 struct rte_flow_error *error) 11877 { 11878 const struct mlx5_priv *priv = dev->data->dev_private; 11879 uint16_t port_id; 11880 11881 if (!priv->sh->config.dv_esw_en) 11882 return rte_flow_error_set(error, EINVAL, 11883 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11884 NULL, 11885 "unable to provide a proxy port" 11886 " without E-Switch configured"); 11887 if (!priv->master && !priv->representor) 11888 return rte_flow_error_set(error, EINVAL, 11889 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11890 NULL, 11891 "unable to provide a proxy port" 11892 " for port which is not a master" 11893 " or a representor port"); 11894 if (priv->master) { 11895 *proxy_port_id = dev->data->port_id; 11896 return 0; 11897 } 11898 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11899 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11900 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11901 11902 if (port_priv->master && 11903 port_priv->domain_id == priv->domain_id) { 11904 *proxy_port_id = port_id; 11905 return 0; 11906 } 11907 } 11908 return rte_flow_error_set(error, ENODEV, 11909 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11910 NULL, "unable to find a proxy port"); 11911 } 11912 11913 /** 11914 * Discover IPv6 traffic class ID support in rdma-core and firmware. 11915 * 11916 * @param dev 11917 * Ethernet device. 11918 * 11919 * @return 11920 * 0, rdma-core is good to work with firmware. 11921 * -EOPNOTSUPP, rdma-core could not work with new IPv6 TC ID. 11922 */ 11923 int 11924 mlx5_flow_discover_ipv6_tc_support(struct rte_eth_dev *dev) 11925 { 11926 struct rte_flow_action_set_dscp set_dscp; 11927 struct rte_flow_attr attr; 11928 struct rte_flow_action actions[2]; 11929 struct rte_flow_item items[3]; 11930 struct rte_flow_error error; 11931 uint32_t flow_idx; 11932 11933 memset(&attr, 0, sizeof(attr)); 11934 memset(actions, 0, sizeof(actions)); 11935 memset(items, 0, sizeof(items)); 11936 attr.group = 1; 11937 attr.egress = 1; 11938 items[0].type = RTE_FLOW_ITEM_TYPE_ETH; 11939 items[1].type = RTE_FLOW_ITEM_TYPE_IPV6; 11940 items[2].type = RTE_FLOW_ITEM_TYPE_END; 11941 /* Random value */ 11942 set_dscp.dscp = 9; 11943 actions[0].type = RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP; 11944 actions[0].conf = &set_dscp; 11945 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 11946 11947 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, items, actions, true, &error); 11948 if (!flow_idx) 11949 return -EOPNOTSUPP; 11950 11951 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 11952 return 0; 11953 } 11954 11955 void * 11956 rte_pmd_mlx5_create_geneve_tlv_parser(uint16_t port_id, 11957 const struct rte_pmd_mlx5_geneve_tlv tlv_list[], 11958 uint8_t nb_options) 11959 { 11960 #ifdef HAVE_MLX5_HWS_SUPPORT 11961 return mlx5_geneve_tlv_parser_create(port_id, tlv_list, nb_options); 11962 #else 11963 (void)port_id; 11964 (void)tlv_list; 11965 (void)nb_options; 11966 DRV_LOG(ERR, "%s is not supported.", __func__); 11967 rte_errno = ENOTSUP; 11968 return NULL; 11969 #endif 11970 } 11971 11972 int 11973 rte_pmd_mlx5_destroy_geneve_tlv_parser(void *handle) 11974 { 11975 #ifdef HAVE_MLX5_HWS_SUPPORT 11976 return mlx5_geneve_tlv_parser_destroy(handle); 11977 #else 11978 (void)handle; 11979 DRV_LOG(ERR, "%s is not supported.", __func__); 11980 rte_errno = ENOTSUP; 11981 return -rte_errno; 11982 #endif 11983 } 11984