1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 struct tunnel_default_miss_ctx { 43 uint16_t *queue; 44 __extension__ 45 union { 46 struct rte_flow_action_rss action_rss; 47 struct rte_flow_action_queue miss_queue; 48 struct rte_flow_action_jump miss_jump; 49 uint8_t raw[0]; 50 }; 51 }; 52 53 void 54 mlx5_indirect_list_handles_release(struct rte_eth_dev *dev) 55 { 56 struct mlx5_priv *priv = dev->data->dev_private; 57 #ifdef HAVE_MLX5_HWS_SUPPORT 58 struct rte_flow_error error; 59 #endif 60 61 while (!LIST_EMPTY(&priv->indirect_list_head)) { 62 struct mlx5_indirect_list *e = 63 LIST_FIRST(&priv->indirect_list_head); 64 65 LIST_REMOVE(e, entry); 66 switch (e->type) { 67 #ifdef HAVE_MLX5_HWS_SUPPORT 68 case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR: 69 mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)e); 70 break; 71 case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY: 72 mlx5_destroy_legacy_indirect(dev, e); 73 break; 74 case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT: 75 mlx5_reformat_action_destroy(dev, 76 (struct rte_flow_action_list_handle *)e, &error); 77 break; 78 #endif 79 default: 80 DRV_LOG(ERR, "invalid indirect list type"); 81 MLX5_ASSERT(false); 82 break; 83 } 84 } 85 } 86 87 static int 88 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 89 struct rte_flow *flow, 90 const struct rte_flow_attr *attr, 91 const struct rte_flow_action *app_actions, 92 uint32_t flow_idx, 93 const struct mlx5_flow_tunnel *tunnel, 94 struct tunnel_default_miss_ctx *ctx, 95 struct rte_flow_error *error); 96 static struct mlx5_flow_tunnel * 97 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 98 static void 99 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 100 static uint32_t 101 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 102 const struct mlx5_flow_tunnel *tunnel, 103 uint32_t group, uint32_t *table, 104 struct rte_flow_error *error); 105 106 /** Device flow drivers. */ 107 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 108 109 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 110 111 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 112 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 113 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 114 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 115 #endif 116 #ifdef HAVE_MLX5_HWS_SUPPORT 117 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 118 #endif 119 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 120 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 121 }; 122 123 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 124 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 125 (const int []){ \ 126 __VA_ARGS__, 0, \ 127 } 128 129 /** Node object of input graph for mlx5_flow_expand_rss(). */ 130 struct mlx5_flow_expand_node { 131 const int *const next; 132 /**< 133 * List of next node indexes. Index 0 is interpreted as a terminator. 134 */ 135 const enum rte_flow_item_type type; 136 /**< Pattern item type of current node. */ 137 uint64_t rss_types; 138 /**< 139 * RSS types bit-field associated with this node 140 * (see RTE_ETH_RSS_* definitions). 141 */ 142 uint64_t node_flags; 143 /**< 144 * Bit-fields that define how the node is used in the expansion. 145 * (see MLX5_EXPANSION_NODE_* definitions). 146 */ 147 }; 148 149 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 150 struct mlx5_flow_expand_sqn { 151 uint32_t entries; /** Number of entries */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 /* Optional expand field. The expansion alg will not go deeper. */ 159 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 160 161 /* The node is not added implicitly as expansion to the flow pattern. 162 * If the node type does not match the flow pattern item type, the 163 * expansion alg will go deeper to its next items. 164 * In the current implementation, the list of next nodes indexes can 165 * have up to one node with this flag set and it has to be the last 166 * node index (before the list terminator). 167 */ 168 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 169 170 /** Object returned by mlx5_flow_expand_rss(). */ 171 struct mlx5_flow_expand_rss { 172 uint32_t entries; 173 /**< Number of entries @p patterns and @p priorities. */ 174 struct { 175 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 176 uint32_t priority; /**< Priority offset for each expansion. */ 177 } entry[]; 178 }; 179 180 static void 181 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 182 183 static const struct mlx5_flow_expand_node * 184 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 185 unsigned int item_idx, 186 const struct mlx5_flow_expand_node graph[], 187 const struct mlx5_flow_expand_node *node); 188 189 static __rte_always_inline int 190 mlx5_need_cache_flow(const struct mlx5_priv *priv, 191 const struct rte_flow_attr *attr) 192 { 193 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 194 (attr ? !attr->group : true) && 195 priv->mode_info.mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY && 196 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 197 } 198 199 static bool 200 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 201 { 202 switch (item->type) { 203 case RTE_FLOW_ITEM_TYPE_ETH: 204 case RTE_FLOW_ITEM_TYPE_VLAN: 205 case RTE_FLOW_ITEM_TYPE_IPV4: 206 case RTE_FLOW_ITEM_TYPE_IPV6: 207 case RTE_FLOW_ITEM_TYPE_UDP: 208 case RTE_FLOW_ITEM_TYPE_TCP: 209 case RTE_FLOW_ITEM_TYPE_ESP: 210 case RTE_FLOW_ITEM_TYPE_ICMP: 211 case RTE_FLOW_ITEM_TYPE_ICMP6: 212 case RTE_FLOW_ITEM_TYPE_VXLAN: 213 case RTE_FLOW_ITEM_TYPE_NVGRE: 214 case RTE_FLOW_ITEM_TYPE_GRE: 215 case RTE_FLOW_ITEM_TYPE_GENEVE: 216 case RTE_FLOW_ITEM_TYPE_MPLS: 217 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 218 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 219 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 220 case RTE_FLOW_ITEM_TYPE_GTP: 221 return true; 222 default: 223 break; 224 } 225 return false; 226 } 227 228 /** 229 * Network Service Header (NSH) and its next protocol values 230 * are described in RFC-8393. 231 */ 232 static enum rte_flow_item_type 233 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 234 { 235 enum rte_flow_item_type type; 236 237 switch (proto_mask & proto_spec) { 238 case 0: 239 type = RTE_FLOW_ITEM_TYPE_VOID; 240 break; 241 case RTE_VXLAN_GPE_TYPE_IPV4: 242 type = RTE_FLOW_ITEM_TYPE_IPV4; 243 break; 244 case RTE_VXLAN_GPE_TYPE_IPV6: 245 type = RTE_VXLAN_GPE_TYPE_IPV6; 246 break; 247 case RTE_VXLAN_GPE_TYPE_ETH: 248 type = RTE_FLOW_ITEM_TYPE_ETH; 249 break; 250 default: 251 type = RTE_FLOW_ITEM_TYPE_END; 252 } 253 return type; 254 } 255 256 static enum rte_flow_item_type 257 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 258 { 259 enum rte_flow_item_type type; 260 261 switch (proto_mask & proto_spec) { 262 case 0: 263 type = RTE_FLOW_ITEM_TYPE_VOID; 264 break; 265 case IPPROTO_UDP: 266 type = RTE_FLOW_ITEM_TYPE_UDP; 267 break; 268 case IPPROTO_TCP: 269 type = RTE_FLOW_ITEM_TYPE_TCP; 270 break; 271 case IPPROTO_IPIP: 272 type = RTE_FLOW_ITEM_TYPE_IPV4; 273 break; 274 case IPPROTO_IPV6: 275 type = RTE_FLOW_ITEM_TYPE_IPV6; 276 break; 277 case IPPROTO_ESP: 278 type = RTE_FLOW_ITEM_TYPE_ESP; 279 break; 280 default: 281 type = RTE_FLOW_ITEM_TYPE_END; 282 } 283 return type; 284 } 285 286 static enum rte_flow_item_type 287 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 288 rte_be16_t type_mask, bool is_tunnel) 289 { 290 enum rte_flow_item_type type; 291 292 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 293 case 0: 294 type = RTE_FLOW_ITEM_TYPE_VOID; 295 break; 296 case RTE_ETHER_TYPE_TEB: 297 type = is_tunnel ? 298 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 299 break; 300 case RTE_ETHER_TYPE_VLAN: 301 type = !is_tunnel ? 302 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 303 break; 304 case RTE_ETHER_TYPE_IPV4: 305 type = RTE_FLOW_ITEM_TYPE_IPV4; 306 break; 307 case RTE_ETHER_TYPE_IPV6: 308 type = RTE_FLOW_ITEM_TYPE_IPV6; 309 break; 310 default: 311 type = RTE_FLOW_ITEM_TYPE_END; 312 } 313 return type; 314 } 315 316 static enum rte_flow_item_type 317 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 318 { 319 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 320 do { \ 321 const void *m = item->mask; \ 322 const void *s = item->spec; \ 323 mask = m ? \ 324 ((const struct rte_flow_item_##type *)m)->fld : \ 325 rte_flow_item_##type##_mask.fld; \ 326 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 327 } while (0) 328 329 enum rte_flow_item_type ret; 330 uint16_t spec, mask; 331 332 if (item == NULL || item->spec == NULL) 333 return RTE_FLOW_ITEM_TYPE_VOID; 334 switch (item->type) { 335 case RTE_FLOW_ITEM_TYPE_ETH: 336 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 337 if (!mask) 338 return RTE_FLOW_ITEM_TYPE_VOID; 339 ret = mlx5_ethertype_to_item_type(spec, mask, false); 340 break; 341 case RTE_FLOW_ITEM_TYPE_VLAN: 342 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 343 if (!mask) 344 return RTE_FLOW_ITEM_TYPE_VOID; 345 ret = mlx5_ethertype_to_item_type(spec, mask, false); 346 break; 347 case RTE_FLOW_ITEM_TYPE_IPV4: 348 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 349 if (!mask) 350 return RTE_FLOW_ITEM_TYPE_VOID; 351 ret = mlx5_inet_proto_to_item_type(spec, mask); 352 break; 353 case RTE_FLOW_ITEM_TYPE_IPV6: 354 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 355 if (!mask) 356 return RTE_FLOW_ITEM_TYPE_VOID; 357 ret = mlx5_inet_proto_to_item_type(spec, mask); 358 break; 359 case RTE_FLOW_ITEM_TYPE_GENEVE: 360 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 361 ret = mlx5_ethertype_to_item_type(spec, mask, true); 362 break; 363 case RTE_FLOW_ITEM_TYPE_GRE: 364 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 365 ret = mlx5_ethertype_to_item_type(spec, mask, true); 366 break; 367 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 368 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 369 ret = mlx5_nsh_proto_to_item_type(spec, mask); 370 break; 371 default: 372 ret = RTE_FLOW_ITEM_TYPE_VOID; 373 break; 374 } 375 return ret; 376 #undef MLX5_XSET_ITEM_MASK_SPEC 377 } 378 379 static const int * 380 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 381 const int *next_node) 382 { 383 const struct mlx5_flow_expand_node *node = NULL; 384 const int *next = next_node; 385 386 while (next && *next) { 387 /* 388 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 389 * flag set, because they were not found in the flow pattern. 390 */ 391 node = &graph[*next]; 392 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 393 break; 394 next = node->next; 395 } 396 return next; 397 } 398 399 #define MLX5_RSS_EXP_ELT_N 32 400 401 /** 402 * Expand RSS flows into several possible flows according to the RSS hash 403 * fields requested and the driver capabilities. 404 * 405 * @param[out] buf 406 * Buffer to store the result expansion. 407 * @param[in] size 408 * Buffer size in bytes. If 0, @p buf can be NULL. 409 * @param[in] pattern 410 * User flow pattern. 411 * @param[in] types 412 * RSS types to expand (see RTE_ETH_RSS_* definitions). 413 * @param[in] graph 414 * Input graph to expand @p pattern according to @p types. 415 * @param[in] graph_root_index 416 * Index of root node in @p graph, typically 0. 417 * 418 * @return 419 * A positive value representing the size of @p buf in bytes regardless of 420 * @p size on success, a negative errno value otherwise and rte_errno is 421 * set, the following errors are defined: 422 * 423 * -E2BIG: graph-depth @p graph is too deep. 424 * -EINVAL: @p size has not enough space for expanded pattern. 425 */ 426 static int 427 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 428 const struct rte_flow_item *pattern, uint64_t types, 429 const struct mlx5_flow_expand_node graph[], 430 int graph_root_index) 431 { 432 const struct rte_flow_item *item; 433 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 434 const int *next_node; 435 const int *stack[MLX5_RSS_EXP_ELT_N]; 436 int stack_pos = 0; 437 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 438 unsigned int i, item_idx, last_expand_item_idx = 0; 439 size_t lsize; 440 size_t user_pattern_size = 0; 441 void *addr = NULL; 442 const struct mlx5_flow_expand_node *next = NULL; 443 struct rte_flow_item missed_item; 444 int missed = 0; 445 int elt = 0; 446 const struct rte_flow_item *last_expand_item = NULL; 447 448 memset(&missed_item, 0, sizeof(missed_item)); 449 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 450 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 451 if (lsize > size) 452 return -EINVAL; 453 buf->entry[0].priority = 0; 454 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 455 buf->entries = 0; 456 addr = buf->entry[0].pattern; 457 for (item = pattern, item_idx = 0; 458 item->type != RTE_FLOW_ITEM_TYPE_END; 459 item++, item_idx++) { 460 if (!mlx5_flow_is_rss_expandable_item(item)) { 461 user_pattern_size += sizeof(*item); 462 continue; 463 } 464 last_expand_item = item; 465 last_expand_item_idx = item_idx; 466 i = 0; 467 while (node->next && node->next[i]) { 468 next = &graph[node->next[i]]; 469 if (next->type == item->type) 470 break; 471 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 472 node = next; 473 i = 0; 474 } else { 475 ++i; 476 } 477 } 478 if (next) 479 node = next; 480 user_pattern_size += sizeof(*item); 481 } 482 user_pattern_size += sizeof(*item); /* Handle END item. */ 483 lsize += user_pattern_size; 484 if (lsize > size) 485 return -EINVAL; 486 /* Copy the user pattern in the first entry of the buffer. */ 487 rte_memcpy(addr, pattern, user_pattern_size); 488 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 489 buf->entries = 1; 490 /* Start expanding. */ 491 memset(flow_items, 0, sizeof(flow_items)); 492 user_pattern_size -= sizeof(*item); 493 /* 494 * Check if the last valid item has spec set, need complete pattern, 495 * and the pattern can be used for expansion. 496 */ 497 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 498 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 499 /* Item type END indicates expansion is not required. */ 500 return lsize; 501 } 502 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 503 next = NULL; 504 missed = 1; 505 i = 0; 506 while (node->next && node->next[i]) { 507 next = &graph[node->next[i]]; 508 if (next->type == missed_item.type) { 509 flow_items[0].type = missed_item.type; 510 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 511 break; 512 } 513 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 514 node = next; 515 i = 0; 516 } else { 517 ++i; 518 } 519 next = NULL; 520 } 521 } 522 if (next && missed) { 523 elt = 2; /* missed item + item end. */ 524 node = next; 525 lsize += elt * sizeof(*item) + user_pattern_size; 526 if (lsize > size) 527 return -EINVAL; 528 if (node->rss_types & types) { 529 buf->entry[buf->entries].priority = 1; 530 buf->entry[buf->entries].pattern = addr; 531 buf->entries++; 532 rte_memcpy(addr, buf->entry[0].pattern, 533 user_pattern_size); 534 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 535 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 536 addr = (void *)(((uintptr_t)addr) + 537 elt * sizeof(*item)); 538 } 539 } else if (last_expand_item != NULL) { 540 node = mlx5_flow_expand_rss_adjust_node(pattern, 541 last_expand_item_idx, graph, node); 542 } 543 memset(flow_items, 0, sizeof(flow_items)); 544 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 545 node->next); 546 stack[stack_pos] = next_node; 547 node = next_node ? &graph[*next_node] : NULL; 548 while (node) { 549 flow_items[stack_pos].type = node->type; 550 if (node->rss_types & types) { 551 size_t n; 552 /* 553 * compute the number of items to copy from the 554 * expansion and copy it. 555 * When the stack_pos is 0, there are 1 element in it, 556 * plus the addition END item. 557 */ 558 elt = stack_pos + 2; 559 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 560 lsize += elt * sizeof(*item) + user_pattern_size; 561 if (lsize > size) 562 return -EINVAL; 563 n = elt * sizeof(*item); 564 MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); 565 buf->entry[buf->entries].priority = 566 stack_pos + 1 + missed; 567 buf->entry[buf->entries].pattern = addr; 568 buf->entries++; 569 rte_memcpy(addr, buf->entry[0].pattern, 570 user_pattern_size); 571 addr = (void *)(((uintptr_t)addr) + 572 user_pattern_size); 573 rte_memcpy(addr, &missed_item, 574 missed * sizeof(*item)); 575 addr = (void *)(((uintptr_t)addr) + 576 missed * sizeof(*item)); 577 rte_memcpy(addr, flow_items, n); 578 addr = (void *)(((uintptr_t)addr) + n); 579 } 580 /* Go deeper. */ 581 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 582 node->next) { 583 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 584 node->next); 585 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 586 rte_errno = E2BIG; 587 return -rte_errno; 588 } 589 stack[stack_pos] = next_node; 590 } else if (*(next_node + 1)) { 591 /* Follow up with the next possibility. */ 592 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 593 ++next_node); 594 } else if (!stack_pos) { 595 /* 596 * Completing the traverse over the different paths. 597 * The next_node is advanced to the terminator. 598 */ 599 ++next_node; 600 } else { 601 /* Move to the next path. */ 602 while (stack_pos) { 603 next_node = stack[--stack_pos]; 604 next_node++; 605 if (*next_node) 606 break; 607 } 608 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 609 next_node); 610 stack[stack_pos] = next_node; 611 } 612 node = next_node && *next_node ? &graph[*next_node] : NULL; 613 }; 614 return lsize; 615 } 616 617 /** 618 * Expand SQN flows into several possible flows according to the Tx queue 619 * number 620 * 621 * @param[in] buf 622 * Buffer to store the result expansion. 623 * @param[in] size 624 * Buffer size in bytes. If 0, @p buf can be NULL. 625 * @param[in] pattern 626 * User flow pattern. 627 * @param[in] sq_specs 628 * Buffer to store sq spec. 629 * 630 * @return 631 * 0 for success and negative value for failure 632 * 633 */ 634 static int 635 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 636 const struct rte_flow_item *pattern, 637 struct mlx5_rte_flow_item_sq *sq_specs) 638 { 639 const struct rte_flow_item *item; 640 bool port_representor = false; 641 size_t user_pattern_size = 0; 642 struct rte_eth_dev *dev; 643 struct mlx5_priv *priv; 644 void *addr = NULL; 645 uint16_t port_id; 646 size_t lsize; 647 int elt = 2; 648 uint16_t i; 649 650 buf->entries = 0; 651 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 652 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 653 const struct rte_flow_item_ethdev *pid_v = item->spec; 654 655 if (!pid_v) 656 return 0; 657 port_id = pid_v->port_id; 658 port_representor = true; 659 } 660 user_pattern_size += sizeof(*item); 661 } 662 if (!port_representor) 663 return 0; 664 dev = &rte_eth_devices[port_id]; 665 priv = dev->data->dev_private; 666 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 667 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 668 sizeof(buf->entry[0]) * priv->txqs_n; 669 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 670 return -EINVAL; 671 addr = buf->entry[0].pattern; 672 for (i = 0; i != priv->txqs_n; ++i) { 673 struct rte_flow_item pattern_add[] = { 674 { 675 .type = (enum rte_flow_item_type) 676 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 677 .spec = &sq_specs[i], 678 }, 679 { 680 .type = RTE_FLOW_ITEM_TYPE_END, 681 }, 682 }; 683 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 684 685 if (txq == NULL) 686 return -EINVAL; 687 buf->entry[i].pattern = addr; 688 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 689 mlx5_txq_release(dev, i); 690 rte_memcpy(addr, pattern, user_pattern_size); 691 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 692 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 693 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 694 buf->entries++; 695 } 696 return 0; 697 } 698 699 enum mlx5_expansion { 700 MLX5_EXPANSION_ROOT, 701 MLX5_EXPANSION_ROOT_OUTER, 702 MLX5_EXPANSION_OUTER_ETH, 703 MLX5_EXPANSION_OUTER_VLAN, 704 MLX5_EXPANSION_OUTER_IPV4, 705 MLX5_EXPANSION_OUTER_IPV4_UDP, 706 MLX5_EXPANSION_OUTER_IPV4_TCP, 707 MLX5_EXPANSION_OUTER_IPV4_ESP, 708 MLX5_EXPANSION_OUTER_IPV4_ICMP, 709 MLX5_EXPANSION_OUTER_IPV6, 710 MLX5_EXPANSION_OUTER_IPV6_UDP, 711 MLX5_EXPANSION_OUTER_IPV6_TCP, 712 MLX5_EXPANSION_OUTER_IPV6_ESP, 713 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 714 MLX5_EXPANSION_VXLAN, 715 MLX5_EXPANSION_STD_VXLAN, 716 MLX5_EXPANSION_L3_VXLAN, 717 MLX5_EXPANSION_VXLAN_GPE, 718 MLX5_EXPANSION_GRE, 719 MLX5_EXPANSION_NVGRE, 720 MLX5_EXPANSION_GRE_KEY, 721 MLX5_EXPANSION_MPLS, 722 MLX5_EXPANSION_ETH, 723 MLX5_EXPANSION_VLAN, 724 MLX5_EXPANSION_IPV4, 725 MLX5_EXPANSION_IPV4_UDP, 726 MLX5_EXPANSION_IPV4_TCP, 727 MLX5_EXPANSION_IPV4_ESP, 728 MLX5_EXPANSION_IPV4_ICMP, 729 MLX5_EXPANSION_IPV6, 730 MLX5_EXPANSION_IPV6_UDP, 731 MLX5_EXPANSION_IPV6_TCP, 732 MLX5_EXPANSION_IPV6_ESP, 733 MLX5_EXPANSION_IPV6_ICMP6, 734 MLX5_EXPANSION_IPV6_FRAG_EXT, 735 MLX5_EXPANSION_GTP, 736 MLX5_EXPANSION_GENEVE, 737 }; 738 739 /** Supported expansion of items. */ 740 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 741 [MLX5_EXPANSION_ROOT] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 743 MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_END, 746 }, 747 [MLX5_EXPANSION_ROOT_OUTER] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 749 MLX5_EXPANSION_OUTER_IPV4, 750 MLX5_EXPANSION_OUTER_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_END, 752 }, 753 [MLX5_EXPANSION_OUTER_ETH] = { 754 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 755 .type = RTE_FLOW_ITEM_TYPE_ETH, 756 .rss_types = 0, 757 }, 758 [MLX5_EXPANSION_OUTER_VLAN] = { 759 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 760 MLX5_EXPANSION_OUTER_IPV6), 761 .type = RTE_FLOW_ITEM_TYPE_VLAN, 762 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 763 }, 764 [MLX5_EXPANSION_OUTER_IPV4] = { 765 .next = MLX5_FLOW_EXPAND_RSS_NEXT 766 (MLX5_EXPANSION_OUTER_IPV4_UDP, 767 MLX5_EXPANSION_OUTER_IPV4_TCP, 768 MLX5_EXPANSION_OUTER_IPV4_ESP, 769 MLX5_EXPANSION_OUTER_IPV4_ICMP, 770 MLX5_EXPANSION_GRE, 771 MLX5_EXPANSION_NVGRE, 772 MLX5_EXPANSION_IPV4, 773 MLX5_EXPANSION_IPV6), 774 .type = RTE_FLOW_ITEM_TYPE_IPV4, 775 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 776 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 777 }, 778 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 779 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 780 MLX5_EXPANSION_VXLAN_GPE, 781 MLX5_EXPANSION_MPLS, 782 MLX5_EXPANSION_GENEVE, 783 MLX5_EXPANSION_GTP), 784 .type = RTE_FLOW_ITEM_TYPE_UDP, 785 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 786 }, 787 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 788 .type = RTE_FLOW_ITEM_TYPE_TCP, 789 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 790 }, 791 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 792 .type = RTE_FLOW_ITEM_TYPE_ESP, 793 .rss_types = RTE_ETH_RSS_ESP, 794 }, 795 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 796 .type = RTE_FLOW_ITEM_TYPE_ICMP, 797 }, 798 [MLX5_EXPANSION_OUTER_IPV6] = { 799 .next = MLX5_FLOW_EXPAND_RSS_NEXT 800 (MLX5_EXPANSION_OUTER_IPV6_UDP, 801 MLX5_EXPANSION_OUTER_IPV6_TCP, 802 MLX5_EXPANSION_OUTER_IPV6_ESP, 803 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 804 MLX5_EXPANSION_IPV4, 805 MLX5_EXPANSION_IPV6, 806 MLX5_EXPANSION_GRE, 807 MLX5_EXPANSION_NVGRE), 808 .type = RTE_FLOW_ITEM_TYPE_IPV6, 809 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 810 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 811 }, 812 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 813 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 814 MLX5_EXPANSION_VXLAN_GPE, 815 MLX5_EXPANSION_MPLS, 816 MLX5_EXPANSION_GENEVE, 817 MLX5_EXPANSION_GTP), 818 .type = RTE_FLOW_ITEM_TYPE_UDP, 819 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 820 }, 821 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 822 .type = RTE_FLOW_ITEM_TYPE_TCP, 823 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 824 }, 825 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 826 .type = RTE_FLOW_ITEM_TYPE_ESP, 827 .rss_types = RTE_ETH_RSS_ESP, 828 }, 829 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 830 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 831 }, 832 [MLX5_EXPANSION_VXLAN] = { 833 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 834 MLX5_EXPANSION_IPV4, 835 MLX5_EXPANSION_IPV6), 836 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 837 }, 838 [MLX5_EXPANSION_STD_VXLAN] = { 839 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 840 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 841 }, 842 [MLX5_EXPANSION_L3_VXLAN] = { 843 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 844 MLX5_EXPANSION_IPV6), 845 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 846 }, 847 [MLX5_EXPANSION_VXLAN_GPE] = { 848 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 849 MLX5_EXPANSION_IPV4, 850 MLX5_EXPANSION_IPV6), 851 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 852 }, 853 [MLX5_EXPANSION_GRE] = { 854 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 855 MLX5_EXPANSION_IPV4, 856 MLX5_EXPANSION_IPV6, 857 MLX5_EXPANSION_GRE_KEY, 858 MLX5_EXPANSION_MPLS), 859 .type = RTE_FLOW_ITEM_TYPE_GRE, 860 }, 861 [MLX5_EXPANSION_GRE_KEY] = { 862 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 863 MLX5_EXPANSION_IPV6, 864 MLX5_EXPANSION_MPLS), 865 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 866 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 867 }, 868 [MLX5_EXPANSION_NVGRE] = { 869 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 870 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 871 }, 872 [MLX5_EXPANSION_MPLS] = { 873 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 874 MLX5_EXPANSION_IPV6, 875 MLX5_EXPANSION_ETH), 876 .type = RTE_FLOW_ITEM_TYPE_MPLS, 877 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 878 }, 879 [MLX5_EXPANSION_ETH] = { 880 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 881 .type = RTE_FLOW_ITEM_TYPE_ETH, 882 }, 883 [MLX5_EXPANSION_VLAN] = { 884 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 885 MLX5_EXPANSION_IPV6), 886 .type = RTE_FLOW_ITEM_TYPE_VLAN, 887 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 888 }, 889 [MLX5_EXPANSION_IPV4] = { 890 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 891 MLX5_EXPANSION_IPV4_TCP, 892 MLX5_EXPANSION_IPV4_ESP, 893 MLX5_EXPANSION_IPV4_ICMP), 894 .type = RTE_FLOW_ITEM_TYPE_IPV4, 895 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 896 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 897 }, 898 [MLX5_EXPANSION_IPV4_UDP] = { 899 .type = RTE_FLOW_ITEM_TYPE_UDP, 900 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 901 }, 902 [MLX5_EXPANSION_IPV4_TCP] = { 903 .type = RTE_FLOW_ITEM_TYPE_TCP, 904 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 905 }, 906 [MLX5_EXPANSION_IPV4_ESP] = { 907 .type = RTE_FLOW_ITEM_TYPE_ESP, 908 .rss_types = RTE_ETH_RSS_ESP, 909 }, 910 [MLX5_EXPANSION_IPV4_ICMP] = { 911 .type = RTE_FLOW_ITEM_TYPE_ICMP, 912 }, 913 [MLX5_EXPANSION_IPV6] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 915 MLX5_EXPANSION_IPV6_TCP, 916 MLX5_EXPANSION_IPV6_ESP, 917 MLX5_EXPANSION_IPV6_ICMP6, 918 MLX5_EXPANSION_IPV6_FRAG_EXT), 919 .type = RTE_FLOW_ITEM_TYPE_IPV6, 920 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 921 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 922 }, 923 [MLX5_EXPANSION_IPV6_UDP] = { 924 .type = RTE_FLOW_ITEM_TYPE_UDP, 925 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 926 }, 927 [MLX5_EXPANSION_IPV6_TCP] = { 928 .type = RTE_FLOW_ITEM_TYPE_TCP, 929 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 930 }, 931 [MLX5_EXPANSION_IPV6_ESP] = { 932 .type = RTE_FLOW_ITEM_TYPE_ESP, 933 .rss_types = RTE_ETH_RSS_ESP, 934 }, 935 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 936 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 937 }, 938 [MLX5_EXPANSION_IPV6_ICMP6] = { 939 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 940 }, 941 [MLX5_EXPANSION_GTP] = { 942 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 943 MLX5_EXPANSION_IPV6), 944 .type = RTE_FLOW_ITEM_TYPE_GTP, 945 }, 946 [MLX5_EXPANSION_GENEVE] = { 947 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 948 MLX5_EXPANSION_IPV4, 949 MLX5_EXPANSION_IPV6), 950 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 951 }, 952 }; 953 954 static struct rte_flow_action_handle * 955 mlx5_action_handle_create(struct rte_eth_dev *dev, 956 const struct rte_flow_indir_action_conf *conf, 957 const struct rte_flow_action *action, 958 struct rte_flow_error *error); 959 static int mlx5_action_handle_destroy 960 (struct rte_eth_dev *dev, 961 struct rte_flow_action_handle *handle, 962 struct rte_flow_error *error); 963 static int mlx5_action_handle_update 964 (struct rte_eth_dev *dev, 965 struct rte_flow_action_handle *handle, 966 const void *update, 967 struct rte_flow_error *error); 968 static int mlx5_action_handle_query 969 (struct rte_eth_dev *dev, 970 const struct rte_flow_action_handle *handle, 971 void *data, 972 struct rte_flow_error *error); 973 static int 974 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 975 struct rte_flow_tunnel *app_tunnel, 976 struct rte_flow_action **actions, 977 uint32_t *num_of_actions, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 981 struct rte_flow_tunnel *app_tunnel, 982 struct rte_flow_item **items, 983 uint32_t *num_of_items, 984 struct rte_flow_error *error); 985 static int 986 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 987 struct rte_flow_item *pmd_items, 988 uint32_t num_items, struct rte_flow_error *err); 989 static int 990 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 991 struct rte_flow_action *pmd_actions, 992 uint32_t num_actions, 993 struct rte_flow_error *err); 994 static int 995 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 996 struct rte_mbuf *m, 997 struct rte_flow_restore_info *info, 998 struct rte_flow_error *err); 999 static struct rte_flow_item_flex_handle * 1000 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 1001 const struct rte_flow_item_flex_conf *conf, 1002 struct rte_flow_error *error); 1003 static int 1004 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 1005 const struct rte_flow_item_flex_handle *handle, 1006 struct rte_flow_error *error); 1007 static int 1008 mlx5_flow_info_get(struct rte_eth_dev *dev, 1009 struct rte_flow_port_info *port_info, 1010 struct rte_flow_queue_info *queue_info, 1011 struct rte_flow_error *error); 1012 static int 1013 mlx5_flow_port_configure(struct rte_eth_dev *dev, 1014 const struct rte_flow_port_attr *port_attr, 1015 uint16_t nb_queue, 1016 const struct rte_flow_queue_attr *queue_attr[], 1017 struct rte_flow_error *err); 1018 1019 static struct rte_flow_pattern_template * 1020 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 1021 const struct rte_flow_pattern_template_attr *attr, 1022 const struct rte_flow_item items[], 1023 struct rte_flow_error *error); 1024 1025 static int 1026 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1027 struct rte_flow_pattern_template *template, 1028 struct rte_flow_error *error); 1029 static struct rte_flow_actions_template * 1030 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1031 const struct rte_flow_actions_template_attr *attr, 1032 const struct rte_flow_action actions[], 1033 const struct rte_flow_action masks[], 1034 struct rte_flow_error *error); 1035 static int 1036 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1037 struct rte_flow_actions_template *template, 1038 struct rte_flow_error *error); 1039 1040 static struct rte_flow_template_table * 1041 mlx5_flow_table_create(struct rte_eth_dev *dev, 1042 const struct rte_flow_template_table_attr *attr, 1043 struct rte_flow_pattern_template *item_templates[], 1044 uint8_t nb_item_templates, 1045 struct rte_flow_actions_template *action_templates[], 1046 uint8_t nb_action_templates, 1047 struct rte_flow_error *error); 1048 static int 1049 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1050 struct rte_flow_template_table *table, 1051 struct rte_flow_error *error); 1052 static int 1053 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 1054 uint32_t group_id, 1055 const struct rte_flow_group_attr *attr, 1056 const struct rte_flow_action actions[], 1057 struct rte_flow_error *error); 1058 1059 static int 1060 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1061 struct rte_flow_action_handle *handle, 1062 const void *update, void *query, 1063 enum rte_flow_query_update_mode qu_mode, 1064 struct rte_flow_error *error); 1065 1066 static struct rte_flow_action_list_handle * 1067 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 1068 const struct rte_flow_indir_action_conf *conf, 1069 const struct rte_flow_action *actions, 1070 struct rte_flow_error *error); 1071 1072 static int 1073 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 1074 struct rte_flow_action_list_handle *handle, 1075 struct rte_flow_error *error); 1076 1077 static int 1078 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 1079 const 1080 struct rte_flow_action_list_handle *handle, 1081 const void **update, void **query, 1082 enum rte_flow_query_update_mode mode, 1083 struct rte_flow_error *error); 1084 1085 static int 1086 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 1087 const struct rte_flow_template_table *table, 1088 const struct rte_flow_item pattern[], 1089 uint8_t pattern_template_index, 1090 uint32_t *hash, struct rte_flow_error *error); 1091 static int 1092 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 1093 const struct rte_flow_item pattern[], 1094 enum rte_flow_encap_hash_field dest_field, 1095 uint8_t *hash, 1096 struct rte_flow_error *error); 1097 1098 static int 1099 mlx5_template_table_resize(struct rte_eth_dev *dev, 1100 struct rte_flow_template_table *table, 1101 uint32_t nb_rules, struct rte_flow_error *error); 1102 static int 1103 mlx5_flow_async_update_resized(struct rte_eth_dev *dev, uint32_t queue, 1104 const struct rte_flow_op_attr *attr, 1105 struct rte_flow *rule, void *user_data, 1106 struct rte_flow_error *error); 1107 static int 1108 mlx5_table_resize_complete(struct rte_eth_dev *dev, 1109 struct rte_flow_template_table *table, 1110 struct rte_flow_error *error); 1111 1112 static const struct rte_flow_ops mlx5_flow_ops = { 1113 .validate = mlx5_flow_validate, 1114 .create = mlx5_flow_create, 1115 .destroy = mlx5_flow_destroy, 1116 .flush = mlx5_flow_flush, 1117 .isolate = mlx5_flow_isolate, 1118 .query = mlx5_flow_query, 1119 .dev_dump = mlx5_flow_dev_dump, 1120 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1121 .get_aged_flows = mlx5_flow_get_aged_flows, 1122 .action_handle_create = mlx5_action_handle_create, 1123 .action_handle_destroy = mlx5_action_handle_destroy, 1124 .action_handle_update = mlx5_action_handle_update, 1125 .action_handle_query = mlx5_action_handle_query, 1126 .action_handle_query_update = mlx5_action_handle_query_update, 1127 .action_list_handle_create = mlx5_action_list_handle_create, 1128 .action_list_handle_destroy = mlx5_action_list_handle_destroy, 1129 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1130 .tunnel_match = mlx5_flow_tunnel_match, 1131 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1132 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1133 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1134 .flex_item_create = mlx5_flow_flex_item_create, 1135 .flex_item_release = mlx5_flow_flex_item_release, 1136 .info_get = mlx5_flow_info_get, 1137 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1138 .configure = mlx5_flow_port_configure, 1139 .pattern_template_create = mlx5_flow_pattern_template_create, 1140 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1141 .actions_template_create = mlx5_flow_actions_template_create, 1142 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1143 .template_table_create = mlx5_flow_table_create, 1144 .template_table_destroy = mlx5_flow_table_destroy, 1145 .group_set_miss_actions = mlx5_flow_group_set_miss_actions, 1146 .action_list_handle_query_update = 1147 mlx5_flow_action_list_handle_query_update, 1148 .flow_calc_table_hash = mlx5_flow_calc_table_hash, 1149 .flow_calc_encap_hash = mlx5_flow_calc_encap_hash, 1150 .flow_template_table_resize = mlx5_template_table_resize, 1151 .flow_update_resized = mlx5_flow_async_update_resized, 1152 .flow_template_table_resize_complete = mlx5_table_resize_complete, 1153 }; 1154 1155 /* Tunnel information. */ 1156 struct mlx5_flow_tunnel_info { 1157 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1158 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1159 }; 1160 1161 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1162 { 1163 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1164 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1165 }, 1166 { 1167 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1168 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1169 }, 1170 { 1171 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1172 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1173 }, 1174 { 1175 .tunnel = MLX5_FLOW_LAYER_GRE, 1176 .ptype = RTE_PTYPE_TUNNEL_GRE, 1177 }, 1178 { 1179 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1180 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1181 }, 1182 { 1183 .tunnel = MLX5_FLOW_LAYER_MPLS, 1184 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1185 }, 1186 { 1187 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1188 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1189 }, 1190 { 1191 .tunnel = MLX5_FLOW_LAYER_IPIP, 1192 .ptype = RTE_PTYPE_TUNNEL_IP, 1193 }, 1194 { 1195 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1196 .ptype = RTE_PTYPE_TUNNEL_IP, 1197 }, 1198 { 1199 .tunnel = MLX5_FLOW_LAYER_GTP, 1200 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1201 }, 1202 }; 1203 1204 1205 1206 /** 1207 * Translate tag ID to register. 1208 * 1209 * @param[in] dev 1210 * Pointer to the Ethernet device structure. 1211 * @param[in] feature 1212 * The feature that request the register. 1213 * @param[in] id 1214 * The request register ID. 1215 * @param[out] error 1216 * Error description in case of any. 1217 * 1218 * @return 1219 * The request register on success, a negative errno 1220 * value otherwise and rte_errno is set. 1221 */ 1222 int 1223 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1224 enum mlx5_feature_name feature, 1225 uint32_t id, 1226 struct rte_flow_error *error) 1227 { 1228 struct mlx5_priv *priv = dev->data->dev_private; 1229 struct mlx5_sh_config *config = &priv->sh->config; 1230 struct mlx5_dev_registers *reg = &priv->sh->registers; 1231 enum modify_reg start_reg; 1232 bool skip_mtr_reg = false; 1233 1234 switch (feature) { 1235 case MLX5_HAIRPIN_RX: 1236 return REG_B; 1237 case MLX5_HAIRPIN_TX: 1238 return REG_A; 1239 case MLX5_METADATA_RX: 1240 switch (config->dv_xmeta_en) { 1241 case MLX5_XMETA_MODE_LEGACY: 1242 return REG_B; 1243 case MLX5_XMETA_MODE_META16: 1244 return REG_C_0; 1245 case MLX5_XMETA_MODE_META32: 1246 return REG_C_1; 1247 case MLX5_XMETA_MODE_META32_HWS: 1248 return REG_C_1; 1249 } 1250 break; 1251 case MLX5_METADATA_TX: 1252 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1253 return REG_C_1; 1254 } else { 1255 return REG_A; 1256 } 1257 case MLX5_METADATA_FDB: 1258 switch (config->dv_xmeta_en) { 1259 case MLX5_XMETA_MODE_LEGACY: 1260 return REG_NON; 1261 case MLX5_XMETA_MODE_META16: 1262 return REG_C_0; 1263 case MLX5_XMETA_MODE_META32: 1264 return REG_C_1; 1265 case MLX5_XMETA_MODE_META32_HWS: 1266 return REG_C_1; 1267 } 1268 break; 1269 case MLX5_FLOW_MARK: 1270 switch (config->dv_xmeta_en) { 1271 case MLX5_XMETA_MODE_LEGACY: 1272 case MLX5_XMETA_MODE_META32_HWS: 1273 return REG_NON; 1274 case MLX5_XMETA_MODE_META16: 1275 return REG_C_1; 1276 case MLX5_XMETA_MODE_META32: 1277 return REG_C_0; 1278 } 1279 break; 1280 case MLX5_MTR_ID: 1281 /* 1282 * If meter color and meter id share one register, flow match 1283 * should use the meter color register for match. 1284 */ 1285 if (priv->mtr_reg_share) 1286 return reg->aso_reg; 1287 else 1288 return reg->aso_reg != REG_C_2 ? REG_C_2 : 1289 REG_C_3; 1290 case MLX5_MTR_COLOR: 1291 case MLX5_ASO_FLOW_HIT: 1292 case MLX5_ASO_CONNTRACK: 1293 case MLX5_SAMPLE_ID: 1294 /* All features use the same REG_C. */ 1295 MLX5_ASSERT(reg->aso_reg != REG_NON); 1296 return reg->aso_reg; 1297 case MLX5_COPY_MARK: 1298 /* 1299 * Metadata COPY_MARK register using is in meter suffix sub 1300 * flow while with meter. It's safe to share the same register. 1301 */ 1302 return reg->aso_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1303 case MLX5_APP_TAG: 1304 /* 1305 * If meter is enable, it will engage the register for color 1306 * match and flow match. If meter color match is not using the 1307 * REG_C_2, need to skip the REG_C_x be used by meter color 1308 * match. 1309 * If meter is disable, free to use all available registers. 1310 */ 1311 start_reg = reg->aso_reg != REG_C_2 ? REG_C_2 : 1312 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1313 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1314 if (id > (uint32_t)(REG_C_7 - start_reg)) 1315 return rte_flow_error_set(error, EINVAL, 1316 RTE_FLOW_ERROR_TYPE_ITEM, 1317 NULL, "invalid tag id"); 1318 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1319 return rte_flow_error_set(error, ENOTSUP, 1320 RTE_FLOW_ERROR_TYPE_ITEM, 1321 NULL, "unsupported tag id"); 1322 /* 1323 * This case means meter is using the REG_C_x great than 2. 1324 * Take care not to conflict with meter color REG_C_x. 1325 * If the available index REG_C_y >= REG_C_x, skip the 1326 * color register. 1327 */ 1328 if (skip_mtr_reg && priv->sh->flow_mreg_c 1329 [id + start_reg - REG_C_0] >= reg->aso_reg) { 1330 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1331 return rte_flow_error_set(error, EINVAL, 1332 RTE_FLOW_ERROR_TYPE_ITEM, 1333 NULL, "invalid tag id"); 1334 if (priv->sh->flow_mreg_c 1335 [id + 1 + start_reg - REG_C_0] != REG_NON) 1336 return priv->sh->flow_mreg_c 1337 [id + 1 + start_reg - REG_C_0]; 1338 return rte_flow_error_set(error, ENOTSUP, 1339 RTE_FLOW_ERROR_TYPE_ITEM, 1340 NULL, "unsupported tag id"); 1341 } 1342 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1343 } 1344 MLX5_ASSERT(false); 1345 return rte_flow_error_set(error, EINVAL, 1346 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1347 NULL, "invalid feature name"); 1348 } 1349 1350 /** 1351 * Check extensive flow metadata register support. 1352 * 1353 * @param dev 1354 * Pointer to rte_eth_dev structure. 1355 * 1356 * @return 1357 * True if device supports extensive flow metadata register, otherwise false. 1358 */ 1359 bool 1360 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1361 { 1362 struct mlx5_priv *priv = dev->data->dev_private; 1363 1364 /* 1365 * Having available reg_c can be regarded inclusively as supporting 1366 * extensive flow metadata register, which could mean, 1367 * - metadata register copy action by modify header. 1368 * - 16 modify header actions is supported. 1369 * - reg_c's are preserved across different domain (FDB and NIC) on 1370 * packet loopback by flow lookup miss. 1371 */ 1372 return priv->sh->flow_mreg_c[2] != REG_NON; 1373 } 1374 1375 /** 1376 * Get the lowest priority. 1377 * 1378 * @param[in] dev 1379 * Pointer to the Ethernet device structure. 1380 * @param[in] attributes 1381 * Pointer to device flow rule attributes. 1382 * 1383 * @return 1384 * The value of lowest priority of flow. 1385 */ 1386 uint32_t 1387 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1388 const struct rte_flow_attr *attr) 1389 { 1390 struct mlx5_priv *priv = dev->data->dev_private; 1391 1392 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1393 return priv->sh->flow_max_priority - 2; 1394 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1395 } 1396 1397 /** 1398 * Calculate matcher priority of the flow. 1399 * 1400 * @param[in] dev 1401 * Pointer to the Ethernet device structure. 1402 * @param[in] attr 1403 * Pointer to device flow rule attributes. 1404 * @param[in] subpriority 1405 * The priority based on the items. 1406 * @param[in] external 1407 * Flow is user flow. 1408 * @return 1409 * The matcher priority of the flow. 1410 */ 1411 uint16_t 1412 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1413 const struct rte_flow_attr *attr, 1414 uint32_t subpriority, bool external) 1415 { 1416 uint16_t priority = (uint16_t)attr->priority; 1417 struct mlx5_priv *priv = dev->data->dev_private; 1418 1419 /* NIC root rules */ 1420 if (!attr->group && !attr->transfer) { 1421 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1422 priority = priv->sh->flow_max_priority - 1; 1423 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1424 /* FDB root rules */ 1425 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1426 attr->group == 0 && 1427 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1428 return (priv->sh->flow_max_priority - 1) * 3; 1429 } 1430 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1431 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1432 return priority * 3 + subpriority; 1433 } 1434 1435 /** 1436 * Verify the @p item specifications (spec, last, mask) are compatible with the 1437 * NIC capabilities. 1438 * 1439 * @param[in] item 1440 * Item specification. 1441 * @param[in] mask 1442 * @p item->mask or flow default bit-masks. 1443 * @param[in] nic_mask 1444 * Bit-masks covering supported fields by the NIC to compare with user mask. 1445 * @param[in] size 1446 * Bit-masks size in bytes. 1447 * @param[in] range_accepted 1448 * True if range of values is accepted for specific fields, false otherwise. 1449 * @param[out] error 1450 * Pointer to error structure. 1451 * 1452 * @return 1453 * 0 on success, a negative errno value otherwise and rte_errno is set. 1454 */ 1455 int 1456 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1457 const uint8_t *mask, 1458 const uint8_t *nic_mask, 1459 unsigned int size, 1460 bool range_accepted, 1461 struct rte_flow_error *error) 1462 { 1463 unsigned int i; 1464 1465 MLX5_ASSERT(nic_mask); 1466 for (i = 0; i < size; ++i) 1467 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1468 return rte_flow_error_set(error, ENOTSUP, 1469 RTE_FLOW_ERROR_TYPE_ITEM, 1470 item, 1471 "mask enables non supported" 1472 " bits"); 1473 if (!item->spec && (item->mask || item->last)) 1474 return rte_flow_error_set(error, EINVAL, 1475 RTE_FLOW_ERROR_TYPE_ITEM, item, 1476 "mask/last without a spec is not" 1477 " supported"); 1478 if (item->spec && item->last && !range_accepted) { 1479 uint8_t spec[size]; 1480 uint8_t last[size]; 1481 unsigned int i; 1482 int ret; 1483 1484 for (i = 0; i < size; ++i) { 1485 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1486 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1487 } 1488 ret = memcmp(spec, last, size); 1489 if (ret != 0) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ITEM, 1492 item, 1493 "range is not valid"); 1494 } 1495 return 0; 1496 } 1497 1498 /** 1499 * Adjust the hash fields according to the @p flow information. 1500 * 1501 * @param[in] dev_flow. 1502 * Pointer to the mlx5_flow. 1503 * @param[in] tunnel 1504 * 1 when the hash field is for a tunnel item. 1505 * @param[in] layer_types 1506 * RTE_ETH_RSS_* types. 1507 * @param[in] hash_fields 1508 * Item hash fields. 1509 * 1510 * @return 1511 * The hash fields that should be used. 1512 */ 1513 uint64_t 1514 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1515 int tunnel __rte_unused, uint64_t layer_types, 1516 uint64_t hash_fields) 1517 { 1518 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1519 int rss_request_inner = rss_desc->level >= 2; 1520 1521 /* Check RSS hash level for tunnel. */ 1522 if (tunnel && rss_request_inner) 1523 hash_fields |= IBV_RX_HASH_INNER; 1524 else if (tunnel || rss_request_inner) 1525 return 0; 1526 #endif 1527 /* Check if requested layer matches RSS hash fields. */ 1528 if (!(rss_desc->types & layer_types)) 1529 return 0; 1530 return hash_fields; 1531 } 1532 1533 /** 1534 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1535 * if several tunnel rules are used on this queue, the tunnel ptype will be 1536 * cleared. 1537 * 1538 * @param rxq_ctrl 1539 * Rx queue to update. 1540 */ 1541 static void 1542 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1543 { 1544 unsigned int i; 1545 uint32_t tunnel_ptype = 0; 1546 1547 /* Look up for the ptype to use. */ 1548 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1549 if (!rxq_ctrl->flow_tunnels_n[i]) 1550 continue; 1551 if (!tunnel_ptype) { 1552 tunnel_ptype = tunnels_info[i].ptype; 1553 } else { 1554 tunnel_ptype = 0; 1555 break; 1556 } 1557 } 1558 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1559 } 1560 1561 /** 1562 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1563 * flow. 1564 * 1565 * @param[in] dev 1566 * Pointer to the Ethernet device structure. 1567 * @param[in] dev_handle 1568 * Pointer to device flow handle structure. 1569 */ 1570 void 1571 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1572 struct mlx5_flow_handle *dev_handle) 1573 { 1574 struct mlx5_priv *priv = dev->data->dev_private; 1575 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1576 struct mlx5_ind_table_obj *ind_tbl = NULL; 1577 unsigned int i; 1578 1579 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1580 struct mlx5_hrxq *hrxq; 1581 1582 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1583 dev_handle->rix_hrxq); 1584 if (hrxq) 1585 ind_tbl = hrxq->ind_table; 1586 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1587 struct mlx5_shared_action_rss *shared_rss; 1588 1589 shared_rss = mlx5_ipool_get 1590 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1591 dev_handle->rix_srss); 1592 if (shared_rss) 1593 ind_tbl = shared_rss->ind_tbl; 1594 } 1595 if (!ind_tbl) 1596 return; 1597 for (i = 0; i != ind_tbl->queues_n; ++i) { 1598 int idx = ind_tbl->queues[i]; 1599 struct mlx5_rxq_ctrl *rxq_ctrl; 1600 1601 if (mlx5_is_external_rxq(dev, idx)) 1602 continue; 1603 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1604 MLX5_ASSERT(rxq_ctrl != NULL); 1605 if (rxq_ctrl == NULL) 1606 continue; 1607 /* 1608 * To support metadata register copy on Tx loopback, 1609 * this must be always enabled (metadata may arive 1610 * from other port - not from local flows only. 1611 */ 1612 if (tunnel) { 1613 unsigned int j; 1614 1615 /* Increase the counter matching the flow. */ 1616 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1617 if ((tunnels_info[j].tunnel & 1618 dev_handle->layers) == 1619 tunnels_info[j].tunnel) { 1620 rxq_ctrl->flow_tunnels_n[j]++; 1621 break; 1622 } 1623 } 1624 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1625 } 1626 } 1627 } 1628 1629 static void 1630 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1631 { 1632 struct mlx5_priv *priv = dev->data->dev_private; 1633 struct mlx5_rxq_ctrl *rxq_ctrl; 1634 uint16_t port_id; 1635 1636 if (priv->sh->shared_mark_enabled) 1637 return; 1638 if (priv->master || priv->representor) { 1639 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1640 struct mlx5_priv *opriv = 1641 rte_eth_devices[port_id].data->dev_private; 1642 1643 if (!opriv || 1644 opriv->sh != priv->sh || 1645 opriv->domain_id != priv->domain_id || 1646 opriv->mark_enabled) 1647 continue; 1648 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1649 rxq_ctrl->rxq.mark = 1; 1650 } 1651 opriv->mark_enabled = 1; 1652 } 1653 } else { 1654 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1655 rxq_ctrl->rxq.mark = 1; 1656 } 1657 priv->mark_enabled = 1; 1658 } 1659 priv->sh->shared_mark_enabled = 1; 1660 } 1661 1662 /** 1663 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1664 * 1665 * @param[in] dev 1666 * Pointer to the Ethernet device structure. 1667 * @param[in] flow 1668 * Pointer to flow structure. 1669 */ 1670 static void 1671 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1672 { 1673 struct mlx5_priv *priv = dev->data->dev_private; 1674 uint32_t handle_idx; 1675 struct mlx5_flow_handle *dev_handle; 1676 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1677 1678 MLX5_ASSERT(wks); 1679 if (wks->mark) 1680 flow_rxq_mark_flag_set(dev); 1681 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1682 handle_idx, dev_handle, next) 1683 flow_drv_rxq_flags_set(dev, dev_handle); 1684 } 1685 1686 /** 1687 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1688 * device flow if no other flow uses it with the same kind of request. 1689 * 1690 * @param dev 1691 * Pointer to Ethernet device. 1692 * @param[in] dev_handle 1693 * Pointer to the device flow handle structure. 1694 */ 1695 static void 1696 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1697 struct mlx5_flow_handle *dev_handle) 1698 { 1699 struct mlx5_priv *priv = dev->data->dev_private; 1700 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1701 struct mlx5_ind_table_obj *ind_tbl = NULL; 1702 unsigned int i; 1703 1704 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1705 struct mlx5_hrxq *hrxq; 1706 1707 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1708 dev_handle->rix_hrxq); 1709 if (hrxq) 1710 ind_tbl = hrxq->ind_table; 1711 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1712 struct mlx5_shared_action_rss *shared_rss; 1713 1714 shared_rss = mlx5_ipool_get 1715 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1716 dev_handle->rix_srss); 1717 if (shared_rss) 1718 ind_tbl = shared_rss->ind_tbl; 1719 } 1720 if (!ind_tbl) 1721 return; 1722 MLX5_ASSERT(dev->data->dev_started); 1723 for (i = 0; i != ind_tbl->queues_n; ++i) { 1724 int idx = ind_tbl->queues[i]; 1725 struct mlx5_rxq_ctrl *rxq_ctrl; 1726 1727 if (mlx5_is_external_rxq(dev, idx)) 1728 continue; 1729 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1730 MLX5_ASSERT(rxq_ctrl != NULL); 1731 if (rxq_ctrl == NULL) 1732 continue; 1733 if (tunnel) { 1734 unsigned int j; 1735 1736 /* Decrease the counter matching the flow. */ 1737 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1738 if ((tunnels_info[j].tunnel & 1739 dev_handle->layers) == 1740 tunnels_info[j].tunnel) { 1741 rxq_ctrl->flow_tunnels_n[j]--; 1742 break; 1743 } 1744 } 1745 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1746 } 1747 } 1748 } 1749 1750 /** 1751 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1752 * @p flow if no other flow uses it with the same kind of request. 1753 * 1754 * @param dev 1755 * Pointer to Ethernet device. 1756 * @param[in] flow 1757 * Pointer to the flow. 1758 */ 1759 static void 1760 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1761 { 1762 struct mlx5_priv *priv = dev->data->dev_private; 1763 uint32_t handle_idx; 1764 struct mlx5_flow_handle *dev_handle; 1765 1766 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1767 handle_idx, dev_handle, next) 1768 flow_drv_rxq_flags_trim(dev, dev_handle); 1769 } 1770 1771 /** 1772 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1773 * 1774 * @param dev 1775 * Pointer to Ethernet device. 1776 */ 1777 static void 1778 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1779 { 1780 struct mlx5_priv *priv = dev->data->dev_private; 1781 unsigned int i; 1782 1783 for (i = 0; i != priv->rxqs_n; ++i) { 1784 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1785 unsigned int j; 1786 1787 if (rxq == NULL || rxq->ctrl == NULL) 1788 continue; 1789 rxq->ctrl->rxq.mark = 0; 1790 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1791 rxq->ctrl->flow_tunnels_n[j] = 0; 1792 rxq->ctrl->rxq.tunnel = 0; 1793 } 1794 priv->mark_enabled = 0; 1795 priv->sh->shared_mark_enabled = 0; 1796 } 1797 1798 static uint64_t mlx5_restore_info_dynflag; 1799 1800 int 1801 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 1802 { 1803 struct mlx5_priv *priv = dev->data->dev_private; 1804 uint64_t supported = 0; 1805 1806 if (!is_tunnel_offload_active(dev)) { 1807 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 1808 supported |= RTE_ETH_RX_METADATA_USER_MARK; 1809 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) { 1810 DRV_LOG(DEBUG, 1811 "tunnel offload was not activated, consider setting dv_xmeta_en=%d", 1812 MLX5_XMETA_MODE_MISS_INFO); 1813 } 1814 } else { 1815 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 1816 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 && 1817 mlx5_restore_info_dynflag == 0) 1818 mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag(); 1819 } 1820 1821 if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) 1822 priv->tunnel_enabled = 1; 1823 else 1824 priv->tunnel_enabled = 0; 1825 1826 *features &= supported; 1827 return 0; 1828 } 1829 1830 /** 1831 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1832 * 1833 * @param[in] dev 1834 * Pointer to the Ethernet device structure. 1835 */ 1836 void 1837 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) 1838 { 1839 struct mlx5_priv *priv = dev->data->dev_private; 1840 uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID; 1841 unsigned int i; 1842 1843 if (priv->tunnel_enabled) 1844 mark_flag |= mlx5_restore_info_dynflag; 1845 1846 for (i = 0; i != priv->rxqs_n; ++i) { 1847 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1848 struct mlx5_rxq_data *data; 1849 1850 if (rxq == NULL || rxq->ctrl == NULL) 1851 continue; 1852 data = &rxq->ctrl->rxq; 1853 if (!rte_flow_dynf_metadata_avail()) { 1854 data->dynf_meta = 0; 1855 data->flow_meta_mask = 0; 1856 data->flow_meta_offset = -1; 1857 data->flow_meta_port_mask = 0; 1858 } else { 1859 data->dynf_meta = 1; 1860 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1861 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1862 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1863 } 1864 data->mark_flag = mark_flag; 1865 } 1866 } 1867 1868 /* 1869 * return a pointer to the desired action in the list of actions. 1870 * 1871 * @param[in] actions 1872 * The list of actions to search the action in. 1873 * @param[in] action 1874 * The action to find. 1875 * 1876 * @return 1877 * Pointer to the action in the list, if found. NULL otherwise. 1878 */ 1879 const struct rte_flow_action * 1880 mlx5_flow_find_action(const struct rte_flow_action *actions, 1881 enum rte_flow_action_type action) 1882 { 1883 if (actions == NULL) 1884 return NULL; 1885 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1886 if (actions->type == action) 1887 return actions; 1888 return NULL; 1889 } 1890 1891 /* 1892 * Validate the flag action. 1893 * 1894 * @param[in] action_flags 1895 * Bit-fields that holds the actions detected until now. 1896 * @param[in] attr 1897 * Attributes of flow that includes this action. 1898 * @param[out] error 1899 * Pointer to error structure. 1900 * 1901 * @return 1902 * 0 on success, a negative errno value otherwise and rte_errno is set. 1903 */ 1904 int 1905 mlx5_flow_validate_action_flag(uint64_t action_flags, 1906 const struct rte_flow_attr *attr, 1907 struct rte_flow_error *error) 1908 { 1909 if (action_flags & MLX5_FLOW_ACTION_MARK) 1910 return rte_flow_error_set(error, EINVAL, 1911 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1912 "can't mark and flag in same flow"); 1913 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1914 return rte_flow_error_set(error, EINVAL, 1915 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1916 "can't have 2 flag" 1917 " actions in same flow"); 1918 if (attr->egress) 1919 return rte_flow_error_set(error, ENOTSUP, 1920 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1921 "flag action not supported for " 1922 "egress"); 1923 return 0; 1924 } 1925 1926 /* 1927 * Validate the mark action. 1928 * 1929 * @param[in] action 1930 * Pointer to the queue action. 1931 * @param[in] action_flags 1932 * Bit-fields that holds the actions detected until now. 1933 * @param[in] attr 1934 * Attributes of flow that includes this action. 1935 * @param[out] error 1936 * Pointer to error structure. 1937 * 1938 * @return 1939 * 0 on success, a negative errno value otherwise and rte_errno is set. 1940 */ 1941 int 1942 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1943 uint64_t action_flags, 1944 const struct rte_flow_attr *attr, 1945 struct rte_flow_error *error) 1946 { 1947 const struct rte_flow_action_mark *mark = action->conf; 1948 1949 if (!mark) 1950 return rte_flow_error_set(error, EINVAL, 1951 RTE_FLOW_ERROR_TYPE_ACTION, 1952 action, 1953 "configuration cannot be null"); 1954 if (mark->id >= MLX5_FLOW_MARK_MAX) 1955 return rte_flow_error_set(error, EINVAL, 1956 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1957 &mark->id, 1958 "mark id must in 0 <= id < " 1959 RTE_STR(MLX5_FLOW_MARK_MAX)); 1960 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1961 return rte_flow_error_set(error, EINVAL, 1962 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1963 "can't flag and mark in same flow"); 1964 if (action_flags & MLX5_FLOW_ACTION_MARK) 1965 return rte_flow_error_set(error, EINVAL, 1966 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1967 "can't have 2 mark actions in same" 1968 " flow"); 1969 if (attr->egress) 1970 return rte_flow_error_set(error, ENOTSUP, 1971 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1972 "mark action not supported for " 1973 "egress"); 1974 return 0; 1975 } 1976 1977 /* 1978 * Validate the drop action. 1979 * 1980 * @param[in] dev 1981 * Pointer to the Ethernet device structure. 1982 * @param[in] is_root 1983 * True if flow is validated for root table. False otherwise. 1984 * @param[in] attr 1985 * Attributes of flow that includes this action. 1986 * @param[out] error 1987 * Pointer to error structure. 1988 * 1989 * @return 1990 * 0 on success, a negative errno value otherwise and rte_errno is set. 1991 */ 1992 int 1993 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 1994 bool is_root, 1995 const struct rte_flow_attr *attr, 1996 struct rte_flow_error *error) 1997 { 1998 struct mlx5_priv *priv = dev->data->dev_private; 1999 2000 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 2001 return rte_flow_error_set(error, ENOTSUP, 2002 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2003 "drop action not supported for " 2004 "egress"); 2005 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 2006 !priv->sh->dr_root_drop_action_en) { 2007 return rte_flow_error_set(error, ENOTSUP, 2008 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 2009 "drop action not supported for " 2010 "egress and transfer on group 0"); 2011 } 2012 return 0; 2013 } 2014 2015 /* 2016 * Validate the queue action. 2017 * 2018 * @param[in] action 2019 * Pointer to the queue action. 2020 * @param[in] action_flags 2021 * Bit-fields that holds the actions detected until now. 2022 * @param[in] dev 2023 * Pointer to the Ethernet device structure. 2024 * @param[in] attr 2025 * Attributes of flow that includes this action. 2026 * @param[out] error 2027 * Pointer to error structure. 2028 * 2029 * @return 2030 * 0 on success, a negative errno value otherwise and rte_errno is set. 2031 */ 2032 int 2033 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 2034 uint64_t action_flags, 2035 struct rte_eth_dev *dev, 2036 const struct rte_flow_attr *attr, 2037 struct rte_flow_error *error) 2038 { 2039 struct mlx5_priv *priv = dev->data->dev_private; 2040 const struct rte_flow_action_queue *queue = action->conf; 2041 2042 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2043 return rte_flow_error_set(error, EINVAL, 2044 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2045 "can't have 2 fate actions in" 2046 " same flow"); 2047 if (attr->egress) 2048 return rte_flow_error_set(error, ENOTSUP, 2049 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2050 "queue action not supported for egress."); 2051 if (mlx5_is_external_rxq(dev, queue->index)) 2052 return 0; 2053 if (!priv->rxqs_n) 2054 return rte_flow_error_set(error, EINVAL, 2055 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2056 NULL, "No Rx queues configured"); 2057 if (queue->index >= priv->rxqs_n) 2058 return rte_flow_error_set(error, EINVAL, 2059 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2060 &queue->index, 2061 "queue index out of range"); 2062 if (mlx5_rxq_get(dev, queue->index) == NULL) 2063 return rte_flow_error_set(error, EINVAL, 2064 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2065 &queue->index, 2066 "queue is not configured"); 2067 return 0; 2068 } 2069 2070 /** 2071 * Validate queue numbers for device RSS. 2072 * 2073 * @param[in] dev 2074 * Configured device. 2075 * @param[in] queues 2076 * Array of queue numbers. 2077 * @param[in] queues_n 2078 * Size of the @p queues array. 2079 * @param[out] error 2080 * On error, filled with a textual error description. 2081 * @param[out] queue_idx 2082 * On error, filled with an offending queue index in @p queues array. 2083 * 2084 * @return 2085 * 0 on success, a negative errno code on error. 2086 */ 2087 static int 2088 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2089 const uint16_t *queues, uint32_t queues_n, 2090 const char **error, uint32_t *queue_idx) 2091 { 2092 const struct mlx5_priv *priv = dev->data->dev_private; 2093 bool is_hairpin = false; 2094 bool is_ext_rss = false; 2095 uint32_t i; 2096 2097 for (i = 0; i != queues_n; ++i) { 2098 struct mlx5_rxq_ctrl *rxq_ctrl; 2099 2100 if (mlx5_is_external_rxq(dev, queues[0])) { 2101 is_ext_rss = true; 2102 continue; 2103 } 2104 if (is_ext_rss) { 2105 *error = "Combining external and regular RSS queues is not supported"; 2106 *queue_idx = i; 2107 return -ENOTSUP; 2108 } 2109 if (queues[i] >= priv->rxqs_n) { 2110 *error = "queue index out of range"; 2111 *queue_idx = i; 2112 return -EINVAL; 2113 } 2114 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2115 if (rxq_ctrl == NULL) { 2116 *error = "queue is not configured"; 2117 *queue_idx = i; 2118 return -EINVAL; 2119 } 2120 if (i == 0 && rxq_ctrl->is_hairpin) 2121 is_hairpin = true; 2122 if (is_hairpin != rxq_ctrl->is_hairpin) { 2123 *error = "combining hairpin and regular RSS queues is not supported"; 2124 *queue_idx = i; 2125 return -ENOTSUP; 2126 } 2127 } 2128 return 0; 2129 } 2130 2131 /* 2132 * Validate the rss action. 2133 * 2134 * @param[in] dev 2135 * Pointer to the Ethernet device structure. 2136 * @param[in] action 2137 * Pointer to the queue action. 2138 * @param[out] error 2139 * Pointer to error structure. 2140 * 2141 * @return 2142 * 0 on success, a negative errno value otherwise and rte_errno is set. 2143 */ 2144 int 2145 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2146 const struct rte_flow_action *action, 2147 struct rte_flow_error *error) 2148 { 2149 struct mlx5_priv *priv = dev->data->dev_private; 2150 const struct rte_flow_action_rss *rss = action->conf; 2151 int ret; 2152 const char *message; 2153 uint32_t queue_idx; 2154 2155 if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 2156 DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT", 2157 dev->data->port_id); 2158 } else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2159 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2160 return rte_flow_error_set(error, ENOTSUP, 2161 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2162 &rss->func, 2163 "RSS hash function not supported"); 2164 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2165 if (rss->level > 2) 2166 #else 2167 if (rss->level > 1) 2168 #endif 2169 return rte_flow_error_set(error, ENOTSUP, 2170 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2171 &rss->level, 2172 "tunnel RSS is not supported"); 2173 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2174 if (rss->key_len == 0 && rss->key != NULL) 2175 return rte_flow_error_set(error, ENOTSUP, 2176 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2177 &rss->key_len, 2178 "RSS hash key length 0"); 2179 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2180 return rte_flow_error_set(error, ENOTSUP, 2181 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2182 &rss->key_len, 2183 "RSS hash key too small"); 2184 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2185 return rte_flow_error_set(error, ENOTSUP, 2186 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2187 &rss->key_len, 2188 "RSS hash key too large"); 2189 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2190 return rte_flow_error_set(error, ENOTSUP, 2191 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2192 &rss->queue_num, 2193 "number of queues too large"); 2194 if (rss->types & MLX5_RSS_HF_MASK) 2195 return rte_flow_error_set(error, ENOTSUP, 2196 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2197 &rss->types, 2198 "some RSS protocols are not" 2199 " supported"); 2200 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2201 !(rss->types & RTE_ETH_RSS_IP)) 2202 return rte_flow_error_set(error, EINVAL, 2203 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2204 "L3 partial RSS requested but L3 RSS" 2205 " type not specified"); 2206 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2207 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2208 return rte_flow_error_set(error, EINVAL, 2209 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2210 "L4 partial RSS requested but L4 RSS" 2211 " type not specified"); 2212 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2213 return rte_flow_error_set(error, EINVAL, 2214 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2215 NULL, "No Rx queues configured"); 2216 if (!rss->queue_num) 2217 return rte_flow_error_set(error, EINVAL, 2218 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2219 NULL, "No queues configured"); 2220 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2221 &message, &queue_idx); 2222 if (ret != 0) { 2223 return rte_flow_error_set(error, -ret, 2224 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2225 &rss->queue[queue_idx], message); 2226 } 2227 return 0; 2228 } 2229 2230 /* 2231 * Validate the rss action. 2232 * 2233 * @param[in] action 2234 * Pointer to the queue action. 2235 * @param[in] action_flags 2236 * Bit-fields that holds the actions detected until now. 2237 * @param[in] dev 2238 * Pointer to the Ethernet device structure. 2239 * @param[in] attr 2240 * Attributes of flow that includes this action. 2241 * @param[in] item_flags 2242 * Items that were detected. 2243 * @param[out] error 2244 * Pointer to error structure. 2245 * 2246 * @return 2247 * 0 on success, a negative errno value otherwise and rte_errno is set. 2248 */ 2249 int 2250 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2251 uint64_t action_flags, 2252 struct rte_eth_dev *dev, 2253 const struct rte_flow_attr *attr, 2254 uint64_t item_flags, 2255 struct rte_flow_error *error) 2256 { 2257 const struct rte_flow_action_rss *rss = action->conf; 2258 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2259 int ret; 2260 2261 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2262 return rte_flow_error_set(error, EINVAL, 2263 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2264 "can't have 2 fate actions" 2265 " in same flow"); 2266 ret = mlx5_validate_action_rss(dev, action, error); 2267 if (ret) 2268 return ret; 2269 if (attr->egress) 2270 return rte_flow_error_set(error, ENOTSUP, 2271 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2272 "rss action not supported for " 2273 "egress"); 2274 if (rss->level > 1 && !tunnel) 2275 return rte_flow_error_set(error, EINVAL, 2276 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2277 "inner RSS is not supported for " 2278 "non-tunnel flows"); 2279 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2280 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2281 return rte_flow_error_set(error, EINVAL, 2282 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2283 "RSS on eCPRI is not supported now"); 2284 } 2285 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2286 !(item_flags & 2287 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2288 rss->level > 1) 2289 return rte_flow_error_set(error, EINVAL, 2290 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2291 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2292 return 0; 2293 } 2294 2295 /* 2296 * Validate the default miss action. 2297 * 2298 * @param[in] action_flags 2299 * Bit-fields that holds the actions detected until now. 2300 * @param[out] error 2301 * Pointer to error structure. 2302 * 2303 * @return 2304 * 0 on success, a negative errno value otherwise and rte_errno is set. 2305 */ 2306 int 2307 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2308 const struct rte_flow_attr *attr, 2309 struct rte_flow_error *error) 2310 { 2311 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2312 return rte_flow_error_set(error, EINVAL, 2313 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2314 "can't have 2 fate actions in" 2315 " same flow"); 2316 if (attr->egress) 2317 return rte_flow_error_set(error, ENOTSUP, 2318 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2319 "default miss action not supported " 2320 "for egress"); 2321 if (attr->group) 2322 return rte_flow_error_set(error, ENOTSUP, 2323 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2324 "only group 0 is supported"); 2325 if (attr->transfer) 2326 return rte_flow_error_set(error, ENOTSUP, 2327 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2328 NULL, "transfer is not supported"); 2329 return 0; 2330 } 2331 2332 /* 2333 * Validate the count action. 2334 * 2335 * @param[in] dev 2336 * Pointer to the Ethernet device structure. 2337 * @param[in] attr 2338 * Attributes of flow that includes this action. 2339 * @param[out] error 2340 * Pointer to error structure. 2341 * 2342 * @return 2343 * 0 on success, a negative errno value otherwise and rte_errno is set. 2344 */ 2345 int 2346 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2347 const struct rte_flow_attr *attr, 2348 struct rte_flow_error *error) 2349 { 2350 if (attr->egress) 2351 return rte_flow_error_set(error, ENOTSUP, 2352 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2353 "count action not supported for " 2354 "egress"); 2355 return 0; 2356 } 2357 2358 /* 2359 * Validate the ASO CT action. 2360 * 2361 * @param[in] dev 2362 * Pointer to the Ethernet device structure. 2363 * @param[in] conntrack 2364 * Pointer to the CT action profile. 2365 * @param[out] error 2366 * Pointer to error structure. 2367 * 2368 * @return 2369 * 0 on success, a negative errno value otherwise and rte_errno is set. 2370 */ 2371 int 2372 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2373 const struct rte_flow_action_conntrack *conntrack, 2374 struct rte_flow_error *error) 2375 { 2376 RTE_SET_USED(dev); 2377 2378 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2379 return rte_flow_error_set(error, EINVAL, 2380 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2381 "Invalid CT state"); 2382 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2383 return rte_flow_error_set(error, EINVAL, 2384 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2385 "Invalid last TCP packet flag"); 2386 return 0; 2387 } 2388 2389 /** 2390 * Validate the level value for modify field action. 2391 * 2392 * @param[in] data 2393 * Pointer to the rte_flow_field_data structure either src or dst. 2394 * @param[out] error 2395 * Pointer to error structure. 2396 * 2397 * @return 2398 * 0 on success, a negative errno value otherwise and rte_errno is set. 2399 */ 2400 int 2401 flow_validate_modify_field_level(const struct rte_flow_field_data *data, 2402 struct rte_flow_error *error) 2403 { 2404 if (data->level == 0 || data->field == RTE_FLOW_FIELD_FLEX_ITEM) 2405 return 0; 2406 if (data->field != RTE_FLOW_FIELD_TAG && 2407 data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) { 2408 if (data->level > 1) 2409 return rte_flow_error_set(error, ENOTSUP, 2410 RTE_FLOW_ERROR_TYPE_ACTION, 2411 NULL, 2412 "inner header fields modification is not supported"); 2413 return 0; 2414 } 2415 if (data->tag_index != 0) 2416 return rte_flow_error_set(error, EINVAL, 2417 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2418 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2419 /* 2420 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2421 * 'tag_index' field. In old API, it was provided using 'level' field 2422 * and it is still supported for backwards compatibility. 2423 */ 2424 DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field."); 2425 return 0; 2426 } 2427 2428 /** 2429 * Validate ICMP6 item. 2430 * 2431 * @param[in] item 2432 * Item specification. 2433 * @param[in] item_flags 2434 * Bit-fields that holds the items detected until now. 2435 * @param[in] ext_vlan_sup 2436 * Whether extended VLAN features are supported or not. 2437 * @param[out] error 2438 * Pointer to error structure. 2439 * 2440 * @return 2441 * 0 on success, a negative errno value otherwise and rte_errno is set. 2442 */ 2443 int 2444 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2445 uint64_t item_flags, 2446 uint8_t target_protocol, 2447 struct rte_flow_error *error) 2448 { 2449 const struct rte_flow_item_icmp6 *mask = item->mask; 2450 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2451 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2452 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2453 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2454 MLX5_FLOW_LAYER_OUTER_L4; 2455 int ret; 2456 2457 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2458 return rte_flow_error_set(error, EINVAL, 2459 RTE_FLOW_ERROR_TYPE_ITEM, item, 2460 "protocol filtering not compatible" 2461 " with ICMP6 layer"); 2462 if (!(item_flags & l3m)) 2463 return rte_flow_error_set(error, EINVAL, 2464 RTE_FLOW_ERROR_TYPE_ITEM, item, 2465 "IPv6 is mandatory to filter on" 2466 " ICMP6"); 2467 if (item_flags & l4m) 2468 return rte_flow_error_set(error, EINVAL, 2469 RTE_FLOW_ERROR_TYPE_ITEM, item, 2470 "multiple L4 layers not supported"); 2471 if (!mask) 2472 mask = &rte_flow_item_icmp6_mask; 2473 ret = mlx5_flow_item_acceptable 2474 (item, (const uint8_t *)mask, 2475 (const uint8_t *)&rte_flow_item_icmp6_mask, 2476 sizeof(struct rte_flow_item_icmp6), 2477 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2478 if (ret < 0) 2479 return ret; 2480 return 0; 2481 } 2482 2483 /** 2484 * Validate ICMP6 echo request/reply item. 2485 * 2486 * @param[in] item 2487 * Item specification. 2488 * @param[in] item_flags 2489 * Bit-fields that holds the items detected until now. 2490 * @param[in] ext_vlan_sup 2491 * Whether extended VLAN features are supported or not. 2492 * @param[out] error 2493 * Pointer to error structure. 2494 * 2495 * @return 2496 * 0 on success, a negative errno value otherwise and rte_errno is set. 2497 */ 2498 int 2499 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2500 uint64_t item_flags, 2501 uint8_t target_protocol, 2502 struct rte_flow_error *error) 2503 { 2504 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2505 const struct rte_flow_item_icmp6_echo nic_mask = { 2506 .hdr.base.type = 0xff, 2507 .hdr.base.code = 0xff, 2508 .hdr.identifier = RTE_BE16(0xffff), 2509 .hdr.sequence = RTE_BE16(0xffff), 2510 }; 2511 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2512 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2513 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2514 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2515 MLX5_FLOW_LAYER_OUTER_L4; 2516 int ret; 2517 2518 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2519 return rte_flow_error_set(error, EINVAL, 2520 RTE_FLOW_ERROR_TYPE_ITEM, item, 2521 "protocol filtering not compatible" 2522 " with ICMP6 layer"); 2523 if (!(item_flags & l3m)) 2524 return rte_flow_error_set(error, EINVAL, 2525 RTE_FLOW_ERROR_TYPE_ITEM, item, 2526 "IPv6 is mandatory to filter on" 2527 " ICMP6"); 2528 if (item_flags & l4m) 2529 return rte_flow_error_set(error, EINVAL, 2530 RTE_FLOW_ERROR_TYPE_ITEM, item, 2531 "multiple L4 layers not supported"); 2532 if (!mask) 2533 mask = &nic_mask; 2534 ret = mlx5_flow_item_acceptable 2535 (item, (const uint8_t *)mask, 2536 (const uint8_t *)&nic_mask, 2537 sizeof(struct rte_flow_item_icmp6_echo), 2538 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2539 if (ret < 0) 2540 return ret; 2541 return 0; 2542 } 2543 2544 /** 2545 * Validate ICMP item. 2546 * 2547 * @param[in] item 2548 * Item specification. 2549 * @param[in] item_flags 2550 * Bit-fields that holds the items detected until now. 2551 * @param[out] error 2552 * Pointer to error structure. 2553 * 2554 * @return 2555 * 0 on success, a negative errno value otherwise and rte_errno is set. 2556 */ 2557 int 2558 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2559 uint64_t item_flags, 2560 uint8_t target_protocol, 2561 struct rte_flow_error *error) 2562 { 2563 const struct rte_flow_item_icmp *mask = item->mask; 2564 const struct rte_flow_item_icmp nic_mask = { 2565 .hdr.icmp_type = 0xff, 2566 .hdr.icmp_code = 0xff, 2567 .hdr.icmp_ident = RTE_BE16(0xffff), 2568 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2569 }; 2570 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2571 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2572 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2573 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2574 MLX5_FLOW_LAYER_OUTER_L4; 2575 int ret; 2576 2577 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2578 return rte_flow_error_set(error, EINVAL, 2579 RTE_FLOW_ERROR_TYPE_ITEM, item, 2580 "protocol filtering not compatible" 2581 " with ICMP layer"); 2582 if (!(item_flags & l3m)) 2583 return rte_flow_error_set(error, EINVAL, 2584 RTE_FLOW_ERROR_TYPE_ITEM, item, 2585 "IPv4 is mandatory to filter" 2586 " on ICMP"); 2587 if (item_flags & l4m) 2588 return rte_flow_error_set(error, EINVAL, 2589 RTE_FLOW_ERROR_TYPE_ITEM, item, 2590 "multiple L4 layers not supported"); 2591 if (!mask) 2592 mask = &nic_mask; 2593 ret = mlx5_flow_item_acceptable 2594 (item, (const uint8_t *)mask, 2595 (const uint8_t *)&nic_mask, 2596 sizeof(struct rte_flow_item_icmp), 2597 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2598 if (ret < 0) 2599 return ret; 2600 return 0; 2601 } 2602 2603 /** 2604 * Validate Ethernet item. 2605 * 2606 * @param[in] item 2607 * Item specification. 2608 * @param[in] item_flags 2609 * Bit-fields that holds the items detected until now. 2610 * @param[out] error 2611 * Pointer to error structure. 2612 * 2613 * @return 2614 * 0 on success, a negative errno value otherwise and rte_errno is set. 2615 */ 2616 int 2617 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2618 uint64_t item_flags, bool ext_vlan_sup, 2619 struct rte_flow_error *error) 2620 { 2621 const struct rte_flow_item_eth *mask = item->mask; 2622 const struct rte_flow_item_eth nic_mask = { 2623 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2624 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2625 .hdr.ether_type = RTE_BE16(0xffff), 2626 .has_vlan = ext_vlan_sup ? 1 : 0, 2627 }; 2628 int ret; 2629 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2630 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2631 MLX5_FLOW_LAYER_OUTER_L2; 2632 2633 if (item_flags & ethm) 2634 return rte_flow_error_set(error, ENOTSUP, 2635 RTE_FLOW_ERROR_TYPE_ITEM, item, 2636 "multiple L2 layers not supported"); 2637 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2638 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2639 return rte_flow_error_set(error, EINVAL, 2640 RTE_FLOW_ERROR_TYPE_ITEM, item, 2641 "L2 layer should not follow " 2642 "L3 layers"); 2643 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2644 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2645 return rte_flow_error_set(error, EINVAL, 2646 RTE_FLOW_ERROR_TYPE_ITEM, item, 2647 "L2 layer should not follow VLAN"); 2648 if (item_flags & MLX5_FLOW_LAYER_GTP) 2649 return rte_flow_error_set(error, EINVAL, 2650 RTE_FLOW_ERROR_TYPE_ITEM, item, 2651 "L2 layer should not follow GTP"); 2652 if (!mask) 2653 mask = &rte_flow_item_eth_mask; 2654 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2655 (const uint8_t *)&nic_mask, 2656 sizeof(struct rte_flow_item_eth), 2657 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2658 return ret; 2659 } 2660 2661 /** 2662 * Validate VLAN item. 2663 * 2664 * @param[in] item 2665 * Item specification. 2666 * @param[in] item_flags 2667 * Bit-fields that holds the items detected until now. 2668 * @param[in] dev 2669 * Ethernet device flow is being created on. 2670 * @param[out] error 2671 * Pointer to error structure. 2672 * 2673 * @return 2674 * 0 on success, a negative errno value otherwise and rte_errno is set. 2675 */ 2676 int 2677 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2678 uint64_t item_flags, 2679 struct rte_eth_dev *dev, 2680 struct rte_flow_error *error) 2681 { 2682 const struct rte_flow_item_vlan *spec = item->spec; 2683 const struct rte_flow_item_vlan *mask = item->mask; 2684 const struct rte_flow_item_vlan nic_mask = { 2685 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2686 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2687 }; 2688 uint16_t vlan_tag = 0; 2689 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2690 int ret; 2691 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2692 MLX5_FLOW_LAYER_INNER_L4) : 2693 (MLX5_FLOW_LAYER_OUTER_L3 | 2694 MLX5_FLOW_LAYER_OUTER_L4); 2695 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2696 MLX5_FLOW_LAYER_OUTER_VLAN; 2697 2698 if (item_flags & vlanm) 2699 return rte_flow_error_set(error, EINVAL, 2700 RTE_FLOW_ERROR_TYPE_ITEM, item, 2701 "multiple VLAN layers not supported"); 2702 else if ((item_flags & l34m) != 0) 2703 return rte_flow_error_set(error, EINVAL, 2704 RTE_FLOW_ERROR_TYPE_ITEM, item, 2705 "VLAN cannot follow L3/L4 layer"); 2706 if (!mask) 2707 mask = &rte_flow_item_vlan_mask; 2708 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2709 (const uint8_t *)&nic_mask, 2710 sizeof(struct rte_flow_item_vlan), 2711 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2712 if (ret) 2713 return ret; 2714 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2715 struct mlx5_priv *priv = dev->data->dev_private; 2716 2717 if (priv->vmwa_context) { 2718 /* 2719 * Non-NULL context means we have a virtual machine 2720 * and SR-IOV enabled, we have to create VLAN interface 2721 * to make hypervisor to setup E-Switch vport 2722 * context correctly. We avoid creating the multiple 2723 * VLAN interfaces, so we cannot support VLAN tag mask. 2724 */ 2725 return rte_flow_error_set(error, EINVAL, 2726 RTE_FLOW_ERROR_TYPE_ITEM, 2727 item, 2728 "VLAN tag mask is not" 2729 " supported in virtual" 2730 " environment"); 2731 } 2732 } 2733 if (spec) { 2734 vlan_tag = spec->hdr.vlan_tci; 2735 vlan_tag &= mask->hdr.vlan_tci; 2736 } 2737 /* 2738 * From verbs perspective an empty VLAN is equivalent 2739 * to a packet without VLAN layer. 2740 */ 2741 if (!vlan_tag) 2742 return rte_flow_error_set(error, EINVAL, 2743 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2744 item->spec, 2745 "VLAN cannot be empty"); 2746 return 0; 2747 } 2748 2749 /** 2750 * Validate IPV4 item. 2751 * 2752 * @param[in] item 2753 * Item specification. 2754 * @param[in] item_flags 2755 * Bit-fields that holds the items detected until now. 2756 * @param[in] last_item 2757 * Previous validated item in the pattern items. 2758 * @param[in] ether_type 2759 * Type in the ethernet layer header (including dot1q). 2760 * @param[in] acc_mask 2761 * Acceptable mask, if NULL default internal default mask 2762 * will be used to check whether item fields are supported. 2763 * @param[in] range_accepted 2764 * True if range of values is accepted for specific fields, false otherwise. 2765 * @param[out] error 2766 * Pointer to error structure. 2767 * 2768 * @return 2769 * 0 on success, a negative errno value otherwise and rte_errno is set. 2770 */ 2771 int 2772 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2773 uint64_t item_flags, 2774 uint64_t last_item, 2775 uint16_t ether_type, 2776 const struct rte_flow_item_ipv4 *acc_mask, 2777 bool range_accepted, 2778 struct rte_flow_error *error) 2779 { 2780 const struct rte_flow_item_ipv4 *mask = item->mask; 2781 const struct rte_flow_item_ipv4 *spec = item->spec; 2782 const struct rte_flow_item_ipv4 nic_mask = { 2783 .hdr = { 2784 .src_addr = RTE_BE32(0xffffffff), 2785 .dst_addr = RTE_BE32(0xffffffff), 2786 .type_of_service = 0xff, 2787 .next_proto_id = 0xff, 2788 }, 2789 }; 2790 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2791 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2792 MLX5_FLOW_LAYER_OUTER_L3; 2793 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2794 MLX5_FLOW_LAYER_OUTER_L4; 2795 int ret; 2796 uint8_t next_proto = 0xFF; 2797 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2798 MLX5_FLOW_LAYER_OUTER_VLAN | 2799 MLX5_FLOW_LAYER_INNER_VLAN); 2800 2801 if ((last_item & l2_vlan) && ether_type && 2802 ether_type != RTE_ETHER_TYPE_IPV4) 2803 return rte_flow_error_set(error, EINVAL, 2804 RTE_FLOW_ERROR_TYPE_ITEM, item, 2805 "IPv4 cannot follow L2/VLAN layer " 2806 "which ether type is not IPv4"); 2807 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2808 if (mask && spec) 2809 next_proto = mask->hdr.next_proto_id & 2810 spec->hdr.next_proto_id; 2811 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2812 return rte_flow_error_set(error, EINVAL, 2813 RTE_FLOW_ERROR_TYPE_ITEM, 2814 item, 2815 "multiple tunnel " 2816 "not supported"); 2817 } 2818 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2819 return rte_flow_error_set(error, EINVAL, 2820 RTE_FLOW_ERROR_TYPE_ITEM, item, 2821 "wrong tunnel type - IPv6 specified " 2822 "but IPv4 item provided"); 2823 if (item_flags & l3m) 2824 return rte_flow_error_set(error, ENOTSUP, 2825 RTE_FLOW_ERROR_TYPE_ITEM, item, 2826 "multiple L3 layers not supported"); 2827 else if (item_flags & l4m) 2828 return rte_flow_error_set(error, EINVAL, 2829 RTE_FLOW_ERROR_TYPE_ITEM, item, 2830 "L3 cannot follow an L4 layer."); 2831 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2832 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2833 return rte_flow_error_set(error, EINVAL, 2834 RTE_FLOW_ERROR_TYPE_ITEM, item, 2835 "L3 cannot follow an NVGRE layer."); 2836 if (!mask) 2837 mask = &rte_flow_item_ipv4_mask; 2838 else if (mask->hdr.next_proto_id != 0 && 2839 mask->hdr.next_proto_id != 0xff) 2840 return rte_flow_error_set(error, EINVAL, 2841 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2842 "partial mask is not supported" 2843 " for protocol"); 2844 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2845 acc_mask ? (const uint8_t *)acc_mask 2846 : (const uint8_t *)&nic_mask, 2847 sizeof(struct rte_flow_item_ipv4), 2848 range_accepted, error); 2849 if (ret < 0) 2850 return ret; 2851 return 0; 2852 } 2853 2854 /** 2855 * Validate IPV6 item. 2856 * 2857 * @param[in] item 2858 * Item specification. 2859 * @param[in] item_flags 2860 * Bit-fields that holds the items detected until now. 2861 * @param[in] last_item 2862 * Previous validated item in the pattern items. 2863 * @param[in] ether_type 2864 * Type in the ethernet layer header (including dot1q). 2865 * @param[in] acc_mask 2866 * Acceptable mask, if NULL default internal default mask 2867 * will be used to check whether item fields are supported. 2868 * @param[out] error 2869 * Pointer to error structure. 2870 * 2871 * @return 2872 * 0 on success, a negative errno value otherwise and rte_errno is set. 2873 */ 2874 int 2875 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2876 uint64_t item_flags, 2877 uint64_t last_item, 2878 uint16_t ether_type, 2879 const struct rte_flow_item_ipv6 *acc_mask, 2880 struct rte_flow_error *error) 2881 { 2882 const struct rte_flow_item_ipv6 *mask = item->mask; 2883 const struct rte_flow_item_ipv6 *spec = item->spec; 2884 const struct rte_flow_item_ipv6 nic_mask = { 2885 .hdr = { 2886 .src_addr = 2887 "\xff\xff\xff\xff\xff\xff\xff\xff" 2888 "\xff\xff\xff\xff\xff\xff\xff\xff", 2889 .dst_addr = 2890 "\xff\xff\xff\xff\xff\xff\xff\xff" 2891 "\xff\xff\xff\xff\xff\xff\xff\xff", 2892 .vtc_flow = RTE_BE32(0xffffffff), 2893 .proto = 0xff, 2894 }, 2895 }; 2896 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2897 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2898 MLX5_FLOW_LAYER_OUTER_L3; 2899 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2900 MLX5_FLOW_LAYER_OUTER_L4; 2901 int ret; 2902 uint8_t next_proto = 0xFF; 2903 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2904 MLX5_FLOW_LAYER_OUTER_VLAN | 2905 MLX5_FLOW_LAYER_INNER_VLAN); 2906 2907 if ((last_item & l2_vlan) && ether_type && 2908 ether_type != RTE_ETHER_TYPE_IPV6) 2909 return rte_flow_error_set(error, EINVAL, 2910 RTE_FLOW_ERROR_TYPE_ITEM, item, 2911 "IPv6 cannot follow L2/VLAN layer " 2912 "which ether type is not IPv6"); 2913 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2914 next_proto = spec->hdr.proto; 2915 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2916 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2917 return rte_flow_error_set(error, EINVAL, 2918 RTE_FLOW_ERROR_TYPE_ITEM, 2919 item, 2920 "multiple tunnel " 2921 "not supported"); 2922 } 2923 if (next_proto == IPPROTO_HOPOPTS || 2924 next_proto == IPPROTO_ROUTING || 2925 next_proto == IPPROTO_FRAGMENT || 2926 next_proto == IPPROTO_ESP || 2927 next_proto == IPPROTO_AH || 2928 next_proto == IPPROTO_DSTOPTS) 2929 return rte_flow_error_set(error, EINVAL, 2930 RTE_FLOW_ERROR_TYPE_ITEM, item, 2931 "IPv6 proto (next header) should " 2932 "not be set as extension header"); 2933 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2934 return rte_flow_error_set(error, EINVAL, 2935 RTE_FLOW_ERROR_TYPE_ITEM, item, 2936 "wrong tunnel type - IPv4 specified " 2937 "but IPv6 item provided"); 2938 if (item_flags & l3m) 2939 return rte_flow_error_set(error, ENOTSUP, 2940 RTE_FLOW_ERROR_TYPE_ITEM, item, 2941 "multiple L3 layers not supported"); 2942 else if (item_flags & l4m) 2943 return rte_flow_error_set(error, EINVAL, 2944 RTE_FLOW_ERROR_TYPE_ITEM, item, 2945 "L3 cannot follow an L4 layer."); 2946 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2947 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2948 return rte_flow_error_set(error, EINVAL, 2949 RTE_FLOW_ERROR_TYPE_ITEM, item, 2950 "L3 cannot follow an NVGRE layer."); 2951 if (!mask) 2952 mask = &rte_flow_item_ipv6_mask; 2953 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2954 acc_mask ? (const uint8_t *)acc_mask 2955 : (const uint8_t *)&nic_mask, 2956 sizeof(struct rte_flow_item_ipv6), 2957 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2958 if (ret < 0) 2959 return ret; 2960 return 0; 2961 } 2962 2963 /** 2964 * Validate UDP item. 2965 * 2966 * @param[in] item 2967 * Item specification. 2968 * @param[in] item_flags 2969 * Bit-fields that holds the items detected until now. 2970 * @param[in] target_protocol 2971 * The next protocol in the previous item. 2972 * @param[in] flow_mask 2973 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2974 * @param[out] error 2975 * Pointer to error structure. 2976 * 2977 * @return 2978 * 0 on success, a negative errno value otherwise and rte_errno is set. 2979 */ 2980 int 2981 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2982 uint64_t item_flags, 2983 uint8_t target_protocol, 2984 struct rte_flow_error *error) 2985 { 2986 const struct rte_flow_item_udp *mask = item->mask; 2987 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2988 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2989 MLX5_FLOW_LAYER_OUTER_L3; 2990 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2991 MLX5_FLOW_LAYER_OUTER_L4; 2992 int ret; 2993 2994 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2995 return rte_flow_error_set(error, EINVAL, 2996 RTE_FLOW_ERROR_TYPE_ITEM, item, 2997 "protocol filtering not compatible" 2998 " with UDP layer"); 2999 if (!(item_flags & l3m)) 3000 return rte_flow_error_set(error, EINVAL, 3001 RTE_FLOW_ERROR_TYPE_ITEM, item, 3002 "L3 is mandatory to filter on L4"); 3003 if (item_flags & l4m) 3004 return rte_flow_error_set(error, EINVAL, 3005 RTE_FLOW_ERROR_TYPE_ITEM, item, 3006 "multiple L4 layers not supported"); 3007 if (!mask) 3008 mask = &rte_flow_item_udp_mask; 3009 ret = mlx5_flow_item_acceptable 3010 (item, (const uint8_t *)mask, 3011 (const uint8_t *)&rte_flow_item_udp_mask, 3012 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3013 error); 3014 if (ret < 0) 3015 return ret; 3016 return 0; 3017 } 3018 3019 /** 3020 * Validate TCP item. 3021 * 3022 * @param[in] item 3023 * Item specification. 3024 * @param[in] item_flags 3025 * Bit-fields that holds the items detected until now. 3026 * @param[in] target_protocol 3027 * The next protocol in the previous item. 3028 * @param[out] error 3029 * Pointer to error structure. 3030 * 3031 * @return 3032 * 0 on success, a negative errno value otherwise and rte_errno is set. 3033 */ 3034 int 3035 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 3036 uint64_t item_flags, 3037 uint8_t target_protocol, 3038 const struct rte_flow_item_tcp *flow_mask, 3039 struct rte_flow_error *error) 3040 { 3041 const struct rte_flow_item_tcp *mask = item->mask; 3042 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3043 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3044 MLX5_FLOW_LAYER_OUTER_L3; 3045 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3046 MLX5_FLOW_LAYER_OUTER_L4; 3047 int ret; 3048 3049 MLX5_ASSERT(flow_mask); 3050 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3051 return rte_flow_error_set(error, EINVAL, 3052 RTE_FLOW_ERROR_TYPE_ITEM, item, 3053 "protocol filtering not compatible" 3054 " with TCP layer"); 3055 if (!(item_flags & l3m)) 3056 return rte_flow_error_set(error, EINVAL, 3057 RTE_FLOW_ERROR_TYPE_ITEM, item, 3058 "L3 is mandatory to filter on L4"); 3059 if (item_flags & l4m) 3060 return rte_flow_error_set(error, EINVAL, 3061 RTE_FLOW_ERROR_TYPE_ITEM, item, 3062 "multiple L4 layers not supported"); 3063 if (!mask) 3064 mask = &rte_flow_item_tcp_mask; 3065 ret = mlx5_flow_item_acceptable 3066 (item, (const uint8_t *)mask, 3067 (const uint8_t *)flow_mask, 3068 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3069 error); 3070 if (ret < 0) 3071 return ret; 3072 return 0; 3073 } 3074 3075 /** 3076 * Validate VXLAN item. 3077 * 3078 * @param[in] dev 3079 * Pointer to the Ethernet device structure. 3080 * @param[in] udp_dport 3081 * UDP destination port 3082 * @param[in] item 3083 * Item specification. 3084 * @param[in] item_flags 3085 * Bit-fields that holds the items detected until now. 3086 * @param root 3087 * Whether action is on root table. 3088 * @param[out] error 3089 * Pointer to error structure. 3090 * 3091 * @return 3092 * 0 on success, a negative errno value otherwise and rte_errno is set. 3093 */ 3094 int 3095 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3096 uint16_t udp_dport, 3097 const struct rte_flow_item *item, 3098 uint64_t item_flags, 3099 bool root, 3100 struct rte_flow_error *error) 3101 { 3102 const struct rte_flow_item_vxlan *spec = item->spec; 3103 const struct rte_flow_item_vxlan *mask = item->mask; 3104 int ret; 3105 struct mlx5_priv *priv = dev->data->dev_private; 3106 union vni { 3107 uint32_t vlan_id; 3108 uint8_t vni[4]; 3109 } id = { .vlan_id = 0, }; 3110 const struct rte_flow_item_vxlan nic_mask = { 3111 .hdr.vni = "\xff\xff\xff", 3112 .hdr.rsvd1 = 0xff, 3113 }; 3114 const struct rte_flow_item_vxlan *valid_mask; 3115 3116 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3117 return rte_flow_error_set(error, ENOTSUP, 3118 RTE_FLOW_ERROR_TYPE_ITEM, item, 3119 "multiple tunnel layers not" 3120 " supported"); 3121 valid_mask = &rte_flow_item_vxlan_mask; 3122 /* 3123 * Verify only UDPv4 is present as defined in 3124 * https://tools.ietf.org/html/rfc7348 3125 */ 3126 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3127 return rte_flow_error_set(error, EINVAL, 3128 RTE_FLOW_ERROR_TYPE_ITEM, item, 3129 "no outer UDP layer found"); 3130 if (!mask) 3131 mask = &rte_flow_item_vxlan_mask; 3132 3133 if (priv->sh->steering_format_version != 3134 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3135 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3136 /* non-root table */ 3137 if (!root && priv->sh->misc5_cap) 3138 valid_mask = &nic_mask; 3139 /* Group zero in NIC domain */ 3140 if (!root && priv->sh->tunnel_header_0_1) 3141 valid_mask = &nic_mask; 3142 } 3143 ret = mlx5_flow_item_acceptable 3144 (item, (const uint8_t *)mask, 3145 (const uint8_t *)valid_mask, 3146 sizeof(struct rte_flow_item_vxlan), 3147 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3148 if (ret < 0) 3149 return ret; 3150 if (spec) { 3151 memcpy(&id.vni[1], spec->hdr.vni, 3); 3152 memcpy(&id.vni[1], mask->hdr.vni, 3); 3153 } 3154 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3155 return rte_flow_error_set(error, ENOTSUP, 3156 RTE_FLOW_ERROR_TYPE_ITEM, item, 3157 "VXLAN tunnel must be fully defined"); 3158 return 0; 3159 } 3160 3161 /** 3162 * Validate VXLAN_GPE item. 3163 * 3164 * @param[in] item 3165 * Item specification. 3166 * @param[in] item_flags 3167 * Bit-fields that holds the items detected until now. 3168 * @param[in] priv 3169 * Pointer to the private data structure. 3170 * @param[in] target_protocol 3171 * The next protocol in the previous item. 3172 * @param[out] error 3173 * Pointer to error structure. 3174 * 3175 * @return 3176 * 0 on success, a negative errno value otherwise and rte_errno is set. 3177 */ 3178 int 3179 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3180 uint64_t item_flags, 3181 struct rte_eth_dev *dev, 3182 struct rte_flow_error *error) 3183 { 3184 struct mlx5_priv *priv = dev->data->dev_private; 3185 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3186 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3187 int ret; 3188 union vni { 3189 uint32_t vlan_id; 3190 uint8_t vni[4]; 3191 } id = { .vlan_id = 0, }; 3192 3193 struct rte_flow_item_vxlan_gpe nic_mask = { 3194 .vni = "\xff\xff\xff", 3195 .protocol = 0xff, 3196 .flags = 0xff, 3197 }; 3198 3199 if (!priv->sh->config.l3_vxlan_en) 3200 return rte_flow_error_set(error, ENOTSUP, 3201 RTE_FLOW_ERROR_TYPE_ITEM, item, 3202 "L3 VXLAN is not enabled by device" 3203 " parameter and/or not configured in" 3204 " firmware"); 3205 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3206 return rte_flow_error_set(error, ENOTSUP, 3207 RTE_FLOW_ERROR_TYPE_ITEM, item, 3208 "multiple tunnel layers not" 3209 " supported"); 3210 /* 3211 * Verify only UDPv4 is present as defined in 3212 * https://tools.ietf.org/html/rfc7348 3213 */ 3214 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3215 return rte_flow_error_set(error, EINVAL, 3216 RTE_FLOW_ERROR_TYPE_ITEM, item, 3217 "no outer UDP layer found"); 3218 if (!mask) 3219 mask = &rte_flow_item_vxlan_gpe_mask; 3220 if (priv->sh->misc5_cap && priv->sh->tunnel_header_0_1) { 3221 nic_mask.rsvd0[0] = 0xff; 3222 nic_mask.rsvd0[1] = 0xff; 3223 nic_mask.rsvd1 = 0xff; 3224 } 3225 ret = mlx5_flow_item_acceptable 3226 (item, (const uint8_t *)mask, 3227 (const uint8_t *)&nic_mask, 3228 sizeof(struct rte_flow_item_vxlan_gpe), 3229 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3230 if (ret < 0) 3231 return ret; 3232 if (spec) { 3233 memcpy(&id.vni[1], spec->hdr.vni, 3); 3234 memcpy(&id.vni[1], mask->hdr.vni, 3); 3235 } 3236 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3237 return rte_flow_error_set(error, ENOTSUP, 3238 RTE_FLOW_ERROR_TYPE_ITEM, item, 3239 "VXLAN-GPE tunnel must be fully" 3240 " defined"); 3241 return 0; 3242 } 3243 /** 3244 * Validate GRE Key item. 3245 * 3246 * @param[in] item 3247 * Item specification. 3248 * @param[in] item_flags 3249 * Bit flags to mark detected items. 3250 * @param[in] gre_item 3251 * Pointer to gre_item 3252 * @param[out] error 3253 * Pointer to error structure. 3254 * 3255 * @return 3256 * 0 on success, a negative errno value otherwise and rte_errno is set. 3257 */ 3258 int 3259 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3260 uint64_t item_flags, 3261 const struct rte_flow_item *gre_item, 3262 struct rte_flow_error *error) 3263 { 3264 const rte_be32_t *mask = item->mask; 3265 int ret = 0; 3266 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3267 const struct rte_flow_item_gre *gre_spec; 3268 const struct rte_flow_item_gre *gre_mask; 3269 3270 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3271 return rte_flow_error_set(error, ENOTSUP, 3272 RTE_FLOW_ERROR_TYPE_ITEM, item, 3273 "Multiple GRE key not support"); 3274 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3275 return rte_flow_error_set(error, ENOTSUP, 3276 RTE_FLOW_ERROR_TYPE_ITEM, item, 3277 "No preceding GRE header"); 3278 if (item_flags & MLX5_FLOW_LAYER_INNER) 3279 return rte_flow_error_set(error, ENOTSUP, 3280 RTE_FLOW_ERROR_TYPE_ITEM, item, 3281 "GRE key following a wrong item"); 3282 gre_mask = gre_item->mask; 3283 if (!gre_mask) 3284 gre_mask = &rte_flow_item_gre_mask; 3285 gre_spec = gre_item->spec; 3286 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3287 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3288 return rte_flow_error_set(error, EINVAL, 3289 RTE_FLOW_ERROR_TYPE_ITEM, item, 3290 "Key bit must be on"); 3291 3292 if (!mask) 3293 mask = &gre_key_default_mask; 3294 ret = mlx5_flow_item_acceptable 3295 (item, (const uint8_t *)mask, 3296 (const uint8_t *)&gre_key_default_mask, 3297 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3298 return ret; 3299 } 3300 3301 /** 3302 * Validate GRE optional item. 3303 * 3304 * @param[in] dev 3305 * Pointer to the Ethernet device structure. 3306 * @param[in] item 3307 * Item specification. 3308 * @param[in] item_flags 3309 * Bit flags to mark detected items. 3310 * @param[in] attr 3311 * Flow rule attributes. 3312 * @param[in] gre_item 3313 * Pointer to gre_item 3314 * @param[out] error 3315 * Pointer to error structure. 3316 * 3317 * @return 3318 * 0 on success, a negative errno value otherwise and rte_errno is set. 3319 */ 3320 int 3321 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3322 const struct rte_flow_item *item, 3323 uint64_t item_flags, 3324 const struct rte_flow_attr *attr, 3325 const struct rte_flow_item *gre_item, 3326 struct rte_flow_error *error) 3327 { 3328 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3329 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3330 const struct rte_flow_item_gre_opt *spec = item->spec; 3331 const struct rte_flow_item_gre_opt *mask = item->mask; 3332 struct mlx5_priv *priv = dev->data->dev_private; 3333 int ret = 0; 3334 struct rte_flow_item_gre_opt nic_mask = { 3335 .checksum_rsvd = { 3336 .checksum = RTE_BE16(UINT16_MAX), 3337 .reserved1 = 0x0, 3338 }, 3339 .key = { 3340 .key = RTE_BE32(UINT32_MAX), 3341 }, 3342 .sequence = { 3343 .sequence = RTE_BE32(UINT32_MAX), 3344 }, 3345 }; 3346 3347 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3348 return rte_flow_error_set(error, ENOTSUP, 3349 RTE_FLOW_ERROR_TYPE_ITEM, item, 3350 "No preceding GRE header"); 3351 if (item_flags & MLX5_FLOW_LAYER_INNER) 3352 return rte_flow_error_set(error, ENOTSUP, 3353 RTE_FLOW_ERROR_TYPE_ITEM, item, 3354 "GRE option following a wrong item"); 3355 if (!spec || !mask) 3356 return rte_flow_error_set(error, EINVAL, 3357 RTE_FLOW_ERROR_TYPE_ITEM, item, 3358 "At least one field gre_option(checksum/key/sequence) must be specified"); 3359 if (!gre_mask) 3360 gre_mask = &rte_flow_item_gre_mask; 3361 if (mask->checksum_rsvd.checksum) 3362 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3363 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3364 return rte_flow_error_set(error, EINVAL, 3365 RTE_FLOW_ERROR_TYPE_ITEM, 3366 item, 3367 "Checksum bit must be on"); 3368 if (mask->key.key) 3369 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3370 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3371 return rte_flow_error_set(error, EINVAL, 3372 RTE_FLOW_ERROR_TYPE_ITEM, 3373 item, "Key bit must be on"); 3374 if (mask->sequence.sequence) 3375 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3376 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3377 return rte_flow_error_set(error, EINVAL, 3378 RTE_FLOW_ERROR_TYPE_ITEM, 3379 item, 3380 "Sequence bit must be on"); 3381 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3382 if (priv->sh->steering_format_version == 3383 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3384 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3385 !priv->sh->misc5_cap) || 3386 (!(priv->sh->tunnel_header_0_1 && 3387 priv->sh->tunnel_header_2_3) && 3388 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3389 return rte_flow_error_set(error, EINVAL, 3390 RTE_FLOW_ERROR_TYPE_ITEM, 3391 item, 3392 "Checksum/Sequence not supported"); 3393 } 3394 ret = mlx5_flow_item_acceptable 3395 (item, (const uint8_t *)mask, 3396 (const uint8_t *)&nic_mask, 3397 sizeof(struct rte_flow_item_gre_opt), 3398 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3399 return ret; 3400 } 3401 3402 /** 3403 * Validate GRE item. 3404 * 3405 * @param[in] item 3406 * Item specification. 3407 * @param[in] item_flags 3408 * Bit flags to mark detected items. 3409 * @param[in] target_protocol 3410 * The next protocol in the previous item. 3411 * @param[out] error 3412 * Pointer to error structure. 3413 * 3414 * @return 3415 * 0 on success, a negative errno value otherwise and rte_errno is set. 3416 */ 3417 int 3418 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3419 uint64_t item_flags, 3420 uint8_t target_protocol, 3421 struct rte_flow_error *error) 3422 { 3423 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3424 const struct rte_flow_item_gre *mask = item->mask; 3425 int ret; 3426 const struct rte_flow_item_gre nic_mask = { 3427 .c_rsvd0_ver = RTE_BE16(0xB000), 3428 .protocol = RTE_BE16(UINT16_MAX), 3429 }; 3430 3431 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3432 return rte_flow_error_set(error, EINVAL, 3433 RTE_FLOW_ERROR_TYPE_ITEM, item, 3434 "protocol filtering not compatible" 3435 " with this GRE layer"); 3436 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3437 return rte_flow_error_set(error, ENOTSUP, 3438 RTE_FLOW_ERROR_TYPE_ITEM, item, 3439 "multiple tunnel layers not" 3440 " supported"); 3441 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3442 return rte_flow_error_set(error, ENOTSUP, 3443 RTE_FLOW_ERROR_TYPE_ITEM, item, 3444 "L3 Layer is missing"); 3445 if (!mask) 3446 mask = &rte_flow_item_gre_mask; 3447 ret = mlx5_flow_item_acceptable 3448 (item, (const uint8_t *)mask, 3449 (const uint8_t *)&nic_mask, 3450 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3451 error); 3452 if (ret < 0) 3453 return ret; 3454 #ifndef HAVE_MLX5DV_DR 3455 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3456 if (spec && (spec->protocol & mask->protocol)) 3457 return rte_flow_error_set(error, ENOTSUP, 3458 RTE_FLOW_ERROR_TYPE_ITEM, item, 3459 "without MPLS support the" 3460 " specification cannot be used for" 3461 " filtering"); 3462 #endif 3463 #endif 3464 return 0; 3465 } 3466 3467 /** 3468 * Validate Geneve item. 3469 * 3470 * @param[in] item 3471 * Item specification. 3472 * @param[in] itemFlags 3473 * Bit-fields that holds the items detected until now. 3474 * @param[in] enPriv 3475 * Pointer to the private data structure. 3476 * @param[out] error 3477 * Pointer to error structure. 3478 * 3479 * @return 3480 * 0 on success, a negative errno value otherwise and rte_errno is set. 3481 */ 3482 3483 int 3484 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3485 uint64_t item_flags, 3486 struct rte_eth_dev *dev, 3487 struct rte_flow_error *error) 3488 { 3489 struct mlx5_priv *priv = dev->data->dev_private; 3490 const struct rte_flow_item_geneve *spec = item->spec; 3491 const struct rte_flow_item_geneve *mask = item->mask; 3492 int ret; 3493 uint16_t gbhdr; 3494 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3495 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3496 const struct rte_flow_item_geneve nic_mask = { 3497 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3498 .vni = "\xff\xff\xff", 3499 .protocol = RTE_BE16(UINT16_MAX), 3500 }; 3501 3502 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3503 return rte_flow_error_set(error, ENOTSUP, 3504 RTE_FLOW_ERROR_TYPE_ITEM, item, 3505 "L3 Geneve is not enabled by device" 3506 " parameter and/or not configured in" 3507 " firmware"); 3508 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3509 return rte_flow_error_set(error, ENOTSUP, 3510 RTE_FLOW_ERROR_TYPE_ITEM, item, 3511 "multiple tunnel layers not" 3512 " supported"); 3513 /* 3514 * Verify only UDPv4 is present as defined in 3515 * https://tools.ietf.org/html/rfc7348 3516 */ 3517 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3518 return rte_flow_error_set(error, EINVAL, 3519 RTE_FLOW_ERROR_TYPE_ITEM, item, 3520 "no outer UDP layer found"); 3521 if (!mask) 3522 mask = &rte_flow_item_geneve_mask; 3523 ret = mlx5_flow_item_acceptable 3524 (item, (const uint8_t *)mask, 3525 (const uint8_t *)&nic_mask, 3526 sizeof(struct rte_flow_item_geneve), 3527 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3528 if (ret) 3529 return ret; 3530 if (spec) { 3531 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3532 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3533 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3534 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3535 return rte_flow_error_set(error, ENOTSUP, 3536 RTE_FLOW_ERROR_TYPE_ITEM, 3537 item, 3538 "Geneve protocol unsupported" 3539 " fields are being used"); 3540 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3541 return rte_flow_error_set 3542 (error, ENOTSUP, 3543 RTE_FLOW_ERROR_TYPE_ITEM, 3544 item, 3545 "Unsupported Geneve options length"); 3546 } 3547 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3548 return rte_flow_error_set 3549 (error, ENOTSUP, 3550 RTE_FLOW_ERROR_TYPE_ITEM, item, 3551 "Geneve tunnel must be fully defined"); 3552 return 0; 3553 } 3554 3555 /** 3556 * Validate Geneve TLV option item. 3557 * 3558 * @param[in] item 3559 * Item specification. 3560 * @param[in] last_item 3561 * Previous validated item in the pattern items. 3562 * @param[in] geneve_item 3563 * Previous GENEVE item specification. 3564 * @param[in] dev 3565 * Pointer to the rte_eth_dev structure. 3566 * @param[out] error 3567 * Pointer to error structure. 3568 * 3569 * @return 3570 * 0 on success, a negative errno value otherwise and rte_errno is set. 3571 */ 3572 int 3573 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3574 uint64_t last_item, 3575 const struct rte_flow_item *geneve_item, 3576 struct rte_eth_dev *dev, 3577 struct rte_flow_error *error) 3578 { 3579 struct mlx5_priv *priv = dev->data->dev_private; 3580 struct mlx5_dev_ctx_shared *sh = priv->sh; 3581 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3582 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3583 uint8_t data_max_supported = 3584 hca_attr->max_geneve_tlv_option_data_len * 4; 3585 const struct rte_flow_item_geneve *geneve_spec; 3586 const struct rte_flow_item_geneve *geneve_mask; 3587 const struct rte_flow_item_geneve_opt *spec = item->spec; 3588 const struct rte_flow_item_geneve_opt *mask = item->mask; 3589 unsigned int i; 3590 unsigned int data_len; 3591 uint8_t tlv_option_len; 3592 uint16_t optlen_m, optlen_v; 3593 const struct rte_flow_item_geneve_opt full_mask = { 3594 .option_class = RTE_BE16(0xffff), 3595 .option_type = 0xff, 3596 .option_len = 0x1f, 3597 }; 3598 3599 if (!mask) 3600 mask = &rte_flow_item_geneve_opt_mask; 3601 if (!spec) 3602 return rte_flow_error_set 3603 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3604 "Geneve TLV opt class/type/length must be specified"); 3605 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3606 return rte_flow_error_set 3607 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3608 "Geneve TLV opt length exceeds the limit (31)"); 3609 /* Check if class type and length masks are full. */ 3610 if (full_mask.option_class != mask->option_class || 3611 full_mask.option_type != mask->option_type || 3612 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3613 return rte_flow_error_set 3614 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3615 "Geneve TLV opt class/type/length masks must be full"); 3616 /* Check if length is supported */ 3617 if ((uint32_t)spec->option_len > 3618 hca_attr->max_geneve_tlv_option_data_len) 3619 return rte_flow_error_set 3620 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3621 "Geneve TLV opt length not supported"); 3622 if (hca_attr->max_geneve_tlv_options > 1) 3623 DRV_LOG(DEBUG, 3624 "max_geneve_tlv_options supports more than 1 option"); 3625 /* Check GENEVE item preceding. */ 3626 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3627 return rte_flow_error_set 3628 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3629 "Geneve opt item must be preceded with Geneve item"); 3630 geneve_spec = geneve_item->spec; 3631 geneve_mask = geneve_item->mask ? geneve_item->mask : 3632 &rte_flow_item_geneve_mask; 3633 /* Check if GENEVE TLV option size doesn't exceed option length */ 3634 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3635 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3636 tlv_option_len = spec->option_len & mask->option_len; 3637 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3638 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3639 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3640 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3641 if ((optlen_v & optlen_m) <= tlv_option_len) 3642 return rte_flow_error_set 3643 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3644 "GENEVE TLV option length exceeds optlen"); 3645 } 3646 /* Check if length is 0 or data is 0. */ 3647 if (spec->data == NULL || spec->option_len == 0) 3648 return rte_flow_error_set 3649 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3650 "Geneve TLV opt with zero data/length not supported"); 3651 /* Check not all data & mask are 0. */ 3652 data_len = spec->option_len * 4; 3653 if (mask->data == NULL) { 3654 for (i = 0; i < data_len; i++) 3655 if (spec->data[i]) 3656 break; 3657 if (i == data_len) 3658 return rte_flow_error_set(error, ENOTSUP, 3659 RTE_FLOW_ERROR_TYPE_ITEM, item, 3660 "Can't match on Geneve option data 0"); 3661 } else { 3662 for (i = 0; i < data_len; i++) 3663 if (spec->data[i] & mask->data[i]) 3664 break; 3665 if (i == data_len) 3666 return rte_flow_error_set(error, ENOTSUP, 3667 RTE_FLOW_ERROR_TYPE_ITEM, item, 3668 "Can't match on Geneve option data and mask 0"); 3669 /* Check data mask supported. */ 3670 for (i = data_max_supported; i < data_len ; i++) 3671 if (mask->data[i]) 3672 return rte_flow_error_set(error, ENOTSUP, 3673 RTE_FLOW_ERROR_TYPE_ITEM, item, 3674 "Data mask is of unsupported size"); 3675 } 3676 /* Check GENEVE option is supported in NIC. */ 3677 if (!hca_attr->geneve_tlv_opt) 3678 return rte_flow_error_set 3679 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3680 "Geneve TLV opt not supported"); 3681 /* Check if we already have geneve option with different type/class. */ 3682 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3683 geneve_opt_resource = sh->geneve_tlv_option_resource; 3684 if (geneve_opt_resource != NULL) 3685 if (geneve_opt_resource->option_class != spec->option_class || 3686 geneve_opt_resource->option_type != spec->option_type || 3687 geneve_opt_resource->length != spec->option_len) { 3688 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3689 return rte_flow_error_set(error, ENOTSUP, 3690 RTE_FLOW_ERROR_TYPE_ITEM, item, 3691 "Only one Geneve TLV option supported"); 3692 } 3693 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3694 return 0; 3695 } 3696 3697 /** 3698 * Validate MPLS item. 3699 * 3700 * @param[in] dev 3701 * Pointer to the rte_eth_dev structure. 3702 * @param[in] item 3703 * Item specification. 3704 * @param[in] item_flags 3705 * Bit-fields that holds the items detected until now. 3706 * @param[in] prev_layer 3707 * The protocol layer indicated in previous item. 3708 * @param[out] error 3709 * Pointer to error structure. 3710 * 3711 * @return 3712 * 0 on success, a negative errno value otherwise and rte_errno is set. 3713 */ 3714 int 3715 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3716 const struct rte_flow_item *item __rte_unused, 3717 uint64_t item_flags __rte_unused, 3718 uint64_t prev_layer __rte_unused, 3719 struct rte_flow_error *error) 3720 { 3721 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3722 const struct rte_flow_item_mpls *mask = item->mask; 3723 struct mlx5_priv *priv = dev->data->dev_private; 3724 int ret; 3725 3726 if (!priv->sh->dev_cap.mpls_en) 3727 return rte_flow_error_set(error, ENOTSUP, 3728 RTE_FLOW_ERROR_TYPE_ITEM, item, 3729 "MPLS not supported or" 3730 " disabled in firmware" 3731 " configuration."); 3732 /* MPLS over UDP, GRE is allowed */ 3733 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3734 MLX5_FLOW_LAYER_GRE | 3735 MLX5_FLOW_LAYER_GRE_KEY))) 3736 return rte_flow_error_set(error, EINVAL, 3737 RTE_FLOW_ERROR_TYPE_ITEM, item, 3738 "protocol filtering not compatible" 3739 " with MPLS layer"); 3740 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3741 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3742 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3743 return rte_flow_error_set(error, ENOTSUP, 3744 RTE_FLOW_ERROR_TYPE_ITEM, item, 3745 "multiple tunnel layers not" 3746 " supported"); 3747 if (!mask) 3748 mask = &rte_flow_item_mpls_mask; 3749 ret = mlx5_flow_item_acceptable 3750 (item, (const uint8_t *)mask, 3751 (const uint8_t *)&rte_flow_item_mpls_mask, 3752 sizeof(struct rte_flow_item_mpls), 3753 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3754 if (ret < 0) 3755 return ret; 3756 return 0; 3757 #else 3758 return rte_flow_error_set(error, ENOTSUP, 3759 RTE_FLOW_ERROR_TYPE_ITEM, item, 3760 "MPLS is not supported by Verbs, please" 3761 " update."); 3762 #endif 3763 } 3764 3765 /** 3766 * Validate NVGRE item. 3767 * 3768 * @param[in] item 3769 * Item specification. 3770 * @param[in] item_flags 3771 * Bit flags to mark detected items. 3772 * @param[in] target_protocol 3773 * The next protocol in the previous item. 3774 * @param[out] error 3775 * Pointer to error structure. 3776 * 3777 * @return 3778 * 0 on success, a negative errno value otherwise and rte_errno is set. 3779 */ 3780 int 3781 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3782 uint64_t item_flags, 3783 uint8_t target_protocol, 3784 struct rte_flow_error *error) 3785 { 3786 const struct rte_flow_item_nvgre *mask = item->mask; 3787 int ret; 3788 3789 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3790 return rte_flow_error_set(error, EINVAL, 3791 RTE_FLOW_ERROR_TYPE_ITEM, item, 3792 "protocol filtering not compatible" 3793 " with this GRE layer"); 3794 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3795 return rte_flow_error_set(error, ENOTSUP, 3796 RTE_FLOW_ERROR_TYPE_ITEM, item, 3797 "multiple tunnel layers not" 3798 " supported"); 3799 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3800 return rte_flow_error_set(error, ENOTSUP, 3801 RTE_FLOW_ERROR_TYPE_ITEM, item, 3802 "L3 Layer is missing"); 3803 if (!mask) 3804 mask = &rte_flow_item_nvgre_mask; 3805 ret = mlx5_flow_item_acceptable 3806 (item, (const uint8_t *)mask, 3807 (const uint8_t *)&rte_flow_item_nvgre_mask, 3808 sizeof(struct rte_flow_item_nvgre), 3809 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3810 if (ret < 0) 3811 return ret; 3812 return 0; 3813 } 3814 3815 /** 3816 * Validate eCPRI item. 3817 * 3818 * @param[in] item 3819 * Item specification. 3820 * @param[in] item_flags 3821 * Bit-fields that holds the items detected until now. 3822 * @param[in] last_item 3823 * Previous validated item in the pattern items. 3824 * @param[in] ether_type 3825 * Type in the ethernet layer header (including dot1q). 3826 * @param[in] acc_mask 3827 * Acceptable mask, if NULL default internal default mask 3828 * will be used to check whether item fields are supported. 3829 * @param[out] error 3830 * Pointer to error structure. 3831 * 3832 * @return 3833 * 0 on success, a negative errno value otherwise and rte_errno is set. 3834 */ 3835 int 3836 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3837 uint64_t item_flags, 3838 uint64_t last_item, 3839 uint16_t ether_type, 3840 const struct rte_flow_item_ecpri *acc_mask, 3841 struct rte_flow_error *error) 3842 { 3843 const struct rte_flow_item_ecpri *mask = item->mask; 3844 const struct rte_flow_item_ecpri nic_mask = { 3845 .hdr = { 3846 .common = { 3847 .u32 = 3848 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3849 .type = 0xFF, 3850 }).u32), 3851 }, 3852 .dummy[0] = 0xFFFFFFFF, 3853 }, 3854 }; 3855 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3856 MLX5_FLOW_LAYER_OUTER_VLAN); 3857 struct rte_flow_item_ecpri mask_lo; 3858 3859 if (!(last_item & outer_l2_vlan) && 3860 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3861 return rte_flow_error_set(error, EINVAL, 3862 RTE_FLOW_ERROR_TYPE_ITEM, item, 3863 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3864 if ((last_item & outer_l2_vlan) && ether_type && 3865 ether_type != RTE_ETHER_TYPE_ECPRI) 3866 return rte_flow_error_set(error, EINVAL, 3867 RTE_FLOW_ERROR_TYPE_ITEM, item, 3868 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3869 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3870 return rte_flow_error_set(error, EINVAL, 3871 RTE_FLOW_ERROR_TYPE_ITEM, item, 3872 "eCPRI with tunnel is not supported right now"); 3873 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3874 return rte_flow_error_set(error, ENOTSUP, 3875 RTE_FLOW_ERROR_TYPE_ITEM, item, 3876 "multiple L3 layers not supported"); 3877 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3878 return rte_flow_error_set(error, EINVAL, 3879 RTE_FLOW_ERROR_TYPE_ITEM, item, 3880 "eCPRI cannot coexist with a TCP layer"); 3881 /* In specification, eCPRI could be over UDP layer. */ 3882 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3883 return rte_flow_error_set(error, EINVAL, 3884 RTE_FLOW_ERROR_TYPE_ITEM, item, 3885 "eCPRI over UDP layer is not yet supported right now"); 3886 /* Mask for type field in common header could be zero. */ 3887 if (!mask) 3888 mask = &rte_flow_item_ecpri_mask; 3889 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3890 /* Input mask is in big-endian format. */ 3891 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3892 return rte_flow_error_set(error, EINVAL, 3893 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3894 "partial mask is not supported for protocol"); 3895 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3896 return rte_flow_error_set(error, EINVAL, 3897 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3898 "message header mask must be after a type mask"); 3899 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3900 acc_mask ? (const uint8_t *)acc_mask 3901 : (const uint8_t *)&nic_mask, 3902 sizeof(struct rte_flow_item_ecpri), 3903 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3904 } 3905 3906 /** 3907 * Validate the NSH item. 3908 * 3909 * @param[in] dev 3910 * Pointer to Ethernet device on which flow rule is being created on. 3911 * @param[out] error 3912 * Pointer to error structure. 3913 * 3914 * @return 3915 * 0 on success, a negative errno value otherwise and rte_errno is set. 3916 */ 3917 int 3918 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev, 3919 const struct rte_flow_item *item, 3920 struct rte_flow_error *error) 3921 { 3922 struct mlx5_priv *priv = dev->data->dev_private; 3923 3924 if (item->mask) { 3925 return rte_flow_error_set(error, ENOTSUP, 3926 RTE_FLOW_ERROR_TYPE_ITEM, item, 3927 "NSH fields matching is not supported"); 3928 } 3929 3930 if (!priv->sh->config.dv_flow_en) { 3931 return rte_flow_error_set(error, ENOTSUP, 3932 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3933 NULL, "NSH support requires DV flow interface"); 3934 } 3935 3936 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) { 3937 return rte_flow_error_set(error, ENOTSUP, 3938 RTE_FLOW_ERROR_TYPE_ITEM, item, 3939 "Current FW does not support matching on NSH"); 3940 } 3941 3942 return 0; 3943 } 3944 3945 static int 3946 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3947 const struct rte_flow_attr *attr __rte_unused, 3948 const struct rte_flow_item items[] __rte_unused, 3949 const struct rte_flow_action actions[] __rte_unused, 3950 bool external __rte_unused, 3951 int hairpin __rte_unused, 3952 struct rte_flow_error *error) 3953 { 3954 return rte_flow_error_set(error, ENOTSUP, 3955 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3956 } 3957 3958 static struct mlx5_flow * 3959 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3960 const struct rte_flow_attr *attr __rte_unused, 3961 const struct rte_flow_item items[] __rte_unused, 3962 const struct rte_flow_action actions[] __rte_unused, 3963 struct rte_flow_error *error) 3964 { 3965 rte_flow_error_set(error, ENOTSUP, 3966 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3967 return NULL; 3968 } 3969 3970 static int 3971 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3972 struct mlx5_flow *dev_flow __rte_unused, 3973 const struct rte_flow_attr *attr __rte_unused, 3974 const struct rte_flow_item items[] __rte_unused, 3975 const struct rte_flow_action actions[] __rte_unused, 3976 struct rte_flow_error *error) 3977 { 3978 return rte_flow_error_set(error, ENOTSUP, 3979 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3980 } 3981 3982 static int 3983 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3984 struct rte_flow *flow __rte_unused, 3985 struct rte_flow_error *error) 3986 { 3987 return rte_flow_error_set(error, ENOTSUP, 3988 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3989 } 3990 3991 static void 3992 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3993 struct rte_flow *flow __rte_unused) 3994 { 3995 } 3996 3997 static void 3998 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3999 struct rte_flow *flow __rte_unused) 4000 { 4001 } 4002 4003 static int 4004 flow_null_query(struct rte_eth_dev *dev __rte_unused, 4005 struct rte_flow *flow __rte_unused, 4006 const struct rte_flow_action *actions __rte_unused, 4007 void *data __rte_unused, 4008 struct rte_flow_error *error) 4009 { 4010 return rte_flow_error_set(error, ENOTSUP, 4011 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4012 } 4013 4014 static int 4015 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 4016 uint32_t domains __rte_unused, 4017 uint32_t flags __rte_unused) 4018 { 4019 return 0; 4020 } 4021 4022 int 4023 flow_null_get_aged_flows(struct rte_eth_dev *dev, 4024 void **context __rte_unused, 4025 uint32_t nb_contexts __rte_unused, 4026 struct rte_flow_error *error __rte_unused) 4027 { 4028 DRV_LOG(ERR, "port %u get aged flows is not supported.", 4029 dev->data->port_id); 4030 return -ENOTSUP; 4031 } 4032 4033 uint32_t 4034 flow_null_counter_allocate(struct rte_eth_dev *dev) 4035 { 4036 DRV_LOG(ERR, "port %u counter allocate is not supported.", 4037 dev->data->port_id); 4038 return 0; 4039 } 4040 4041 void 4042 flow_null_counter_free(struct rte_eth_dev *dev, 4043 uint32_t counter __rte_unused) 4044 { 4045 DRV_LOG(ERR, "port %u counter free is not supported.", 4046 dev->data->port_id); 4047 } 4048 4049 int 4050 flow_null_counter_query(struct rte_eth_dev *dev, 4051 uint32_t counter __rte_unused, 4052 bool clear __rte_unused, 4053 uint64_t *pkts __rte_unused, 4054 uint64_t *bytes __rte_unused, 4055 void **action __rte_unused) 4056 { 4057 DRV_LOG(ERR, "port %u counter query is not supported.", 4058 dev->data->port_id); 4059 return -ENOTSUP; 4060 } 4061 4062 /* Void driver to protect from null pointer reference. */ 4063 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 4064 .validate = flow_null_validate, 4065 .prepare = flow_null_prepare, 4066 .translate = flow_null_translate, 4067 .apply = flow_null_apply, 4068 .remove = flow_null_remove, 4069 .destroy = flow_null_destroy, 4070 .query = flow_null_query, 4071 .sync_domain = flow_null_sync_domain, 4072 .get_aged_flows = flow_null_get_aged_flows, 4073 .counter_alloc = flow_null_counter_allocate, 4074 .counter_free = flow_null_counter_free, 4075 .counter_query = flow_null_counter_query 4076 }; 4077 4078 /** 4079 * Select flow driver type according to flow attributes and device 4080 * configuration. 4081 * 4082 * @param[in] dev 4083 * Pointer to the dev structure. 4084 * @param[in] attr 4085 * Pointer to the flow attributes. 4086 * 4087 * @return 4088 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4089 */ 4090 static enum mlx5_flow_drv_type 4091 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4092 { 4093 struct mlx5_priv *priv = dev->data->dev_private; 4094 /* The OS can determine first a specific flow type (DV, VERBS) */ 4095 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4096 4097 if (type != MLX5_FLOW_TYPE_MAX) 4098 return type; 4099 /* 4100 * Currently when dv_flow_en == 2, only HW steering engine is 4101 * supported. New engines can also be chosen here if ready. 4102 */ 4103 if (priv->sh->config.dv_flow_en == 2) 4104 return MLX5_FLOW_TYPE_HW; 4105 if (!attr) 4106 return MLX5_FLOW_TYPE_MIN; 4107 /* If no OS specific type - continue with DV/VERBS selection */ 4108 if (attr->transfer && priv->sh->config.dv_esw_en) 4109 type = MLX5_FLOW_TYPE_DV; 4110 if (!attr->transfer) 4111 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4112 MLX5_FLOW_TYPE_VERBS; 4113 return type; 4114 } 4115 4116 #define flow_get_drv_ops(type) flow_drv_ops[type] 4117 4118 /** 4119 * Flow driver validation API. This abstracts calling driver specific functions. 4120 * The type of flow driver is determined according to flow attributes. 4121 * 4122 * @param[in] dev 4123 * Pointer to the dev structure. 4124 * @param[in] attr 4125 * Pointer to the flow attributes. 4126 * @param[in] items 4127 * Pointer to the list of items. 4128 * @param[in] actions 4129 * Pointer to the list of actions. 4130 * @param[in] external 4131 * This flow rule is created by request external to PMD. 4132 * @param[in] hairpin 4133 * Number of hairpin TX actions, 0 means classic flow. 4134 * @param[out] error 4135 * Pointer to the error structure. 4136 * 4137 * @return 4138 * 0 on success, a negative errno value otherwise and rte_errno is set. 4139 */ 4140 static inline int 4141 flow_drv_validate(struct rte_eth_dev *dev, 4142 const struct rte_flow_attr *attr, 4143 const struct rte_flow_item items[], 4144 const struct rte_flow_action actions[], 4145 bool external, int hairpin, struct rte_flow_error *error) 4146 { 4147 const struct mlx5_flow_driver_ops *fops; 4148 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4149 4150 fops = flow_get_drv_ops(type); 4151 return fops->validate(dev, attr, items, actions, external, 4152 hairpin, error); 4153 } 4154 4155 /** 4156 * Flow driver preparation API. This abstracts calling driver specific 4157 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4158 * calculates the size of memory required for device flow, allocates the memory, 4159 * initializes the device flow and returns the pointer. 4160 * 4161 * @note 4162 * This function initializes device flow structure such as dv or verbs in 4163 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4164 * rest. For example, adding returning device flow to flow->dev_flow list and 4165 * setting backward reference to the flow should be done out of this function. 4166 * layers field is not filled either. 4167 * 4168 * @param[in] dev 4169 * Pointer to the dev structure. 4170 * @param[in] attr 4171 * Pointer to the flow attributes. 4172 * @param[in] items 4173 * Pointer to the list of items. 4174 * @param[in] actions 4175 * Pointer to the list of actions. 4176 * @param[in] flow_idx 4177 * This memory pool index to the flow. 4178 * @param[out] error 4179 * Pointer to the error structure. 4180 * 4181 * @return 4182 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4183 */ 4184 static inline struct mlx5_flow * 4185 flow_drv_prepare(struct rte_eth_dev *dev, 4186 const struct rte_flow *flow, 4187 const struct rte_flow_attr *attr, 4188 const struct rte_flow_item items[], 4189 const struct rte_flow_action actions[], 4190 uint32_t flow_idx, 4191 struct rte_flow_error *error) 4192 { 4193 const struct mlx5_flow_driver_ops *fops; 4194 enum mlx5_flow_drv_type type = flow->drv_type; 4195 struct mlx5_flow *mlx5_flow = NULL; 4196 4197 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4198 fops = flow_get_drv_ops(type); 4199 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4200 if (mlx5_flow) 4201 mlx5_flow->flow_idx = flow_idx; 4202 return mlx5_flow; 4203 } 4204 4205 /** 4206 * Flow driver translation API. This abstracts calling driver specific 4207 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4208 * translates a generic flow into a driver flow. flow_drv_prepare() must 4209 * precede. 4210 * 4211 * @note 4212 * dev_flow->layers could be filled as a result of parsing during translation 4213 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4214 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4215 * flow->actions could be overwritten even though all the expanded dev_flows 4216 * have the same actions. 4217 * 4218 * @param[in] dev 4219 * Pointer to the rte dev structure. 4220 * @param[in, out] dev_flow 4221 * Pointer to the mlx5 flow. 4222 * @param[in] attr 4223 * Pointer to the flow attributes. 4224 * @param[in] items 4225 * Pointer to the list of items. 4226 * @param[in] actions 4227 * Pointer to the list of actions. 4228 * @param[out] error 4229 * Pointer to the error structure. 4230 * 4231 * @return 4232 * 0 on success, a negative errno value otherwise and rte_errno is set. 4233 */ 4234 static inline int 4235 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4236 const struct rte_flow_attr *attr, 4237 const struct rte_flow_item items[], 4238 const struct rte_flow_action actions[], 4239 struct rte_flow_error *error) 4240 { 4241 const struct mlx5_flow_driver_ops *fops; 4242 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4243 4244 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4245 fops = flow_get_drv_ops(type); 4246 return fops->translate(dev, dev_flow, attr, items, actions, error); 4247 } 4248 4249 /** 4250 * Flow driver apply API. This abstracts calling driver specific functions. 4251 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4252 * translated driver flows on to device. flow_drv_translate() must precede. 4253 * 4254 * @param[in] dev 4255 * Pointer to Ethernet device structure. 4256 * @param[in, out] flow 4257 * Pointer to flow structure. 4258 * @param[out] error 4259 * Pointer to error structure. 4260 * 4261 * @return 4262 * 0 on success, a negative errno value otherwise and rte_errno is set. 4263 */ 4264 static inline int 4265 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4266 struct rte_flow_error *error) 4267 { 4268 const struct mlx5_flow_driver_ops *fops; 4269 enum mlx5_flow_drv_type type = flow->drv_type; 4270 4271 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4272 fops = flow_get_drv_ops(type); 4273 return fops->apply(dev, flow, error); 4274 } 4275 4276 /** 4277 * Flow driver destroy API. This abstracts calling driver specific functions. 4278 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4279 * on device and releases resources of the flow. 4280 * 4281 * @param[in] dev 4282 * Pointer to Ethernet device. 4283 * @param[in, out] flow 4284 * Pointer to flow structure. 4285 */ 4286 static inline void 4287 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4288 { 4289 const struct mlx5_flow_driver_ops *fops; 4290 enum mlx5_flow_drv_type type = flow->drv_type; 4291 4292 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4293 fops = flow_get_drv_ops(type); 4294 fops->destroy(dev, flow); 4295 } 4296 4297 /** 4298 * Flow driver find RSS policy tbl API. This abstracts calling driver 4299 * specific functions. Parent flow (rte_flow) should have driver 4300 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4301 * 4302 * @param[in] dev 4303 * Pointer to Ethernet device. 4304 * @param[in, out] flow 4305 * Pointer to flow structure. 4306 * @param[in] policy 4307 * Pointer to meter policy table. 4308 * @param[in] rss_desc 4309 * Pointer to rss_desc 4310 */ 4311 static struct mlx5_flow_meter_sub_policy * 4312 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4313 struct rte_flow *flow, 4314 struct mlx5_flow_meter_policy *policy, 4315 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4316 { 4317 const struct mlx5_flow_driver_ops *fops; 4318 enum mlx5_flow_drv_type type = flow->drv_type; 4319 4320 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4321 fops = flow_get_drv_ops(type); 4322 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4323 } 4324 4325 /** 4326 * Flow driver color tag rule API. This abstracts calling driver 4327 * specific functions. Parent flow (rte_flow) should have driver 4328 * type (drv_type). It will create the color tag rules in hierarchy meter. 4329 * 4330 * @param[in] dev 4331 * Pointer to Ethernet device. 4332 * @param[in, out] flow 4333 * Pointer to flow structure. 4334 * @param[in] fm 4335 * Pointer to flow meter structure. 4336 * @param[in] src_port 4337 * The src port this extra rule should use. 4338 * @param[in] item 4339 * The src port id match item. 4340 * @param[out] error 4341 * Pointer to error structure. 4342 */ 4343 static int 4344 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4345 struct rte_flow *flow, 4346 struct mlx5_flow_meter_info *fm, 4347 int32_t src_port, 4348 const struct rte_flow_item *item, 4349 struct rte_flow_error *error) 4350 { 4351 const struct mlx5_flow_driver_ops *fops; 4352 enum mlx5_flow_drv_type type = flow->drv_type; 4353 4354 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4355 fops = flow_get_drv_ops(type); 4356 return fops->meter_hierarchy_rule_create(dev, fm, 4357 src_port, item, error); 4358 } 4359 4360 /** 4361 * Get RSS action from the action list. 4362 * 4363 * @param[in] dev 4364 * Pointer to Ethernet device. 4365 * @param[in] actions 4366 * Pointer to the list of actions. 4367 * @param[in] flow 4368 * Parent flow structure pointer. 4369 * 4370 * @return 4371 * Pointer to the RSS action if exist, else return NULL. 4372 */ 4373 static const struct rte_flow_action_rss* 4374 flow_get_rss_action(struct rte_eth_dev *dev, 4375 const struct rte_flow_action actions[]) 4376 { 4377 struct mlx5_priv *priv = dev->data->dev_private; 4378 const struct rte_flow_action_rss *rss = NULL; 4379 struct mlx5_meter_policy_action_container *acg; 4380 struct mlx5_meter_policy_action_container *acy; 4381 4382 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4383 switch (actions->type) { 4384 case RTE_FLOW_ACTION_TYPE_RSS: 4385 rss = actions->conf; 4386 break; 4387 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4388 { 4389 const struct rte_flow_action_sample *sample = 4390 actions->conf; 4391 const struct rte_flow_action *act = sample->actions; 4392 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4393 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4394 rss = act->conf; 4395 break; 4396 } 4397 case RTE_FLOW_ACTION_TYPE_METER: 4398 { 4399 uint32_t mtr_idx; 4400 struct mlx5_flow_meter_info *fm; 4401 struct mlx5_flow_meter_policy *policy; 4402 const struct rte_flow_action_meter *mtr = actions->conf; 4403 4404 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4405 if (fm && !fm->def_policy) { 4406 policy = mlx5_flow_meter_policy_find(dev, 4407 fm->policy_id, NULL); 4408 MLX5_ASSERT(policy); 4409 if (policy->is_hierarchy) { 4410 policy = 4411 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4412 policy); 4413 if (!policy) 4414 return NULL; 4415 } 4416 if (policy->is_rss) { 4417 acg = 4418 &policy->act_cnt[RTE_COLOR_GREEN]; 4419 acy = 4420 &policy->act_cnt[RTE_COLOR_YELLOW]; 4421 if (acg->fate_action == 4422 MLX5_FLOW_FATE_SHARED_RSS) 4423 rss = acg->rss->conf; 4424 else if (acy->fate_action == 4425 MLX5_FLOW_FATE_SHARED_RSS) 4426 rss = acy->rss->conf; 4427 } 4428 } 4429 break; 4430 } 4431 default: 4432 break; 4433 } 4434 } 4435 return rss; 4436 } 4437 4438 /** 4439 * Get ASO age action by index. 4440 * 4441 * @param[in] dev 4442 * Pointer to the Ethernet device structure. 4443 * @param[in] age_idx 4444 * Index to the ASO age action. 4445 * 4446 * @return 4447 * The specified ASO age action. 4448 */ 4449 struct mlx5_aso_age_action* 4450 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4451 { 4452 uint16_t pool_idx = age_idx & UINT16_MAX; 4453 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4454 struct mlx5_priv *priv = dev->data->dev_private; 4455 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4456 struct mlx5_aso_age_pool *pool; 4457 4458 rte_rwlock_read_lock(&mng->resize_rwl); 4459 pool = mng->pools[pool_idx]; 4460 rte_rwlock_read_unlock(&mng->resize_rwl); 4461 return &pool->actions[offset - 1]; 4462 } 4463 4464 /* maps indirect action to translated direct in some actions array */ 4465 struct mlx5_translated_action_handle { 4466 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4467 int index; /**< Index in related array of rte_flow_action. */ 4468 }; 4469 4470 /** 4471 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4472 * direct action if translation possible. 4473 * This functionality used to run same execution path for both direct and 4474 * indirect actions on flow create. All necessary preparations for indirect 4475 * action handling should be performed on *handle* actions list returned 4476 * from this call. 4477 * 4478 * @param[in] dev 4479 * Pointer to Ethernet device. 4480 * @param[in] actions 4481 * List of actions to translate. 4482 * @param[out] handle 4483 * List to store translated indirect action object handles. 4484 * @param[in, out] indir_n 4485 * Size of *handle* array. On return should be updated with number of 4486 * indirect actions retrieved from the *actions* list. 4487 * @param[out] translated_actions 4488 * List of actions where all indirect actions were translated to direct 4489 * if possible. NULL if no translation took place. 4490 * @param[out] error 4491 * Pointer to the error structure. 4492 * 4493 * @return 4494 * 0 on success, a negative errno value otherwise and rte_errno is set. 4495 */ 4496 static int 4497 flow_action_handles_translate(struct rte_eth_dev *dev, 4498 const struct rte_flow_action actions[], 4499 struct mlx5_translated_action_handle *handle, 4500 int *indir_n, 4501 struct rte_flow_action **translated_actions, 4502 struct rte_flow_error *error) 4503 { 4504 struct mlx5_priv *priv = dev->data->dev_private; 4505 struct rte_flow_action *translated = NULL; 4506 size_t actions_size; 4507 int n; 4508 int copied_n = 0; 4509 struct mlx5_translated_action_handle *handle_end = NULL; 4510 4511 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4512 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4513 continue; 4514 if (copied_n == *indir_n) { 4515 return rte_flow_error_set 4516 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4517 NULL, "too many shared actions"); 4518 } 4519 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4520 sizeof(actions[n].conf)); 4521 handle[copied_n].index = n; 4522 copied_n++; 4523 } 4524 n++; 4525 *indir_n = copied_n; 4526 if (!copied_n) 4527 return 0; 4528 actions_size = sizeof(struct rte_flow_action) * n; 4529 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4530 if (!translated) { 4531 rte_errno = ENOMEM; 4532 return -ENOMEM; 4533 } 4534 memcpy(translated, actions, actions_size); 4535 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4536 struct mlx5_shared_action_rss *shared_rss; 4537 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4538 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4539 uint32_t idx = act_idx & 4540 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4541 4542 switch (type) { 4543 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4544 shared_rss = mlx5_ipool_get 4545 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4546 translated[handle->index].type = 4547 RTE_FLOW_ACTION_TYPE_RSS; 4548 translated[handle->index].conf = 4549 &shared_rss->origin; 4550 break; 4551 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4552 translated[handle->index].type = 4553 (enum rte_flow_action_type) 4554 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4555 translated[handle->index].conf = (void *)(uintptr_t)idx; 4556 break; 4557 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4558 translated[handle->index].type = 4559 (enum rte_flow_action_type) 4560 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4561 translated[handle->index].conf = (void *)(uintptr_t)idx; 4562 break; 4563 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4564 if (priv->sh->flow_hit_aso_en) { 4565 translated[handle->index].type = 4566 (enum rte_flow_action_type) 4567 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4568 translated[handle->index].conf = 4569 (void *)(uintptr_t)idx; 4570 break; 4571 } 4572 /* Fall-through */ 4573 case MLX5_INDIRECT_ACTION_TYPE_CT: 4574 if (priv->sh->ct_aso_en) { 4575 translated[handle->index].type = 4576 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4577 translated[handle->index].conf = 4578 (void *)(uintptr_t)idx; 4579 break; 4580 } 4581 /* Fall-through */ 4582 default: 4583 mlx5_free(translated); 4584 return rte_flow_error_set 4585 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4586 NULL, "invalid indirect action type"); 4587 } 4588 } 4589 *translated_actions = translated; 4590 return 0; 4591 } 4592 4593 /** 4594 * Get Shared RSS action from the action list. 4595 * 4596 * @param[in] dev 4597 * Pointer to Ethernet device. 4598 * @param[in] shared 4599 * Pointer to the list of actions. 4600 * @param[in] shared_n 4601 * Actions list length. 4602 * 4603 * @return 4604 * The MLX5 RSS action ID if exists, otherwise return 0. 4605 */ 4606 static uint32_t 4607 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4608 struct mlx5_translated_action_handle *handle, 4609 int shared_n) 4610 { 4611 struct mlx5_translated_action_handle *handle_end; 4612 struct mlx5_priv *priv = dev->data->dev_private; 4613 struct mlx5_shared_action_rss *shared_rss; 4614 4615 4616 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4617 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4618 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4619 uint32_t idx = act_idx & 4620 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4621 switch (type) { 4622 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4623 shared_rss = mlx5_ipool_get 4624 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4625 idx); 4626 __atomic_fetch_add(&shared_rss->refcnt, 1, 4627 __ATOMIC_RELAXED); 4628 return idx; 4629 default: 4630 break; 4631 } 4632 } 4633 return 0; 4634 } 4635 4636 static unsigned int 4637 find_graph_root(uint32_t rss_level) 4638 { 4639 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4640 MLX5_EXPANSION_ROOT_OUTER; 4641 } 4642 4643 /** 4644 * Get layer flags from the prefix flow. 4645 * 4646 * Some flows may be split to several subflows, the prefix subflow gets the 4647 * match items and the suffix sub flow gets the actions. 4648 * Some actions need the user defined match item flags to get the detail for 4649 * the action. 4650 * This function helps the suffix flow to get the item layer flags from prefix 4651 * subflow. 4652 * 4653 * @param[in] dev_flow 4654 * Pointer the created prefix subflow. 4655 * 4656 * @return 4657 * The layers get from prefix subflow. 4658 */ 4659 static inline uint64_t 4660 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4661 { 4662 uint64_t layers = 0; 4663 4664 /* 4665 * Layers bits could be localization, but usually the compiler will 4666 * help to do the optimization work for source code. 4667 * If no decap actions, use the layers directly. 4668 */ 4669 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4670 return dev_flow->handle->layers; 4671 /* Convert L3 layers with decap action. */ 4672 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4673 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4674 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4675 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4676 /* Convert L4 layers with decap action. */ 4677 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4678 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4679 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4680 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4681 return layers; 4682 } 4683 4684 /** 4685 * Get metadata split action information. 4686 * 4687 * @param[in] actions 4688 * Pointer to the list of actions. 4689 * @param[out] qrss 4690 * Pointer to the return pointer. 4691 * @param[out] qrss_type 4692 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4693 * if no QUEUE/RSS is found. 4694 * @param[out] encap_idx 4695 * Pointer to the index of the encap action if exists, otherwise the last 4696 * action index. 4697 * 4698 * @return 4699 * Total number of actions. 4700 */ 4701 static int 4702 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4703 const struct rte_flow_action **qrss, 4704 int *encap_idx) 4705 { 4706 const struct rte_flow_action_raw_encap *raw_encap; 4707 int actions_n = 0; 4708 int raw_decap_idx = -1; 4709 4710 *encap_idx = -1; 4711 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4712 switch (actions->type) { 4713 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4714 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4715 *encap_idx = actions_n; 4716 break; 4717 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4718 raw_decap_idx = actions_n; 4719 break; 4720 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4721 raw_encap = actions->conf; 4722 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4723 *encap_idx = raw_decap_idx != -1 ? 4724 raw_decap_idx : actions_n; 4725 break; 4726 case RTE_FLOW_ACTION_TYPE_QUEUE: 4727 case RTE_FLOW_ACTION_TYPE_RSS: 4728 *qrss = actions; 4729 break; 4730 default: 4731 break; 4732 } 4733 actions_n++; 4734 } 4735 if (*encap_idx == -1) 4736 *encap_idx = actions_n; 4737 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4738 return actions_n + 1; 4739 } 4740 4741 /** 4742 * Check if the action will change packet. 4743 * 4744 * @param dev 4745 * Pointer to Ethernet device. 4746 * @param[in] type 4747 * action type. 4748 * 4749 * @return 4750 * true if action will change packet, false otherwise. 4751 */ 4752 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4753 enum rte_flow_action_type type) 4754 { 4755 struct mlx5_priv *priv = dev->data->dev_private; 4756 4757 switch (type) { 4758 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4759 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4760 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4761 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4762 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4763 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4764 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4765 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4766 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4767 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4768 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4769 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4770 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4771 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4772 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4773 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4774 case RTE_FLOW_ACTION_TYPE_SET_META: 4775 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4776 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4777 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4778 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4779 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4780 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4781 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4782 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4783 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4784 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4785 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4786 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4787 return true; 4788 case RTE_FLOW_ACTION_TYPE_FLAG: 4789 case RTE_FLOW_ACTION_TYPE_MARK: 4790 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4791 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4792 return true; 4793 else 4794 return false; 4795 default: 4796 return false; 4797 } 4798 } 4799 4800 /** 4801 * Check meter action from the action list. 4802 * 4803 * @param dev 4804 * Pointer to Ethernet device. 4805 * @param[in] actions 4806 * Pointer to the list of actions. 4807 * @param[out] has_mtr 4808 * Pointer to the meter exist flag. 4809 * @param[out] has_modify 4810 * Pointer to the flag showing there's packet change action. 4811 * @param[out] meter_id 4812 * Pointer to the meter id. 4813 * 4814 * @return 4815 * Total number of actions. 4816 */ 4817 static int 4818 flow_check_meter_action(struct rte_eth_dev *dev, 4819 const struct rte_flow_action actions[], 4820 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4821 { 4822 const struct rte_flow_action_meter *mtr = NULL; 4823 int actions_n = 0; 4824 4825 MLX5_ASSERT(has_mtr); 4826 *has_mtr = false; 4827 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4828 switch (actions->type) { 4829 case RTE_FLOW_ACTION_TYPE_METER: 4830 mtr = actions->conf; 4831 *meter_id = mtr->mtr_id; 4832 *has_mtr = true; 4833 break; 4834 default: 4835 break; 4836 } 4837 if (!*has_mtr) 4838 *has_modify |= flow_check_modify_action_type(dev, 4839 actions->type); 4840 actions_n++; 4841 } 4842 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4843 return actions_n + 1; 4844 } 4845 4846 /** 4847 * Check if the flow should be split due to hairpin. 4848 * The reason for the split is that in current HW we can't 4849 * support encap and push-vlan on Rx, so if a flow contains 4850 * these actions we move it to Tx. 4851 * 4852 * @param dev 4853 * Pointer to Ethernet device. 4854 * @param[in] attr 4855 * Flow rule attributes. 4856 * @param[in] actions 4857 * Associated actions (list terminated by the END action). 4858 * 4859 * @return 4860 * > 0 the number of actions and the flow should be split, 4861 * 0 when no split required. 4862 */ 4863 static int 4864 flow_check_hairpin_split(struct rte_eth_dev *dev, 4865 const struct rte_flow_attr *attr, 4866 const struct rte_flow_action actions[]) 4867 { 4868 int queue_action = 0; 4869 int action_n = 0; 4870 int split = 0; 4871 int push_vlan = 0; 4872 const struct rte_flow_action_queue *queue; 4873 const struct rte_flow_action_rss *rss; 4874 const struct rte_flow_action_raw_encap *raw_encap; 4875 const struct rte_eth_hairpin_conf *conf; 4876 4877 if (!attr->ingress) 4878 return 0; 4879 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4880 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4881 push_vlan = 1; 4882 switch (actions->type) { 4883 case RTE_FLOW_ACTION_TYPE_QUEUE: 4884 queue = actions->conf; 4885 if (queue == NULL) 4886 return 0; 4887 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4888 if (conf == NULL || conf->tx_explicit != 0) 4889 return 0; 4890 queue_action = 1; 4891 action_n++; 4892 break; 4893 case RTE_FLOW_ACTION_TYPE_RSS: 4894 rss = actions->conf; 4895 if (rss == NULL || rss->queue_num == 0) 4896 return 0; 4897 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4898 if (conf == NULL || conf->tx_explicit != 0) 4899 return 0; 4900 queue_action = 1; 4901 action_n++; 4902 break; 4903 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4904 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4905 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4906 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4907 split++; 4908 action_n++; 4909 break; 4910 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4911 if (push_vlan) 4912 split++; 4913 action_n++; 4914 break; 4915 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4916 raw_encap = actions->conf; 4917 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4918 split++; 4919 action_n++; 4920 break; 4921 default: 4922 action_n++; 4923 break; 4924 } 4925 } 4926 if (split && queue_action) 4927 return action_n; 4928 return 0; 4929 } 4930 4931 /* Declare flow create/destroy prototype in advance. */ 4932 static uint32_t 4933 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4934 const struct rte_flow_attr *attr, 4935 const struct rte_flow_item items[], 4936 const struct rte_flow_action actions[], 4937 bool external, struct rte_flow_error *error); 4938 4939 static void 4940 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4941 uint32_t flow_idx); 4942 4943 int 4944 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4945 struct mlx5_list_entry *entry, void *cb_ctx) 4946 { 4947 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4948 struct mlx5_flow_mreg_copy_resource *mcp_res = 4949 container_of(entry, typeof(*mcp_res), hlist_ent); 4950 4951 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4952 } 4953 4954 struct mlx5_list_entry * 4955 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4956 { 4957 struct rte_eth_dev *dev = tool_ctx; 4958 struct mlx5_priv *priv = dev->data->dev_private; 4959 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4960 struct mlx5_flow_mreg_copy_resource *mcp_res; 4961 struct rte_flow_error *error = ctx->error; 4962 uint32_t idx = 0; 4963 int ret; 4964 uint32_t mark_id = *(uint32_t *)(ctx->data); 4965 struct rte_flow_attr attr = { 4966 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4967 .ingress = 1, 4968 }; 4969 struct mlx5_rte_flow_item_tag tag_spec = { 4970 .data = mark_id, 4971 }; 4972 struct rte_flow_item items[] = { 4973 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4974 }; 4975 struct rte_flow_action_mark ftag = { 4976 .id = mark_id, 4977 }; 4978 struct mlx5_flow_action_copy_mreg cp_mreg = { 4979 .dst = REG_B, 4980 .src = REG_NON, 4981 }; 4982 struct rte_flow_action_jump jump = { 4983 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4984 }; 4985 struct rte_flow_action actions[] = { 4986 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4987 }; 4988 4989 /* Fill the register fields in the flow. */ 4990 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4991 if (ret < 0) 4992 return NULL; 4993 tag_spec.id = ret; 4994 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4995 if (ret < 0) 4996 return NULL; 4997 cp_mreg.src = ret; 4998 /* Provide the full width of FLAG specific value. */ 4999 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 5000 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 5001 /* Build a new flow. */ 5002 if (mark_id != MLX5_DEFAULT_COPY_ID) { 5003 items[0] = (struct rte_flow_item){ 5004 .type = (enum rte_flow_item_type) 5005 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5006 .spec = &tag_spec, 5007 }; 5008 items[1] = (struct rte_flow_item){ 5009 .type = RTE_FLOW_ITEM_TYPE_END, 5010 }; 5011 actions[0] = (struct rte_flow_action){ 5012 .type = (enum rte_flow_action_type) 5013 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 5014 .conf = &ftag, 5015 }; 5016 actions[1] = (struct rte_flow_action){ 5017 .type = (enum rte_flow_action_type) 5018 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5019 .conf = &cp_mreg, 5020 }; 5021 actions[2] = (struct rte_flow_action){ 5022 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5023 .conf = &jump, 5024 }; 5025 actions[3] = (struct rte_flow_action){ 5026 .type = RTE_FLOW_ACTION_TYPE_END, 5027 }; 5028 } else { 5029 /* Default rule, wildcard match. */ 5030 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 5031 items[0] = (struct rte_flow_item){ 5032 .type = RTE_FLOW_ITEM_TYPE_END, 5033 }; 5034 actions[0] = (struct rte_flow_action){ 5035 .type = (enum rte_flow_action_type) 5036 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5037 .conf = &cp_mreg, 5038 }; 5039 actions[1] = (struct rte_flow_action){ 5040 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5041 .conf = &jump, 5042 }; 5043 actions[2] = (struct rte_flow_action){ 5044 .type = RTE_FLOW_ACTION_TYPE_END, 5045 }; 5046 } 5047 /* Build a new entry. */ 5048 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5049 if (!mcp_res) { 5050 rte_errno = ENOMEM; 5051 return NULL; 5052 } 5053 mcp_res->idx = idx; 5054 mcp_res->mark_id = mark_id; 5055 /* 5056 * The copy Flows are not included in any list. There 5057 * ones are referenced from other Flows and can not 5058 * be applied, removed, deleted in arbitrary order 5059 * by list traversing. 5060 */ 5061 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 5062 &attr, items, actions, false, error); 5063 if (!mcp_res->rix_flow) { 5064 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 5065 return NULL; 5066 } 5067 return &mcp_res->hlist_ent; 5068 } 5069 5070 struct mlx5_list_entry * 5071 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5072 void *cb_ctx __rte_unused) 5073 { 5074 struct rte_eth_dev *dev = tool_ctx; 5075 struct mlx5_priv *priv = dev->data->dev_private; 5076 struct mlx5_flow_mreg_copy_resource *mcp_res; 5077 uint32_t idx = 0; 5078 5079 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5080 if (!mcp_res) { 5081 rte_errno = ENOMEM; 5082 return NULL; 5083 } 5084 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 5085 mcp_res->idx = idx; 5086 return &mcp_res->hlist_ent; 5087 } 5088 5089 void 5090 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5091 { 5092 struct mlx5_flow_mreg_copy_resource *mcp_res = 5093 container_of(entry, typeof(*mcp_res), hlist_ent); 5094 struct rte_eth_dev *dev = tool_ctx; 5095 struct mlx5_priv *priv = dev->data->dev_private; 5096 5097 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5098 } 5099 5100 /** 5101 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5102 * 5103 * As mark_id is unique, if there's already a registered flow for the mark_id, 5104 * return by increasing the reference counter of the resource. Otherwise, create 5105 * the resource (mcp_res) and flow. 5106 * 5107 * Flow looks like, 5108 * - If ingress port is ANY and reg_c[1] is mark_id, 5109 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5110 * 5111 * For default flow (zero mark_id), flow is like, 5112 * - If ingress port is ANY, 5113 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5114 * 5115 * @param dev 5116 * Pointer to Ethernet device. 5117 * @param mark_id 5118 * ID of MARK action, zero means default flow for META. 5119 * @param[out] error 5120 * Perform verbose error reporting if not NULL. 5121 * 5122 * @return 5123 * Associated resource on success, NULL otherwise and rte_errno is set. 5124 */ 5125 static struct mlx5_flow_mreg_copy_resource * 5126 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5127 struct rte_flow_error *error) 5128 { 5129 struct mlx5_priv *priv = dev->data->dev_private; 5130 struct mlx5_list_entry *entry; 5131 struct mlx5_flow_cb_ctx ctx = { 5132 .dev = dev, 5133 .error = error, 5134 .data = &mark_id, 5135 }; 5136 5137 /* Check if already registered. */ 5138 MLX5_ASSERT(priv->mreg_cp_tbl); 5139 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5140 if (!entry) 5141 return NULL; 5142 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5143 hlist_ent); 5144 } 5145 5146 void 5147 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5148 { 5149 struct mlx5_flow_mreg_copy_resource *mcp_res = 5150 container_of(entry, typeof(*mcp_res), hlist_ent); 5151 struct rte_eth_dev *dev = tool_ctx; 5152 struct mlx5_priv *priv = dev->data->dev_private; 5153 5154 MLX5_ASSERT(mcp_res->rix_flow); 5155 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5156 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5157 } 5158 5159 /** 5160 * Release flow in RX_CP_TBL. 5161 * 5162 * @param dev 5163 * Pointer to Ethernet device. 5164 * @flow 5165 * Parent flow for wich copying is provided. 5166 */ 5167 static void 5168 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5169 struct rte_flow *flow) 5170 { 5171 struct mlx5_flow_mreg_copy_resource *mcp_res; 5172 struct mlx5_priv *priv = dev->data->dev_private; 5173 5174 if (!flow->rix_mreg_copy) 5175 return; 5176 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5177 flow->rix_mreg_copy); 5178 if (!mcp_res || !priv->mreg_cp_tbl) 5179 return; 5180 MLX5_ASSERT(mcp_res->rix_flow); 5181 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5182 flow->rix_mreg_copy = 0; 5183 } 5184 5185 /** 5186 * Remove the default copy action from RX_CP_TBL. 5187 * 5188 * This functions is called in the mlx5_dev_start(). No thread safe 5189 * is guaranteed. 5190 * 5191 * @param dev 5192 * Pointer to Ethernet device. 5193 */ 5194 static void 5195 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5196 { 5197 struct mlx5_list_entry *entry; 5198 struct mlx5_priv *priv = dev->data->dev_private; 5199 struct mlx5_flow_cb_ctx ctx; 5200 uint32_t mark_id; 5201 5202 /* Check if default flow is registered. */ 5203 if (!priv->mreg_cp_tbl) 5204 return; 5205 mark_id = MLX5_DEFAULT_COPY_ID; 5206 ctx.data = &mark_id; 5207 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5208 if (!entry) 5209 return; 5210 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5211 } 5212 5213 /** 5214 * Add the default copy action in RX_CP_TBL. 5215 * 5216 * This functions is called in the mlx5_dev_start(). No thread safe 5217 * is guaranteed. 5218 * 5219 * @param dev 5220 * Pointer to Ethernet device. 5221 * @param[out] error 5222 * Perform verbose error reporting if not NULL. 5223 * 5224 * @return 5225 * 0 for success, negative value otherwise and rte_errno is set. 5226 */ 5227 static int 5228 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5229 struct rte_flow_error *error) 5230 { 5231 struct mlx5_priv *priv = dev->data->dev_private; 5232 struct mlx5_flow_mreg_copy_resource *mcp_res; 5233 struct mlx5_flow_cb_ctx ctx; 5234 uint32_t mark_id; 5235 5236 /* Check whether extensive metadata feature is engaged. */ 5237 if (!priv->sh->config.dv_flow_en || 5238 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5239 !mlx5_flow_ext_mreg_supported(dev) || 5240 !priv->sh->dv_regc0_mask) 5241 return 0; 5242 /* 5243 * Add default mreg copy flow may be called multiple time, but 5244 * only be called once in stop. Avoid register it twice. 5245 */ 5246 mark_id = MLX5_DEFAULT_COPY_ID; 5247 ctx.data = &mark_id; 5248 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5249 return 0; 5250 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5251 if (!mcp_res) 5252 return -rte_errno; 5253 return 0; 5254 } 5255 5256 /** 5257 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5258 * 5259 * All the flow having Q/RSS action should be split by 5260 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5261 * performs the following, 5262 * - CQE->flow_tag := reg_c[1] (MARK) 5263 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5264 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5265 * but there should be a flow per each MARK ID set by MARK action. 5266 * 5267 * For the aforementioned reason, if there's a MARK action in flow's action 5268 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5269 * the MARK ID to CQE's flow_tag like, 5270 * - If reg_c[1] is mark_id, 5271 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5272 * 5273 * For SET_META action which stores value in reg_c[0], as the destination is 5274 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5275 * MARK ID means the default flow. The default flow looks like, 5276 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5277 * 5278 * @param dev 5279 * Pointer to Ethernet device. 5280 * @param flow 5281 * Pointer to flow structure. 5282 * @param[in] actions 5283 * Pointer to the list of actions. 5284 * @param[out] error 5285 * Perform verbose error reporting if not NULL. 5286 * 5287 * @return 5288 * 0 on success, negative value otherwise and rte_errno is set. 5289 */ 5290 static int 5291 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5292 struct rte_flow *flow, 5293 const struct rte_flow_action *actions, 5294 struct rte_flow_error *error) 5295 { 5296 struct mlx5_priv *priv = dev->data->dev_private; 5297 struct mlx5_sh_config *config = &priv->sh->config; 5298 struct mlx5_flow_mreg_copy_resource *mcp_res; 5299 const struct rte_flow_action_mark *mark; 5300 5301 /* Check whether extensive metadata feature is engaged. */ 5302 if (!config->dv_flow_en || 5303 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5304 !mlx5_flow_ext_mreg_supported(dev) || 5305 !priv->sh->dv_regc0_mask) 5306 return 0; 5307 /* Find MARK action. */ 5308 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5309 switch (actions->type) { 5310 case RTE_FLOW_ACTION_TYPE_FLAG: 5311 mcp_res = flow_mreg_add_copy_action 5312 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5313 if (!mcp_res) 5314 return -rte_errno; 5315 flow->rix_mreg_copy = mcp_res->idx; 5316 return 0; 5317 case RTE_FLOW_ACTION_TYPE_MARK: 5318 mark = (const struct rte_flow_action_mark *) 5319 actions->conf; 5320 mcp_res = 5321 flow_mreg_add_copy_action(dev, mark->id, error); 5322 if (!mcp_res) 5323 return -rte_errno; 5324 flow->rix_mreg_copy = mcp_res->idx; 5325 return 0; 5326 default: 5327 break; 5328 } 5329 } 5330 return 0; 5331 } 5332 5333 #define MLX5_MAX_SPLIT_ACTIONS 24 5334 #define MLX5_MAX_SPLIT_ITEMS 24 5335 5336 /** 5337 * Split the hairpin flow. 5338 * Since HW can't support encap and push-vlan on Rx, we move these 5339 * actions to Tx. 5340 * If the count action is after the encap then we also 5341 * move the count action. in this case the count will also measure 5342 * the outer bytes. 5343 * 5344 * @param dev 5345 * Pointer to Ethernet device. 5346 * @param[in] actions 5347 * Associated actions (list terminated by the END action). 5348 * @param[out] actions_rx 5349 * Rx flow actions. 5350 * @param[out] actions_tx 5351 * Tx flow actions.. 5352 * @param[out] pattern_tx 5353 * The pattern items for the Tx flow. 5354 * @param[out] flow_id 5355 * The flow ID connected to this flow. 5356 * 5357 * @return 5358 * 0 on success. 5359 */ 5360 static int 5361 flow_hairpin_split(struct rte_eth_dev *dev, 5362 const struct rte_flow_action actions[], 5363 struct rte_flow_action actions_rx[], 5364 struct rte_flow_action actions_tx[], 5365 struct rte_flow_item pattern_tx[], 5366 uint32_t flow_id) 5367 { 5368 const struct rte_flow_action_raw_encap *raw_encap; 5369 const struct rte_flow_action_raw_decap *raw_decap; 5370 struct mlx5_rte_flow_action_set_tag *set_tag; 5371 struct rte_flow_action *tag_action; 5372 struct mlx5_rte_flow_item_tag *tag_item; 5373 struct rte_flow_item *item; 5374 char *addr; 5375 int push_vlan = 0; 5376 int encap = 0; 5377 5378 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5379 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5380 push_vlan = 1; 5381 switch (actions->type) { 5382 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5383 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5384 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5385 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5386 rte_memcpy(actions_tx, actions, 5387 sizeof(struct rte_flow_action)); 5388 actions_tx++; 5389 break; 5390 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5391 if (push_vlan) { 5392 rte_memcpy(actions_tx, actions, 5393 sizeof(struct rte_flow_action)); 5394 actions_tx++; 5395 } else { 5396 rte_memcpy(actions_rx, actions, 5397 sizeof(struct rte_flow_action)); 5398 actions_rx++; 5399 } 5400 break; 5401 case RTE_FLOW_ACTION_TYPE_COUNT: 5402 if (encap) { 5403 rte_memcpy(actions_tx, actions, 5404 sizeof(struct rte_flow_action)); 5405 actions_tx++; 5406 } else { 5407 rte_memcpy(actions_rx, actions, 5408 sizeof(struct rte_flow_action)); 5409 actions_rx++; 5410 } 5411 break; 5412 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5413 raw_encap = actions->conf; 5414 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5415 memcpy(actions_tx, actions, 5416 sizeof(struct rte_flow_action)); 5417 actions_tx++; 5418 encap = 1; 5419 } else { 5420 rte_memcpy(actions_rx, actions, 5421 sizeof(struct rte_flow_action)); 5422 actions_rx++; 5423 } 5424 break; 5425 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5426 raw_decap = actions->conf; 5427 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5428 memcpy(actions_tx, actions, 5429 sizeof(struct rte_flow_action)); 5430 actions_tx++; 5431 } else { 5432 rte_memcpy(actions_rx, actions, 5433 sizeof(struct rte_flow_action)); 5434 actions_rx++; 5435 } 5436 break; 5437 default: 5438 rte_memcpy(actions_rx, actions, 5439 sizeof(struct rte_flow_action)); 5440 actions_rx++; 5441 break; 5442 } 5443 } 5444 /* Add set meta action and end action for the Rx flow. */ 5445 tag_action = actions_rx; 5446 tag_action->type = (enum rte_flow_action_type) 5447 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5448 actions_rx++; 5449 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5450 actions_rx++; 5451 set_tag = (void *)actions_rx; 5452 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5453 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5454 .data = flow_id, 5455 }; 5456 MLX5_ASSERT(set_tag->id > REG_NON); 5457 tag_action->conf = set_tag; 5458 /* Create Tx item list. */ 5459 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5460 addr = (void *)&pattern_tx[2]; 5461 item = pattern_tx; 5462 item->type = (enum rte_flow_item_type) 5463 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5464 tag_item = (void *)addr; 5465 tag_item->data = flow_id; 5466 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5467 MLX5_ASSERT(set_tag->id > REG_NON); 5468 item->spec = tag_item; 5469 addr += sizeof(struct mlx5_rte_flow_item_tag); 5470 tag_item = (void *)addr; 5471 tag_item->data = UINT32_MAX; 5472 tag_item->id = UINT16_MAX; 5473 item->mask = tag_item; 5474 item->last = NULL; 5475 item++; 5476 item->type = RTE_FLOW_ITEM_TYPE_END; 5477 return 0; 5478 } 5479 5480 /** 5481 * The last stage of splitting chain, just creates the subflow 5482 * without any modification. 5483 * 5484 * @param[in] dev 5485 * Pointer to Ethernet device. 5486 * @param[in] flow 5487 * Parent flow structure pointer. 5488 * @param[in, out] sub_flow 5489 * Pointer to return the created subflow, may be NULL. 5490 * @param[in] attr 5491 * Flow rule attributes. 5492 * @param[in] items 5493 * Pattern specification (list terminated by the END pattern item). 5494 * @param[in] actions 5495 * Associated actions (list terminated by the END action). 5496 * @param[in] flow_split_info 5497 * Pointer to flow split info structure. 5498 * @param[out] error 5499 * Perform verbose error reporting if not NULL. 5500 * @return 5501 * 0 on success, negative value otherwise 5502 */ 5503 static int 5504 flow_create_split_inner(struct rte_eth_dev *dev, 5505 struct rte_flow *flow, 5506 struct mlx5_flow **sub_flow, 5507 const struct rte_flow_attr *attr, 5508 const struct rte_flow_item items[], 5509 const struct rte_flow_action actions[], 5510 struct mlx5_flow_split_info *flow_split_info, 5511 struct rte_flow_error *error) 5512 { 5513 struct mlx5_flow *dev_flow; 5514 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5515 5516 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5517 flow_split_info->flow_idx, error); 5518 if (!dev_flow) 5519 return -rte_errno; 5520 dev_flow->flow = flow; 5521 dev_flow->external = flow_split_info->external; 5522 dev_flow->skip_scale = flow_split_info->skip_scale; 5523 /* Subflow object was created, we must include one in the list. */ 5524 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5525 dev_flow->handle, next); 5526 /* 5527 * If dev_flow is as one of the suffix flow, some actions in suffix 5528 * flow may need some user defined item layer flags, and pass the 5529 * Metadata rxq mark flag to suffix flow as well. 5530 */ 5531 if (flow_split_info->prefix_layers) 5532 dev_flow->handle->layers = flow_split_info->prefix_layers; 5533 if (flow_split_info->prefix_mark) { 5534 MLX5_ASSERT(wks); 5535 wks->mark = 1; 5536 } 5537 if (sub_flow) 5538 *sub_flow = dev_flow; 5539 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5540 dev_flow->dv.table_id = flow_split_info->table_id; 5541 #endif 5542 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5543 } 5544 5545 /** 5546 * Get the sub policy of a meter. 5547 * 5548 * @param[in] dev 5549 * Pointer to Ethernet device. 5550 * @param[in] flow 5551 * Parent flow structure pointer. 5552 * @param wks 5553 * Pointer to thread flow work space. 5554 * @param[in] attr 5555 * Flow rule attributes. 5556 * @param[in] items 5557 * Pattern specification (list terminated by the END pattern item). 5558 * @param[out] error 5559 * Perform verbose error reporting if not NULL. 5560 * 5561 * @return 5562 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5563 */ 5564 static struct mlx5_flow_meter_sub_policy * 5565 get_meter_sub_policy(struct rte_eth_dev *dev, 5566 struct rte_flow *flow, 5567 struct mlx5_flow_workspace *wks, 5568 const struct rte_flow_attr *attr, 5569 const struct rte_flow_item items[], 5570 struct rte_flow_error *error) 5571 { 5572 struct mlx5_flow_meter_policy *policy; 5573 struct mlx5_flow_meter_policy *final_policy; 5574 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5575 5576 policy = wks->policy; 5577 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5578 if (final_policy->is_rss || final_policy->is_queue) { 5579 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5580 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5581 uint32_t i; 5582 5583 /* 5584 * This is a tmp dev_flow, 5585 * no need to register any matcher for it in translate. 5586 */ 5587 wks->skip_matcher_reg = 1; 5588 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5589 struct mlx5_flow dev_flow = {0}; 5590 struct mlx5_flow_handle dev_handle = { {0} }; 5591 uint8_t fate = final_policy->act_cnt[i].fate_action; 5592 5593 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5594 const struct rte_flow_action_rss *rss_act = 5595 final_policy->act_cnt[i].rss->conf; 5596 struct rte_flow_action rss_actions[2] = { 5597 [0] = { 5598 .type = RTE_FLOW_ACTION_TYPE_RSS, 5599 .conf = rss_act, 5600 }, 5601 [1] = { 5602 .type = RTE_FLOW_ACTION_TYPE_END, 5603 .conf = NULL, 5604 } 5605 }; 5606 5607 dev_flow.handle = &dev_handle; 5608 dev_flow.ingress = attr->ingress; 5609 dev_flow.flow = flow; 5610 dev_flow.external = 0; 5611 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5612 dev_flow.dv.transfer = attr->transfer; 5613 #endif 5614 /** 5615 * Translate RSS action to get rss hash fields. 5616 */ 5617 if (flow_drv_translate(dev, &dev_flow, attr, 5618 items, rss_actions, error)) 5619 goto exit; 5620 rss_desc_v[i] = wks->rss_desc; 5621 rss_desc_v[i].symmetric_hash_function = 5622 dev_flow.symmetric_hash_function; 5623 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5624 rss_desc_v[i].hash_fields = 5625 dev_flow.hash_fields; 5626 rss_desc_v[i].queue_num = 5627 rss_desc_v[i].hash_fields ? 5628 rss_desc_v[i].queue_num : 1; 5629 rss_desc_v[i].tunnel = 5630 !!(dev_flow.handle->layers & 5631 MLX5_FLOW_LAYER_TUNNEL); 5632 /* Use the RSS queues in the containers. */ 5633 rss_desc_v[i].queue = 5634 (uint16_t *)(uintptr_t)rss_act->queue; 5635 rss_desc[i] = &rss_desc_v[i]; 5636 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5637 /* This is queue action. */ 5638 rss_desc_v[i] = wks->rss_desc; 5639 rss_desc_v[i].key_len = 0; 5640 rss_desc_v[i].hash_fields = 0; 5641 rss_desc_v[i].queue = 5642 &final_policy->act_cnt[i].queue; 5643 rss_desc_v[i].queue_num = 1; 5644 rss_desc[i] = &rss_desc_v[i]; 5645 } else { 5646 rss_desc[i] = NULL; 5647 } 5648 } 5649 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5650 flow, policy, rss_desc); 5651 } else { 5652 enum mlx5_meter_domain mtr_domain = 5653 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5654 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5655 MLX5_MTR_DOMAIN_INGRESS); 5656 sub_policy = policy->sub_policys[mtr_domain][0]; 5657 } 5658 if (!sub_policy) 5659 rte_flow_error_set(error, EINVAL, 5660 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5661 "Failed to get meter sub-policy."); 5662 exit: 5663 return sub_policy; 5664 } 5665 5666 /** 5667 * Split the meter flow. 5668 * 5669 * As meter flow will split to three sub flow, other than meter 5670 * action, the other actions make sense to only meter accepts 5671 * the packet. If it need to be dropped, no other additional 5672 * actions should be take. 5673 * 5674 * One kind of special action which decapsulates the L3 tunnel 5675 * header will be in the prefix sub flow, as not to take the 5676 * L3 tunnel header into account. 5677 * 5678 * @param[in] dev 5679 * Pointer to Ethernet device. 5680 * @param[in] flow 5681 * Parent flow structure pointer. 5682 * @param wks 5683 * Pointer to thread flow work space. 5684 * @param[in] attr 5685 * Flow rule attributes. 5686 * @param[in] items 5687 * Pattern specification (list terminated by the END pattern item). 5688 * @param[out] sfx_items 5689 * Suffix flow match items (list terminated by the END pattern item). 5690 * @param[in] actions 5691 * Associated actions (list terminated by the END action). 5692 * @param[out] actions_sfx 5693 * Suffix flow actions. 5694 * @param[out] actions_pre 5695 * Prefix flow actions. 5696 * @param[out] mtr_flow_id 5697 * Pointer to meter flow id. 5698 * @param[out] error 5699 * Perform verbose error reporting if not NULL. 5700 * 5701 * @return 5702 * 0 on success, a negative errno value otherwise and rte_errno is set. 5703 */ 5704 static int 5705 flow_meter_split_prep(struct rte_eth_dev *dev, 5706 struct rte_flow *flow, 5707 struct mlx5_flow_workspace *wks, 5708 const struct rte_flow_attr *attr, 5709 const struct rte_flow_item items[], 5710 struct rte_flow_item sfx_items[], 5711 const struct rte_flow_action actions[], 5712 struct rte_flow_action actions_sfx[], 5713 struct rte_flow_action actions_pre[], 5714 uint32_t *mtr_flow_id, 5715 struct rte_flow_error *error) 5716 { 5717 struct mlx5_priv *priv = dev->data->dev_private; 5718 struct mlx5_flow_meter_info *fm = wks->fm; 5719 struct rte_flow_action *tag_action = NULL; 5720 struct rte_flow_item *tag_item; 5721 struct mlx5_rte_flow_action_set_tag *set_tag; 5722 const struct rte_flow_action_raw_encap *raw_encap; 5723 const struct rte_flow_action_raw_decap *raw_decap; 5724 struct mlx5_rte_flow_item_tag *tag_item_spec; 5725 struct mlx5_rte_flow_item_tag *tag_item_mask; 5726 uint32_t tag_id = 0; 5727 bool vlan_actions; 5728 struct rte_flow_item *orig_sfx_items = sfx_items; 5729 const struct rte_flow_item *orig_items = items; 5730 struct rte_flow_action *hw_mtr_action; 5731 struct rte_flow_action *action_pre_head = NULL; 5732 uint16_t flow_src_port = priv->representor_id; 5733 bool mtr_first; 5734 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5735 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5736 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5737 uint32_t flow_id = 0; 5738 uint32_t flow_id_reversed = 0; 5739 uint8_t flow_id_bits = 0; 5740 bool after_meter = false; 5741 int shift; 5742 5743 /* Prepare the suffix subflow items. */ 5744 tag_item = sfx_items++; 5745 tag_item->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5746 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5747 int item_type = items->type; 5748 5749 switch (item_type) { 5750 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5751 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5752 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5753 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5754 return -rte_errno; 5755 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5756 flow_src_port != priv->representor_id) { 5757 if (flow_drv_mtr_hierarchy_rule_create(dev, 5758 flow, fm, 5759 flow_src_port, 5760 items, 5761 error)) 5762 return -rte_errno; 5763 } 5764 memcpy(sfx_items, items, sizeof(*sfx_items)); 5765 sfx_items++; 5766 break; 5767 case RTE_FLOW_ITEM_TYPE_VLAN: 5768 /* 5769 * Copy VLAN items in case VLAN actions are performed. 5770 * If there are no VLAN actions, these items will be VOID. 5771 */ 5772 memcpy(sfx_items, items, sizeof(*sfx_items)); 5773 sfx_items->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5774 sfx_items++; 5775 break; 5776 default: 5777 break; 5778 } 5779 } 5780 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5781 sfx_items++; 5782 mtr_first = priv->sh->meter_aso_en && 5783 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5784 /* For ASO meter, meter must be before tag in TX direction. */ 5785 if (mtr_first) { 5786 action_pre_head = actions_pre++; 5787 /* Leave space for tag action. */ 5788 tag_action = actions_pre++; 5789 } 5790 /* Prepare the actions for prefix and suffix flow. */ 5791 vlan_actions = false; 5792 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5793 struct rte_flow_action *action_cur = NULL; 5794 5795 switch (actions->type) { 5796 case RTE_FLOW_ACTION_TYPE_METER: 5797 if (mtr_first) { 5798 action_cur = action_pre_head; 5799 } else { 5800 /* Leave space for tag action. */ 5801 tag_action = actions_pre++; 5802 action_cur = actions_pre++; 5803 } 5804 after_meter = true; 5805 break; 5806 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5807 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5808 action_cur = actions_pre++; 5809 break; 5810 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5811 raw_encap = actions->conf; 5812 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5813 action_cur = actions_pre++; 5814 break; 5815 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5816 raw_decap = actions->conf; 5817 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5818 action_cur = actions_pre++; 5819 break; 5820 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5821 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5822 vlan_actions = true; 5823 break; 5824 case RTE_FLOW_ACTION_TYPE_COUNT: 5825 if (fm->def_policy) 5826 action_cur = after_meter ? 5827 actions_sfx++ : actions_pre++; 5828 break; 5829 default: 5830 break; 5831 } 5832 if (!action_cur) 5833 action_cur = (fm->def_policy) ? 5834 actions_sfx++ : actions_pre++; 5835 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5836 } 5837 /* If there are no VLAN actions, convert VLAN items to VOID in suffix flow items. */ 5838 if (!vlan_actions) { 5839 struct rte_flow_item *it = orig_sfx_items; 5840 5841 for (; it->type != RTE_FLOW_ITEM_TYPE_END; it++) 5842 if (it->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 5843 it->type = RTE_FLOW_ITEM_TYPE_VOID; 5844 } 5845 /* Add end action to the actions. */ 5846 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5847 if (priv->sh->meter_aso_en) { 5848 /** 5849 * For ASO meter, need to add an extra jump action explicitly, 5850 * to jump from meter to policer table. 5851 */ 5852 struct mlx5_flow_meter_sub_policy *sub_policy; 5853 struct mlx5_flow_tbl_data_entry *tbl_data; 5854 5855 if (!fm->def_policy) { 5856 sub_policy = get_meter_sub_policy(dev, flow, wks, 5857 attr, orig_items, 5858 error); 5859 if (!sub_policy) 5860 return -rte_errno; 5861 } else { 5862 enum mlx5_meter_domain mtr_domain = 5863 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5864 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5865 MLX5_MTR_DOMAIN_INGRESS); 5866 5867 sub_policy = 5868 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5869 } 5870 tbl_data = container_of(sub_policy->tbl_rsc, 5871 struct mlx5_flow_tbl_data_entry, tbl); 5872 hw_mtr_action = actions_pre++; 5873 hw_mtr_action->type = (enum rte_flow_action_type) 5874 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5875 hw_mtr_action->conf = tbl_data->jump.action; 5876 } 5877 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5878 actions_pre++; 5879 if (!tag_action) 5880 return rte_flow_error_set(error, ENOMEM, 5881 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5882 NULL, "No tag action space."); 5883 if (!mtr_flow_id) { 5884 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5885 goto exit; 5886 } 5887 /* Only default-policy Meter creates mtr flow id. */ 5888 if (fm->def_policy) { 5889 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5890 if (!tag_id) 5891 return rte_flow_error_set(error, ENOMEM, 5892 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5893 "Failed to allocate meter flow id."); 5894 flow_id = tag_id - 1; 5895 flow_id_bits = (!flow_id) ? 1 : 5896 (MLX5_REG_BITS - rte_clz32(flow_id)); 5897 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5898 mtr_reg_bits) { 5899 mlx5_ipool_free(fm->flow_ipool, tag_id); 5900 return rte_flow_error_set(error, EINVAL, 5901 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5902 "Meter flow id exceeds max limit."); 5903 } 5904 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5905 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5906 } 5907 /* Build tag actions and items for meter_id/meter flow_id. */ 5908 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5909 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5910 tag_item_mask = tag_item_spec + 1; 5911 /* Both flow_id and meter_id share the same register. */ 5912 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5913 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5914 0, error), 5915 .offset = mtr_id_offset, 5916 .length = mtr_reg_bits, 5917 .data = flow->meter, 5918 }; 5919 /* 5920 * The color Reg bits used by flow_id are growing from 5921 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5922 */ 5923 for (shift = 0; shift < flow_id_bits; shift++) 5924 flow_id_reversed = (flow_id_reversed << 1) | 5925 ((flow_id >> shift) & 0x1); 5926 set_tag->data |= 5927 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5928 tag_item_spec->id = set_tag->id; 5929 tag_item_spec->data = set_tag->data << mtr_id_offset; 5930 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5931 tag_action->type = (enum rte_flow_action_type) 5932 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5933 tag_action->conf = set_tag; 5934 tag_item->spec = tag_item_spec; 5935 tag_item->last = NULL; 5936 tag_item->mask = tag_item_mask; 5937 exit: 5938 if (mtr_flow_id) 5939 *mtr_flow_id = tag_id; 5940 return 0; 5941 } 5942 5943 /** 5944 * Split action list having QUEUE/RSS for metadata register copy. 5945 * 5946 * Once Q/RSS action is detected in user's action list, the flow action 5947 * should be split in order to copy metadata registers, which will happen in 5948 * RX_CP_TBL like, 5949 * - CQE->flow_tag := reg_c[1] (MARK) 5950 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5951 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5952 * This is because the last action of each flow must be a terminal action 5953 * (QUEUE, RSS or DROP). 5954 * 5955 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5956 * stored and kept in the mlx5_flow structure per each sub_flow. 5957 * 5958 * The Q/RSS action is replaced with, 5959 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5960 * And the following JUMP action is added at the end, 5961 * - JUMP, to RX_CP_TBL. 5962 * 5963 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5964 * flow_create_split_metadata() routine. The flow will look like, 5965 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5966 * 5967 * @param dev 5968 * Pointer to Ethernet device. 5969 * @param[out] split_actions 5970 * Pointer to store split actions to jump to CP_TBL. 5971 * @param[in] actions 5972 * Pointer to the list of original flow actions. 5973 * @param[in] qrss 5974 * Pointer to the Q/RSS action. 5975 * @param[in] actions_n 5976 * Number of original actions. 5977 * @param[in] mtr_sfx 5978 * Check if it is in meter suffix table. 5979 * @param[out] error 5980 * Perform verbose error reporting if not NULL. 5981 * 5982 * @return 5983 * non-zero unique flow_id on success, otherwise 0 and 5984 * error/rte_error are set. 5985 */ 5986 static uint32_t 5987 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5988 struct rte_flow_action *split_actions, 5989 const struct rte_flow_action *actions, 5990 const struct rte_flow_action *qrss, 5991 int actions_n, int mtr_sfx, 5992 struct rte_flow_error *error) 5993 { 5994 struct mlx5_priv *priv = dev->data->dev_private; 5995 struct mlx5_rte_flow_action_set_tag *set_tag; 5996 struct rte_flow_action_jump *jump; 5997 const int qrss_idx = qrss - actions; 5998 uint32_t flow_id = 0; 5999 int ret = 0; 6000 6001 /* 6002 * Given actions will be split 6003 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 6004 * - Add jump to mreg CP_TBL. 6005 * As a result, there will be one more action. 6006 */ 6007 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 6008 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 6009 ++actions_n; 6010 set_tag = (void *)(split_actions + actions_n); 6011 /* 6012 * If we are not the meter suffix flow, add the tag action. 6013 * Since meter suffix flow already has the tag added. 6014 */ 6015 if (!mtr_sfx) { 6016 /* 6017 * Allocate the new subflow ID. This one is unique within 6018 * device and not shared with representors. Otherwise, 6019 * we would have to resolve multi-thread access synch 6020 * issue. Each flow on the shared device is appended 6021 * with source vport identifier, so the resulting 6022 * flows will be unique in the shared (by master and 6023 * representors) domain even if they have coinciding 6024 * IDs. 6025 */ 6026 mlx5_ipool_malloc(priv->sh->ipool 6027 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 6028 if (!flow_id) 6029 return rte_flow_error_set(error, ENOMEM, 6030 RTE_FLOW_ERROR_TYPE_ACTION, 6031 NULL, "can't allocate id " 6032 "for split Q/RSS subflow"); 6033 /* Internal SET_TAG action to set flow ID. */ 6034 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 6035 .data = flow_id, 6036 }; 6037 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 6038 if (ret < 0) 6039 return ret; 6040 set_tag->id = ret; 6041 /* Construct new actions array. */ 6042 /* Replace QUEUE/RSS action. */ 6043 split_actions[qrss_idx] = (struct rte_flow_action){ 6044 .type = (enum rte_flow_action_type) 6045 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6046 .conf = set_tag, 6047 }; 6048 } else { 6049 /* 6050 * If we are the suffix flow of meter, tag already exist. 6051 * Set the QUEUE/RSS action to void. 6052 */ 6053 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 6054 } 6055 /* JUMP action to jump to mreg copy table (CP_TBL). */ 6056 jump = (void *)(set_tag + 1); 6057 *jump = (struct rte_flow_action_jump){ 6058 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6059 }; 6060 split_actions[actions_n - 2] = (struct rte_flow_action){ 6061 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6062 .conf = jump, 6063 }; 6064 split_actions[actions_n - 1] = (struct rte_flow_action){ 6065 .type = RTE_FLOW_ACTION_TYPE_END, 6066 }; 6067 return flow_id; 6068 } 6069 6070 /** 6071 * Extend the given action list for Tx metadata copy. 6072 * 6073 * Copy the given action list to the ext_actions and add flow metadata register 6074 * copy action in order to copy reg_a set by WQE to reg_c[0]. 6075 * 6076 * @param[out] ext_actions 6077 * Pointer to the extended action list. 6078 * @param[in] actions 6079 * Pointer to the list of actions. 6080 * @param[in] actions_n 6081 * Number of actions in the list. 6082 * @param[out] error 6083 * Perform verbose error reporting if not NULL. 6084 * @param[in] encap_idx 6085 * The encap action index. 6086 * 6087 * @return 6088 * 0 on success, negative value otherwise 6089 */ 6090 static int 6091 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6092 struct rte_flow_action *ext_actions, 6093 const struct rte_flow_action *actions, 6094 int actions_n, struct rte_flow_error *error, 6095 int encap_idx) 6096 { 6097 struct mlx5_flow_action_copy_mreg *cp_mreg = 6098 (struct mlx5_flow_action_copy_mreg *) 6099 (ext_actions + actions_n + 1); 6100 int ret; 6101 6102 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6103 if (ret < 0) 6104 return ret; 6105 cp_mreg->dst = ret; 6106 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6107 if (ret < 0) 6108 return ret; 6109 cp_mreg->src = ret; 6110 if (encap_idx != 0) 6111 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6112 if (encap_idx == actions_n - 1) { 6113 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6114 .type = (enum rte_flow_action_type) 6115 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6116 .conf = cp_mreg, 6117 }; 6118 ext_actions[actions_n] = (struct rte_flow_action){ 6119 .type = RTE_FLOW_ACTION_TYPE_END, 6120 }; 6121 } else { 6122 ext_actions[encap_idx] = (struct rte_flow_action){ 6123 .type = (enum rte_flow_action_type) 6124 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6125 .conf = cp_mreg, 6126 }; 6127 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6128 sizeof(*ext_actions) * (actions_n - encap_idx)); 6129 } 6130 return 0; 6131 } 6132 6133 /** 6134 * Check the match action from the action list. 6135 * 6136 * @param[in] actions 6137 * Pointer to the list of actions. 6138 * @param[in] attr 6139 * Flow rule attributes. 6140 * @param[in] action 6141 * The action to be check if exist. 6142 * @param[out] match_action_pos 6143 * Pointer to the position of the matched action if exists, otherwise is -1. 6144 * @param[out] qrss_action_pos 6145 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6146 * @param[out] modify_after_mirror 6147 * Pointer to the flag of modify action after FDB mirroring. 6148 * 6149 * @return 6150 * > 0 the total number of actions. 6151 * 0 if not found match action in action list. 6152 */ 6153 static int 6154 flow_check_match_action(const struct rte_flow_action actions[], 6155 const struct rte_flow_attr *attr, 6156 enum rte_flow_action_type action, 6157 int *match_action_pos, int *qrss_action_pos, 6158 int *modify_after_mirror) 6159 { 6160 const struct rte_flow_action_sample *sample; 6161 const struct rte_flow_action_raw_decap *decap; 6162 const struct rte_flow_action *action_cur = NULL; 6163 int actions_n = 0; 6164 uint32_t ratio = 0; 6165 int sub_type = 0; 6166 int flag = 0; 6167 int fdb_mirror = 0; 6168 6169 *match_action_pos = -1; 6170 *qrss_action_pos = -1; 6171 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6172 if (actions->type == action) { 6173 flag = 1; 6174 *match_action_pos = actions_n; 6175 } 6176 switch (actions->type) { 6177 case RTE_FLOW_ACTION_TYPE_QUEUE: 6178 case RTE_FLOW_ACTION_TYPE_RSS: 6179 *qrss_action_pos = actions_n; 6180 break; 6181 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6182 sample = actions->conf; 6183 ratio = sample->ratio; 6184 sub_type = ((const struct rte_flow_action *) 6185 (sample->actions))->type; 6186 if (ratio == 1 && attr->transfer && 6187 sub_type != RTE_FLOW_ACTION_TYPE_END) 6188 fdb_mirror = 1; 6189 break; 6190 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6191 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6192 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6193 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6194 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6195 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6196 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6197 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6198 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6199 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6200 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6201 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6202 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6203 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6204 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6205 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6206 case RTE_FLOW_ACTION_TYPE_FLAG: 6207 case RTE_FLOW_ACTION_TYPE_MARK: 6208 case RTE_FLOW_ACTION_TYPE_SET_META: 6209 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6210 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6211 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6212 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6213 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6214 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6215 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6216 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6217 case RTE_FLOW_ACTION_TYPE_METER: 6218 if (fdb_mirror) 6219 *modify_after_mirror = 1; 6220 break; 6221 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6222 decap = actions->conf; 6223 action_cur = actions; 6224 while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID) 6225 ; 6226 if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6227 const struct rte_flow_action_raw_encap *encap = 6228 action_cur->conf; 6229 if (decap->size <= 6230 MLX5_ENCAPSULATION_DECISION_SIZE && 6231 encap->size > 6232 MLX5_ENCAPSULATION_DECISION_SIZE) 6233 /* L3 encap. */ 6234 break; 6235 } 6236 if (fdb_mirror) 6237 *modify_after_mirror = 1; 6238 break; 6239 default: 6240 break; 6241 } 6242 actions_n++; 6243 } 6244 if (flag && fdb_mirror && !*modify_after_mirror) { 6245 /* FDB mirroring uses the destination array to implement 6246 * instead of FLOW_SAMPLER object. 6247 */ 6248 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6249 flag = 0; 6250 } 6251 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6252 return flag ? actions_n + 1 : 0; 6253 } 6254 6255 #define SAMPLE_SUFFIX_ITEM 3 6256 6257 /** 6258 * Split the sample flow. 6259 * 6260 * As sample flow will split to two sub flow, sample flow with 6261 * sample action, the other actions will move to new suffix flow. 6262 * 6263 * Also add unique tag id with tag action in the sample flow, 6264 * the same tag id will be as match in the suffix flow. 6265 * 6266 * @param dev 6267 * Pointer to Ethernet device. 6268 * @param[in] add_tag 6269 * Add extra tag action flag. 6270 * @param[out] sfx_items 6271 * Suffix flow match items (list terminated by the END pattern item). 6272 * @param[in] actions 6273 * Associated actions (list terminated by the END action). 6274 * @param[out] actions_sfx 6275 * Suffix flow actions. 6276 * @param[out] actions_pre 6277 * Prefix flow actions. 6278 * @param[in] actions_n 6279 * The total number of actions. 6280 * @param[in] sample_action_pos 6281 * The sample action position. 6282 * @param[in] qrss_action_pos 6283 * The Queue/RSS action position. 6284 * @param[in] jump_table 6285 * Add extra jump action flag. 6286 * @param[out] error 6287 * Perform verbose error reporting if not NULL. 6288 * 6289 * @return 6290 * 0 on success, or unique flow_id, a negative errno value 6291 * otherwise and rte_errno is set. 6292 */ 6293 static int 6294 flow_sample_split_prep(struct rte_eth_dev *dev, 6295 int add_tag, 6296 const struct rte_flow_item items[], 6297 struct rte_flow_item sfx_items[], 6298 const struct rte_flow_action actions[], 6299 struct rte_flow_action actions_sfx[], 6300 struct rte_flow_action actions_pre[], 6301 int actions_n, 6302 int sample_action_pos, 6303 int qrss_action_pos, 6304 int jump_table, 6305 struct rte_flow_error *error) 6306 { 6307 struct mlx5_priv *priv = dev->data->dev_private; 6308 struct mlx5_rte_flow_action_set_tag *set_tag; 6309 struct mlx5_rte_flow_item_tag *tag_spec; 6310 struct mlx5_rte_flow_item_tag *tag_mask; 6311 struct rte_flow_action_jump *jump_action; 6312 uint32_t tag_id = 0; 6313 int append_index = 0; 6314 int set_tag_idx = -1; 6315 int index; 6316 int ret; 6317 6318 if (sample_action_pos < 0) 6319 return rte_flow_error_set(error, EINVAL, 6320 RTE_FLOW_ERROR_TYPE_ACTION, 6321 NULL, "invalid position of sample " 6322 "action in list"); 6323 /* Prepare the actions for prefix and suffix flow. */ 6324 if (add_tag) { 6325 /* Update the new added tag action index preceding 6326 * the PUSH_VLAN or ENCAP action. 6327 */ 6328 const struct rte_flow_action_raw_encap *raw_encap; 6329 const struct rte_flow_action *action = actions; 6330 int encap_idx; 6331 int action_idx = 0; 6332 int raw_decap_idx = -1; 6333 int push_vlan_idx = -1; 6334 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6335 switch (action->type) { 6336 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6337 raw_decap_idx = action_idx; 6338 break; 6339 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6340 raw_encap = action->conf; 6341 if (raw_encap->size > 6342 MLX5_ENCAPSULATION_DECISION_SIZE) { 6343 encap_idx = raw_decap_idx != -1 ? 6344 raw_decap_idx : action_idx; 6345 if (encap_idx < sample_action_pos && 6346 push_vlan_idx == -1) 6347 set_tag_idx = encap_idx; 6348 } 6349 break; 6350 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6351 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6352 encap_idx = action_idx; 6353 if (encap_idx < sample_action_pos && 6354 push_vlan_idx == -1) 6355 set_tag_idx = encap_idx; 6356 break; 6357 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6358 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6359 if (action_idx < sample_action_pos && 6360 push_vlan_idx == -1) { 6361 set_tag_idx = action_idx; 6362 push_vlan_idx = action_idx; 6363 } 6364 break; 6365 default: 6366 break; 6367 } 6368 action_idx++; 6369 } 6370 } 6371 /* Prepare the actions for prefix and suffix flow. */ 6372 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6373 index = qrss_action_pos; 6374 /* Put the preceding the Queue/RSS action into prefix flow. */ 6375 if (index != 0) 6376 memcpy(actions_pre, actions, 6377 sizeof(struct rte_flow_action) * index); 6378 /* Put others preceding the sample action into prefix flow. */ 6379 if (sample_action_pos > index + 1) 6380 memcpy(actions_pre + index, actions + index + 1, 6381 sizeof(struct rte_flow_action) * 6382 (sample_action_pos - index - 1)); 6383 index = sample_action_pos - 1; 6384 /* Put Queue/RSS action into Suffix flow. */ 6385 memcpy(actions_sfx, actions + qrss_action_pos, 6386 sizeof(struct rte_flow_action)); 6387 actions_sfx++; 6388 } else if (add_tag && set_tag_idx >= 0) { 6389 if (set_tag_idx > 0) 6390 memcpy(actions_pre, actions, 6391 sizeof(struct rte_flow_action) * set_tag_idx); 6392 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6393 sizeof(struct rte_flow_action) * 6394 (sample_action_pos - set_tag_idx)); 6395 index = sample_action_pos; 6396 } else { 6397 index = sample_action_pos; 6398 if (index != 0) 6399 memcpy(actions_pre, actions, 6400 sizeof(struct rte_flow_action) * index); 6401 } 6402 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6403 * For CX6DX and above, metadata registers Cx preserve their value, 6404 * add an extra tag action for NIC-RX and E-Switch Domain. 6405 */ 6406 if (add_tag) { 6407 /* Prepare the prefix tag action. */ 6408 append_index++; 6409 set_tag = (void *)(actions_pre + actions_n + append_index); 6410 /* Trust VF/SF on CX5 not supported meter so that the reserved 6411 * metadata regC is REG_NON, back to use application tag 6412 * index 0. 6413 */ 6414 if (unlikely(priv->sh->registers.aso_reg == REG_NON)) 6415 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6416 else 6417 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6418 if (ret < 0) 6419 return ret; 6420 mlx5_ipool_malloc(priv->sh->ipool 6421 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6422 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6423 .id = ret, 6424 .data = tag_id, 6425 }; 6426 /* Prepare the suffix subflow items. */ 6427 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6428 tag_spec->data = tag_id; 6429 tag_spec->id = set_tag->id; 6430 tag_mask = tag_spec + 1; 6431 tag_mask->data = UINT32_MAX; 6432 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6433 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6434 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6435 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6436 memcpy(sfx_items, items, sizeof(*sfx_items)); 6437 sfx_items++; 6438 break; 6439 } 6440 } 6441 sfx_items[0] = (struct rte_flow_item){ 6442 .type = (enum rte_flow_item_type) 6443 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6444 .spec = tag_spec, 6445 .last = NULL, 6446 .mask = tag_mask, 6447 }; 6448 sfx_items[1] = (struct rte_flow_item){ 6449 .type = (enum rte_flow_item_type) 6450 RTE_FLOW_ITEM_TYPE_END, 6451 }; 6452 /* Prepare the tag action in prefix subflow. */ 6453 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6454 actions_pre[set_tag_idx] = 6455 (struct rte_flow_action){ 6456 .type = (enum rte_flow_action_type) 6457 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6458 .conf = set_tag, 6459 }; 6460 /* Update next sample position due to add one tag action */ 6461 index += 1; 6462 } 6463 /* Copy the sample action into prefix flow. */ 6464 memcpy(actions_pre + index, actions + sample_action_pos, 6465 sizeof(struct rte_flow_action)); 6466 index += 1; 6467 /* For the modify action after the sample action in E-Switch mirroring, 6468 * Add the extra jump action in prefix subflow and jump into the next 6469 * table, then do the modify action in the new table. 6470 */ 6471 if (jump_table) { 6472 /* Prepare the prefix jump action. */ 6473 append_index++; 6474 jump_action = (void *)(actions_pre + actions_n + append_index); 6475 jump_action->group = jump_table; 6476 actions_pre[index++] = 6477 (struct rte_flow_action){ 6478 .type = (enum rte_flow_action_type) 6479 RTE_FLOW_ACTION_TYPE_JUMP, 6480 .conf = jump_action, 6481 }; 6482 } 6483 actions_pre[index] = (struct rte_flow_action){ 6484 .type = (enum rte_flow_action_type) 6485 RTE_FLOW_ACTION_TYPE_END, 6486 }; 6487 /* Put the actions after sample into Suffix flow. */ 6488 memcpy(actions_sfx, actions + sample_action_pos + 1, 6489 sizeof(struct rte_flow_action) * 6490 (actions_n - sample_action_pos - 1)); 6491 return tag_id; 6492 } 6493 6494 /** 6495 * The splitting for metadata feature. 6496 * 6497 * - Q/RSS action on NIC Rx should be split in order to pass by 6498 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6499 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6500 * 6501 * - All the actions on NIC Tx should have a mreg copy action to 6502 * copy reg_a from WQE to reg_c[0]. 6503 * 6504 * @param dev 6505 * Pointer to Ethernet device. 6506 * @param[in] flow 6507 * Parent flow structure pointer. 6508 * @param[in] attr 6509 * Flow rule attributes. 6510 * @param[in] items 6511 * Pattern specification (list terminated by the END pattern item). 6512 * @param[in] actions 6513 * Associated actions (list terminated by the END action). 6514 * @param[in] flow_split_info 6515 * Pointer to flow split info structure. 6516 * @param[out] error 6517 * Perform verbose error reporting if not NULL. 6518 * @return 6519 * 0 on success, negative value otherwise 6520 */ 6521 static int 6522 flow_create_split_metadata(struct rte_eth_dev *dev, 6523 struct rte_flow *flow, 6524 const struct rte_flow_attr *attr, 6525 const struct rte_flow_item items[], 6526 const struct rte_flow_action actions[], 6527 struct mlx5_flow_split_info *flow_split_info, 6528 struct rte_flow_error *error) 6529 { 6530 struct mlx5_priv *priv = dev->data->dev_private; 6531 struct mlx5_sh_config *config = &priv->sh->config; 6532 const struct rte_flow_action *qrss = NULL; 6533 struct rte_flow_action *ext_actions = NULL; 6534 struct mlx5_flow *dev_flow = NULL; 6535 uint32_t qrss_id = 0; 6536 int mtr_sfx = 0; 6537 size_t act_size; 6538 int actions_n; 6539 int encap_idx; 6540 int ret; 6541 6542 /* Check whether extensive metadata feature is engaged. */ 6543 if (!config->dv_flow_en || 6544 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6545 !mlx5_flow_ext_mreg_supported(dev)) 6546 return flow_create_split_inner(dev, flow, NULL, attr, items, 6547 actions, flow_split_info, error); 6548 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6549 &encap_idx); 6550 if (qrss) { 6551 /* Exclude hairpin flows from splitting. */ 6552 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6553 const struct rte_flow_action_queue *queue; 6554 6555 queue = qrss->conf; 6556 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6557 qrss = NULL; 6558 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6559 const struct rte_flow_action_rss *rss; 6560 6561 rss = qrss->conf; 6562 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6563 qrss = NULL; 6564 } 6565 } 6566 if (qrss) { 6567 /* Check if it is in meter suffix table. */ 6568 mtr_sfx = attr->group == 6569 ((attr->transfer && priv->fdb_def_rule) ? 6570 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6571 MLX5_FLOW_TABLE_LEVEL_METER); 6572 /* 6573 * Q/RSS action on NIC Rx should be split in order to pass by 6574 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6575 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6576 */ 6577 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6578 sizeof(struct rte_flow_action_set_tag) + 6579 sizeof(struct rte_flow_action_jump); 6580 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6581 SOCKET_ID_ANY); 6582 if (!ext_actions) 6583 return rte_flow_error_set(error, ENOMEM, 6584 RTE_FLOW_ERROR_TYPE_ACTION, 6585 NULL, "no memory to split " 6586 "metadata flow"); 6587 /* 6588 * Create the new actions list with removed Q/RSS action 6589 * and appended set tag and jump to register copy table 6590 * (RX_CP_TBL). We should preallocate unique tag ID here 6591 * in advance, because it is needed for set tag action. 6592 */ 6593 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6594 qrss, actions_n, 6595 mtr_sfx, error); 6596 if (!mtr_sfx && !qrss_id) { 6597 ret = -rte_errno; 6598 goto exit; 6599 } 6600 } else if (attr->egress) { 6601 /* 6602 * All the actions on NIC Tx should have a metadata register 6603 * copy action to copy reg_a from WQE to reg_c[meta] 6604 */ 6605 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6606 sizeof(struct mlx5_flow_action_copy_mreg); 6607 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6608 SOCKET_ID_ANY); 6609 if (!ext_actions) 6610 return rte_flow_error_set(error, ENOMEM, 6611 RTE_FLOW_ERROR_TYPE_ACTION, 6612 NULL, "no memory to split " 6613 "metadata flow"); 6614 /* Create the action list appended with copy register. */ 6615 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6616 actions_n, error, encap_idx); 6617 if (ret < 0) 6618 goto exit; 6619 } 6620 /* Add the unmodified original or prefix subflow. */ 6621 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6622 items, ext_actions ? ext_actions : 6623 actions, flow_split_info, error); 6624 if (ret < 0) 6625 goto exit; 6626 MLX5_ASSERT(dev_flow); 6627 if (qrss) { 6628 const struct rte_flow_attr q_attr = { 6629 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6630 .ingress = 1, 6631 }; 6632 /* Internal PMD action to set register. */ 6633 struct mlx5_rte_flow_item_tag q_tag_spec = { 6634 .data = qrss_id, 6635 .id = REG_NON, 6636 }; 6637 struct rte_flow_item q_items[] = { 6638 { 6639 .type = (enum rte_flow_item_type) 6640 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6641 .spec = &q_tag_spec, 6642 .last = NULL, 6643 .mask = NULL, 6644 }, 6645 { 6646 .type = RTE_FLOW_ITEM_TYPE_END, 6647 }, 6648 }; 6649 struct rte_flow_action q_actions[] = { 6650 { 6651 .type = qrss->type, 6652 .conf = qrss->conf, 6653 }, 6654 { 6655 .type = RTE_FLOW_ACTION_TYPE_END, 6656 }, 6657 }; 6658 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6659 6660 /* 6661 * Configure the tag item only if there is no meter subflow. 6662 * Since tag is already marked in the meter suffix subflow 6663 * we can just use the meter suffix items as is. 6664 */ 6665 if (qrss_id) { 6666 /* Not meter subflow. */ 6667 MLX5_ASSERT(!mtr_sfx); 6668 /* 6669 * Put unique id in prefix flow due to it is destroyed 6670 * after suffix flow and id will be freed after there 6671 * is no actual flows with this id and identifier 6672 * reallocation becomes possible (for example, for 6673 * other flows in other threads). 6674 */ 6675 dev_flow->handle->split_flow_id = qrss_id; 6676 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6677 error); 6678 if (ret < 0) 6679 goto exit; 6680 q_tag_spec.id = ret; 6681 } 6682 dev_flow = NULL; 6683 /* Add suffix subflow to execute Q/RSS. */ 6684 flow_split_info->prefix_layers = layers; 6685 flow_split_info->prefix_mark = 0; 6686 flow_split_info->table_id = 0; 6687 ret = flow_create_split_inner(dev, flow, &dev_flow, 6688 &q_attr, mtr_sfx ? items : 6689 q_items, q_actions, 6690 flow_split_info, error); 6691 if (ret < 0) 6692 goto exit; 6693 /* qrss ID should be freed if failed. */ 6694 qrss_id = 0; 6695 MLX5_ASSERT(dev_flow); 6696 } 6697 6698 exit: 6699 /* 6700 * We do not destroy the partially created sub_flows in case of error. 6701 * These ones are included into parent flow list and will be destroyed 6702 * by flow_drv_destroy. 6703 */ 6704 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6705 qrss_id); 6706 mlx5_free(ext_actions); 6707 return ret; 6708 } 6709 6710 /** 6711 * Create meter internal drop flow with the original pattern. 6712 * 6713 * @param dev 6714 * Pointer to Ethernet device. 6715 * @param[in] flow 6716 * Parent flow structure pointer. 6717 * @param[in] attr 6718 * Flow rule attributes. 6719 * @param[in] items 6720 * Pattern specification (list terminated by the END pattern item). 6721 * @param[in] flow_split_info 6722 * Pointer to flow split info structure. 6723 * @param[in] fm 6724 * Pointer to flow meter structure. 6725 * @param[out] error 6726 * Perform verbose error reporting if not NULL. 6727 * @return 6728 * 0 on success, negative value otherwise 6729 */ 6730 static uint32_t 6731 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6732 struct rte_flow *flow, 6733 const struct rte_flow_attr *attr, 6734 const struct rte_flow_item items[], 6735 struct mlx5_flow_split_info *flow_split_info, 6736 struct mlx5_flow_meter_info *fm, 6737 struct rte_flow_error *error) 6738 { 6739 struct mlx5_flow *dev_flow = NULL; 6740 struct rte_flow_attr drop_attr = *attr; 6741 struct rte_flow_action drop_actions[3]; 6742 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6743 6744 MLX5_ASSERT(fm->drop_cnt); 6745 drop_actions[0].type = 6746 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6747 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6748 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6749 drop_actions[1].conf = NULL; 6750 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6751 drop_actions[2].conf = NULL; 6752 drop_split_info.external = false; 6753 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6754 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6755 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6756 return flow_create_split_inner(dev, flow, &dev_flow, 6757 &drop_attr, items, drop_actions, 6758 &drop_split_info, error); 6759 } 6760 6761 static int 6762 flow_count_vlan_items(const struct rte_flow_item items[]) 6763 { 6764 int items_n = 0; 6765 6766 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6767 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN || 6768 items->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 6769 items_n++; 6770 } 6771 return items_n; 6772 } 6773 6774 /** 6775 * The splitting for meter feature. 6776 * 6777 * - The meter flow will be split to two flows as prefix and 6778 * suffix flow. The packets make sense only it pass the prefix 6779 * meter action. 6780 * 6781 * - Reg_C_5 is used for the packet to match betweend prefix and 6782 * suffix flow. 6783 * 6784 * @param dev 6785 * Pointer to Ethernet device. 6786 * @param[in] flow 6787 * Parent flow structure pointer. 6788 * @param[in] attr 6789 * Flow rule attributes. 6790 * @param[in] items 6791 * Pattern specification (list terminated by the END pattern item). 6792 * @param[in] actions 6793 * Associated actions (list terminated by the END action). 6794 * @param[in] flow_split_info 6795 * Pointer to flow split info structure. 6796 * @param[out] error 6797 * Perform verbose error reporting if not NULL. 6798 * @return 6799 * 0 on success, negative value otherwise 6800 */ 6801 static int 6802 flow_create_split_meter(struct rte_eth_dev *dev, 6803 struct rte_flow *flow, 6804 const struct rte_flow_attr *attr, 6805 const struct rte_flow_item items[], 6806 const struct rte_flow_action actions[], 6807 struct mlx5_flow_split_info *flow_split_info, 6808 struct rte_flow_error *error) 6809 { 6810 struct mlx5_priv *priv = dev->data->dev_private; 6811 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6812 struct rte_flow_action *sfx_actions = NULL; 6813 struct rte_flow_action *pre_actions = NULL; 6814 struct rte_flow_item *sfx_items = NULL; 6815 struct mlx5_flow *dev_flow = NULL; 6816 struct rte_flow_attr sfx_attr = *attr; 6817 struct mlx5_flow_meter_info *fm = NULL; 6818 uint8_t skip_scale_restore; 6819 bool has_mtr = false; 6820 bool has_modify = false; 6821 bool set_mtr_reg = true; 6822 bool is_mtr_hierarchy = false; 6823 uint32_t meter_id = 0; 6824 uint32_t mtr_idx = 0; 6825 uint32_t mtr_flow_id = 0; 6826 size_t act_size; 6827 size_t item_size; 6828 int actions_n = 0; 6829 int vlan_items_n = 0; 6830 int ret = 0; 6831 6832 if (priv->mtr_en) 6833 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6834 &has_modify, &meter_id); 6835 if (has_mtr) { 6836 if (flow->meter) { 6837 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6838 if (!fm) 6839 return rte_flow_error_set(error, EINVAL, 6840 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6841 NULL, "Meter not found."); 6842 } else { 6843 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6844 if (!fm) 6845 return rte_flow_error_set(error, EINVAL, 6846 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6847 NULL, "Meter not found."); 6848 ret = mlx5_flow_meter_attach(priv, fm, 6849 &sfx_attr, error); 6850 if (ret) 6851 return -rte_errno; 6852 flow->meter = mtr_idx; 6853 } 6854 MLX5_ASSERT(wks); 6855 wks->fm = fm; 6856 if (!fm->def_policy) { 6857 wks->policy = mlx5_flow_meter_policy_find(dev, 6858 fm->policy_id, 6859 NULL); 6860 MLX5_ASSERT(wks->policy); 6861 if (wks->policy->mark) 6862 wks->mark = 1; 6863 if (wks->policy->is_hierarchy) { 6864 wks->final_policy = 6865 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6866 wks->policy); 6867 if (!wks->final_policy) 6868 return rte_flow_error_set(error, 6869 EINVAL, 6870 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6871 "Failed to find terminal policy of hierarchy."); 6872 is_mtr_hierarchy = true; 6873 } 6874 } 6875 /* 6876 * If it isn't default-policy Meter, and 6877 * 1. Not meter hierarchy and there's no action in flow to change 6878 * packet (modify/encap/decap etc.), OR 6879 * 2. No drop count needed for this meter. 6880 * Then no need to use regC to save meter id anymore. 6881 */ 6882 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6883 set_mtr_reg = false; 6884 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6885 #define METER_PREFIX_ACTION 7 6886 act_size = (sizeof(struct rte_flow_action) * 6887 (actions_n + METER_PREFIX_ACTION)) + 6888 sizeof(struct mlx5_rte_flow_action_set_tag); 6889 /* Flow can have multiple VLAN items. Account for them in suffix items. */ 6890 vlan_items_n = flow_count_vlan_items(items); 6891 /* Suffix items: tag, [vlans], port id, end. */ 6892 #define METER_SUFFIX_ITEM 3 6893 item_size = sizeof(struct rte_flow_item) * (METER_SUFFIX_ITEM + vlan_items_n) + 6894 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6895 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6896 0, SOCKET_ID_ANY); 6897 if (!sfx_actions) 6898 return rte_flow_error_set(error, ENOMEM, 6899 RTE_FLOW_ERROR_TYPE_ACTION, 6900 NULL, "no memory to split " 6901 "meter flow"); 6902 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6903 act_size); 6904 /* There's no suffix flow for meter of non-default policy. */ 6905 if (!fm->def_policy) 6906 pre_actions = sfx_actions + 1; 6907 else 6908 pre_actions = sfx_actions + actions_n; 6909 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6910 items, sfx_items, actions, 6911 sfx_actions, pre_actions, 6912 (set_mtr_reg ? &mtr_flow_id : NULL), 6913 error); 6914 if (ret) { 6915 ret = -rte_errno; 6916 goto exit; 6917 } 6918 /* Add the prefix subflow. */ 6919 skip_scale_restore = flow_split_info->skip_scale; 6920 flow_split_info->skip_scale |= 6921 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6922 ret = flow_create_split_inner(dev, flow, &dev_flow, 6923 attr, items, pre_actions, 6924 flow_split_info, error); 6925 flow_split_info->skip_scale = skip_scale_restore; 6926 if (ret) { 6927 if (mtr_flow_id) 6928 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6929 ret = -rte_errno; 6930 goto exit; 6931 } 6932 if (mtr_flow_id) { 6933 dev_flow->handle->split_flow_id = mtr_flow_id; 6934 dev_flow->handle->is_meter_flow_id = 1; 6935 } 6936 if (!fm->def_policy) { 6937 if (!set_mtr_reg && fm->drop_cnt) 6938 ret = 6939 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6940 &sfx_attr, items, 6941 flow_split_info, 6942 fm, error); 6943 goto exit; 6944 } 6945 /* Setting the sfx group atrr. */ 6946 sfx_attr.group = sfx_attr.transfer ? 6947 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6948 MLX5_FLOW_TABLE_LEVEL_METER; 6949 flow_split_info->prefix_layers = 6950 flow_get_prefix_layer_flags(dev_flow); 6951 flow_split_info->prefix_mark |= wks->mark; 6952 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6953 } 6954 /* Add the prefix subflow. */ 6955 ret = flow_create_split_metadata(dev, flow, 6956 &sfx_attr, sfx_items ? 6957 sfx_items : items, 6958 sfx_actions ? sfx_actions : actions, 6959 flow_split_info, error); 6960 exit: 6961 if (sfx_actions) 6962 mlx5_free(sfx_actions); 6963 return ret; 6964 } 6965 6966 /** 6967 * The splitting for sample feature. 6968 * 6969 * Once Sample action is detected in the action list, the flow actions should 6970 * be split into prefix sub flow and suffix sub flow. 6971 * 6972 * The original items remain in the prefix sub flow, all actions preceding the 6973 * sample action and the sample action itself will be copied to the prefix 6974 * sub flow, the actions following the sample action will be copied to the 6975 * suffix sub flow, Queue action always be located in the suffix sub flow. 6976 * 6977 * In order to make the packet from prefix sub flow matches with suffix sub 6978 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6979 * flow uses tag item with the unique flow id. 6980 * 6981 * @param dev 6982 * Pointer to Ethernet device. 6983 * @param[in] flow 6984 * Parent flow structure pointer. 6985 * @param[in] attr 6986 * Flow rule attributes. 6987 * @param[in] items 6988 * Pattern specification (list terminated by the END pattern item). 6989 * @param[in] actions 6990 * Associated actions (list terminated by the END action). 6991 * @param[in] flow_split_info 6992 * Pointer to flow split info structure. 6993 * @param[out] error 6994 * Perform verbose error reporting if not NULL. 6995 * @return 6996 * 0 on success, negative value otherwise 6997 */ 6998 static int 6999 flow_create_split_sample(struct rte_eth_dev *dev, 7000 struct rte_flow *flow, 7001 const struct rte_flow_attr *attr, 7002 const struct rte_flow_item items[], 7003 const struct rte_flow_action actions[], 7004 struct mlx5_flow_split_info *flow_split_info, 7005 struct rte_flow_error *error) 7006 { 7007 struct mlx5_priv *priv = dev->data->dev_private; 7008 struct rte_flow_action *sfx_actions = NULL; 7009 struct rte_flow_action *pre_actions = NULL; 7010 struct rte_flow_item *sfx_items = NULL; 7011 struct mlx5_flow *dev_flow = NULL; 7012 struct rte_flow_attr sfx_attr = *attr; 7013 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7014 struct mlx5_flow_dv_sample_resource *sample_res; 7015 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 7016 struct mlx5_flow_tbl_resource *sfx_tbl; 7017 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 7018 #endif 7019 size_t act_size; 7020 size_t item_size; 7021 uint32_t fdb_tx = 0; 7022 int32_t tag_id = 0; 7023 int actions_n = 0; 7024 int sample_action_pos; 7025 int qrss_action_pos; 7026 int add_tag = 0; 7027 int modify_after_mirror = 0; 7028 uint16_t jump_table = 0; 7029 const uint32_t next_ft_step = 1; 7030 int ret = 0; 7031 struct mlx5_priv *item_port_priv = NULL; 7032 const struct rte_flow_item *item; 7033 7034 if (priv->sampler_en) 7035 actions_n = flow_check_match_action(actions, attr, 7036 RTE_FLOW_ACTION_TYPE_SAMPLE, 7037 &sample_action_pos, &qrss_action_pos, 7038 &modify_after_mirror); 7039 if (actions_n) { 7040 /* The prefix actions must includes sample, tag, end. */ 7041 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 7042 + sizeof(struct mlx5_rte_flow_action_set_tag); 7043 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 7044 sizeof(struct mlx5_rte_flow_item_tag) * 2; 7045 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 7046 item_size), 0, SOCKET_ID_ANY); 7047 if (!sfx_actions) 7048 return rte_flow_error_set(error, ENOMEM, 7049 RTE_FLOW_ERROR_TYPE_ACTION, 7050 NULL, "no memory to split " 7051 "sample flow"); 7052 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 7053 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 7054 const struct rte_flow_item_port_id *spec; 7055 7056 spec = (const struct rte_flow_item_port_id *)item->spec; 7057 if (spec) 7058 item_port_priv = 7059 mlx5_port_to_eswitch_info(spec->id, true); 7060 break; 7061 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 7062 const struct rte_flow_item_ethdev *spec; 7063 7064 spec = (const struct rte_flow_item_ethdev *)item->spec; 7065 if (spec) 7066 item_port_priv = 7067 mlx5_port_to_eswitch_info(spec->port_id, true); 7068 break; 7069 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 7070 const struct rte_flow_item_ethdev *spec; 7071 7072 spec = (const struct rte_flow_item_ethdev *)item->spec; 7073 if (spec) 7074 item_port_priv = 7075 mlx5_port_to_eswitch_info(spec->port_id, true); 7076 break; 7077 } 7078 } 7079 /* The representor_id is UINT16_MAX for uplink. */ 7080 fdb_tx = (attr->transfer && 7081 flow_source_vport_representor(priv, item_port_priv)); 7082 /* 7083 * When reg_c_preserve is set, metadata registers Cx preserve 7084 * their value even through packet duplication. 7085 */ 7086 add_tag = (!fdb_tx || 7087 priv->sh->cdev->config.hca_attr.reg_c_preserve); 7088 if (add_tag) 7089 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 7090 + act_size); 7091 if (modify_after_mirror) 7092 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 7093 next_ft_step; 7094 pre_actions = sfx_actions + actions_n; 7095 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 7096 actions, sfx_actions, 7097 pre_actions, actions_n, 7098 sample_action_pos, 7099 qrss_action_pos, jump_table, 7100 error); 7101 if (tag_id < 0 || (add_tag && !tag_id)) { 7102 ret = -rte_errno; 7103 goto exit; 7104 } 7105 if (modify_after_mirror) 7106 flow_split_info->skip_scale = 7107 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 7108 /* Add the prefix subflow. */ 7109 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7110 items, pre_actions, 7111 flow_split_info, error); 7112 if (ret) { 7113 ret = -rte_errno; 7114 goto exit; 7115 } 7116 dev_flow->handle->split_flow_id = tag_id; 7117 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7118 if (!modify_after_mirror) { 7119 /* Set the sfx group attr. */ 7120 sample_res = (struct mlx5_flow_dv_sample_resource *) 7121 dev_flow->dv.sample_res; 7122 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7123 sample_res->normal_path_tbl; 7124 sfx_tbl_data = container_of(sfx_tbl, 7125 struct mlx5_flow_tbl_data_entry, 7126 tbl); 7127 sfx_attr.group = sfx_attr.transfer ? 7128 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7129 } else { 7130 MLX5_ASSERT(attr->transfer); 7131 sfx_attr.group = jump_table; 7132 } 7133 flow_split_info->prefix_layers = 7134 flow_get_prefix_layer_flags(dev_flow); 7135 MLX5_ASSERT(wks); 7136 flow_split_info->prefix_mark |= wks->mark; 7137 /* Suffix group level already be scaled with factor, set 7138 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7139 * again in translation. 7140 */ 7141 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7142 #endif 7143 } 7144 /* Add the suffix subflow. */ 7145 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7146 sfx_items ? sfx_items : items, 7147 sfx_actions ? sfx_actions : actions, 7148 flow_split_info, error); 7149 exit: 7150 if (sfx_actions) 7151 mlx5_free(sfx_actions); 7152 return ret; 7153 } 7154 7155 /** 7156 * Split the flow to subflow set. The splitters might be linked 7157 * in the chain, like this: 7158 * flow_create_split_outer() calls: 7159 * flow_create_split_meter() calls: 7160 * flow_create_split_metadata(meter_subflow_0) calls: 7161 * flow_create_split_inner(metadata_subflow_0) 7162 * flow_create_split_inner(metadata_subflow_1) 7163 * flow_create_split_inner(metadata_subflow_2) 7164 * flow_create_split_metadata(meter_subflow_1) calls: 7165 * flow_create_split_inner(metadata_subflow_0) 7166 * flow_create_split_inner(metadata_subflow_1) 7167 * flow_create_split_inner(metadata_subflow_2) 7168 * 7169 * This provide flexible way to add new levels of flow splitting. 7170 * The all of successfully created subflows are included to the 7171 * parent flow dev_flow list. 7172 * 7173 * @param dev 7174 * Pointer to Ethernet device. 7175 * @param[in] flow 7176 * Parent flow structure pointer. 7177 * @param[in] attr 7178 * Flow rule attributes. 7179 * @param[in] items 7180 * Pattern specification (list terminated by the END pattern item). 7181 * @param[in] actions 7182 * Associated actions (list terminated by the END action). 7183 * @param[in] flow_split_info 7184 * Pointer to flow split info structure. 7185 * @param[out] error 7186 * Perform verbose error reporting if not NULL. 7187 * @return 7188 * 0 on success, negative value otherwise 7189 */ 7190 static int 7191 flow_create_split_outer(struct rte_eth_dev *dev, 7192 struct rte_flow *flow, 7193 const struct rte_flow_attr *attr, 7194 const struct rte_flow_item items[], 7195 const struct rte_flow_action actions[], 7196 struct mlx5_flow_split_info *flow_split_info, 7197 struct rte_flow_error *error) 7198 { 7199 int ret; 7200 7201 ret = flow_create_split_sample(dev, flow, attr, items, 7202 actions, flow_split_info, error); 7203 MLX5_ASSERT(ret <= 0); 7204 return ret; 7205 } 7206 7207 static inline struct mlx5_flow_tunnel * 7208 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7209 { 7210 struct mlx5_flow_tunnel *tunnel; 7211 7212 #pragma GCC diagnostic push 7213 #pragma GCC diagnostic ignored "-Wcast-qual" 7214 tunnel = (typeof(tunnel))flow->tunnel; 7215 #pragma GCC diagnostic pop 7216 7217 return tunnel; 7218 } 7219 7220 /** 7221 * Create a flow and add it to @p list. 7222 * 7223 * @param dev 7224 * Pointer to Ethernet device. 7225 * @param list 7226 * Pointer to a TAILQ flow list. If this parameter NULL, 7227 * no list insertion occurred, flow is just created, 7228 * this is caller's responsibility to track the 7229 * created flow. 7230 * @param[in] attr 7231 * Flow rule attributes. 7232 * @param[in] items 7233 * Pattern specification (list terminated by the END pattern item). 7234 * @param[in] actions 7235 * Associated actions (list terminated by the END action). 7236 * @param[in] external 7237 * This flow rule is created by request external to PMD. 7238 * @param[out] error 7239 * Perform verbose error reporting if not NULL. 7240 * 7241 * @return 7242 * A flow index on success, 0 otherwise and rte_errno is set. 7243 */ 7244 static uint32_t 7245 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7246 const struct rte_flow_attr *attr, 7247 const struct rte_flow_item items[], 7248 const struct rte_flow_action original_actions[], 7249 bool external, struct rte_flow_error *error) 7250 { 7251 struct mlx5_priv *priv = dev->data->dev_private; 7252 struct rte_flow *flow = NULL; 7253 struct mlx5_flow *dev_flow; 7254 const struct rte_flow_action_rss *rss = NULL; 7255 struct mlx5_translated_action_handle 7256 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7257 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7258 union { 7259 struct mlx5_flow_expand_rss buf; 7260 uint8_t buffer[8192]; 7261 } expand_buffer; 7262 union { 7263 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7264 uint8_t buffer[2048]; 7265 } actions_rx; 7266 union { 7267 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7268 uint8_t buffer[2048]; 7269 } actions_hairpin_tx; 7270 union { 7271 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7272 uint8_t buffer[2048]; 7273 } items_tx; 7274 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7275 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7276 struct mlx5_flow_rss_desc *rss_desc; 7277 const struct rte_flow_action *p_actions_rx; 7278 uint32_t i; 7279 uint32_t idx = 0; 7280 int hairpin_flow; 7281 struct rte_flow_attr attr_tx = { .priority = 0 }; 7282 const struct rte_flow_action *actions; 7283 struct rte_flow_action *translated_actions = NULL; 7284 struct mlx5_flow_tunnel *tunnel; 7285 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7286 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7287 struct mlx5_flow_split_info flow_split_info = { 7288 .external = !!external, 7289 .skip_scale = 0, 7290 .flow_idx = 0, 7291 .prefix_mark = 0, 7292 .prefix_layers = 0, 7293 .table_id = 0 7294 }; 7295 int ret; 7296 7297 MLX5_ASSERT(wks); 7298 rss_desc = &wks->rss_desc; 7299 ret = flow_action_handles_translate(dev, original_actions, 7300 indir_actions, 7301 &indir_actions_n, 7302 &translated_actions, error); 7303 if (ret < 0) { 7304 MLX5_ASSERT(translated_actions == NULL); 7305 return 0; 7306 } 7307 actions = translated_actions ? translated_actions : original_actions; 7308 p_actions_rx = actions; 7309 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7310 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7311 external, hairpin_flow, error); 7312 if (ret < 0) 7313 goto error_before_hairpin_split; 7314 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7315 if (!flow) { 7316 rte_errno = ENOMEM; 7317 goto error_before_hairpin_split; 7318 } 7319 if (hairpin_flow > 0) { 7320 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7321 rte_errno = EINVAL; 7322 goto error_before_hairpin_split; 7323 } 7324 flow_hairpin_split(dev, actions, actions_rx.actions, 7325 actions_hairpin_tx.actions, items_tx.items, 7326 idx); 7327 p_actions_rx = actions_rx.actions; 7328 } 7329 flow_split_info.flow_idx = idx; 7330 flow->drv_type = flow_get_drv_type(dev, attr); 7331 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7332 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7333 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7334 /* RSS Action only works on NIC RX domain */ 7335 if (attr->ingress) 7336 rss = flow_get_rss_action(dev, p_actions_rx); 7337 if (rss) { 7338 MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); 7339 rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func); 7340 /* 7341 * The following information is required by 7342 * mlx5_flow_hashfields_adjust() in advance. 7343 */ 7344 rss_desc->level = rss->level; 7345 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7346 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7347 } 7348 flow->dev_handles = 0; 7349 if (rss && rss->types) { 7350 unsigned int graph_root; 7351 7352 graph_root = find_graph_root(rss->level); 7353 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7354 items, rss->types, 7355 mlx5_support_expansion, graph_root); 7356 MLX5_ASSERT(ret > 0 && 7357 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7358 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7359 for (i = 0; i < buf->entries; ++i) 7360 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7361 } 7362 } else { 7363 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7364 sizeof(expand_buffer.buffer), 7365 items, sq_specs); 7366 if (ret) { 7367 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7368 NULL, "not enough memory for rte_flow"); 7369 goto error; 7370 } 7371 if (buf->entries == 0) { 7372 buf->entries = 1; 7373 buf->entry[0].pattern = (void *)(uintptr_t)items; 7374 } 7375 } 7376 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7377 indir_actions_n); 7378 for (i = 0; i < buf->entries; ++i) { 7379 /* Initialize flow split data. */ 7380 flow_split_info.prefix_layers = 0; 7381 flow_split_info.prefix_mark = 0; 7382 flow_split_info.skip_scale = 0; 7383 /* 7384 * The splitter may create multiple dev_flows, 7385 * depending on configuration. In the simplest 7386 * case it just creates unmodified original flow. 7387 */ 7388 ret = flow_create_split_outer(dev, flow, attr, 7389 buf->entry[i].pattern, 7390 p_actions_rx, &flow_split_info, 7391 error); 7392 if (ret < 0) 7393 goto error; 7394 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7395 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7396 p_actions_rx, 7397 idx, 7398 wks->flows[0].tunnel, 7399 &default_miss_ctx, 7400 error); 7401 if (ret < 0) { 7402 mlx5_free(default_miss_ctx.queue); 7403 goto error; 7404 } 7405 } 7406 } 7407 /* Create the tx flow. */ 7408 if (hairpin_flow) { 7409 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7410 attr_tx.ingress = 0; 7411 attr_tx.egress = 1; 7412 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7413 actions_hairpin_tx.actions, 7414 idx, error); 7415 if (!dev_flow) 7416 goto error; 7417 dev_flow->flow = flow; 7418 dev_flow->external = 0; 7419 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7420 dev_flow->handle, next); 7421 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7422 items_tx.items, 7423 actions_hairpin_tx.actions, error); 7424 if (ret < 0) 7425 goto error; 7426 } 7427 /* 7428 * Update the metadata register copy table. If extensive 7429 * metadata feature is enabled and registers are supported 7430 * we might create the extra rte_flow for each unique 7431 * MARK/FLAG action ID. 7432 * 7433 * The table is updated for ingress and transfer flows only, because 7434 * the egress Flows belong to the different device and 7435 * copy table should be updated in peer NIC Rx domain. 7436 */ 7437 if ((attr->ingress || attr->transfer) && 7438 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7439 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7440 if (ret) 7441 goto error; 7442 } 7443 /* 7444 * If the flow is external (from application) OR device is started, 7445 * OR mreg discover, then apply immediately. 7446 */ 7447 if (external || dev->data->dev_started || 7448 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7449 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7450 ret = flow_drv_apply(dev, flow, error); 7451 if (ret < 0) 7452 goto error; 7453 } 7454 flow->type = type; 7455 flow_rxq_flags_set(dev, flow); 7456 rte_free(translated_actions); 7457 tunnel = flow_tunnel_from_rule(wks->flows); 7458 if (tunnel) { 7459 flow->tunnel = 1; 7460 flow->tunnel_id = tunnel->tunnel_id; 7461 __atomic_fetch_add(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7462 mlx5_free(default_miss_ctx.queue); 7463 } 7464 mlx5_flow_pop_thread_workspace(); 7465 return idx; 7466 error: 7467 MLX5_ASSERT(flow); 7468 ret = rte_errno; /* Save rte_errno before cleanup. */ 7469 flow_mreg_del_copy_action(dev, flow); 7470 flow_drv_destroy(dev, flow); 7471 if (rss_desc->shared_rss) 7472 __atomic_fetch_sub(&((struct mlx5_shared_action_rss *) 7473 mlx5_ipool_get 7474 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7475 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7476 mlx5_ipool_free(priv->flows[type], idx); 7477 rte_errno = ret; /* Restore rte_errno. */ 7478 ret = rte_errno; 7479 rte_errno = ret; 7480 error_before_hairpin_split: 7481 mlx5_flow_pop_thread_workspace(); 7482 rte_free(translated_actions); 7483 return 0; 7484 } 7485 7486 /** 7487 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7488 * incoming packets to table 1. 7489 * 7490 * Other flow rules, requested for group n, will be created in 7491 * e-switch table n+1. 7492 * Jump action to e-switch group n will be created to group n+1. 7493 * 7494 * Used when working in switchdev mode, to utilise advantages of table 1 7495 * and above. 7496 * 7497 * @param dev 7498 * Pointer to Ethernet device. 7499 * 7500 * @return 7501 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7502 */ 7503 struct rte_flow * 7504 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7505 { 7506 const struct rte_flow_attr attr = { 7507 .group = 0, 7508 .priority = 0, 7509 .ingress = 0, 7510 .egress = 0, 7511 .transfer = 1, 7512 }; 7513 const struct rte_flow_item pattern = { 7514 .type = RTE_FLOW_ITEM_TYPE_END, 7515 }; 7516 struct rte_flow_action_jump jump = { 7517 .group = 1, 7518 }; 7519 const struct rte_flow_action actions[] = { 7520 { 7521 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7522 .conf = &jump, 7523 }, 7524 { 7525 .type = RTE_FLOW_ACTION_TYPE_END, 7526 }, 7527 }; 7528 struct rte_flow_error error; 7529 7530 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7531 &attr, &pattern, 7532 actions, false, &error); 7533 } 7534 7535 /** 7536 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7537 * and sq number, directs all packets to peer vport. 7538 * 7539 * @param dev 7540 * Pointer to Ethernet device. 7541 * @param sq_num 7542 * SQ number. 7543 * 7544 * @return 7545 * Flow ID on success, 0 otherwise and rte_errno is set. 7546 */ 7547 uint32_t 7548 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7549 { 7550 struct rte_flow_attr attr = { 7551 .group = 0, 7552 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7553 .ingress = 0, 7554 .egress = 0, 7555 .transfer = 1, 7556 }; 7557 struct rte_flow_item_port_id port_spec = { 7558 .id = MLX5_PORT_ESW_MGR, 7559 }; 7560 struct mlx5_rte_flow_item_sq sq_spec = { 7561 .queue = sq_num, 7562 }; 7563 struct rte_flow_item pattern[] = { 7564 { 7565 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7566 .spec = &port_spec, 7567 }, 7568 { 7569 .type = (enum rte_flow_item_type) 7570 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7571 .spec = &sq_spec, 7572 }, 7573 { 7574 .type = RTE_FLOW_ITEM_TYPE_END, 7575 }, 7576 }; 7577 struct rte_flow_action_jump jump = { 7578 .group = 1, 7579 }; 7580 struct rte_flow_action_port_id port = { 7581 .id = dev->data->port_id, 7582 }; 7583 struct rte_flow_action actions[] = { 7584 { 7585 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7586 .conf = &jump, 7587 }, 7588 { 7589 .type = RTE_FLOW_ACTION_TYPE_END, 7590 }, 7591 }; 7592 struct rte_flow_error error; 7593 7594 /* 7595 * Creates group 0, highest priority jump flow. 7596 * Matches txq to bypass kernel packets. 7597 */ 7598 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7599 false, &error) == 0) 7600 return 0; 7601 /* Create group 1, lowest priority redirect flow for txq. */ 7602 attr.group = 1; 7603 actions[0].conf = &port; 7604 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7605 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7606 actions, false, &error); 7607 } 7608 7609 /** 7610 * Validate a flow supported by the NIC. 7611 * 7612 * @see rte_flow_validate() 7613 * @see rte_flow_ops 7614 */ 7615 int 7616 mlx5_flow_validate(struct rte_eth_dev *dev, 7617 const struct rte_flow_attr *attr, 7618 const struct rte_flow_item items[], 7619 const struct rte_flow_action original_actions[], 7620 struct rte_flow_error *error) 7621 { 7622 int hairpin_flow; 7623 struct mlx5_translated_action_handle 7624 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7625 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7626 const struct rte_flow_action *actions; 7627 struct rte_flow_action *translated_actions = NULL; 7628 int ret = flow_action_handles_translate(dev, original_actions, 7629 indir_actions, 7630 &indir_actions_n, 7631 &translated_actions, error); 7632 7633 if (ret) 7634 return ret; 7635 actions = translated_actions ? translated_actions : original_actions; 7636 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7637 ret = flow_drv_validate(dev, attr, items, actions, 7638 true, hairpin_flow, error); 7639 rte_free(translated_actions); 7640 return ret; 7641 } 7642 7643 static int 7644 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7645 const struct rte_flow_attr *attr, 7646 const uint32_t orig_prio, 7647 const struct rte_flow_item *items, 7648 const struct rte_flow_action *actions, 7649 uint32_t flow_idx) 7650 { 7651 struct mlx5_priv *priv = dev->data->dev_private; 7652 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7653 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7654 struct rte_flow_error error; 7655 int len, ret; 7656 7657 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7658 if (!flow_info) { 7659 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7660 return -1; 7661 } 7662 flow_info->orig_prio = orig_prio; 7663 flow_info->attr = *attr; 7664 /* Standby mode rule awlays saves it in low priority entry. */ 7665 flow_info->flow_idx_low_prio = flow_idx; 7666 7667 /* Store matching items. */ 7668 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7669 if (ret <= 0) { 7670 DRV_LOG(ERR, "Can't get items length."); 7671 goto end; 7672 } 7673 len = RTE_ALIGN(ret, 16); 7674 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7675 if (!flow_info->items) { 7676 DRV_LOG(ERR, "No enough memory for items caching."); 7677 goto end; 7678 } 7679 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7680 if (ret <= 0) { 7681 DRV_LOG(ERR, "Can't duplicate items."); 7682 goto end; 7683 } 7684 7685 /* Store flow actions. */ 7686 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7687 if (ret <= 0) { 7688 DRV_LOG(ERR, "Can't get actions length."); 7689 goto end; 7690 } 7691 len = RTE_ALIGN(ret, 16); 7692 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7693 if (!flow_info->actions) { 7694 DRV_LOG(ERR, "No enough memory for actions caching."); 7695 goto end; 7696 } 7697 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7698 if (ret <= 0) { 7699 DRV_LOG(ERR, "Can't duplicate actions."); 7700 goto end; 7701 } 7702 7703 /* Insert to the list end. */ 7704 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7705 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7706 } else { 7707 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7708 while (LIST_NEXT(tmp_info, next)) 7709 tmp_info = LIST_NEXT(tmp_info, next); 7710 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7711 } 7712 return 0; 7713 end: 7714 if (flow_info->items) 7715 mlx5_free(flow_info->items); 7716 if (flow_info->actions) 7717 mlx5_free(flow_info->actions); 7718 mlx5_free(flow_info); 7719 return -1; 7720 } 7721 7722 static int 7723 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7724 { 7725 struct mlx5_priv *priv = dev->data->dev_private; 7726 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7727 struct mlx5_dv_flow_info *flow_info; 7728 struct rte_flow_attr attr; 7729 struct rte_flow_error error; 7730 struct rte_flow *high, *low; 7731 7732 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7733 while (flow_info) { 7734 /* DUP flow may have the same priority. */ 7735 if (flow_info->orig_prio != flow_info->attr.priority) { 7736 attr = flow_info->attr; 7737 if (orig_prio) 7738 attr.priority = flow_info->orig_prio; 7739 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7740 &attr, flow_info->items, flow_info->actions, 7741 true, &error); 7742 if (!flow_info->flow_idx_high_prio) { 7743 DRV_LOG(ERR, "Priority toggle failed internally."); 7744 goto err; 7745 } 7746 } 7747 flow_info = LIST_NEXT(flow_info, next); 7748 } 7749 /* Delete the low priority rules and swap the flow handle. */ 7750 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7751 while (flow_info) { 7752 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7753 if (flow_info->orig_prio != flow_info->attr.priority) { 7754 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7755 flow_info->flow_idx_high_prio); 7756 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7757 flow_info->flow_idx_low_prio); 7758 if (high && low) { 7759 RTE_SWAP(*low, *high); 7760 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7761 flow_info->flow_idx_low_prio); 7762 flow_info->flow_idx_high_prio = 0; 7763 } 7764 } 7765 flow_info = LIST_NEXT(flow_info, next); 7766 } 7767 return 0; 7768 err: 7769 /* Destroy preceding successful high priority rules. */ 7770 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7771 while (flow_info) { 7772 if (flow_info->orig_prio != flow_info->attr.priority) { 7773 if (flow_info->flow_idx_high_prio) 7774 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7775 flow_info->flow_idx_high_prio); 7776 else 7777 break; 7778 flow_info->flow_idx_high_prio = 0; 7779 } 7780 flow_info = LIST_NEXT(flow_info, next); 7781 } 7782 return -1; 7783 } 7784 7785 /** 7786 * Set the mode of the flow engine of a process to active or standby during live migration. 7787 * 7788 * @param[in] mode 7789 * MLX5 flow engine mode, @see `enum rte_pmd_mlx5_flow_engine_mode`. 7790 * @param[in] flags 7791 * Flow engine mode specific flags. 7792 * 7793 * @return 7794 * Negative value on error, positive on success. 7795 */ 7796 int 7797 rte_pmd_mlx5_flow_engine_set_mode(enum rte_pmd_mlx5_flow_engine_mode mode, uint32_t flags) 7798 { 7799 struct mlx5_priv *priv; 7800 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info; 7801 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7802 uint16_t port, port_id; 7803 uint16_t toggle_num = 0; 7804 struct rte_eth_dev *dev; 7805 enum rte_pmd_mlx5_flow_engine_mode orig_mode; 7806 uint32_t orig_flags; 7807 bool need_toggle = false; 7808 7809 /* Check if flags combinations are supported. */ 7810 if (flags && flags != RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7811 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7812 return -1; 7813 } 7814 MLX5_ETH_FOREACH_DEV(port, NULL) { 7815 dev = &rte_eth_devices[port]; 7816 priv = dev->data->dev_private; 7817 mode_info = &priv->mode_info; 7818 /* No mode change. Assume all devices hold the same mode. */ 7819 if (mode_info->mode == mode) { 7820 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7821 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7822 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7823 port, mode_info->mode_flag); 7824 orig_mode = mode_info->mode; 7825 orig_flags = mode_info->mode_flag; 7826 goto err; 7827 } 7828 mode_info->mode_flag = flags; 7829 toggle_num++; 7830 continue; 7831 } 7832 /* Active -> standby. */ 7833 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY) { 7834 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7835 DRV_LOG(ERR, "Cached rule existed"); 7836 orig_mode = mode_info->mode; 7837 orig_flags = mode_info->mode_flag; 7838 goto err; 7839 } 7840 mode_info->mode_flag = flags; 7841 mode_info->mode = mode; 7842 toggle_num++; 7843 /* Standby -> active. */ 7844 } else if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7845 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7846 DRV_LOG(INFO, "No cached rule existed"); 7847 } else { 7848 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7849 orig_mode = mode_info->mode; 7850 orig_flags = mode_info->mode_flag; 7851 need_toggle = true; 7852 goto err; 7853 } 7854 } 7855 toggle_num++; 7856 } 7857 } 7858 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7859 /* Clear cache flow rules. */ 7860 MLX5_ETH_FOREACH_DEV(port, NULL) { 7861 priv = rte_eth_devices[port].data->dev_private; 7862 mode_info = &priv->mode_info; 7863 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7864 while (flow_info) { 7865 tmp_info = LIST_NEXT(flow_info, next); 7866 LIST_REMOVE(flow_info, next); 7867 mlx5_free(flow_info->actions); 7868 mlx5_free(flow_info->items); 7869 mlx5_free(flow_info); 7870 flow_info = tmp_info; 7871 } 7872 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7873 } 7874 } 7875 return toggle_num; 7876 err: 7877 /* Rollback all preceding successful ports. */ 7878 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7879 if (port_id == port) 7880 break; 7881 priv = rte_eth_devices[port_id].data->dev_private; 7882 mode_info = &priv->mode_info; 7883 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7884 mlx5_flow_cache_flow_toggle(dev, false)) 7885 return -EPERM; 7886 mode_info->mode = orig_mode; 7887 mode_info->mode_flag = orig_flags; 7888 } 7889 return -EINVAL; 7890 } 7891 /** 7892 * Create a flow. 7893 * 7894 * @see rte_flow_create() 7895 * @see rte_flow_ops 7896 */ 7897 struct rte_flow * 7898 mlx5_flow_create(struct rte_eth_dev *dev, 7899 const struct rte_flow_attr *attr, 7900 const struct rte_flow_item items[], 7901 const struct rte_flow_action actions[], 7902 struct rte_flow_error *error) 7903 { 7904 struct mlx5_priv *priv = dev->data->dev_private; 7905 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7906 uint32_t prio = attr->priority; 7907 uint32_t flow_idx; 7908 7909 if (priv->sh->config.dv_flow_en == 2) { 7910 rte_flow_error_set(error, ENOTSUP, 7911 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7912 NULL, 7913 "Flow non-Q creation not supported"); 7914 return NULL; 7915 } 7916 /* 7917 * If the device is not started yet, it is not allowed to created a 7918 * flow from application. PMD default flows and traffic control flows 7919 * are not affected. 7920 */ 7921 if (unlikely(!dev->data->dev_started)) { 7922 DRV_LOG(DEBUG, "port %u is not started when " 7923 "inserting a flow", dev->data->port_id); 7924 rte_flow_error_set(error, ENODEV, 7925 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7926 NULL, 7927 "port not started"); 7928 return NULL; 7929 } 7930 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7931 if (attr->transfer || 7932 (attr->ingress && !(priv->mode_info.mode_flag & 7933 RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 7934 new_attr->priority += 1; 7935 } 7936 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 7937 if (!flow_idx) 7938 return NULL; 7939 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7940 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 7941 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7942 flow_idx = 0; 7943 } 7944 } 7945 return (void *)(uintptr_t)flow_idx; 7946 } 7947 7948 /** 7949 * Destroy a flow in a list. 7950 * 7951 * @param dev 7952 * Pointer to Ethernet device. 7953 * @param[in] flow_idx 7954 * Index of flow to destroy. 7955 */ 7956 static void 7957 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7958 uint32_t flow_idx) 7959 { 7960 struct mlx5_priv *priv = dev->data->dev_private; 7961 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7962 7963 if (!flow) 7964 return; 7965 MLX5_ASSERT(flow->type == type); 7966 /* 7967 * Update RX queue flags only if port is started, otherwise it is 7968 * already clean. 7969 */ 7970 if (dev->data->dev_started) 7971 flow_rxq_flags_trim(dev, flow); 7972 flow_drv_destroy(dev, flow); 7973 if (flow->tunnel) { 7974 struct mlx5_flow_tunnel *tunnel; 7975 7976 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7977 RTE_VERIFY(tunnel); 7978 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 7979 mlx5_flow_tunnel_free(dev, tunnel); 7980 } 7981 flow_mreg_del_copy_action(dev, flow); 7982 mlx5_ipool_free(priv->flows[type], flow_idx); 7983 } 7984 7985 /** 7986 * Destroy all flows. 7987 * 7988 * @param dev 7989 * Pointer to Ethernet device. 7990 * @param type 7991 * Flow type to be flushed. 7992 * @param active 7993 * If flushing is called actively. 7994 */ 7995 void 7996 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7997 bool active) 7998 { 7999 struct mlx5_priv *priv = dev->data->dev_private; 8000 uint32_t num_flushed = 0, fidx = 1; 8001 struct rte_flow *flow; 8002 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8003 struct mlx5_dv_flow_info *flow_info; 8004 8005 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8006 if (priv->sh->config.dv_flow_en == 2 && 8007 type == MLX5_FLOW_TYPE_GEN) { 8008 flow_hw_q_flow_flush(dev, NULL); 8009 return; 8010 } 8011 #endif 8012 8013 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 8014 flow_list_destroy(dev, type, fidx); 8015 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 8016 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8017 while (flow_info) { 8018 /* Romove the cache flow info. */ 8019 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 8020 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8021 LIST_REMOVE(flow_info, next); 8022 mlx5_free(flow_info->items); 8023 mlx5_free(flow_info->actions); 8024 mlx5_free(flow_info); 8025 break; 8026 } 8027 flow_info = LIST_NEXT(flow_info, next); 8028 } 8029 } 8030 num_flushed++; 8031 } 8032 if (active) { 8033 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 8034 dev->data->port_id, num_flushed); 8035 } 8036 } 8037 8038 /** 8039 * Stop all default actions for flows. 8040 * 8041 * @param dev 8042 * Pointer to Ethernet device. 8043 */ 8044 void 8045 mlx5_flow_stop_default(struct rte_eth_dev *dev) 8046 { 8047 flow_mreg_del_default_copy_action(dev); 8048 flow_rxq_flags_clear(dev); 8049 } 8050 8051 /** 8052 * Set rxq flag. 8053 * 8054 * @param[in] dev 8055 * Pointer to the rte_eth_dev structure. 8056 * @param[in] enable 8057 * Flag to enable or not. 8058 */ 8059 void 8060 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 8061 { 8062 struct mlx5_priv *priv = dev->data->dev_private; 8063 unsigned int i; 8064 8065 if ((!priv->mark_enabled && !enable) || 8066 (priv->mark_enabled && enable)) 8067 return; 8068 for (i = 0; i < priv->rxqs_n; ++i) { 8069 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 8070 8071 /* With RXQ start/stop feature, RXQ might be stopped. */ 8072 if (!rxq_ctrl) 8073 continue; 8074 rxq_ctrl->rxq.mark = enable; 8075 } 8076 priv->mark_enabled = enable; 8077 } 8078 8079 /** 8080 * Start all default actions for flows. 8081 * 8082 * @param dev 8083 * Pointer to Ethernet device. 8084 * @return 8085 * 0 on success, a negative errno value otherwise and rte_errno is set. 8086 */ 8087 int 8088 mlx5_flow_start_default(struct rte_eth_dev *dev) 8089 { 8090 struct rte_flow_error error; 8091 8092 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8093 return flow_mreg_add_default_copy_action(dev, &error); 8094 } 8095 8096 /** 8097 * Release key of thread specific flow workspace data. 8098 */ 8099 void 8100 flow_release_workspace(void *data) 8101 { 8102 struct mlx5_flow_workspace *wks = data; 8103 struct mlx5_flow_workspace *next; 8104 8105 while (wks) { 8106 next = wks->next; 8107 free(wks); 8108 wks = next; 8109 } 8110 } 8111 8112 /** 8113 * Get thread specific current flow workspace. 8114 * 8115 * @return pointer to thread specific flow workspace data, NULL on error. 8116 */ 8117 struct mlx5_flow_workspace* 8118 mlx5_flow_get_thread_workspace(void) 8119 { 8120 struct mlx5_flow_workspace *data; 8121 8122 data = mlx5_flow_os_get_specific_workspace(); 8123 MLX5_ASSERT(data && data->inuse); 8124 if (!data || !data->inuse) 8125 DRV_LOG(ERR, "flow workspace not initialized."); 8126 return data; 8127 } 8128 8129 /** 8130 * Allocate and init new flow workspace. 8131 * 8132 * @return pointer to flow workspace data, NULL on error. 8133 */ 8134 static struct mlx5_flow_workspace* 8135 flow_alloc_thread_workspace(void) 8136 { 8137 size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); 8138 size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; 8139 struct mlx5_flow_workspace *data = calloc(1, data_size + 8140 rss_queue_array_size); 8141 8142 if (!data) { 8143 DRV_LOG(ERR, "Failed to allocate flow workspace memory."); 8144 return NULL; 8145 } 8146 data->rss_desc.queue = RTE_PTR_ADD(data, data_size); 8147 return data; 8148 } 8149 8150 /** 8151 * Get new thread specific flow workspace. 8152 * 8153 * If current workspace inuse, create new one and set as current. 8154 * 8155 * @return pointer to thread specific flow workspace data, NULL on error. 8156 */ 8157 struct mlx5_flow_workspace* 8158 mlx5_flow_push_thread_workspace(void) 8159 { 8160 struct mlx5_flow_workspace *curr; 8161 struct mlx5_flow_workspace *data; 8162 8163 curr = mlx5_flow_os_get_specific_workspace(); 8164 if (!curr) { 8165 data = flow_alloc_thread_workspace(); 8166 if (!data) 8167 return NULL; 8168 mlx5_flow_os_workspace_gc_add(data); 8169 } else if (!curr->inuse) { 8170 data = curr; 8171 } else if (curr->next) { 8172 data = curr->next; 8173 } else { 8174 data = flow_alloc_thread_workspace(); 8175 if (!data) 8176 return NULL; 8177 curr->next = data; 8178 data->prev = curr; 8179 } 8180 data->inuse = 1; 8181 data->flow_idx = 0; 8182 /* Set as current workspace */ 8183 if (mlx5_flow_os_set_specific_workspace(data)) 8184 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8185 return data; 8186 } 8187 8188 /** 8189 * Close current thread specific flow workspace. 8190 * 8191 * If previous workspace available, set it as current. 8192 * 8193 * @return pointer to thread specific flow workspace data, NULL on error. 8194 */ 8195 void 8196 mlx5_flow_pop_thread_workspace(void) 8197 { 8198 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8199 8200 if (!data) 8201 return; 8202 if (!data->inuse) { 8203 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8204 return; 8205 } 8206 data->inuse = 0; 8207 if (!data->prev) 8208 return; 8209 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8210 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8211 } 8212 8213 /** 8214 * Verify the flow list is empty 8215 * 8216 * @param dev 8217 * Pointer to Ethernet device. 8218 * 8219 * @return the number of flows not released. 8220 */ 8221 int 8222 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8223 { 8224 struct mlx5_priv *priv = dev->data->dev_private; 8225 struct rte_flow *flow; 8226 uint32_t idx = 0; 8227 int ret = 0, i; 8228 8229 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8230 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8231 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8232 dev->data->port_id, (void *)flow); 8233 ret++; 8234 } 8235 } 8236 return ret; 8237 } 8238 8239 /** 8240 * Enable default hairpin egress flow. 8241 * 8242 * @param dev 8243 * Pointer to Ethernet device. 8244 * @param sq_num 8245 * The SQ hw number. 8246 * 8247 * @return 8248 * 0 on success, a negative errno value otherwise and rte_errno is set. 8249 */ 8250 int 8251 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8252 uint32_t sq_num) 8253 { 8254 const struct rte_flow_attr attr = { 8255 .egress = 1, 8256 .priority = 0, 8257 }; 8258 struct mlx5_rte_flow_item_sq queue_spec = { 8259 .queue = sq_num, 8260 }; 8261 struct mlx5_rte_flow_item_sq queue_mask = { 8262 .queue = UINT32_MAX, 8263 }; 8264 struct rte_flow_item items[] = { 8265 { 8266 .type = (enum rte_flow_item_type) 8267 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8268 .spec = &queue_spec, 8269 .last = NULL, 8270 .mask = &queue_mask, 8271 }, 8272 { 8273 .type = RTE_FLOW_ITEM_TYPE_END, 8274 }, 8275 }; 8276 struct rte_flow_action_jump jump = { 8277 .group = MLX5_HAIRPIN_TX_TABLE, 8278 }; 8279 struct rte_flow_action actions[2]; 8280 uint32_t flow_idx; 8281 struct rte_flow_error error; 8282 8283 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8284 actions[0].conf = &jump; 8285 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8286 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8287 &attr, items, actions, false, &error); 8288 if (!flow_idx) { 8289 DRV_LOG(DEBUG, 8290 "Failed to create ctrl flow: rte_errno(%d)," 8291 " type(%d), message(%s)", 8292 rte_errno, error.type, 8293 error.message ? error.message : " (no stated reason)"); 8294 return -rte_errno; 8295 } 8296 return 0; 8297 } 8298 8299 /** 8300 * Enable a control flow configured from the control plane. 8301 * 8302 * @param dev 8303 * Pointer to Ethernet device. 8304 * @param eth_spec 8305 * An Ethernet flow spec to apply. 8306 * @param eth_mask 8307 * An Ethernet flow mask to apply. 8308 * @param vlan_spec 8309 * A VLAN flow spec to apply. 8310 * @param vlan_mask 8311 * A VLAN flow mask to apply. 8312 * 8313 * @return 8314 * 0 on success, a negative errno value otherwise and rte_errno is set. 8315 */ 8316 int 8317 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8318 struct rte_flow_item_eth *eth_spec, 8319 struct rte_flow_item_eth *eth_mask, 8320 struct rte_flow_item_vlan *vlan_spec, 8321 struct rte_flow_item_vlan *vlan_mask) 8322 { 8323 struct mlx5_priv *priv = dev->data->dev_private; 8324 const struct rte_flow_attr attr = { 8325 .ingress = 1, 8326 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8327 }; 8328 struct rte_flow_item items[] = { 8329 { 8330 .type = RTE_FLOW_ITEM_TYPE_ETH, 8331 .spec = eth_spec, 8332 .last = NULL, 8333 .mask = eth_mask, 8334 }, 8335 { 8336 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8337 RTE_FLOW_ITEM_TYPE_END, 8338 .spec = vlan_spec, 8339 .last = NULL, 8340 .mask = vlan_mask, 8341 }, 8342 { 8343 .type = RTE_FLOW_ITEM_TYPE_END, 8344 }, 8345 }; 8346 uint16_t queue[priv->reta_idx_n]; 8347 struct rte_flow_action_rss action_rss = { 8348 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8349 .level = 0, 8350 .types = priv->rss_conf.rss_hf, 8351 .key_len = priv->rss_conf.rss_key_len, 8352 .queue_num = priv->reta_idx_n, 8353 .key = priv->rss_conf.rss_key, 8354 .queue = queue, 8355 }; 8356 struct rte_flow_action actions[] = { 8357 { 8358 .type = RTE_FLOW_ACTION_TYPE_RSS, 8359 .conf = &action_rss, 8360 }, 8361 { 8362 .type = RTE_FLOW_ACTION_TYPE_END, 8363 }, 8364 }; 8365 uint32_t flow_idx; 8366 struct rte_flow_error error; 8367 unsigned int i; 8368 8369 if (!priv->reta_idx_n || !priv->rxqs_n) { 8370 return 0; 8371 } 8372 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8373 action_rss.types = 0; 8374 for (i = 0; i != priv->reta_idx_n; ++i) 8375 queue[i] = (*priv->reta_idx)[i]; 8376 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8377 &attr, items, actions, false, &error); 8378 if (!flow_idx) 8379 return -rte_errno; 8380 return 0; 8381 } 8382 8383 /** 8384 * Enable a flow control configured from the control plane. 8385 * 8386 * @param dev 8387 * Pointer to Ethernet device. 8388 * @param eth_spec 8389 * An Ethernet flow spec to apply. 8390 * @param eth_mask 8391 * An Ethernet flow mask to apply. 8392 * 8393 * @return 8394 * 0 on success, a negative errno value otherwise and rte_errno is set. 8395 */ 8396 int 8397 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8398 struct rte_flow_item_eth *eth_spec, 8399 struct rte_flow_item_eth *eth_mask) 8400 { 8401 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8402 } 8403 8404 /** 8405 * Create default miss flow rule matching lacp traffic 8406 * 8407 * @param dev 8408 * Pointer to Ethernet device. 8409 * @param eth_spec 8410 * An Ethernet flow spec to apply. 8411 * 8412 * @return 8413 * 0 on success, a negative errno value otherwise and rte_errno is set. 8414 */ 8415 int 8416 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8417 { 8418 /* 8419 * The LACP matching is done by only using ether type since using 8420 * a multicast dst mac causes kernel to give low priority to this flow. 8421 */ 8422 static const struct rte_flow_item_eth lacp_spec = { 8423 .hdr.ether_type = RTE_BE16(0x8809), 8424 }; 8425 static const struct rte_flow_item_eth lacp_mask = { 8426 .hdr.ether_type = 0xffff, 8427 }; 8428 const struct rte_flow_attr attr = { 8429 .ingress = 1, 8430 }; 8431 struct rte_flow_item items[] = { 8432 { 8433 .type = RTE_FLOW_ITEM_TYPE_ETH, 8434 .spec = &lacp_spec, 8435 .mask = &lacp_mask, 8436 }, 8437 { 8438 .type = RTE_FLOW_ITEM_TYPE_END, 8439 }, 8440 }; 8441 struct rte_flow_action actions[] = { 8442 { 8443 .type = (enum rte_flow_action_type) 8444 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8445 }, 8446 { 8447 .type = RTE_FLOW_ACTION_TYPE_END, 8448 }, 8449 }; 8450 struct rte_flow_error error; 8451 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8452 &attr, items, actions, 8453 false, &error); 8454 8455 if (!flow_idx) 8456 return -rte_errno; 8457 return 0; 8458 } 8459 8460 /** 8461 * Destroy a flow. 8462 * 8463 * @see rte_flow_destroy() 8464 * @see rte_flow_ops 8465 */ 8466 int 8467 mlx5_flow_destroy(struct rte_eth_dev *dev, 8468 struct rte_flow *flow, 8469 struct rte_flow_error *error __rte_unused) 8470 { 8471 struct mlx5_priv *priv = dev->data->dev_private; 8472 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8473 struct mlx5_dv_flow_info *flow_info; 8474 8475 if (priv->sh->config.dv_flow_en == 2) 8476 return rte_flow_error_set(error, ENOTSUP, 8477 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8478 NULL, 8479 "Flow non-Q destruction not supported"); 8480 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8481 (uintptr_t)(void *)flow); 8482 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8483 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8484 while (flow_info) { 8485 /* Romove the cache flow info. */ 8486 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8487 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8488 LIST_REMOVE(flow_info, next); 8489 mlx5_free(flow_info->items); 8490 mlx5_free(flow_info->actions); 8491 mlx5_free(flow_info); 8492 break; 8493 } 8494 flow_info = LIST_NEXT(flow_info, next); 8495 } 8496 } 8497 return 0; 8498 } 8499 8500 /** 8501 * Destroy all flows. 8502 * 8503 * @see rte_flow_flush() 8504 * @see rte_flow_ops 8505 */ 8506 int 8507 mlx5_flow_flush(struct rte_eth_dev *dev, 8508 struct rte_flow_error *error __rte_unused) 8509 { 8510 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8511 return 0; 8512 } 8513 8514 /** 8515 * Isolated mode. 8516 * 8517 * @see rte_flow_isolate() 8518 * @see rte_flow_ops 8519 */ 8520 int 8521 mlx5_flow_isolate(struct rte_eth_dev *dev, 8522 int enable, 8523 struct rte_flow_error *error) 8524 { 8525 struct mlx5_priv *priv = dev->data->dev_private; 8526 8527 if (dev->data->dev_started) { 8528 rte_flow_error_set(error, EBUSY, 8529 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8530 NULL, 8531 "port must be stopped first"); 8532 return -rte_errno; 8533 } 8534 if (!enable && !priv->sh->config.repr_matching) 8535 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8536 "isolated mode cannot be disabled when " 8537 "representor matching is disabled"); 8538 priv->isolated = !!enable; 8539 if (enable) 8540 dev->dev_ops = &mlx5_dev_ops_isolate; 8541 else 8542 dev->dev_ops = &mlx5_dev_ops; 8543 8544 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8545 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8546 8547 return 0; 8548 } 8549 8550 /** 8551 * Query a flow. 8552 * 8553 * @see rte_flow_query() 8554 * @see rte_flow_ops 8555 */ 8556 static int 8557 flow_drv_query(struct rte_eth_dev *dev, 8558 struct rte_flow *eflow, 8559 const struct rte_flow_action *actions, 8560 void *data, 8561 struct rte_flow_error *error) 8562 { 8563 struct mlx5_priv *priv = dev->data->dev_private; 8564 const struct mlx5_flow_driver_ops *fops; 8565 struct rte_flow *flow = NULL; 8566 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8567 8568 if (priv->sh->config.dv_flow_en == 2) { 8569 #ifdef HAVE_MLX5_HWS_SUPPORT 8570 flow = eflow; 8571 ftype = MLX5_FLOW_TYPE_HW; 8572 #endif 8573 } else { 8574 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8575 (uintptr_t)(void *)eflow); 8576 } 8577 if (!flow) { 8578 return rte_flow_error_set(error, ENOENT, 8579 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8580 NULL, 8581 "invalid flow handle"); 8582 } 8583 if (ftype == MLX5_FLOW_TYPE_MIN) 8584 ftype = flow->drv_type; 8585 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8586 fops = flow_get_drv_ops(ftype); 8587 8588 return fops->query(dev, flow, actions, data, error); 8589 } 8590 8591 /** 8592 * Query a flow. 8593 * 8594 * @see rte_flow_query() 8595 * @see rte_flow_ops 8596 */ 8597 int 8598 mlx5_flow_query(struct rte_eth_dev *dev, 8599 struct rte_flow *flow, 8600 const struct rte_flow_action *actions, 8601 void *data, 8602 struct rte_flow_error *error) 8603 { 8604 int ret; 8605 8606 ret = flow_drv_query(dev, flow, actions, data, 8607 error); 8608 if (ret < 0) 8609 return ret; 8610 return 0; 8611 } 8612 8613 /** 8614 * Get rte_flow callbacks. 8615 * 8616 * @param dev 8617 * Pointer to Ethernet device structure. 8618 * @param ops 8619 * Pointer to operation-specific structure. 8620 * 8621 * @return 0 8622 */ 8623 int 8624 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8625 const struct rte_flow_ops **ops) 8626 { 8627 *ops = &mlx5_flow_ops; 8628 return 0; 8629 } 8630 8631 /** 8632 * Validate meter policy actions. 8633 * Dispatcher for action type specific validation. 8634 * 8635 * @param[in] dev 8636 * Pointer to the Ethernet device structure. 8637 * @param[in] action 8638 * The meter policy action object to validate. 8639 * @param[in] attr 8640 * Attributes of flow to determine steering domain. 8641 * @param[out] is_rss 8642 * Is RSS or not. 8643 * @param[out] domain_bitmap 8644 * Domain bitmap. 8645 * @param[out] is_def_policy 8646 * Is default policy or not. 8647 * @param[out] error 8648 * Perform verbose error reporting if not NULL. Initialized in case of 8649 * error only. 8650 * 8651 * @return 8652 * 0 on success, otherwise negative errno value. 8653 */ 8654 int 8655 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8656 const struct rte_flow_action *actions[RTE_COLORS], 8657 struct rte_flow_attr *attr, 8658 bool *is_rss, 8659 uint8_t *domain_bitmap, 8660 uint8_t *policy_mode, 8661 struct rte_mtr_error *error) 8662 { 8663 const struct mlx5_flow_driver_ops *fops; 8664 8665 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8666 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8667 domain_bitmap, policy_mode, error); 8668 } 8669 8670 /** 8671 * Destroy the meter table set. 8672 * 8673 * @param[in] dev 8674 * Pointer to Ethernet device. 8675 * @param[in] mtr_policy 8676 * Meter policy struct. 8677 */ 8678 void 8679 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8680 struct mlx5_flow_meter_policy *mtr_policy) 8681 { 8682 const struct mlx5_flow_driver_ops *fops; 8683 8684 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8685 fops->destroy_mtr_acts(dev, mtr_policy); 8686 } 8687 8688 /** 8689 * Create policy action, lock free, 8690 * (mutex should be acquired by caller). 8691 * Dispatcher for action type specific call. 8692 * 8693 * @param[in] dev 8694 * Pointer to the Ethernet device structure. 8695 * @param[in] mtr_policy 8696 * Meter policy struct. 8697 * @param[in] action 8698 * Action specification used to create meter actions. 8699 * @param[in] attr 8700 * Flow rule attributes. 8701 * @param[out] error 8702 * Perform verbose error reporting if not NULL. Initialized in case of 8703 * error only. 8704 * 8705 * @return 8706 * 0 on success, otherwise negative errno value. 8707 */ 8708 int 8709 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8710 struct mlx5_flow_meter_policy *mtr_policy, 8711 const struct rte_flow_action *actions[RTE_COLORS], 8712 struct rte_flow_attr *attr, 8713 struct rte_mtr_error *error) 8714 { 8715 const struct mlx5_flow_driver_ops *fops; 8716 8717 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8718 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8719 } 8720 8721 /** 8722 * Create policy rules, lock free, 8723 * (mutex should be acquired by caller). 8724 * Dispatcher for action type specific call. 8725 * 8726 * @param[in] dev 8727 * Pointer to the Ethernet device structure. 8728 * @param[in] mtr_policy 8729 * Meter policy struct. 8730 * 8731 * @return 8732 * 0 on success, -1 otherwise. 8733 */ 8734 int 8735 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8736 struct mlx5_flow_meter_policy *mtr_policy) 8737 { 8738 const struct mlx5_flow_driver_ops *fops; 8739 8740 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8741 return fops->create_policy_rules(dev, mtr_policy); 8742 } 8743 8744 /** 8745 * Destroy policy rules, lock free, 8746 * (mutex should be acquired by caller). 8747 * Dispatcher for action type specific call. 8748 * 8749 * @param[in] dev 8750 * Pointer to the Ethernet device structure. 8751 * @param[in] mtr_policy 8752 * Meter policy struct. 8753 */ 8754 void 8755 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8756 struct mlx5_flow_meter_policy *mtr_policy) 8757 { 8758 const struct mlx5_flow_driver_ops *fops; 8759 8760 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8761 fops->destroy_policy_rules(dev, mtr_policy); 8762 } 8763 8764 /** 8765 * Destroy the default policy table set. 8766 * 8767 * @param[in] dev 8768 * Pointer to Ethernet device. 8769 */ 8770 void 8771 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8772 { 8773 const struct mlx5_flow_driver_ops *fops; 8774 8775 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8776 fops->destroy_def_policy(dev); 8777 } 8778 8779 /** 8780 * Destroy the default policy table set. 8781 * 8782 * @param[in] dev 8783 * Pointer to Ethernet device. 8784 * 8785 * @return 8786 * 0 on success, -1 otherwise. 8787 */ 8788 int 8789 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8790 { 8791 const struct mlx5_flow_driver_ops *fops; 8792 8793 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8794 return fops->create_def_policy(dev); 8795 } 8796 8797 /** 8798 * Create the needed meter and suffix tables. 8799 * 8800 * @param[in] dev 8801 * Pointer to Ethernet device. 8802 * 8803 * @return 8804 * 0 on success, -1 otherwise. 8805 */ 8806 int 8807 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8808 struct mlx5_flow_meter_info *fm, 8809 uint32_t mtr_idx, 8810 uint8_t domain_bitmap) 8811 { 8812 const struct mlx5_flow_driver_ops *fops; 8813 8814 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8815 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8816 } 8817 8818 /** 8819 * Destroy the meter table set. 8820 * 8821 * @param[in] dev 8822 * Pointer to Ethernet device. 8823 * @param[in] tbl 8824 * Pointer to the meter table set. 8825 */ 8826 void 8827 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8828 struct mlx5_flow_meter_info *fm) 8829 { 8830 const struct mlx5_flow_driver_ops *fops; 8831 8832 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8833 fops->destroy_mtr_tbls(dev, fm); 8834 } 8835 8836 /** 8837 * Destroy the global meter drop table. 8838 * 8839 * @param[in] dev 8840 * Pointer to Ethernet device. 8841 */ 8842 void 8843 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8844 { 8845 const struct mlx5_flow_driver_ops *fops; 8846 8847 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8848 fops->destroy_mtr_drop_tbls(dev); 8849 } 8850 8851 /** 8852 * Destroy the sub policy table with RX queue. 8853 * 8854 * @param[in] dev 8855 * Pointer to Ethernet device. 8856 * @param[in] mtr_policy 8857 * Pointer to meter policy table. 8858 */ 8859 void 8860 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8861 struct mlx5_flow_meter_policy *mtr_policy) 8862 { 8863 const struct mlx5_flow_driver_ops *fops; 8864 8865 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8866 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8867 } 8868 8869 /** 8870 * Allocate the needed aso flow meter id. 8871 * 8872 * @param[in] dev 8873 * Pointer to Ethernet device. 8874 * 8875 * @return 8876 * Index to aso flow meter on success, NULL otherwise. 8877 */ 8878 uint32_t 8879 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8880 { 8881 const struct mlx5_flow_driver_ops *fops; 8882 8883 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8884 return fops->create_meter(dev); 8885 } 8886 8887 /** 8888 * Free the aso flow meter id. 8889 * 8890 * @param[in] dev 8891 * Pointer to Ethernet device. 8892 * @param[in] mtr_idx 8893 * Index to aso flow meter to be free. 8894 * 8895 * @return 8896 * 0 on success. 8897 */ 8898 void 8899 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8900 { 8901 const struct mlx5_flow_driver_ops *fops; 8902 8903 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8904 fops->free_meter(dev, mtr_idx); 8905 } 8906 8907 /** 8908 * Allocate a counter. 8909 * 8910 * @param[in] dev 8911 * Pointer to Ethernet device structure. 8912 * 8913 * @return 8914 * Index to allocated counter on success, 0 otherwise. 8915 */ 8916 uint32_t 8917 mlx5_counter_alloc(struct rte_eth_dev *dev) 8918 { 8919 struct rte_flow_attr attr = { .transfer = 0 }; 8920 8921 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8922 (dev); 8923 } 8924 8925 /** 8926 * Free a counter. 8927 * 8928 * @param[in] dev 8929 * Pointer to Ethernet device structure. 8930 * @param[in] cnt 8931 * Index to counter to be free. 8932 */ 8933 void 8934 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8935 { 8936 struct rte_flow_attr attr = { .transfer = 0 }; 8937 8938 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8939 } 8940 8941 /** 8942 * Query counter statistics. 8943 * 8944 * @param[in] dev 8945 * Pointer to Ethernet device structure. 8946 * @param[in] cnt 8947 * Index to counter to query. 8948 * @param[in] clear 8949 * Set to clear counter statistics. 8950 * @param[out] pkts 8951 * The counter hits packets number to save. 8952 * @param[out] bytes 8953 * The counter hits bytes number to save. 8954 * 8955 * @return 8956 * 0 on success, a negative errno value otherwise. 8957 */ 8958 int 8959 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8960 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8961 { 8962 struct rte_flow_attr attr = { .transfer = 0 }; 8963 8964 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8965 (dev, cnt, clear, pkts, bytes, action); 8966 } 8967 8968 /** 8969 * Get information about HWS pre-configurable resources. 8970 * 8971 * @param[in] dev 8972 * Pointer to the rte_eth_dev structure. 8973 * @param[out] port_info 8974 * Pointer to port information. 8975 * @param[out] queue_info 8976 * Pointer to queue information. 8977 * @param[out] error 8978 * Pointer to error structure. 8979 * 8980 * @return 8981 * 0 on success, a negative errno value otherwise and rte_errno is set. 8982 */ 8983 static int 8984 mlx5_flow_info_get(struct rte_eth_dev *dev, 8985 struct rte_flow_port_info *port_info, 8986 struct rte_flow_queue_info *queue_info, 8987 struct rte_flow_error *error) 8988 { 8989 const struct mlx5_flow_driver_ops *fops; 8990 struct rte_flow_attr attr = {0}; 8991 8992 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8993 return rte_flow_error_set(error, ENOTSUP, 8994 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8995 NULL, 8996 "info get with incorrect steering mode"); 8997 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8998 return fops->info_get(dev, port_info, queue_info, error); 8999 } 9000 9001 /** 9002 * Configure port HWS resources. 9003 * 9004 * @param[in] dev 9005 * Pointer to the rte_eth_dev structure. 9006 * @param[in] port_attr 9007 * Port configuration attributes. 9008 * @param[in] nb_queue 9009 * Number of queue. 9010 * @param[in] queue_attr 9011 * Array that holds attributes for each flow queue. 9012 * @param[out] error 9013 * Pointer to error structure. 9014 * 9015 * @return 9016 * 0 on success, a negative errno value otherwise and rte_errno is set. 9017 */ 9018 static int 9019 mlx5_flow_port_configure(struct rte_eth_dev *dev, 9020 const struct rte_flow_port_attr *port_attr, 9021 uint16_t nb_queue, 9022 const struct rte_flow_queue_attr *queue_attr[], 9023 struct rte_flow_error *error) 9024 { 9025 const struct mlx5_flow_driver_ops *fops; 9026 struct rte_flow_attr attr = {0}; 9027 9028 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9029 return rte_flow_error_set(error, ENOTSUP, 9030 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9031 NULL, 9032 "port configure with incorrect steering mode"); 9033 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9034 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 9035 } 9036 9037 /** 9038 * Validate item template. 9039 * 9040 * @param[in] dev 9041 * Pointer to the rte_eth_dev structure. 9042 * @param[in] attr 9043 * Pointer to the item template attributes. 9044 * @param[in] items 9045 * The template item pattern. 9046 * @param[out] error 9047 * Pointer to error structure. 9048 * 9049 * @return 9050 * 0 on success, a negative errno value otherwise and rte_errno is set. 9051 */ 9052 int 9053 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 9054 const struct rte_flow_pattern_template_attr *attr, 9055 const struct rte_flow_item items[], 9056 struct rte_flow_error *error) 9057 { 9058 const struct mlx5_flow_driver_ops *fops; 9059 struct rte_flow_attr fattr = {0}; 9060 9061 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9062 rte_flow_error_set(error, ENOTSUP, 9063 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9064 "pattern validate with incorrect steering mode"); 9065 return -ENOTSUP; 9066 } 9067 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9068 return fops->pattern_validate(dev, attr, items, error); 9069 } 9070 9071 /** 9072 * Create flow item template. 9073 * 9074 * @param[in] dev 9075 * Pointer to the rte_eth_dev structure. 9076 * @param[in] attr 9077 * Pointer to the item template attributes. 9078 * @param[in] items 9079 * The template item pattern. 9080 * @param[out] error 9081 * Pointer to error structure. 9082 * 9083 * @return 9084 * 0 on success, a negative errno value otherwise and rte_errno is set. 9085 */ 9086 static struct rte_flow_pattern_template * 9087 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9088 const struct rte_flow_pattern_template_attr *attr, 9089 const struct rte_flow_item items[], 9090 struct rte_flow_error *error) 9091 { 9092 const struct mlx5_flow_driver_ops *fops; 9093 struct rte_flow_attr fattr = {0}; 9094 9095 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9096 rte_flow_error_set(error, ENOTSUP, 9097 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9098 NULL, 9099 "pattern create with incorrect steering mode"); 9100 return NULL; 9101 } 9102 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9103 return fops->pattern_template_create(dev, attr, items, error); 9104 } 9105 9106 /** 9107 * Destroy flow item template. 9108 * 9109 * @param[in] dev 9110 * Pointer to the rte_eth_dev structure. 9111 * @param[in] template 9112 * Pointer to the item template to be destroyed. 9113 * @param[out] error 9114 * Pointer to error structure. 9115 * 9116 * @return 9117 * 0 on success, a negative errno value otherwise and rte_errno is set. 9118 */ 9119 static int 9120 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9121 struct rte_flow_pattern_template *template, 9122 struct rte_flow_error *error) 9123 { 9124 const struct mlx5_flow_driver_ops *fops; 9125 struct rte_flow_attr attr = {0}; 9126 9127 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9128 return rte_flow_error_set(error, ENOTSUP, 9129 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9130 NULL, 9131 "pattern destroy with incorrect steering mode"); 9132 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9133 return fops->pattern_template_destroy(dev, template, error); 9134 } 9135 9136 /** 9137 * Validate flow actions template. 9138 * 9139 * @param[in] dev 9140 * Pointer to the rte_eth_dev structure. 9141 * @param[in] attr 9142 * Pointer to the action template attributes. 9143 * @param[in] actions 9144 * Associated actions (list terminated by the END action). 9145 * @param[in] masks 9146 * List of actions that marks which of the action's member is constant. 9147 * @param[out] error 9148 * Pointer to error structure. 9149 * 9150 * @return 9151 * 0 on success, a negative errno value otherwise and rte_errno is set. 9152 */ 9153 int 9154 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9155 const struct rte_flow_actions_template_attr *attr, 9156 const struct rte_flow_action actions[], 9157 const struct rte_flow_action masks[], 9158 struct rte_flow_error *error) 9159 { 9160 const struct mlx5_flow_driver_ops *fops; 9161 struct rte_flow_attr fattr = {0}; 9162 9163 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9164 rte_flow_error_set(error, ENOTSUP, 9165 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9166 "actions validate with incorrect steering mode"); 9167 return -ENOTSUP; 9168 } 9169 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9170 return fops->actions_validate(dev, attr, actions, masks, error); 9171 } 9172 9173 /** 9174 * Create flow item template. 9175 * 9176 * @param[in] dev 9177 * Pointer to the rte_eth_dev structure. 9178 * @param[in] attr 9179 * Pointer to the action template attributes. 9180 * @param[in] actions 9181 * Associated actions (list terminated by the END action). 9182 * @param[in] masks 9183 * List of actions that marks which of the action's member is constant. 9184 * @param[out] error 9185 * Pointer to error structure. 9186 * 9187 * @return 9188 * 0 on success, a negative errno value otherwise and rte_errno is set. 9189 */ 9190 static struct rte_flow_actions_template * 9191 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9192 const struct rte_flow_actions_template_attr *attr, 9193 const struct rte_flow_action actions[], 9194 const struct rte_flow_action masks[], 9195 struct rte_flow_error *error) 9196 { 9197 const struct mlx5_flow_driver_ops *fops; 9198 struct rte_flow_attr fattr = {0}; 9199 9200 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9201 rte_flow_error_set(error, ENOTSUP, 9202 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9203 NULL, 9204 "action create with incorrect steering mode"); 9205 return NULL; 9206 } 9207 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9208 return fops->actions_template_create(dev, attr, actions, masks, error); 9209 } 9210 9211 /** 9212 * Destroy flow action template. 9213 * 9214 * @param[in] dev 9215 * Pointer to the rte_eth_dev structure. 9216 * @param[in] template 9217 * Pointer to the action template to be destroyed. 9218 * @param[out] error 9219 * Pointer to error structure. 9220 * 9221 * @return 9222 * 0 on success, a negative errno value otherwise and rte_errno is set. 9223 */ 9224 static int 9225 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9226 struct rte_flow_actions_template *template, 9227 struct rte_flow_error *error) 9228 { 9229 const struct mlx5_flow_driver_ops *fops; 9230 struct rte_flow_attr attr = {0}; 9231 9232 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9233 return rte_flow_error_set(error, ENOTSUP, 9234 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9235 NULL, 9236 "action destroy with incorrect steering mode"); 9237 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9238 return fops->actions_template_destroy(dev, template, error); 9239 } 9240 9241 /** 9242 * Create flow table. 9243 * 9244 * @param[in] dev 9245 * Pointer to the rte_eth_dev structure. 9246 * @param[in] attr 9247 * Pointer to the table attributes. 9248 * @param[in] item_templates 9249 * Item template array to be binded to the table. 9250 * @param[in] nb_item_templates 9251 * Number of item template. 9252 * @param[in] action_templates 9253 * Action template array to be binded to the table. 9254 * @param[in] nb_action_templates 9255 * Number of action template. 9256 * @param[out] error 9257 * Pointer to error structure. 9258 * 9259 * @return 9260 * Table on success, NULL otherwise and rte_errno is set. 9261 */ 9262 static struct rte_flow_template_table * 9263 mlx5_flow_table_create(struct rte_eth_dev *dev, 9264 const struct rte_flow_template_table_attr *attr, 9265 struct rte_flow_pattern_template *item_templates[], 9266 uint8_t nb_item_templates, 9267 struct rte_flow_actions_template *action_templates[], 9268 uint8_t nb_action_templates, 9269 struct rte_flow_error *error) 9270 { 9271 const struct mlx5_flow_driver_ops *fops; 9272 struct rte_flow_attr fattr = {0}; 9273 9274 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9275 rte_flow_error_set(error, ENOTSUP, 9276 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9277 NULL, 9278 "table create with incorrect steering mode"); 9279 return NULL; 9280 } 9281 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9282 return fops->template_table_create(dev, 9283 attr, 9284 item_templates, 9285 nb_item_templates, 9286 action_templates, 9287 nb_action_templates, 9288 error); 9289 } 9290 9291 /** 9292 * PMD destroy flow table. 9293 * 9294 * @param[in] dev 9295 * Pointer to the rte_eth_dev structure. 9296 * @param[in] table 9297 * Pointer to the table to be destroyed. 9298 * @param[out] error 9299 * Pointer to error structure. 9300 * 9301 * @return 9302 * 0 on success, a negative errno value otherwise and rte_errno is set. 9303 */ 9304 static int 9305 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9306 struct rte_flow_template_table *table, 9307 struct rte_flow_error *error) 9308 { 9309 const struct mlx5_flow_driver_ops *fops; 9310 struct rte_flow_attr attr = {0}; 9311 9312 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9313 return rte_flow_error_set(error, ENOTSUP, 9314 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9315 NULL, 9316 "table destroy with incorrect steering mode"); 9317 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9318 return fops->template_table_destroy(dev, table, error); 9319 } 9320 9321 /** 9322 * PMD group set miss actions. 9323 * 9324 * @param[in] dev 9325 * Pointer to the rte_eth_dev structure. 9326 * @param[in] attr 9327 * Pointer to group attributes 9328 * @param[in] actions 9329 * Array of actions 9330 * @param[out] error 9331 * Pointer to error structure. 9332 * 9333 * @return 9334 * 0 on success, a negative errno value otherwise and rte_errno is set. 9335 */ 9336 static int 9337 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 9338 uint32_t group_id, 9339 const struct rte_flow_group_attr *attr, 9340 const struct rte_flow_action actions[], 9341 struct rte_flow_error *error) 9342 { 9343 const struct mlx5_flow_driver_ops *fops; 9344 struct rte_flow_attr fattr = {0}; 9345 9346 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9347 return rte_flow_error_set(error, ENOTSUP, 9348 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9349 NULL, 9350 "group set miss actions with incorrect steering mode"); 9351 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9352 return fops->group_set_miss_actions(dev, group_id, attr, actions, error); 9353 } 9354 9355 /** 9356 * Allocate a new memory for the counter values wrapped by all the needed 9357 * management. 9358 * 9359 * @param[in] sh 9360 * Pointer to mlx5_dev_ctx_shared object. 9361 * 9362 * @return 9363 * 0 on success, a negative errno value otherwise. 9364 */ 9365 static int 9366 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9367 { 9368 struct mlx5_counter_stats_mem_mng *mem_mng; 9369 volatile struct flow_counter_stats *raw_data; 9370 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9371 int size = (sizeof(struct flow_counter_stats) * 9372 MLX5_COUNTERS_PER_POOL + 9373 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9374 sizeof(struct mlx5_counter_stats_mem_mng); 9375 size_t pgsize = rte_mem_page_size(); 9376 uint8_t *mem; 9377 int ret; 9378 int i; 9379 9380 if (pgsize == (size_t)-1) { 9381 DRV_LOG(ERR, "Failed to get mem page size"); 9382 rte_errno = ENOMEM; 9383 return -ENOMEM; 9384 } 9385 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9386 if (!mem) { 9387 rte_errno = ENOMEM; 9388 return -ENOMEM; 9389 } 9390 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9391 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9392 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9393 sh->cdev->pdn, mem, size, 9394 &mem_mng->wm); 9395 if (ret) { 9396 rte_errno = errno; 9397 mlx5_free(mem); 9398 return -rte_errno; 9399 } 9400 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9401 raw_data = (volatile struct flow_counter_stats *)mem; 9402 for (i = 0; i < raws_n; ++i) { 9403 mem_mng->raws[i].mem_mng = mem_mng; 9404 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9405 } 9406 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9407 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9408 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9409 next); 9410 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9411 sh->sws_cmng.mem_mng = mem_mng; 9412 return 0; 9413 } 9414 9415 /** 9416 * Set the statistic memory to the new counter pool. 9417 * 9418 * @param[in] sh 9419 * Pointer to mlx5_dev_ctx_shared object. 9420 * @param[in] pool 9421 * Pointer to the pool to set the statistic memory. 9422 * 9423 * @return 9424 * 0 on success, a negative errno value otherwise. 9425 */ 9426 static int 9427 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9428 struct mlx5_flow_counter_pool *pool) 9429 { 9430 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9431 /* Resize statistic memory once used out. */ 9432 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9433 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9434 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9435 return -1; 9436 } 9437 rte_spinlock_lock(&pool->sl); 9438 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9439 rte_spinlock_unlock(&pool->sl); 9440 pool->raw_hw = NULL; 9441 return 0; 9442 } 9443 9444 #define MLX5_POOL_QUERY_FREQ_US 1000000 9445 9446 /** 9447 * Set the periodic procedure for triggering asynchronous batch queries for all 9448 * the counter pools. 9449 * 9450 * @param[in] sh 9451 * Pointer to mlx5_dev_ctx_shared object. 9452 */ 9453 void 9454 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9455 { 9456 uint32_t pools_n, us; 9457 9458 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9459 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9460 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9461 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9462 sh->sws_cmng.query_thread_on = 0; 9463 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9464 } else { 9465 sh->sws_cmng.query_thread_on = 1; 9466 } 9467 } 9468 9469 /** 9470 * The periodic procedure for triggering asynchronous batch queries for all the 9471 * counter pools. This function is probably called by the host thread. 9472 * 9473 * @param[in] arg 9474 * The parameter for the alarm process. 9475 */ 9476 void 9477 mlx5_flow_query_alarm(void *arg) 9478 { 9479 struct mlx5_dev_ctx_shared *sh = arg; 9480 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9481 uint16_t pool_index = cmng->pool_index; 9482 struct mlx5_flow_counter_pool *pool; 9483 uint16_t n_valid; 9484 int ret; 9485 9486 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9487 goto set_alarm; 9488 rte_spinlock_lock(&cmng->pool_update_sl); 9489 pool = cmng->pools[pool_index]; 9490 n_valid = cmng->n_valid; 9491 rte_spinlock_unlock(&cmng->pool_update_sl); 9492 /* Set the statistic memory to the new created pool. */ 9493 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9494 goto set_alarm; 9495 if (pool->raw_hw) 9496 /* There is a pool query in progress. */ 9497 goto set_alarm; 9498 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9499 if (!pool->raw_hw) 9500 /* No free counter statistics raw memory. */ 9501 goto set_alarm; 9502 /* 9503 * Identify the counters released between query trigger and query 9504 * handle more efficiently. The counter released in this gap period 9505 * should wait for a new round of query as the new arrived packets 9506 * will not be taken into account. 9507 */ 9508 pool->query_gen++; 9509 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9510 MLX5_COUNTERS_PER_POOL, 9511 NULL, NULL, 9512 pool->raw_hw->mem_mng->wm.lkey, 9513 (void *)(uintptr_t) 9514 pool->raw_hw->data, 9515 sh->devx_comp, 9516 (uint64_t)(uintptr_t)pool); 9517 if (ret) { 9518 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9519 " %d", pool->min_dcs->id); 9520 pool->raw_hw = NULL; 9521 goto set_alarm; 9522 } 9523 LIST_REMOVE(pool->raw_hw, next); 9524 cmng->pending_queries++; 9525 pool_index++; 9526 if (pool_index >= n_valid) 9527 pool_index = 0; 9528 set_alarm: 9529 cmng->pool_index = pool_index; 9530 mlx5_set_query_alarm(sh); 9531 } 9532 9533 /** 9534 * Check and callback event for new aged flow in the counter pool 9535 * 9536 * @param[in] sh 9537 * Pointer to mlx5_dev_ctx_shared object. 9538 * @param[in] pool 9539 * Pointer to Current counter pool. 9540 */ 9541 static void 9542 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9543 struct mlx5_flow_counter_pool *pool) 9544 { 9545 struct mlx5_priv *priv; 9546 struct mlx5_flow_counter *cnt; 9547 struct mlx5_age_info *age_info; 9548 struct mlx5_age_param *age_param; 9549 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9550 struct mlx5_counter_stats_raw *prev = pool->raw; 9551 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9552 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9553 uint16_t expected = AGE_CANDIDATE; 9554 uint32_t i; 9555 9556 pool->time_of_last_age_check = curr_time; 9557 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9558 cnt = MLX5_POOL_GET_CNT(pool, i); 9559 age_param = MLX5_CNT_TO_AGE(cnt); 9560 if (__atomic_load_n(&age_param->state, 9561 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9562 continue; 9563 if (cur->data[i].hits != prev->data[i].hits) { 9564 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9565 __ATOMIC_RELAXED); 9566 continue; 9567 } 9568 if (__atomic_fetch_add(&age_param->sec_since_last_hit, 9569 time_delta, 9570 __ATOMIC_RELAXED) + time_delta <= age_param->timeout) 9571 continue; 9572 /** 9573 * Hold the lock first, or if between the 9574 * state AGE_TMOUT and tailq operation the 9575 * release happened, the release procedure 9576 * may delete a non-existent tailq node. 9577 */ 9578 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9579 age_info = GET_PORT_AGE_INFO(priv); 9580 rte_spinlock_lock(&age_info->aged_sl); 9581 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9582 AGE_TMOUT, false, 9583 __ATOMIC_RELAXED, 9584 __ATOMIC_RELAXED)) { 9585 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9586 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9587 } 9588 rte_spinlock_unlock(&age_info->aged_sl); 9589 } 9590 mlx5_age_event_prepare(sh); 9591 } 9592 9593 /** 9594 * Handler for the HW respond about ready values from an asynchronous batch 9595 * query. This function is probably called by the host thread. 9596 * 9597 * @param[in] sh 9598 * The pointer to the shared device context. 9599 * @param[in] async_id 9600 * The Devx async ID. 9601 * @param[in] status 9602 * The status of the completion. 9603 */ 9604 void 9605 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9606 uint64_t async_id, int status) 9607 { 9608 struct mlx5_flow_counter_pool *pool = 9609 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9610 struct mlx5_counter_stats_raw *raw_to_free; 9611 uint8_t query_gen = pool->query_gen ^ 1; 9612 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9613 enum mlx5_counter_type cnt_type = 9614 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9615 MLX5_COUNTER_TYPE_ORIGIN; 9616 9617 if (unlikely(status)) { 9618 raw_to_free = pool->raw_hw; 9619 } else { 9620 raw_to_free = pool->raw; 9621 if (pool->is_aged) 9622 mlx5_flow_aging_check(sh, pool); 9623 rte_spinlock_lock(&pool->sl); 9624 pool->raw = pool->raw_hw; 9625 rte_spinlock_unlock(&pool->sl); 9626 /* Be sure the new raw counters data is updated in memory. */ 9627 rte_io_wmb(); 9628 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9629 rte_spinlock_lock(&cmng->csl[cnt_type]); 9630 TAILQ_CONCAT(&cmng->counters[cnt_type], 9631 &pool->counters[query_gen], next); 9632 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9633 } 9634 } 9635 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9636 pool->raw_hw = NULL; 9637 sh->sws_cmng.pending_queries--; 9638 } 9639 9640 static int 9641 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9642 const struct flow_grp_info *grp_info, 9643 struct rte_flow_error *error) 9644 { 9645 if (grp_info->transfer && grp_info->external && 9646 grp_info->fdb_def_rule) { 9647 if (group == UINT32_MAX) 9648 return rte_flow_error_set 9649 (error, EINVAL, 9650 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9651 NULL, 9652 "group index not supported"); 9653 *table = group + 1; 9654 } else { 9655 *table = group; 9656 } 9657 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9658 return 0; 9659 } 9660 9661 /** 9662 * Translate the rte_flow group index to HW table value. 9663 * 9664 * If tunnel offload is disabled, all group ids converted to flow table 9665 * id using the standard method. 9666 * If tunnel offload is enabled, group id can be converted using the 9667 * standard or tunnel conversion method. Group conversion method 9668 * selection depends on flags in `grp_info` parameter: 9669 * - Internal (grp_info.external == 0) groups conversion uses the 9670 * standard method. 9671 * - Group ids in JUMP action converted with the tunnel conversion. 9672 * - Group id in rule attribute conversion depends on a rule type and 9673 * group id value: 9674 * ** non zero group attributes converted with the tunnel method 9675 * ** zero group attribute in non-tunnel rule is converted using the 9676 * standard method - there's only one root table 9677 * ** zero group attribute in steer tunnel rule is converted with the 9678 * standard method - single root table 9679 * ** zero group attribute in match tunnel rule is a special OvS 9680 * case: that value is used for portability reasons. That group 9681 * id is converted with the tunnel conversion method. 9682 * 9683 * @param[in] dev 9684 * Port device 9685 * @param[in] tunnel 9686 * PMD tunnel offload object 9687 * @param[in] group 9688 * rte_flow group index value. 9689 * @param[out] table 9690 * HW table value. 9691 * @param[in] grp_info 9692 * flags used for conversion 9693 * @param[out] error 9694 * Pointer to error structure. 9695 * 9696 * @return 9697 * 0 on success, a negative errno value otherwise and rte_errno is set. 9698 */ 9699 int 9700 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9701 const struct mlx5_flow_tunnel *tunnel, 9702 uint32_t group, uint32_t *table, 9703 const struct flow_grp_info *grp_info, 9704 struct rte_flow_error *error) 9705 { 9706 int ret; 9707 bool standard_translation; 9708 9709 if (!grp_info->skip_scale && grp_info->external && 9710 group < MLX5_MAX_TABLES_EXTERNAL) 9711 group *= MLX5_FLOW_TABLE_FACTOR; 9712 if (is_tunnel_offload_active(dev)) { 9713 standard_translation = !grp_info->external || 9714 grp_info->std_tbl_fix; 9715 } else { 9716 standard_translation = true; 9717 } 9718 DRV_LOG(DEBUG, 9719 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9720 dev->data->port_id, group, grp_info->transfer, 9721 grp_info->external, grp_info->fdb_def_rule, 9722 standard_translation ? "STANDARD" : "TUNNEL"); 9723 if (standard_translation) 9724 ret = flow_group_to_table(dev->data->port_id, group, table, 9725 grp_info, error); 9726 else 9727 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9728 table, error); 9729 9730 return ret; 9731 } 9732 9733 /** 9734 * Discover availability of metadata reg_c's. 9735 * 9736 * Iteratively use test flows to check availability. 9737 * 9738 * @param[in] dev 9739 * Pointer to the Ethernet device structure. 9740 * 9741 * @return 9742 * 0 on success, a negative errno value otherwise and rte_errno is set. 9743 */ 9744 int 9745 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9746 { 9747 struct mlx5_priv *priv = dev->data->dev_private; 9748 enum modify_reg idx; 9749 int n = 0; 9750 9751 /* reg_c[0] and reg_c[1] are reserved. */ 9752 priv->sh->flow_mreg_c[n++] = REG_C_0; 9753 priv->sh->flow_mreg_c[n++] = REG_C_1; 9754 /* Discover availability of other reg_c's. */ 9755 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9756 struct rte_flow_attr attr = { 9757 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9758 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9759 .ingress = 1, 9760 }; 9761 struct rte_flow_item items[] = { 9762 [0] = { 9763 .type = RTE_FLOW_ITEM_TYPE_END, 9764 }, 9765 }; 9766 struct rte_flow_action actions[] = { 9767 [0] = { 9768 .type = (enum rte_flow_action_type) 9769 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9770 .conf = &(struct mlx5_flow_action_copy_mreg){ 9771 .src = REG_C_1, 9772 .dst = idx, 9773 }, 9774 }, 9775 [1] = { 9776 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9777 .conf = &(struct rte_flow_action_jump){ 9778 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9779 }, 9780 }, 9781 [2] = { 9782 .type = RTE_FLOW_ACTION_TYPE_END, 9783 }, 9784 }; 9785 uint32_t flow_idx; 9786 struct rte_flow *flow; 9787 struct rte_flow_error error; 9788 9789 if (!priv->sh->config.dv_flow_en) 9790 break; 9791 /* Create internal flow, validation skips copy action. */ 9792 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9793 items, actions, false, &error); 9794 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9795 flow_idx); 9796 if (!flow) 9797 continue; 9798 priv->sh->flow_mreg_c[n++] = idx; 9799 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9800 } 9801 for (; n < MLX5_MREG_C_NUM; ++n) 9802 priv->sh->flow_mreg_c[n] = REG_NON; 9803 priv->sh->metadata_regc_check_flag = 1; 9804 return 0; 9805 } 9806 9807 int 9808 save_dump_file(const uint8_t *data, uint32_t size, 9809 uint32_t type, uint64_t id, void *arg, FILE *file) 9810 { 9811 char line[BUF_SIZE]; 9812 uint32_t out = 0; 9813 uint32_t k; 9814 uint32_t actions_num; 9815 struct rte_flow_query_count *count; 9816 9817 memset(line, 0, BUF_SIZE); 9818 switch (type) { 9819 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9820 actions_num = *(uint32_t *)(arg); 9821 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9822 type, id, actions_num); 9823 break; 9824 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9825 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9826 type, id); 9827 break; 9828 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9829 count = (struct rte_flow_query_count *)arg; 9830 fprintf(file, 9831 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9832 type, id, count->hits, count->bytes); 9833 return 0; 9834 default: 9835 return -1; 9836 } 9837 9838 for (k = 0; k < size; k++) { 9839 /* Make sure we do not overrun the line buffer length. */ 9840 if (out >= BUF_SIZE - 4) { 9841 line[out] = '\0'; 9842 break; 9843 } 9844 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9845 (data[k]) & 0xff); 9846 } 9847 fprintf(file, "%s\n", line); 9848 return 0; 9849 } 9850 9851 int 9852 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9853 struct rte_flow_query_count *count, struct rte_flow_error *error) 9854 { 9855 struct rte_flow_action action[2]; 9856 enum mlx5_flow_drv_type ftype; 9857 const struct mlx5_flow_driver_ops *fops; 9858 9859 if (!flow) { 9860 return rte_flow_error_set(error, ENOENT, 9861 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9862 NULL, 9863 "invalid flow handle"); 9864 } 9865 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9866 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9867 if (flow->counter) { 9868 memset(count, 0, sizeof(struct rte_flow_query_count)); 9869 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9870 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9871 ftype < MLX5_FLOW_TYPE_MAX); 9872 fops = flow_get_drv_ops(ftype); 9873 return fops->query(dev, flow, action, count, error); 9874 } 9875 return -1; 9876 } 9877 9878 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9879 /** 9880 * Dump flow ipool data to file 9881 * 9882 * @param[in] dev 9883 * The pointer to Ethernet device. 9884 * @param[in] file 9885 * A pointer to a file for output. 9886 * @param[out] error 9887 * Perform verbose error reporting if not NULL. PMDs initialize this 9888 * structure in case of error only. 9889 * @return 9890 * 0 on success, a negative value otherwise. 9891 */ 9892 int 9893 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9894 struct rte_flow *flow, FILE *file, 9895 struct rte_flow_error *error) 9896 { 9897 struct mlx5_priv *priv = dev->data->dev_private; 9898 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9899 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9900 uint32_t handle_idx; 9901 struct mlx5_flow_handle *dh; 9902 struct rte_flow_query_count count; 9903 uint32_t actions_num; 9904 const uint8_t *data; 9905 size_t size; 9906 uint64_t id; 9907 uint32_t type; 9908 void *action = NULL; 9909 9910 if (!flow) { 9911 return rte_flow_error_set(error, ENOENT, 9912 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9913 NULL, 9914 "invalid flow handle"); 9915 } 9916 handle_idx = flow->dev_handles; 9917 /* query counter */ 9918 if (flow->counter && 9919 (!mlx5_counter_query(dev, flow->counter, false, 9920 &count.hits, &count.bytes, &action)) && action) { 9921 id = (uint64_t)(uintptr_t)action; 9922 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9923 save_dump_file(NULL, 0, type, 9924 id, (void *)&count, file); 9925 } 9926 9927 while (handle_idx) { 9928 dh = mlx5_ipool_get(priv->sh->ipool 9929 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9930 if (!dh) 9931 continue; 9932 handle_idx = dh->next.next; 9933 9934 /* Get modify_hdr and encap_decap buf from ipools. */ 9935 encap_decap = NULL; 9936 modify_hdr = dh->dvh.modify_hdr; 9937 9938 if (dh->dvh.rix_encap_decap) { 9939 encap_decap = mlx5_ipool_get(priv->sh->ipool 9940 [MLX5_IPOOL_DECAP_ENCAP], 9941 dh->dvh.rix_encap_decap); 9942 } 9943 if (modify_hdr) { 9944 data = (const uint8_t *)modify_hdr->actions; 9945 size = (size_t)(modify_hdr->actions_num) * 8; 9946 id = (uint64_t)(uintptr_t)modify_hdr->action; 9947 actions_num = modify_hdr->actions_num; 9948 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9949 save_dump_file(data, size, type, id, 9950 (void *)(&actions_num), file); 9951 } 9952 if (encap_decap) { 9953 data = encap_decap->buf; 9954 size = encap_decap->size; 9955 id = (uint64_t)(uintptr_t)encap_decap->action; 9956 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9957 save_dump_file(data, size, type, 9958 id, NULL, file); 9959 } 9960 } 9961 return 0; 9962 } 9963 9964 /** 9965 * Dump all flow's encap_decap/modify_hdr/counter data to file 9966 * 9967 * @param[in] dev 9968 * The pointer to Ethernet device. 9969 * @param[in] file 9970 * A pointer to a file for output. 9971 * @param[out] error 9972 * Perform verbose error reporting if not NULL. PMDs initialize this 9973 * structure in case of error only. 9974 * @return 9975 * 0 on success, a negative value otherwise. 9976 */ 9977 static int 9978 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9979 FILE *file, struct rte_flow_error *error __rte_unused) 9980 { 9981 struct mlx5_priv *priv = dev->data->dev_private; 9982 struct mlx5_dev_ctx_shared *sh = priv->sh; 9983 struct mlx5_hlist *h; 9984 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9985 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9986 struct rte_flow_query_count count; 9987 uint32_t actions_num; 9988 const uint8_t *data; 9989 size_t size; 9990 uint64_t id; 9991 uint32_t type; 9992 uint32_t i; 9993 uint32_t j; 9994 struct mlx5_list_inconst *l_inconst; 9995 struct mlx5_list_entry *e; 9996 int lcore_index; 9997 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 9998 uint32_t max; 9999 void *action; 10000 10001 /* encap_decap hlist is lcore_share, get global core cache. */ 10002 i = MLX5_LIST_GLOBAL; 10003 h = sh->encaps_decaps; 10004 if (h) { 10005 for (j = 0; j <= h->mask; j++) { 10006 l_inconst = &h->buckets[j].l; 10007 if (!l_inconst || !l_inconst->cache[i]) 10008 continue; 10009 10010 e = LIST_FIRST(&l_inconst->cache[i]->h); 10011 while (e) { 10012 encap_decap = 10013 (struct mlx5_flow_dv_encap_decap_resource *)e; 10014 data = encap_decap->buf; 10015 size = encap_decap->size; 10016 id = (uint64_t)(uintptr_t)encap_decap->action; 10017 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10018 save_dump_file(data, size, type, 10019 id, NULL, file); 10020 e = LIST_NEXT(e, next); 10021 } 10022 } 10023 } 10024 10025 /* get modify_hdr */ 10026 h = sh->modify_cmds; 10027 if (h) { 10028 lcore_index = rte_lcore_index(rte_lcore_id()); 10029 if (unlikely(lcore_index == -1)) { 10030 lcore_index = MLX5_LIST_NLCORE; 10031 rte_spinlock_lock(&h->l_const.lcore_lock); 10032 } 10033 i = lcore_index; 10034 10035 if (lcore_index == MLX5_LIST_NLCORE) { 10036 for (i = 0; i <= (uint32_t)lcore_index; i++) { 10037 for (j = 0; j <= h->mask; j++) { 10038 l_inconst = &h->buckets[j].l; 10039 if (!l_inconst || !l_inconst->cache[i]) 10040 continue; 10041 10042 e = LIST_FIRST(&l_inconst->cache[i]->h); 10043 while (e) { 10044 modify_hdr = 10045 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10046 data = (const uint8_t *)modify_hdr->actions; 10047 size = (size_t)(modify_hdr->actions_num) * 8; 10048 actions_num = modify_hdr->actions_num; 10049 id = (uint64_t)(uintptr_t)modify_hdr->action; 10050 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10051 save_dump_file(data, size, type, id, 10052 (void *)(&actions_num), file); 10053 e = LIST_NEXT(e, next); 10054 } 10055 } 10056 } 10057 } else { 10058 for (j = 0; j <= h->mask; j++) { 10059 l_inconst = &h->buckets[j].l; 10060 if (!l_inconst || !l_inconst->cache[i]) 10061 continue; 10062 10063 e = LIST_FIRST(&l_inconst->cache[i]->h); 10064 while (e) { 10065 modify_hdr = 10066 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10067 data = (const uint8_t *)modify_hdr->actions; 10068 size = (size_t)(modify_hdr->actions_num) * 8; 10069 actions_num = modify_hdr->actions_num; 10070 id = (uint64_t)(uintptr_t)modify_hdr->action; 10071 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10072 save_dump_file(data, size, type, id, 10073 (void *)(&actions_num), file); 10074 e = LIST_NEXT(e, next); 10075 } 10076 } 10077 } 10078 10079 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10080 rte_spinlock_unlock(&h->l_const.lcore_lock); 10081 } 10082 10083 /* get counter */ 10084 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10085 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10086 for (j = 1; j <= max; j++) { 10087 action = NULL; 10088 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10089 &count.bytes, &action)) && action) { 10090 id = (uint64_t)(uintptr_t)action; 10091 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10092 save_dump_file(NULL, 0, type, 10093 id, (void *)&count, file); 10094 } 10095 } 10096 return 0; 10097 } 10098 #endif 10099 10100 /** 10101 * Dump flow raw hw data to file 10102 * 10103 * @param[in] dev 10104 * The pointer to Ethernet device. 10105 * @param[in] file 10106 * A pointer to a file for output. 10107 * @param[out] error 10108 * Perform verbose error reporting if not NULL. PMDs initialize this 10109 * structure in case of error only. 10110 * @return 10111 * 0 on success, a negative value otherwise. 10112 */ 10113 int 10114 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10115 FILE *file, 10116 struct rte_flow_error *error __rte_unused) 10117 { 10118 struct mlx5_priv *priv = dev->data->dev_private; 10119 struct mlx5_dev_ctx_shared *sh = priv->sh; 10120 uint32_t handle_idx; 10121 int ret; 10122 struct mlx5_flow_handle *dh; 10123 struct rte_flow *flow; 10124 10125 if (!sh->config.dv_flow_en) { 10126 if (fputs("device dv flow disabled\n", file) <= 0) 10127 return -errno; 10128 return -ENOTSUP; 10129 } 10130 10131 /* dump all */ 10132 if (!flow_idx) { 10133 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10134 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10135 return -EINVAL; 10136 10137 if (sh->config.dv_flow_en == 2) 10138 return mlx5dr_debug_dump(priv->dr_ctx, file); 10139 #endif 10140 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10141 sh->rx_domain, 10142 sh->tx_domain, file); 10143 } 10144 /* dump one */ 10145 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10146 (uintptr_t)(void *)flow_idx); 10147 if (!flow) 10148 return -EINVAL; 10149 10150 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10151 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10152 #endif 10153 handle_idx = flow->dev_handles; 10154 while (handle_idx) { 10155 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10156 handle_idx); 10157 if (!dh) 10158 return -ENOENT; 10159 if (dh->drv_flow) { 10160 if (sh->config.dv_flow_en == 2) 10161 return -ENOTSUP; 10162 10163 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10164 file); 10165 if (ret) 10166 return -ENOENT; 10167 } 10168 handle_idx = dh->next.next; 10169 } 10170 return 0; 10171 } 10172 10173 /** 10174 * Get aged-out flows. 10175 * 10176 * @param[in] dev 10177 * Pointer to the Ethernet device structure. 10178 * @param[in] context 10179 * The address of an array of pointers to the aged-out flows contexts. 10180 * @param[in] nb_countexts 10181 * The length of context array pointers. 10182 * @param[out] error 10183 * Perform verbose error reporting if not NULL. Initialized in case of 10184 * error only. 10185 * 10186 * @return 10187 * how many contexts get in success, otherwise negative errno value. 10188 * if nb_contexts is 0, return the amount of all aged contexts. 10189 * if nb_contexts is not 0 , return the amount of aged flows reported 10190 * in the context array. 10191 */ 10192 int 10193 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10194 uint32_t nb_contexts, struct rte_flow_error *error) 10195 { 10196 struct rte_flow_attr attr = { .transfer = 0 }; 10197 10198 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10199 (dev, contexts, nb_contexts, error); 10200 } 10201 10202 /** 10203 * Get aged-out flows per HWS queue. 10204 * 10205 * @param[in] dev 10206 * Pointer to the Ethernet device structure. 10207 * @param[in] queue_id 10208 * Flow queue to query. 10209 * @param[in] context 10210 * The address of an array of pointers to the aged-out flows contexts. 10211 * @param[in] nb_countexts 10212 * The length of context array pointers. 10213 * @param[out] error 10214 * Perform verbose error reporting if not NULL. Initialized in case of 10215 * error only. 10216 * 10217 * @return 10218 * how many contexts get in success, otherwise negative errno value. 10219 * if nb_contexts is 0, return the amount of all aged contexts. 10220 * if nb_contexts is not 0 , return the amount of aged flows reported 10221 * in the context array. 10222 */ 10223 int 10224 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10225 void **contexts, uint32_t nb_contexts, 10226 struct rte_flow_error *error) 10227 { 10228 const struct mlx5_flow_driver_ops *fops; 10229 struct rte_flow_attr attr = { 0 }; 10230 10231 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10232 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10233 return fops->get_q_aged_flows(dev, queue_id, contexts, 10234 nb_contexts, error); 10235 } 10236 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10237 dev->data->port_id, queue_id); 10238 return rte_flow_error_set(error, ENOTSUP, 10239 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10240 "get Q aged flows with incorrect steering mode"); 10241 } 10242 10243 /* Wrapper for driver action_validate op callback */ 10244 static int 10245 flow_drv_action_validate(struct rte_eth_dev *dev, 10246 const struct rte_flow_indir_action_conf *conf, 10247 const struct rte_flow_action *action, 10248 const struct mlx5_flow_driver_ops *fops, 10249 struct rte_flow_error *error) 10250 { 10251 static const char err_msg[] = "indirect action validation unsupported"; 10252 10253 if (!fops->action_validate) { 10254 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10255 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10256 NULL, err_msg); 10257 return -rte_errno; 10258 } 10259 return fops->action_validate(dev, conf, action, error); 10260 } 10261 10262 /** 10263 * Destroys the shared action by handle. 10264 * 10265 * @param dev 10266 * Pointer to Ethernet device structure. 10267 * @param[in] handle 10268 * Handle for the indirect action object to be destroyed. 10269 * @param[out] error 10270 * Perform verbose error reporting if not NULL. PMDs initialize this 10271 * structure in case of error only. 10272 * 10273 * @return 10274 * 0 on success, a negative errno value otherwise and rte_errno is set. 10275 * 10276 * @note: wrapper for driver action_create op callback. 10277 */ 10278 static int 10279 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10280 struct rte_flow_action_handle *handle, 10281 struct rte_flow_error *error) 10282 { 10283 static const char err_msg[] = "indirect action destruction unsupported"; 10284 struct rte_flow_attr attr = { .transfer = 0 }; 10285 const struct mlx5_flow_driver_ops *fops = 10286 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10287 10288 if (!fops->action_destroy) { 10289 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10290 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10291 NULL, err_msg); 10292 return -rte_errno; 10293 } 10294 return fops->action_destroy(dev, handle, error); 10295 } 10296 10297 /* Wrapper for driver action_destroy op callback */ 10298 static int 10299 flow_drv_action_update(struct rte_eth_dev *dev, 10300 struct rte_flow_action_handle *handle, 10301 const void *update, 10302 const struct mlx5_flow_driver_ops *fops, 10303 struct rte_flow_error *error) 10304 { 10305 static const char err_msg[] = "indirect action update unsupported"; 10306 10307 if (!fops->action_update) { 10308 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10309 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10310 NULL, err_msg); 10311 return -rte_errno; 10312 } 10313 return fops->action_update(dev, handle, update, error); 10314 } 10315 10316 /* Wrapper for driver action_destroy op callback */ 10317 static int 10318 flow_drv_action_query(struct rte_eth_dev *dev, 10319 const struct rte_flow_action_handle *handle, 10320 void *data, 10321 const struct mlx5_flow_driver_ops *fops, 10322 struct rte_flow_error *error) 10323 { 10324 static const char err_msg[] = "indirect action query unsupported"; 10325 10326 if (!fops->action_query) { 10327 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10328 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10329 NULL, err_msg); 10330 return -rte_errno; 10331 } 10332 return fops->action_query(dev, handle, data, error); 10333 } 10334 10335 /** 10336 * Create indirect action for reuse in multiple flow rules. 10337 * 10338 * @param dev 10339 * Pointer to Ethernet device structure. 10340 * @param conf 10341 * Pointer to indirect action object configuration. 10342 * @param[in] action 10343 * Action configuration for indirect action object creation. 10344 * @param[out] error 10345 * Perform verbose error reporting if not NULL. PMDs initialize this 10346 * structure in case of error only. 10347 * @return 10348 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10349 */ 10350 static struct rte_flow_action_handle * 10351 mlx5_action_handle_create(struct rte_eth_dev *dev, 10352 const struct rte_flow_indir_action_conf *conf, 10353 const struct rte_flow_action *action, 10354 struct rte_flow_error *error) 10355 { 10356 static const char err_msg[] = "indirect action creation unsupported"; 10357 struct rte_flow_attr attr = { .transfer = 0 }; 10358 const struct mlx5_flow_driver_ops *fops = 10359 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10360 10361 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10362 return NULL; 10363 if (!fops->action_create) { 10364 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10365 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10366 NULL, err_msg); 10367 return NULL; 10368 } 10369 return fops->action_create(dev, conf, action, error); 10370 } 10371 10372 /** 10373 * Updates inplace the indirect action configuration pointed by *handle* 10374 * with the configuration provided as *update* argument. 10375 * The update of the indirect action configuration effects all flow rules 10376 * reusing the action via handle. 10377 * 10378 * @param dev 10379 * Pointer to Ethernet device structure. 10380 * @param[in] handle 10381 * Handle for the indirect action to be updated. 10382 * @param[in] update 10383 * Action specification used to modify the action pointed by handle. 10384 * *update* could be of same type with the action pointed by the *handle* 10385 * handle argument, or some other structures like a wrapper, depending on 10386 * the indirect action type. 10387 * @param[out] error 10388 * Perform verbose error reporting if not NULL. PMDs initialize this 10389 * structure in case of error only. 10390 * 10391 * @return 10392 * 0 on success, a negative errno value otherwise and rte_errno is set. 10393 */ 10394 static int 10395 mlx5_action_handle_update(struct rte_eth_dev *dev, 10396 struct rte_flow_action_handle *handle, 10397 const void *update, 10398 struct rte_flow_error *error) 10399 { 10400 struct rte_flow_attr attr = { .transfer = 0 }; 10401 const struct mlx5_flow_driver_ops *fops = 10402 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10403 int ret; 10404 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 10405 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 10406 10407 switch (type) { 10408 case MLX5_INDIRECT_ACTION_TYPE_CT: 10409 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 10410 ret = 0; 10411 break; 10412 default: 10413 ret = flow_drv_action_validate(dev, NULL, 10414 (const struct rte_flow_action *)update, 10415 fops, error); 10416 } 10417 if (ret) 10418 return ret; 10419 return flow_drv_action_update(dev, handle, update, fops, 10420 error); 10421 } 10422 10423 /** 10424 * Query the indirect action by handle. 10425 * 10426 * This function allows retrieving action-specific data such as counters. 10427 * Data is gathered by special action which may be present/referenced in 10428 * more than one flow rule definition. 10429 * 10430 * see @RTE_FLOW_ACTION_TYPE_COUNT 10431 * 10432 * @param dev 10433 * Pointer to Ethernet device structure. 10434 * @param[in] handle 10435 * Handle for the indirect action to query. 10436 * @param[in, out] data 10437 * Pointer to storage for the associated query data type. 10438 * @param[out] error 10439 * Perform verbose error reporting if not NULL. PMDs initialize this 10440 * structure in case of error only. 10441 * 10442 * @return 10443 * 0 on success, a negative errno value otherwise and rte_errno is set. 10444 */ 10445 static int 10446 mlx5_action_handle_query(struct rte_eth_dev *dev, 10447 const struct rte_flow_action_handle *handle, 10448 void *data, 10449 struct rte_flow_error *error) 10450 { 10451 struct rte_flow_attr attr = { .transfer = 0 }; 10452 const struct mlx5_flow_driver_ops *fops = 10453 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10454 10455 return flow_drv_action_query(dev, handle, data, fops, error); 10456 } 10457 10458 static int 10459 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10460 struct rte_flow_action_handle *handle, 10461 const void *update, void *query, 10462 enum rte_flow_query_update_mode qu_mode, 10463 struct rte_flow_error *error) 10464 { 10465 struct rte_flow_attr attr = { .transfer = 0 }; 10466 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10467 const struct mlx5_flow_driver_ops *fops; 10468 10469 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10470 return rte_flow_error_set(error, ENOTSUP, 10471 RTE_FLOW_ERROR_TYPE_ACTION, 10472 NULL, "invalid driver type"); 10473 fops = flow_get_drv_ops(drv_type); 10474 if (!fops || !fops->action_query_update) 10475 return rte_flow_error_set(error, ENOTSUP, 10476 RTE_FLOW_ERROR_TYPE_ACTION, 10477 NULL, "no query_update handler"); 10478 return fops->action_query_update(dev, handle, update, 10479 query, qu_mode, error); 10480 } 10481 10482 10483 #define MLX5_DRV_FOPS_OR_ERR(dev, fops, drv_cb, ret) \ 10484 { \ 10485 struct rte_flow_attr attr = { .transfer = 0 }; \ 10486 enum mlx5_flow_drv_type drv_type = flow_get_drv_type((dev), &attr); \ 10487 if (drv_type == MLX5_FLOW_TYPE_MIN || \ 10488 drv_type == MLX5_FLOW_TYPE_MAX) { \ 10489 rte_flow_error_set(error, ENOTSUP, \ 10490 RTE_FLOW_ERROR_TYPE_ACTION, \ 10491 NULL, "invalid driver type"); \ 10492 return ret; \ 10493 } \ 10494 (fops) = flow_get_drv_ops(drv_type); \ 10495 if (!(fops) || !(fops)->drv_cb) { \ 10496 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, \ 10497 NULL, "no action_list handler"); \ 10498 return ret; \ 10499 } \ 10500 } 10501 10502 static struct rte_flow_action_list_handle * 10503 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 10504 const struct rte_flow_indir_action_conf *conf, 10505 const struct rte_flow_action *actions, 10506 struct rte_flow_error *error) 10507 { 10508 const struct mlx5_flow_driver_ops *fops; 10509 10510 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_create, NULL); 10511 return fops->action_list_handle_create(dev, conf, actions, error); 10512 } 10513 10514 static int 10515 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 10516 struct rte_flow_action_list_handle *handle, 10517 struct rte_flow_error *error) 10518 { 10519 const struct mlx5_flow_driver_ops *fops; 10520 10521 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_destroy, ENOTSUP); 10522 return fops->action_list_handle_destroy(dev, handle, error); 10523 } 10524 10525 static int 10526 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 10527 const 10528 struct rte_flow_action_list_handle *handle, 10529 const void **update, void **query, 10530 enum rte_flow_query_update_mode mode, 10531 struct rte_flow_error *error) 10532 { 10533 const struct mlx5_flow_driver_ops *fops; 10534 10535 MLX5_DRV_FOPS_OR_ERR(dev, fops, 10536 action_list_handle_query_update, ENOTSUP); 10537 return fops->action_list_handle_query_update(dev, handle, update, query, 10538 mode, error); 10539 } 10540 static int 10541 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 10542 const struct rte_flow_template_table *table, 10543 const struct rte_flow_item pattern[], 10544 uint8_t pattern_template_index, 10545 uint32_t *hash, struct rte_flow_error *error) 10546 { 10547 struct rte_flow_attr attr = { .transfer = 0 }; 10548 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10549 const struct mlx5_flow_driver_ops *fops; 10550 10551 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10552 return rte_flow_error_set(error, ENOTSUP, 10553 RTE_FLOW_ERROR_TYPE_ACTION, 10554 NULL, "invalid driver type"); 10555 fops = flow_get_drv_ops(drv_type); 10556 if (!fops || !fops->action_query_update) 10557 return rte_flow_error_set(error, ENOTSUP, 10558 RTE_FLOW_ERROR_TYPE_ACTION, 10559 NULL, "no query_update handler"); 10560 return fops->flow_calc_table_hash(dev, table, pattern, pattern_template_index, 10561 hash, error); 10562 } 10563 10564 static int 10565 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 10566 const struct rte_flow_item pattern[], 10567 enum rte_flow_encap_hash_field dest_field, 10568 uint8_t *hash, 10569 struct rte_flow_error *error) 10570 { 10571 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, NULL); 10572 const struct mlx5_flow_driver_ops *fops; 10573 10574 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10575 return rte_flow_error_set(error, ENOTSUP, 10576 RTE_FLOW_ERROR_TYPE_ACTION, 10577 NULL, "invalid driver type"); 10578 fops = flow_get_drv_ops(drv_type); 10579 if (!fops || !fops->flow_calc_encap_hash) 10580 return rte_flow_error_set(error, ENOTSUP, 10581 RTE_FLOW_ERROR_TYPE_ACTION, 10582 NULL, "no calc encap hash handler"); 10583 return fops->flow_calc_encap_hash(dev, pattern, dest_field, hash, error); 10584 } 10585 10586 static int 10587 mlx5_template_table_resize(struct rte_eth_dev *dev, 10588 struct rte_flow_template_table *table, 10589 uint32_t nb_rules, struct rte_flow_error *error) 10590 { 10591 const struct mlx5_flow_driver_ops *fops; 10592 10593 MLX5_DRV_FOPS_OR_ERR(dev, fops, table_resize, ENOTSUP); 10594 return fops->table_resize(dev, table, nb_rules, error); 10595 } 10596 10597 static int 10598 mlx5_table_resize_complete(struct rte_eth_dev *dev, 10599 struct rte_flow_template_table *table, 10600 struct rte_flow_error *error) 10601 { 10602 const struct mlx5_flow_driver_ops *fops; 10603 10604 MLX5_DRV_FOPS_OR_ERR(dev, fops, table_resize_complete, ENOTSUP); 10605 return fops->table_resize_complete(dev, table, error); 10606 } 10607 10608 static int 10609 mlx5_flow_async_update_resized(struct rte_eth_dev *dev, uint32_t queue, 10610 const struct rte_flow_op_attr *op_attr, 10611 struct rte_flow *rule, void *user_data, 10612 struct rte_flow_error *error) 10613 { 10614 const struct mlx5_flow_driver_ops *fops; 10615 10616 MLX5_DRV_FOPS_OR_ERR(dev, fops, flow_update_resized, ENOTSUP); 10617 return fops->flow_update_resized(dev, queue, op_attr, rule, user_data, error); 10618 } 10619 10620 /** 10621 * Destroy all indirect actions (shared RSS). 10622 * 10623 * @param dev 10624 * Pointer to Ethernet device. 10625 * 10626 * @return 10627 * 0 on success, a negative errno value otherwise and rte_errno is set. 10628 */ 10629 int 10630 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10631 { 10632 struct rte_flow_error error; 10633 struct mlx5_priv *priv = dev->data->dev_private; 10634 struct mlx5_shared_action_rss *shared_rss; 10635 int ret = 0; 10636 uint32_t idx; 10637 10638 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10639 priv->rss_shared_actions, idx, shared_rss, next) { 10640 ret |= mlx5_action_handle_destroy(dev, 10641 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10642 } 10643 return ret; 10644 } 10645 10646 /** 10647 * Validate existing indirect actions against current device configuration 10648 * and attach them to device resources. 10649 * 10650 * @param dev 10651 * Pointer to Ethernet device. 10652 * 10653 * @return 10654 * 0 on success, a negative errno value otherwise and rte_errno is set. 10655 */ 10656 int 10657 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10658 { 10659 struct mlx5_priv *priv = dev->data->dev_private; 10660 int ret = 0; 10661 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10662 10663 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10664 const char *message; 10665 uint32_t queue_idx; 10666 10667 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10668 ind_tbl->queues_n, 10669 &message, &queue_idx); 10670 if (ret != 0) { 10671 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10672 dev->data->port_id, ind_tbl->queues[queue_idx], 10673 message); 10674 break; 10675 } 10676 } 10677 if (ret != 0) 10678 return ret; 10679 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10680 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10681 if (ret != 0) { 10682 DRV_LOG(ERR, "Port %u could not attach " 10683 "indirection table obj %p", 10684 dev->data->port_id, (void *)ind_tbl); 10685 goto error; 10686 } 10687 } 10688 10689 return 0; 10690 error: 10691 ind_tbl_last = ind_tbl; 10692 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10693 if (ind_tbl == ind_tbl_last) 10694 break; 10695 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10696 DRV_LOG(CRIT, "Port %u could not detach " 10697 "indirection table obj %p on rollback", 10698 dev->data->port_id, (void *)ind_tbl); 10699 } 10700 return ret; 10701 } 10702 10703 /** 10704 * Detach indirect actions of the device from its resources. 10705 * 10706 * @param dev 10707 * Pointer to Ethernet device. 10708 * 10709 * @return 10710 * 0 on success, a negative errno value otherwise and rte_errno is set. 10711 */ 10712 int 10713 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10714 { 10715 struct mlx5_priv *priv = dev->data->dev_private; 10716 int ret = 0; 10717 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10718 10719 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10720 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10721 if (ret != 0) { 10722 DRV_LOG(ERR, "Port %u could not detach " 10723 "indirection table obj %p", 10724 dev->data->port_id, (void *)ind_tbl); 10725 goto error; 10726 } 10727 } 10728 return 0; 10729 error: 10730 ind_tbl_last = ind_tbl; 10731 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10732 if (ind_tbl == ind_tbl_last) 10733 break; 10734 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10735 DRV_LOG(CRIT, "Port %u could not attach " 10736 "indirection table obj %p on rollback", 10737 dev->data->port_id, (void *)ind_tbl); 10738 } 10739 return ret; 10740 } 10741 10742 #ifndef HAVE_MLX5DV_DR 10743 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10744 #else 10745 #define MLX5_DOMAIN_SYNC_FLOW \ 10746 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10747 #endif 10748 10749 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10750 { 10751 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10752 const struct mlx5_flow_driver_ops *fops; 10753 int ret; 10754 struct rte_flow_attr attr = { .transfer = 0 }; 10755 10756 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10757 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10758 if (ret > 0) 10759 ret = -ret; 10760 return ret; 10761 } 10762 10763 const struct mlx5_flow_tunnel * 10764 mlx5_get_tof(const struct rte_flow_item *item, 10765 const struct rte_flow_action *action, 10766 enum mlx5_tof_rule_type *rule_type) 10767 { 10768 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10769 if (item->type == (typeof(item->type)) 10770 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10771 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10772 return flow_items_to_tunnel(item); 10773 } 10774 } 10775 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10776 if (action->type == (typeof(action->type)) 10777 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10778 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10779 return flow_actions_to_tunnel(action); 10780 } 10781 } 10782 return NULL; 10783 } 10784 10785 /** 10786 * tunnel offload functionality is defined for DV environment only 10787 */ 10788 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10789 __extension__ 10790 union tunnel_offload_mark { 10791 uint32_t val; 10792 struct { 10793 uint32_t app_reserve:8; 10794 uint32_t table_id:15; 10795 uint32_t transfer:1; 10796 uint32_t _unused_:8; 10797 }; 10798 }; 10799 10800 static bool 10801 mlx5_access_tunnel_offload_db 10802 (struct rte_eth_dev *dev, 10803 bool (*match)(struct rte_eth_dev *, 10804 struct mlx5_flow_tunnel *, const void *), 10805 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10806 void (*miss)(struct rte_eth_dev *, void *), 10807 void *ctx, bool lock_op); 10808 10809 static int 10810 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10811 struct rte_flow *flow, 10812 const struct rte_flow_attr *attr, 10813 const struct rte_flow_action *app_actions, 10814 uint32_t flow_idx, 10815 const struct mlx5_flow_tunnel *tunnel, 10816 struct tunnel_default_miss_ctx *ctx, 10817 struct rte_flow_error *error) 10818 { 10819 struct mlx5_priv *priv = dev->data->dev_private; 10820 struct mlx5_flow *dev_flow; 10821 struct rte_flow_attr miss_attr = *attr; 10822 const struct rte_flow_item miss_items[2] = { 10823 { 10824 .type = RTE_FLOW_ITEM_TYPE_ETH, 10825 .spec = NULL, 10826 .last = NULL, 10827 .mask = NULL 10828 }, 10829 { 10830 .type = RTE_FLOW_ITEM_TYPE_END, 10831 .spec = NULL, 10832 .last = NULL, 10833 .mask = NULL 10834 } 10835 }; 10836 union tunnel_offload_mark mark_id; 10837 struct rte_flow_action_mark miss_mark; 10838 struct rte_flow_action miss_actions[3] = { 10839 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10840 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10841 }; 10842 const struct rte_flow_action_jump *jump_data; 10843 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10844 struct flow_grp_info grp_info = { 10845 .external = 1, 10846 .transfer = attr->transfer, 10847 .fdb_def_rule = !!priv->fdb_def_rule, 10848 .std_tbl_fix = 0, 10849 }; 10850 int ret; 10851 10852 if (!attr->transfer) { 10853 uint32_t q_size; 10854 10855 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10856 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10857 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10858 0, SOCKET_ID_ANY); 10859 if (!ctx->queue) 10860 return rte_flow_error_set 10861 (error, ENOMEM, 10862 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10863 NULL, "invalid default miss RSS"); 10864 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10865 ctx->action_rss.level = 0, 10866 ctx->action_rss.types = priv->rss_conf.rss_hf, 10867 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10868 ctx->action_rss.queue_num = priv->reta_idx_n, 10869 ctx->action_rss.key = priv->rss_conf.rss_key, 10870 ctx->action_rss.queue = ctx->queue; 10871 if (!priv->reta_idx_n || !priv->rxqs_n) 10872 return rte_flow_error_set 10873 (error, EINVAL, 10874 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10875 NULL, "invalid port configuration"); 10876 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10877 ctx->action_rss.types = 0; 10878 for (i = 0; i != priv->reta_idx_n; ++i) 10879 ctx->queue[i] = (*priv->reta_idx)[i]; 10880 } else { 10881 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10882 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10883 } 10884 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10885 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10886 jump_data = app_actions->conf; 10887 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10888 miss_attr.group = jump_data->group; 10889 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10890 &flow_table, &grp_info, error); 10891 if (ret) 10892 return rte_flow_error_set(error, EINVAL, 10893 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10894 NULL, "invalid tunnel id"); 10895 mark_id.app_reserve = 0; 10896 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10897 mark_id.transfer = !!attr->transfer; 10898 mark_id._unused_ = 0; 10899 miss_mark.id = mark_id.val; 10900 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10901 miss_items, miss_actions, flow_idx, error); 10902 if (!dev_flow) 10903 return -rte_errno; 10904 dev_flow->flow = flow; 10905 dev_flow->external = true; 10906 dev_flow->tunnel = tunnel; 10907 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10908 /* Subflow object was created, we must include one in the list. */ 10909 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10910 dev_flow->handle, next); 10911 DRV_LOG(DEBUG, 10912 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10913 dev->data->port_id, tunnel->app_tunnel.type, 10914 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10915 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10916 miss_actions, error); 10917 if (!ret) 10918 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10919 error); 10920 10921 return ret; 10922 } 10923 10924 static const struct mlx5_flow_tbl_data_entry * 10925 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10926 { 10927 struct mlx5_priv *priv = dev->data->dev_private; 10928 struct mlx5_dev_ctx_shared *sh = priv->sh; 10929 struct mlx5_list_entry *he; 10930 union tunnel_offload_mark mbits = { .val = mark }; 10931 union mlx5_flow_tbl_key table_key = { 10932 { 10933 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10934 .id = 0, 10935 .reserved = 0, 10936 .dummy = 0, 10937 .is_fdb = !!mbits.transfer, 10938 .is_egress = 0, 10939 } 10940 }; 10941 struct mlx5_flow_cb_ctx ctx = { 10942 .data = &table_key.v64, 10943 }; 10944 10945 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10946 return he ? 10947 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10948 } 10949 10950 static void 10951 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10952 struct mlx5_list_entry *entry) 10953 { 10954 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10955 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10956 10957 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10958 tunnel_flow_tbl_to_id(tte->flow_table)); 10959 mlx5_free(tte); 10960 } 10961 10962 static int 10963 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10964 struct mlx5_list_entry *entry, void *cb_ctx) 10965 { 10966 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10967 union tunnel_tbl_key tbl = { 10968 .val = *(uint64_t *)(ctx->data), 10969 }; 10970 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10971 10972 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10973 } 10974 10975 static struct mlx5_list_entry * 10976 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10977 { 10978 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10979 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10980 struct tunnel_tbl_entry *tte; 10981 union tunnel_tbl_key tbl = { 10982 .val = *(uint64_t *)(ctx->data), 10983 }; 10984 10985 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10986 sizeof(*tte), 0, 10987 SOCKET_ID_ANY); 10988 if (!tte) 10989 goto err; 10990 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10991 &tte->flow_table); 10992 if (tte->flow_table >= MLX5_MAX_TABLES) { 10993 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10994 tte->flow_table); 10995 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10996 tte->flow_table); 10997 goto err; 10998 } else if (!tte->flow_table) { 10999 goto err; 11000 } 11001 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 11002 tte->tunnel_id = tbl.tunnel_id; 11003 tte->group = tbl.group; 11004 return &tte->hash; 11005 err: 11006 if (tte) 11007 mlx5_free(tte); 11008 return NULL; 11009 } 11010 11011 static struct mlx5_list_entry * 11012 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 11013 struct mlx5_list_entry *oentry, 11014 void *cb_ctx __rte_unused) 11015 { 11016 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 11017 0, SOCKET_ID_ANY); 11018 11019 if (!tte) 11020 return NULL; 11021 memcpy(tte, oentry, sizeof(*tte)); 11022 return &tte->hash; 11023 } 11024 11025 static void 11026 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 11027 struct mlx5_list_entry *entry) 11028 { 11029 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11030 11031 mlx5_free(tte); 11032 } 11033 11034 static uint32_t 11035 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 11036 const struct mlx5_flow_tunnel *tunnel, 11037 uint32_t group, uint32_t *table, 11038 struct rte_flow_error *error) 11039 { 11040 struct mlx5_list_entry *he; 11041 struct tunnel_tbl_entry *tte; 11042 union tunnel_tbl_key key = { 11043 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 11044 .group = group 11045 }; 11046 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11047 struct mlx5_hlist *group_hash; 11048 struct mlx5_flow_cb_ctx ctx = { 11049 .data = &key.val, 11050 }; 11051 11052 group_hash = tunnel ? tunnel->groups : thub->groups; 11053 he = mlx5_hlist_register(group_hash, key.val, &ctx); 11054 if (!he) 11055 return rte_flow_error_set(error, EINVAL, 11056 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 11057 NULL, 11058 "tunnel group index not supported"); 11059 tte = container_of(he, typeof(*tte), hash); 11060 *table = tte->flow_table; 11061 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 11062 dev->data->port_id, key.tunnel_id, group, *table); 11063 return 0; 11064 } 11065 11066 static void 11067 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 11068 struct mlx5_flow_tunnel *tunnel) 11069 { 11070 struct mlx5_priv *priv = dev->data->dev_private; 11071 struct mlx5_indexed_pool *ipool; 11072 11073 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11074 dev->data->port_id, tunnel->tunnel_id); 11075 LIST_REMOVE(tunnel, chain); 11076 mlx5_hlist_destroy(tunnel->groups); 11077 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11078 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11079 } 11080 11081 static bool 11082 mlx5_access_tunnel_offload_db 11083 (struct rte_eth_dev *dev, 11084 bool (*match)(struct rte_eth_dev *, 11085 struct mlx5_flow_tunnel *, const void *), 11086 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11087 void (*miss)(struct rte_eth_dev *, void *), 11088 void *ctx, bool lock_op) 11089 { 11090 bool verdict = false; 11091 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11092 struct mlx5_flow_tunnel *tunnel; 11093 11094 rte_spinlock_lock(&thub->sl); 11095 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11096 verdict = match(dev, tunnel, (const void *)ctx); 11097 if (verdict) 11098 break; 11099 } 11100 if (!lock_op) 11101 rte_spinlock_unlock(&thub->sl); 11102 if (verdict && hit) 11103 hit(dev, tunnel, ctx); 11104 if (!verdict && miss) 11105 miss(dev, ctx); 11106 if (lock_op) 11107 rte_spinlock_unlock(&thub->sl); 11108 11109 return verdict; 11110 } 11111 11112 struct tunnel_db_find_tunnel_id_ctx { 11113 uint32_t tunnel_id; 11114 struct mlx5_flow_tunnel *tunnel; 11115 }; 11116 11117 static bool 11118 find_tunnel_id_match(struct rte_eth_dev *dev, 11119 struct mlx5_flow_tunnel *tunnel, const void *x) 11120 { 11121 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11122 11123 RTE_SET_USED(dev); 11124 return tunnel->tunnel_id == ctx->tunnel_id; 11125 } 11126 11127 static void 11128 find_tunnel_id_hit(struct rte_eth_dev *dev, 11129 struct mlx5_flow_tunnel *tunnel, void *x) 11130 { 11131 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11132 RTE_SET_USED(dev); 11133 ctx->tunnel = tunnel; 11134 } 11135 11136 static struct mlx5_flow_tunnel * 11137 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11138 { 11139 struct tunnel_db_find_tunnel_id_ctx ctx = { 11140 .tunnel_id = id, 11141 }; 11142 11143 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11144 find_tunnel_id_hit, NULL, &ctx, true); 11145 11146 return ctx.tunnel; 11147 } 11148 11149 static struct mlx5_flow_tunnel * 11150 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11151 const struct rte_flow_tunnel *app_tunnel) 11152 { 11153 struct mlx5_priv *priv = dev->data->dev_private; 11154 struct mlx5_indexed_pool *ipool; 11155 struct mlx5_flow_tunnel *tunnel; 11156 uint32_t id; 11157 11158 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11159 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11160 if (!tunnel) 11161 return NULL; 11162 if (id >= MLX5_MAX_TUNNELS) { 11163 mlx5_ipool_free(ipool, id); 11164 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11165 return NULL; 11166 } 11167 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11168 priv->sh, 11169 mlx5_flow_tunnel_grp2tbl_create_cb, 11170 mlx5_flow_tunnel_grp2tbl_match_cb, 11171 mlx5_flow_tunnel_grp2tbl_remove_cb, 11172 mlx5_flow_tunnel_grp2tbl_clone_cb, 11173 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11174 if (!tunnel->groups) { 11175 mlx5_ipool_free(ipool, id); 11176 return NULL; 11177 } 11178 /* initiate new PMD tunnel */ 11179 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11180 tunnel->tunnel_id = id; 11181 tunnel->action.type = (typeof(tunnel->action.type)) 11182 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11183 tunnel->action.conf = tunnel; 11184 tunnel->item.type = (typeof(tunnel->item.type)) 11185 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11186 tunnel->item.spec = tunnel; 11187 tunnel->item.last = NULL; 11188 tunnel->item.mask = NULL; 11189 11190 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11191 dev->data->port_id, tunnel->tunnel_id); 11192 11193 return tunnel; 11194 } 11195 11196 struct tunnel_db_get_tunnel_ctx { 11197 const struct rte_flow_tunnel *app_tunnel; 11198 struct mlx5_flow_tunnel *tunnel; 11199 }; 11200 11201 static bool get_tunnel_match(struct rte_eth_dev *dev, 11202 struct mlx5_flow_tunnel *tunnel, const void *x) 11203 { 11204 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11205 11206 RTE_SET_USED(dev); 11207 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11208 sizeof(*ctx->app_tunnel)); 11209 } 11210 11211 static void get_tunnel_hit(struct rte_eth_dev *dev, 11212 struct mlx5_flow_tunnel *tunnel, void *x) 11213 { 11214 /* called under tunnel spinlock protection */ 11215 struct tunnel_db_get_tunnel_ctx *ctx = x; 11216 11217 RTE_SET_USED(dev); 11218 tunnel->refctn++; 11219 ctx->tunnel = tunnel; 11220 } 11221 11222 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11223 { 11224 /* called under tunnel spinlock protection */ 11225 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11226 struct tunnel_db_get_tunnel_ctx *ctx = x; 11227 11228 rte_spinlock_unlock(&thub->sl); 11229 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11230 rte_spinlock_lock(&thub->sl); 11231 if (ctx->tunnel) { 11232 ctx->tunnel->refctn = 1; 11233 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11234 } 11235 } 11236 11237 11238 static int 11239 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11240 const struct rte_flow_tunnel *app_tunnel, 11241 struct mlx5_flow_tunnel **tunnel) 11242 { 11243 struct tunnel_db_get_tunnel_ctx ctx = { 11244 .app_tunnel = app_tunnel, 11245 }; 11246 11247 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11248 get_tunnel_miss, &ctx, true); 11249 *tunnel = ctx.tunnel; 11250 return ctx.tunnel ? 0 : -ENOMEM; 11251 } 11252 11253 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11254 { 11255 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11256 11257 if (!thub) 11258 return; 11259 if (!LIST_EMPTY(&thub->tunnels)) 11260 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11261 mlx5_hlist_destroy(thub->groups); 11262 mlx5_free(thub); 11263 } 11264 11265 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11266 { 11267 int err; 11268 struct mlx5_flow_tunnel_hub *thub; 11269 11270 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11271 0, SOCKET_ID_ANY); 11272 if (!thub) 11273 return -ENOMEM; 11274 LIST_INIT(&thub->tunnels); 11275 rte_spinlock_init(&thub->sl); 11276 thub->groups = mlx5_hlist_create("flow groups", 64, 11277 false, true, sh, 11278 mlx5_flow_tunnel_grp2tbl_create_cb, 11279 mlx5_flow_tunnel_grp2tbl_match_cb, 11280 mlx5_flow_tunnel_grp2tbl_remove_cb, 11281 mlx5_flow_tunnel_grp2tbl_clone_cb, 11282 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11283 if (!thub->groups) { 11284 err = -rte_errno; 11285 goto err; 11286 } 11287 sh->tunnel_hub = thub; 11288 11289 return 0; 11290 11291 err: 11292 if (thub->groups) 11293 mlx5_hlist_destroy(thub->groups); 11294 if (thub) 11295 mlx5_free(thub); 11296 return err; 11297 } 11298 11299 static inline int 11300 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11301 struct rte_flow_tunnel *tunnel, 11302 struct rte_flow_error *error) 11303 { 11304 struct mlx5_priv *priv = dev->data->dev_private; 11305 11306 if (!priv->sh->config.dv_flow_en) 11307 return rte_flow_error_set(error, ENOTSUP, 11308 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11309 "flow DV interface is off"); 11310 if (!is_tunnel_offload_active(dev)) 11311 return rte_flow_error_set(error, ENOTSUP, 11312 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11313 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11314 if (!tunnel) 11315 return rte_flow_error_set(error, EINVAL, 11316 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11317 "no application tunnel"); 11318 switch (tunnel->type) { 11319 default: 11320 return rte_flow_error_set(error, EINVAL, 11321 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11322 "unsupported tunnel type"); 11323 case RTE_FLOW_ITEM_TYPE_VXLAN: 11324 case RTE_FLOW_ITEM_TYPE_GRE: 11325 case RTE_FLOW_ITEM_TYPE_NVGRE: 11326 case RTE_FLOW_ITEM_TYPE_GENEVE: 11327 break; 11328 } 11329 return 0; 11330 } 11331 11332 static int 11333 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11334 struct rte_flow_tunnel *app_tunnel, 11335 struct rte_flow_action **actions, 11336 uint32_t *num_of_actions, 11337 struct rte_flow_error *error) 11338 { 11339 struct mlx5_flow_tunnel *tunnel; 11340 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11341 11342 if (ret) 11343 return ret; 11344 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11345 if (ret < 0) { 11346 return rte_flow_error_set(error, ret, 11347 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11348 "failed to initialize pmd tunnel"); 11349 } 11350 *actions = &tunnel->action; 11351 *num_of_actions = 1; 11352 return 0; 11353 } 11354 11355 static int 11356 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11357 struct rte_flow_tunnel *app_tunnel, 11358 struct rte_flow_item **items, 11359 uint32_t *num_of_items, 11360 struct rte_flow_error *error) 11361 { 11362 struct mlx5_flow_tunnel *tunnel; 11363 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11364 11365 if (ret) 11366 return ret; 11367 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11368 if (ret < 0) { 11369 return rte_flow_error_set(error, ret, 11370 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11371 "failed to initialize pmd tunnel"); 11372 } 11373 *items = &tunnel->item; 11374 *num_of_items = 1; 11375 return 0; 11376 } 11377 11378 struct tunnel_db_element_release_ctx { 11379 struct rte_flow_item *items; 11380 struct rte_flow_action *actions; 11381 uint32_t num_elements; 11382 struct rte_flow_error *error; 11383 int ret; 11384 }; 11385 11386 static bool 11387 tunnel_element_release_match(struct rte_eth_dev *dev, 11388 struct mlx5_flow_tunnel *tunnel, const void *x) 11389 { 11390 const struct tunnel_db_element_release_ctx *ctx = x; 11391 11392 RTE_SET_USED(dev); 11393 if (ctx->num_elements != 1) 11394 return false; 11395 else if (ctx->items) 11396 return ctx->items == &tunnel->item; 11397 else if (ctx->actions) 11398 return ctx->actions == &tunnel->action; 11399 11400 return false; 11401 } 11402 11403 static void 11404 tunnel_element_release_hit(struct rte_eth_dev *dev, 11405 struct mlx5_flow_tunnel *tunnel, void *x) 11406 { 11407 struct tunnel_db_element_release_ctx *ctx = x; 11408 ctx->ret = 0; 11409 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 11410 mlx5_flow_tunnel_free(dev, tunnel); 11411 } 11412 11413 static void 11414 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11415 { 11416 struct tunnel_db_element_release_ctx *ctx = x; 11417 RTE_SET_USED(dev); 11418 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11419 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11420 "invalid argument"); 11421 } 11422 11423 static int 11424 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11425 struct rte_flow_item *pmd_items, 11426 uint32_t num_items, struct rte_flow_error *err) 11427 { 11428 struct tunnel_db_element_release_ctx ctx = { 11429 .items = pmd_items, 11430 .actions = NULL, 11431 .num_elements = num_items, 11432 .error = err, 11433 }; 11434 11435 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11436 tunnel_element_release_hit, 11437 tunnel_element_release_miss, &ctx, false); 11438 11439 return ctx.ret; 11440 } 11441 11442 static int 11443 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11444 struct rte_flow_action *pmd_actions, 11445 uint32_t num_actions, struct rte_flow_error *err) 11446 { 11447 struct tunnel_db_element_release_ctx ctx = { 11448 .items = NULL, 11449 .actions = pmd_actions, 11450 .num_elements = num_actions, 11451 .error = err, 11452 }; 11453 11454 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11455 tunnel_element_release_hit, 11456 tunnel_element_release_miss, &ctx, false); 11457 11458 return ctx.ret; 11459 } 11460 11461 static int 11462 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11463 struct rte_mbuf *m, 11464 struct rte_flow_restore_info *info, 11465 struct rte_flow_error *err) 11466 { 11467 uint64_t ol_flags = m->ol_flags; 11468 const struct mlx5_flow_tbl_data_entry *tble; 11469 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11470 struct mlx5_priv *priv = dev->data->dev_private; 11471 11472 if (priv->tunnel_enabled == 0) 11473 goto err; 11474 if ((ol_flags & mask) != mask) 11475 goto err; 11476 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11477 if (!tble) { 11478 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11479 dev->data->port_id, m->hash.fdir.hi); 11480 goto err; 11481 } 11482 MLX5_ASSERT(tble->tunnel); 11483 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11484 info->group_id = tble->group_id; 11485 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11486 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11487 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11488 11489 return 0; 11490 11491 err: 11492 return rte_flow_error_set(err, EINVAL, 11493 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11494 "failed to get restore info"); 11495 } 11496 11497 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11498 static int 11499 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11500 __rte_unused struct rte_flow_tunnel *app_tunnel, 11501 __rte_unused struct rte_flow_action **actions, 11502 __rte_unused uint32_t *num_of_actions, 11503 __rte_unused struct rte_flow_error *error) 11504 { 11505 return -ENOTSUP; 11506 } 11507 11508 static int 11509 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11510 __rte_unused struct rte_flow_tunnel *app_tunnel, 11511 __rte_unused struct rte_flow_item **items, 11512 __rte_unused uint32_t *num_of_items, 11513 __rte_unused struct rte_flow_error *error) 11514 { 11515 return -ENOTSUP; 11516 } 11517 11518 static int 11519 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11520 __rte_unused struct rte_flow_item *pmd_items, 11521 __rte_unused uint32_t num_items, 11522 __rte_unused struct rte_flow_error *err) 11523 { 11524 return -ENOTSUP; 11525 } 11526 11527 static int 11528 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11529 __rte_unused struct rte_flow_action *pmd_action, 11530 __rte_unused uint32_t num_actions, 11531 __rte_unused struct rte_flow_error *err) 11532 { 11533 return -ENOTSUP; 11534 } 11535 11536 static int 11537 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11538 __rte_unused struct rte_mbuf *m, 11539 __rte_unused struct rte_flow_restore_info *i, 11540 __rte_unused struct rte_flow_error *err) 11541 { 11542 return -ENOTSUP; 11543 } 11544 11545 static int 11546 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11547 __rte_unused struct rte_flow *flow, 11548 __rte_unused const struct rte_flow_attr *attr, 11549 __rte_unused const struct rte_flow_action *actions, 11550 __rte_unused uint32_t flow_idx, 11551 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11552 __rte_unused struct tunnel_default_miss_ctx *ctx, 11553 __rte_unused struct rte_flow_error *error) 11554 { 11555 return -ENOTSUP; 11556 } 11557 11558 static struct mlx5_flow_tunnel * 11559 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11560 __rte_unused uint32_t id) 11561 { 11562 return NULL; 11563 } 11564 11565 static void 11566 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11567 __rte_unused struct mlx5_flow_tunnel *tunnel) 11568 { 11569 } 11570 11571 static uint32_t 11572 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11573 __rte_unused const struct mlx5_flow_tunnel *t, 11574 __rte_unused uint32_t group, 11575 __rte_unused uint32_t *table, 11576 struct rte_flow_error *error) 11577 { 11578 return rte_flow_error_set(error, ENOTSUP, 11579 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11580 "tunnel offload requires DV support"); 11581 } 11582 11583 void 11584 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11585 __rte_unused uint16_t port_id) 11586 { 11587 } 11588 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11589 11590 /* Flex flow item API */ 11591 static struct rte_flow_item_flex_handle * 11592 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11593 const struct rte_flow_item_flex_conf *conf, 11594 struct rte_flow_error *error) 11595 { 11596 static const char err_msg[] = "flex item creation unsupported"; 11597 struct mlx5_priv *priv = dev->data->dev_private; 11598 struct rte_flow_attr attr = { .transfer = 0 }; 11599 const struct mlx5_flow_driver_ops *fops = 11600 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11601 11602 if (!priv->pci_dev) { 11603 rte_flow_error_set(error, ENOTSUP, 11604 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11605 "create flex item on PF only"); 11606 return NULL; 11607 } 11608 switch (priv->pci_dev->id.device_id) { 11609 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11610 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11611 break; 11612 default: 11613 rte_flow_error_set(error, ENOTSUP, 11614 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11615 "flex item available on BlueField ports only"); 11616 return NULL; 11617 } 11618 if (!fops->item_create) { 11619 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11620 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11621 NULL, err_msg); 11622 return NULL; 11623 } 11624 return fops->item_create(dev, conf, error); 11625 } 11626 11627 static int 11628 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11629 const struct rte_flow_item_flex_handle *handle, 11630 struct rte_flow_error *error) 11631 { 11632 static const char err_msg[] = "flex item release unsupported"; 11633 struct rte_flow_attr attr = { .transfer = 0 }; 11634 const struct mlx5_flow_driver_ops *fops = 11635 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11636 11637 if (!fops->item_release) { 11638 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11639 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11640 NULL, err_msg); 11641 return -rte_errno; 11642 } 11643 return fops->item_release(dev, handle, error); 11644 } 11645 11646 static void 11647 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11648 { 11649 int ret; 11650 struct rte_flow_error error; 11651 11652 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11653 char *item_name; 11654 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11655 sizeof(item_name), 11656 (void *)(uintptr_t)item->type, &error); 11657 if (ret > 0) 11658 printf("%s ", item_name); 11659 else 11660 printf("%d\n", (int)item->type); 11661 } 11662 printf("END\n"); 11663 } 11664 11665 static int 11666 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11667 { 11668 const struct rte_flow_item_udp *spec = udp_item->spec; 11669 const struct rte_flow_item_udp *mask = udp_item->mask; 11670 uint16_t udp_dport = 0; 11671 11672 if (spec != NULL) { 11673 if (!mask) 11674 mask = &rte_flow_item_udp_mask; 11675 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11676 mask->hdr.dst_port); 11677 } 11678 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11679 } 11680 11681 static const struct mlx5_flow_expand_node * 11682 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11683 unsigned int item_idx, 11684 const struct mlx5_flow_expand_node graph[], 11685 const struct mlx5_flow_expand_node *node) 11686 { 11687 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11688 11689 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11690 node != NULL && 11691 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11692 /* 11693 * The expansion node is VXLAN and it is also the last 11694 * expandable item in the pattern, so need to continue 11695 * expansion of the inner tunnel. 11696 */ 11697 MLX5_ASSERT(item_idx > 0); 11698 prev_item = pattern + item_idx - 1; 11699 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11700 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11701 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11702 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11703 } 11704 return node; 11705 } 11706 11707 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11708 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11709 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11710 }; 11711 11712 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11713 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11714 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11715 { 9, 10, 11 }, { 12, 13, 14 }, 11716 }; 11717 11718 /** 11719 * Discover the number of available flow priorities. 11720 * 11721 * @param dev 11722 * Ethernet device. 11723 * 11724 * @return 11725 * On success, number of available flow priorities. 11726 * On failure, a negative errno-style code and rte_errno is set. 11727 */ 11728 int 11729 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11730 { 11731 static const uint16_t vprio[] = {8, 16}; 11732 const struct mlx5_priv *priv = dev->data->dev_private; 11733 const struct mlx5_flow_driver_ops *fops; 11734 enum mlx5_flow_drv_type type; 11735 int ret; 11736 11737 type = mlx5_flow_os_get_type(); 11738 if (type == MLX5_FLOW_TYPE_MAX) { 11739 type = MLX5_FLOW_TYPE_VERBS; 11740 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11741 type = MLX5_FLOW_TYPE_DV; 11742 } 11743 fops = flow_get_drv_ops(type); 11744 if (fops->discover_priorities == NULL) { 11745 DRV_LOG(ERR, "Priority discovery not supported"); 11746 rte_errno = ENOTSUP; 11747 return -rte_errno; 11748 } 11749 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11750 if (ret < 0) 11751 return ret; 11752 switch (ret) { 11753 case 8: 11754 ret = RTE_DIM(priority_map_3); 11755 break; 11756 case 16: 11757 ret = RTE_DIM(priority_map_5); 11758 break; 11759 default: 11760 rte_errno = ENOTSUP; 11761 DRV_LOG(ERR, 11762 "port %u maximum priority: %d expected 8/16", 11763 dev->data->port_id, ret); 11764 return -rte_errno; 11765 } 11766 DRV_LOG(INFO, "port %u supported flow priorities:" 11767 " 0-%d for ingress or egress root table," 11768 " 0-%d for non-root table or transfer root table.", 11769 dev->data->port_id, ret - 2, 11770 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11771 return ret; 11772 } 11773 11774 /** 11775 * Adjust flow priority based on the highest layer and the request priority. 11776 * 11777 * @param[in] dev 11778 * Pointer to the Ethernet device structure. 11779 * @param[in] priority 11780 * The rule base priority. 11781 * @param[in] subpriority 11782 * The priority based on the items. 11783 * 11784 * @return 11785 * The new priority. 11786 */ 11787 uint32_t 11788 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11789 uint32_t subpriority) 11790 { 11791 uint32_t res = 0; 11792 struct mlx5_priv *priv = dev->data->dev_private; 11793 11794 switch (priv->sh->flow_max_priority) { 11795 case RTE_DIM(priority_map_3): 11796 res = priority_map_3[priority][subpriority]; 11797 break; 11798 case RTE_DIM(priority_map_5): 11799 res = priority_map_5[priority][subpriority]; 11800 break; 11801 } 11802 return res; 11803 } 11804 11805 /** 11806 * Get the priority for sending traffic to kernel table. 11807 * 11808 * @param[in] dev 11809 * Pointer to the Ethernet device structure. 11810 * 11811 * @return 11812 * On success: the value of priority for sending traffic to kernel table 11813 * On failure: -1 11814 */ 11815 uint32_t 11816 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11817 { 11818 struct mlx5_priv *priv = dev->data->dev_private; 11819 uint32_t res; 11820 11821 switch (priv->sh->flow_max_priority) { 11822 case RTE_DIM(priority_map_5): 11823 res = 15; 11824 break; 11825 case RTE_DIM(priority_map_3): 11826 res = 7; 11827 break; 11828 default: 11829 DRV_LOG(ERR, 11830 "port %u maximum priority: %d expected 8/16", 11831 dev->data->port_id, priv->sh->flow_max_priority); 11832 res = (uint32_t)-1; 11833 } 11834 return res; 11835 } 11836 11837 /** 11838 * Get the E-Switch Manager vport id. 11839 * 11840 * @param[in] dev 11841 * Pointer to the Ethernet device structure. 11842 * 11843 * @return 11844 * The vport id. 11845 */ 11846 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11847 { 11848 struct mlx5_priv *priv = dev->data->dev_private; 11849 struct mlx5_common_device *cdev = priv->sh->cdev; 11850 11851 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11852 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11853 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11854 11855 if (priv->pci_dev == NULL) 11856 return 0; 11857 switch (priv->pci_dev->id.device_id) { 11858 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 11859 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11860 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11861 /* 11862 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11863 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11864 * ECPF vport ID is selected and not the host port (0) in any BF case. 11865 */ 11866 return (int16_t)MLX5_ECPF_VPORT_ID; 11867 default: 11868 return MLX5_PF_VPORT_ID; 11869 } 11870 } 11871 11872 /** 11873 * Parse item to get the vport id. 11874 * 11875 * @param[in] dev 11876 * Pointer to the Ethernet device structure. 11877 * @param[in] item 11878 * The src port id match item. 11879 * @param[out] vport_id 11880 * Pointer to put the vport id. 11881 * @param[out] all_ports 11882 * Indicate if the item matches all ports. 11883 * @param[out] error 11884 * Pointer to error structure. 11885 * 11886 * @return 11887 * 0 on success, a negative errno value otherwise and rte_errno is set. 11888 */ 11889 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11890 const struct rte_flow_item *item, 11891 uint16_t *vport_id, 11892 bool *all_ports, 11893 struct rte_flow_error *error) 11894 { 11895 struct mlx5_priv *port_priv; 11896 const struct rte_flow_item_port_id *pid_v = NULL; 11897 const struct rte_flow_item_ethdev *dev_v = NULL; 11898 uint32_t esw_mgr_port; 11899 uint32_t src_port; 11900 11901 if (all_ports) 11902 *all_ports = false; 11903 switch (item->type) { 11904 case RTE_FLOW_ITEM_TYPE_PORT_ID: 11905 pid_v = item->spec; 11906 if (!pid_v) 11907 return 0; 11908 src_port = pid_v->id; 11909 esw_mgr_port = MLX5_PORT_ESW_MGR; 11910 break; 11911 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 11912 dev_v = item->spec; 11913 if (!dev_v) { 11914 if (all_ports) 11915 *all_ports = true; 11916 return 0; 11917 } 11918 src_port = dev_v->port_id; 11919 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11920 break; 11921 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 11922 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11923 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11924 break; 11925 default: 11926 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11927 NULL, "Incorrect item type."); 11928 } 11929 if (src_port == esw_mgr_port) { 11930 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11931 } else { 11932 port_priv = mlx5_port_to_eswitch_info(src_port, false); 11933 if (!port_priv) 11934 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11935 NULL, "Failed to get port info."); 11936 *vport_id = port_priv->representor_id; 11937 } 11938 11939 return 0; 11940 } 11941 11942 int 11943 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11944 uint16_t *proxy_port_id, 11945 struct rte_flow_error *error) 11946 { 11947 const struct mlx5_priv *priv = dev->data->dev_private; 11948 uint16_t port_id; 11949 11950 if (!priv->sh->config.dv_esw_en) 11951 return rte_flow_error_set(error, EINVAL, 11952 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11953 NULL, 11954 "unable to provide a proxy port" 11955 " without E-Switch configured"); 11956 if (!priv->master && !priv->representor) 11957 return rte_flow_error_set(error, EINVAL, 11958 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11959 NULL, 11960 "unable to provide a proxy port" 11961 " for port which is not a master" 11962 " or a representor port"); 11963 if (priv->master) { 11964 *proxy_port_id = dev->data->port_id; 11965 return 0; 11966 } 11967 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11968 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11969 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11970 11971 if (port_priv->master && 11972 port_priv->domain_id == priv->domain_id) { 11973 *proxy_port_id = port_id; 11974 return 0; 11975 } 11976 } 11977 return rte_flow_error_set(error, ENODEV, 11978 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11979 NULL, "unable to find a proxy port"); 11980 } 11981 11982 /** 11983 * Discover IPv6 traffic class ID support in rdma-core and firmware. 11984 * 11985 * @param dev 11986 * Ethernet device. 11987 * 11988 * @return 11989 * 0, rdma-core is good to work with firmware. 11990 * -EOPNOTSUPP, rdma-core could not work with new IPv6 TC ID. 11991 */ 11992 int 11993 mlx5_flow_discover_ipv6_tc_support(struct rte_eth_dev *dev) 11994 { 11995 struct rte_flow_action_set_dscp set_dscp; 11996 struct rte_flow_attr attr; 11997 struct rte_flow_action actions[2]; 11998 struct rte_flow_item items[3]; 11999 struct rte_flow_error error; 12000 uint32_t flow_idx; 12001 12002 memset(&attr, 0, sizeof(attr)); 12003 memset(actions, 0, sizeof(actions)); 12004 memset(items, 0, sizeof(items)); 12005 attr.group = 1; 12006 attr.egress = 1; 12007 items[0].type = RTE_FLOW_ITEM_TYPE_ETH; 12008 items[1].type = RTE_FLOW_ITEM_TYPE_IPV6; 12009 items[2].type = RTE_FLOW_ITEM_TYPE_END; 12010 /* Random value */ 12011 set_dscp.dscp = 9; 12012 actions[0].type = RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP; 12013 actions[0].conf = &set_dscp; 12014 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 12015 12016 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, items, actions, true, &error); 12017 if (!flow_idx) 12018 return -EOPNOTSUPP; 12019 12020 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 12021 return 0; 12022 } 12023 12024 void * 12025 rte_pmd_mlx5_create_geneve_tlv_parser(uint16_t port_id, 12026 const struct rte_pmd_mlx5_geneve_tlv tlv_list[], 12027 uint8_t nb_options) 12028 { 12029 #ifdef HAVE_MLX5_HWS_SUPPORT 12030 return mlx5_geneve_tlv_parser_create(port_id, tlv_list, nb_options); 12031 #else 12032 (void)port_id; 12033 (void)tlv_list; 12034 (void)nb_options; 12035 DRV_LOG(ERR, "%s is not supported.", __func__); 12036 rte_errno = ENOTSUP; 12037 return NULL; 12038 #endif 12039 } 12040 12041 int 12042 rte_pmd_mlx5_destroy_geneve_tlv_parser(void *handle) 12043 { 12044 #ifdef HAVE_MLX5_HWS_SUPPORT 12045 return mlx5_geneve_tlv_parser_destroy(handle); 12046 #else 12047 (void)handle; 12048 DRV_LOG(ERR, "%s is not supported.", __func__); 12049 rte_errno = ENOTSUP; 12050 return -rte_errno; 12051 #endif 12052 } 12053