1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 void 66 mlx5_indirect_list_handles_release(struct rte_eth_dev *dev) 67 { 68 struct mlx5_priv *priv = dev->data->dev_private; 69 #ifdef HAVE_MLX5_HWS_SUPPORT 70 struct rte_flow_error error; 71 #endif 72 73 while (!LIST_EMPTY(&priv->indirect_list_head)) { 74 struct mlx5_indirect_list *e = 75 LIST_FIRST(&priv->indirect_list_head); 76 77 LIST_REMOVE(e, entry); 78 switch (e->type) { 79 #ifdef HAVE_MLX5_HWS_SUPPORT 80 case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR: 81 mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)e); 82 break; 83 case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY: 84 mlx5_destroy_legacy_indirect(dev, e); 85 break; 86 case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT: 87 mlx5_reformat_action_destroy(dev, 88 (struct rte_flow_action_list_handle *)e, &error); 89 break; 90 #endif 91 default: 92 DRV_LOG(ERR, "invalid indirect list type"); 93 MLX5_ASSERT(false); 94 break; 95 } 96 } 97 } 98 99 static int 100 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 101 struct rte_flow *flow, 102 const struct rte_flow_attr *attr, 103 const struct rte_flow_action *app_actions, 104 uint32_t flow_idx, 105 const struct mlx5_flow_tunnel *tunnel, 106 struct tunnel_default_miss_ctx *ctx, 107 struct rte_flow_error *error); 108 static struct mlx5_flow_tunnel * 109 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 110 static void 111 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 112 static uint32_t 113 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 114 const struct mlx5_flow_tunnel *tunnel, 115 uint32_t group, uint32_t *table, 116 struct rte_flow_error *error); 117 118 /** Device flow drivers. */ 119 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 120 121 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 122 123 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 124 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 125 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 126 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 127 #endif 128 #ifdef HAVE_MLX5_HWS_SUPPORT 129 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 130 #endif 131 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 132 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 133 }; 134 135 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 136 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 137 (const int []){ \ 138 __VA_ARGS__, 0, \ 139 } 140 141 /** Node object of input graph for mlx5_flow_expand_rss(). */ 142 struct mlx5_flow_expand_node { 143 const int *const next; 144 /**< 145 * List of next node indexes. Index 0 is interpreted as a terminator. 146 */ 147 const enum rte_flow_item_type type; 148 /**< Pattern item type of current node. */ 149 uint64_t rss_types; 150 /**< 151 * RSS types bit-field associated with this node 152 * (see RTE_ETH_RSS_* definitions). 153 */ 154 uint64_t node_flags; 155 /**< 156 * Bit-fields that define how the node is used in the expansion. 157 * (see MLX5_EXPANSION_NODE_* definitions). 158 */ 159 }; 160 161 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 162 struct mlx5_flow_expand_sqn { 163 uint32_t entries; /** Number of entries */ 164 struct { 165 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 166 uint32_t priority; /**< Priority offset for each expansion. */ 167 } entry[]; 168 }; 169 170 /* Optional expand field. The expansion alg will not go deeper. */ 171 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 172 173 /* The node is not added implicitly as expansion to the flow pattern. 174 * If the node type does not match the flow pattern item type, the 175 * expansion alg will go deeper to its next items. 176 * In the current implementation, the list of next nodes indexes can 177 * have up to one node with this flag set and it has to be the last 178 * node index (before the list terminator). 179 */ 180 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 181 182 /** Object returned by mlx5_flow_expand_rss(). */ 183 struct mlx5_flow_expand_rss { 184 uint32_t entries; 185 /**< Number of entries @p patterns and @p priorities. */ 186 struct { 187 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 188 uint32_t priority; /**< Priority offset for each expansion. */ 189 } entry[]; 190 }; 191 192 static void 193 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 194 195 static const struct mlx5_flow_expand_node * 196 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 197 unsigned int item_idx, 198 const struct mlx5_flow_expand_node graph[], 199 const struct mlx5_flow_expand_node *node); 200 201 static __rte_always_inline int 202 mlx5_need_cache_flow(const struct mlx5_priv *priv, 203 const struct rte_flow_attr *attr) 204 { 205 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 206 (attr ? !attr->group : true) && 207 priv->mode_info.mode == MLX5_FLOW_ENGINE_MODE_STANDBY && 208 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 209 } 210 211 static bool 212 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 213 { 214 switch (item->type) { 215 case RTE_FLOW_ITEM_TYPE_ETH: 216 case RTE_FLOW_ITEM_TYPE_VLAN: 217 case RTE_FLOW_ITEM_TYPE_IPV4: 218 case RTE_FLOW_ITEM_TYPE_IPV6: 219 case RTE_FLOW_ITEM_TYPE_UDP: 220 case RTE_FLOW_ITEM_TYPE_TCP: 221 case RTE_FLOW_ITEM_TYPE_ESP: 222 case RTE_FLOW_ITEM_TYPE_ICMP: 223 case RTE_FLOW_ITEM_TYPE_ICMP6: 224 case RTE_FLOW_ITEM_TYPE_VXLAN: 225 case RTE_FLOW_ITEM_TYPE_NVGRE: 226 case RTE_FLOW_ITEM_TYPE_GRE: 227 case RTE_FLOW_ITEM_TYPE_GENEVE: 228 case RTE_FLOW_ITEM_TYPE_MPLS: 229 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 230 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 231 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 232 case RTE_FLOW_ITEM_TYPE_GTP: 233 return true; 234 default: 235 break; 236 } 237 return false; 238 } 239 240 /** 241 * Network Service Header (NSH) and its next protocol values 242 * are described in RFC-8393. 243 */ 244 static enum rte_flow_item_type 245 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 246 { 247 enum rte_flow_item_type type; 248 249 switch (proto_mask & proto_spec) { 250 case 0: 251 type = RTE_FLOW_ITEM_TYPE_VOID; 252 break; 253 case RTE_VXLAN_GPE_TYPE_IPV4: 254 type = RTE_FLOW_ITEM_TYPE_IPV4; 255 break; 256 case RTE_VXLAN_GPE_TYPE_IPV6: 257 type = RTE_VXLAN_GPE_TYPE_IPV6; 258 break; 259 case RTE_VXLAN_GPE_TYPE_ETH: 260 type = RTE_FLOW_ITEM_TYPE_ETH; 261 break; 262 default: 263 type = RTE_FLOW_ITEM_TYPE_END; 264 } 265 return type; 266 } 267 268 static enum rte_flow_item_type 269 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 270 { 271 enum rte_flow_item_type type; 272 273 switch (proto_mask & proto_spec) { 274 case 0: 275 type = RTE_FLOW_ITEM_TYPE_VOID; 276 break; 277 case IPPROTO_UDP: 278 type = RTE_FLOW_ITEM_TYPE_UDP; 279 break; 280 case IPPROTO_TCP: 281 type = RTE_FLOW_ITEM_TYPE_TCP; 282 break; 283 case IPPROTO_IPIP: 284 type = RTE_FLOW_ITEM_TYPE_IPV4; 285 break; 286 case IPPROTO_IPV6: 287 type = RTE_FLOW_ITEM_TYPE_IPV6; 288 break; 289 case IPPROTO_ESP: 290 type = RTE_FLOW_ITEM_TYPE_ESP; 291 break; 292 default: 293 type = RTE_FLOW_ITEM_TYPE_END; 294 } 295 return type; 296 } 297 298 static enum rte_flow_item_type 299 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 300 rte_be16_t type_mask, bool is_tunnel) 301 { 302 enum rte_flow_item_type type; 303 304 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 305 case 0: 306 type = RTE_FLOW_ITEM_TYPE_VOID; 307 break; 308 case RTE_ETHER_TYPE_TEB: 309 type = is_tunnel ? 310 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 311 break; 312 case RTE_ETHER_TYPE_VLAN: 313 type = !is_tunnel ? 314 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 315 break; 316 case RTE_ETHER_TYPE_IPV4: 317 type = RTE_FLOW_ITEM_TYPE_IPV4; 318 break; 319 case RTE_ETHER_TYPE_IPV6: 320 type = RTE_FLOW_ITEM_TYPE_IPV6; 321 break; 322 default: 323 type = RTE_FLOW_ITEM_TYPE_END; 324 } 325 return type; 326 } 327 328 static enum rte_flow_item_type 329 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 330 { 331 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 332 do { \ 333 const void *m = item->mask; \ 334 const void *s = item->spec; \ 335 mask = m ? \ 336 ((const struct rte_flow_item_##type *)m)->fld : \ 337 rte_flow_item_##type##_mask.fld; \ 338 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 339 } while (0) 340 341 enum rte_flow_item_type ret; 342 uint16_t spec, mask; 343 344 if (item == NULL || item->spec == NULL) 345 return RTE_FLOW_ITEM_TYPE_VOID; 346 switch (item->type) { 347 case RTE_FLOW_ITEM_TYPE_ETH: 348 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 349 if (!mask) 350 return RTE_FLOW_ITEM_TYPE_VOID; 351 ret = mlx5_ethertype_to_item_type(spec, mask, false); 352 break; 353 case RTE_FLOW_ITEM_TYPE_VLAN: 354 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 355 if (!mask) 356 return RTE_FLOW_ITEM_TYPE_VOID; 357 ret = mlx5_ethertype_to_item_type(spec, mask, false); 358 break; 359 case RTE_FLOW_ITEM_TYPE_IPV4: 360 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 361 if (!mask) 362 return RTE_FLOW_ITEM_TYPE_VOID; 363 ret = mlx5_inet_proto_to_item_type(spec, mask); 364 break; 365 case RTE_FLOW_ITEM_TYPE_IPV6: 366 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 367 if (!mask) 368 return RTE_FLOW_ITEM_TYPE_VOID; 369 ret = mlx5_inet_proto_to_item_type(spec, mask); 370 break; 371 case RTE_FLOW_ITEM_TYPE_GENEVE: 372 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 373 ret = mlx5_ethertype_to_item_type(spec, mask, true); 374 break; 375 case RTE_FLOW_ITEM_TYPE_GRE: 376 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 377 ret = mlx5_ethertype_to_item_type(spec, mask, true); 378 break; 379 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 380 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 381 ret = mlx5_nsh_proto_to_item_type(spec, mask); 382 break; 383 default: 384 ret = RTE_FLOW_ITEM_TYPE_VOID; 385 break; 386 } 387 return ret; 388 #undef MLX5_XSET_ITEM_MASK_SPEC 389 } 390 391 static const int * 392 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 393 const int *next_node) 394 { 395 const struct mlx5_flow_expand_node *node = NULL; 396 const int *next = next_node; 397 398 while (next && *next) { 399 /* 400 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 401 * flag set, because they were not found in the flow pattern. 402 */ 403 node = &graph[*next]; 404 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 405 break; 406 next = node->next; 407 } 408 return next; 409 } 410 411 #define MLX5_RSS_EXP_ELT_N 32 412 413 /** 414 * Expand RSS flows into several possible flows according to the RSS hash 415 * fields requested and the driver capabilities. 416 * 417 * @param[out] buf 418 * Buffer to store the result expansion. 419 * @param[in] size 420 * Buffer size in bytes. If 0, @p buf can be NULL. 421 * @param[in] pattern 422 * User flow pattern. 423 * @param[in] types 424 * RSS types to expand (see RTE_ETH_RSS_* definitions). 425 * @param[in] graph 426 * Input graph to expand @p pattern according to @p types. 427 * @param[in] graph_root_index 428 * Index of root node in @p graph, typically 0. 429 * 430 * @return 431 * A positive value representing the size of @p buf in bytes regardless of 432 * @p size on success, a negative errno value otherwise and rte_errno is 433 * set, the following errors are defined: 434 * 435 * -E2BIG: graph-depth @p graph is too deep. 436 * -EINVAL: @p size has not enough space for expanded pattern. 437 */ 438 static int 439 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 440 const struct rte_flow_item *pattern, uint64_t types, 441 const struct mlx5_flow_expand_node graph[], 442 int graph_root_index) 443 { 444 const struct rte_flow_item *item; 445 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 446 const int *next_node; 447 const int *stack[MLX5_RSS_EXP_ELT_N]; 448 int stack_pos = 0; 449 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 450 unsigned int i, item_idx, last_expand_item_idx = 0; 451 size_t lsize; 452 size_t user_pattern_size = 0; 453 void *addr = NULL; 454 const struct mlx5_flow_expand_node *next = NULL; 455 struct rte_flow_item missed_item; 456 int missed = 0; 457 int elt = 0; 458 const struct rte_flow_item *last_expand_item = NULL; 459 460 memset(&missed_item, 0, sizeof(missed_item)); 461 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 462 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 463 if (lsize > size) 464 return -EINVAL; 465 buf->entry[0].priority = 0; 466 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 467 buf->entries = 0; 468 addr = buf->entry[0].pattern; 469 for (item = pattern, item_idx = 0; 470 item->type != RTE_FLOW_ITEM_TYPE_END; 471 item++, item_idx++) { 472 if (!mlx5_flow_is_rss_expandable_item(item)) { 473 user_pattern_size += sizeof(*item); 474 continue; 475 } 476 last_expand_item = item; 477 last_expand_item_idx = item_idx; 478 i = 0; 479 while (node->next && node->next[i]) { 480 next = &graph[node->next[i]]; 481 if (next->type == item->type) 482 break; 483 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 484 node = next; 485 i = 0; 486 } else { 487 ++i; 488 } 489 } 490 if (next) 491 node = next; 492 user_pattern_size += sizeof(*item); 493 } 494 user_pattern_size += sizeof(*item); /* Handle END item. */ 495 lsize += user_pattern_size; 496 if (lsize > size) 497 return -EINVAL; 498 /* Copy the user pattern in the first entry of the buffer. */ 499 rte_memcpy(addr, pattern, user_pattern_size); 500 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 501 buf->entries = 1; 502 /* Start expanding. */ 503 memset(flow_items, 0, sizeof(flow_items)); 504 user_pattern_size -= sizeof(*item); 505 /* 506 * Check if the last valid item has spec set, need complete pattern, 507 * and the pattern can be used for expansion. 508 */ 509 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 510 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 511 /* Item type END indicates expansion is not required. */ 512 return lsize; 513 } 514 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 515 next = NULL; 516 missed = 1; 517 i = 0; 518 while (node->next && node->next[i]) { 519 next = &graph[node->next[i]]; 520 if (next->type == missed_item.type) { 521 flow_items[0].type = missed_item.type; 522 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 523 break; 524 } 525 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 526 node = next; 527 i = 0; 528 } else { 529 ++i; 530 } 531 next = NULL; 532 } 533 } 534 if (next && missed) { 535 elt = 2; /* missed item + item end. */ 536 node = next; 537 lsize += elt * sizeof(*item) + user_pattern_size; 538 if (lsize > size) 539 return -EINVAL; 540 if (node->rss_types & types) { 541 buf->entry[buf->entries].priority = 1; 542 buf->entry[buf->entries].pattern = addr; 543 buf->entries++; 544 rte_memcpy(addr, buf->entry[0].pattern, 545 user_pattern_size); 546 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 547 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 548 addr = (void *)(((uintptr_t)addr) + 549 elt * sizeof(*item)); 550 } 551 } else if (last_expand_item != NULL) { 552 node = mlx5_flow_expand_rss_adjust_node(pattern, 553 last_expand_item_idx, graph, node); 554 } 555 memset(flow_items, 0, sizeof(flow_items)); 556 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 557 node->next); 558 stack[stack_pos] = next_node; 559 node = next_node ? &graph[*next_node] : NULL; 560 while (node) { 561 flow_items[stack_pos].type = node->type; 562 if (node->rss_types & types) { 563 size_t n; 564 /* 565 * compute the number of items to copy from the 566 * expansion and copy it. 567 * When the stack_pos is 0, there are 1 element in it, 568 * plus the addition END item. 569 */ 570 elt = stack_pos + 2; 571 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 572 lsize += elt * sizeof(*item) + user_pattern_size; 573 if (lsize > size) 574 return -EINVAL; 575 n = elt * sizeof(*item); 576 MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); 577 buf->entry[buf->entries].priority = 578 stack_pos + 1 + missed; 579 buf->entry[buf->entries].pattern = addr; 580 buf->entries++; 581 rte_memcpy(addr, buf->entry[0].pattern, 582 user_pattern_size); 583 addr = (void *)(((uintptr_t)addr) + 584 user_pattern_size); 585 rte_memcpy(addr, &missed_item, 586 missed * sizeof(*item)); 587 addr = (void *)(((uintptr_t)addr) + 588 missed * sizeof(*item)); 589 rte_memcpy(addr, flow_items, n); 590 addr = (void *)(((uintptr_t)addr) + n); 591 } 592 /* Go deeper. */ 593 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 594 node->next) { 595 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 596 node->next); 597 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 598 rte_errno = E2BIG; 599 return -rte_errno; 600 } 601 stack[stack_pos] = next_node; 602 } else if (*(next_node + 1)) { 603 /* Follow up with the next possibility. */ 604 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 605 ++next_node); 606 } else if (!stack_pos) { 607 /* 608 * Completing the traverse over the different paths. 609 * The next_node is advanced to the terminator. 610 */ 611 ++next_node; 612 } else { 613 /* Move to the next path. */ 614 while (stack_pos) { 615 next_node = stack[--stack_pos]; 616 next_node++; 617 if (*next_node) 618 break; 619 } 620 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 621 next_node); 622 stack[stack_pos] = next_node; 623 } 624 node = next_node && *next_node ? &graph[*next_node] : NULL; 625 }; 626 return lsize; 627 } 628 629 /** 630 * Expand SQN flows into several possible flows according to the Tx queue 631 * number 632 * 633 * @param[in] buf 634 * Buffer to store the result expansion. 635 * @param[in] size 636 * Buffer size in bytes. If 0, @p buf can be NULL. 637 * @param[in] pattern 638 * User flow pattern. 639 * @param[in] sq_specs 640 * Buffer to store sq spec. 641 * 642 * @return 643 * 0 for success and negative value for failure 644 * 645 */ 646 static int 647 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 648 const struct rte_flow_item *pattern, 649 struct mlx5_rte_flow_item_sq *sq_specs) 650 { 651 const struct rte_flow_item *item; 652 bool port_representor = false; 653 size_t user_pattern_size = 0; 654 struct rte_eth_dev *dev; 655 struct mlx5_priv *priv; 656 void *addr = NULL; 657 uint16_t port_id; 658 size_t lsize; 659 int elt = 2; 660 uint16_t i; 661 662 buf->entries = 0; 663 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 664 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 665 const struct rte_flow_item_ethdev *pid_v = item->spec; 666 667 if (!pid_v) 668 return 0; 669 port_id = pid_v->port_id; 670 port_representor = true; 671 } 672 user_pattern_size += sizeof(*item); 673 } 674 if (!port_representor) 675 return 0; 676 dev = &rte_eth_devices[port_id]; 677 priv = dev->data->dev_private; 678 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 679 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 680 sizeof(buf->entry[0]) * priv->txqs_n; 681 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 682 return -EINVAL; 683 addr = buf->entry[0].pattern; 684 for (i = 0; i != priv->txqs_n; ++i) { 685 struct rte_flow_item pattern_add[] = { 686 { 687 .type = (enum rte_flow_item_type) 688 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 689 .spec = &sq_specs[i], 690 }, 691 { 692 .type = RTE_FLOW_ITEM_TYPE_END, 693 }, 694 }; 695 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 696 697 if (txq == NULL) 698 return -EINVAL; 699 buf->entry[i].pattern = addr; 700 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 701 mlx5_txq_release(dev, i); 702 rte_memcpy(addr, pattern, user_pattern_size); 703 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 704 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 705 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 706 buf->entries++; 707 } 708 return 0; 709 } 710 711 enum mlx5_expansion { 712 MLX5_EXPANSION_ROOT, 713 MLX5_EXPANSION_ROOT_OUTER, 714 MLX5_EXPANSION_OUTER_ETH, 715 MLX5_EXPANSION_OUTER_VLAN, 716 MLX5_EXPANSION_OUTER_IPV4, 717 MLX5_EXPANSION_OUTER_IPV4_UDP, 718 MLX5_EXPANSION_OUTER_IPV4_TCP, 719 MLX5_EXPANSION_OUTER_IPV4_ESP, 720 MLX5_EXPANSION_OUTER_IPV4_ICMP, 721 MLX5_EXPANSION_OUTER_IPV6, 722 MLX5_EXPANSION_OUTER_IPV6_UDP, 723 MLX5_EXPANSION_OUTER_IPV6_TCP, 724 MLX5_EXPANSION_OUTER_IPV6_ESP, 725 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 726 MLX5_EXPANSION_VXLAN, 727 MLX5_EXPANSION_STD_VXLAN, 728 MLX5_EXPANSION_L3_VXLAN, 729 MLX5_EXPANSION_VXLAN_GPE, 730 MLX5_EXPANSION_GRE, 731 MLX5_EXPANSION_NVGRE, 732 MLX5_EXPANSION_GRE_KEY, 733 MLX5_EXPANSION_MPLS, 734 MLX5_EXPANSION_ETH, 735 MLX5_EXPANSION_VLAN, 736 MLX5_EXPANSION_IPV4, 737 MLX5_EXPANSION_IPV4_UDP, 738 MLX5_EXPANSION_IPV4_TCP, 739 MLX5_EXPANSION_IPV4_ESP, 740 MLX5_EXPANSION_IPV4_ICMP, 741 MLX5_EXPANSION_IPV6, 742 MLX5_EXPANSION_IPV6_UDP, 743 MLX5_EXPANSION_IPV6_TCP, 744 MLX5_EXPANSION_IPV6_ESP, 745 MLX5_EXPANSION_IPV6_ICMP6, 746 MLX5_EXPANSION_IPV6_FRAG_EXT, 747 MLX5_EXPANSION_GTP, 748 MLX5_EXPANSION_GENEVE, 749 }; 750 751 /** Supported expansion of items. */ 752 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 753 [MLX5_EXPANSION_ROOT] = { 754 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 755 MLX5_EXPANSION_IPV4, 756 MLX5_EXPANSION_IPV6), 757 .type = RTE_FLOW_ITEM_TYPE_END, 758 }, 759 [MLX5_EXPANSION_ROOT_OUTER] = { 760 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 761 MLX5_EXPANSION_OUTER_IPV4, 762 MLX5_EXPANSION_OUTER_IPV6), 763 .type = RTE_FLOW_ITEM_TYPE_END, 764 }, 765 [MLX5_EXPANSION_OUTER_ETH] = { 766 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 767 .type = RTE_FLOW_ITEM_TYPE_ETH, 768 .rss_types = 0, 769 }, 770 [MLX5_EXPANSION_OUTER_VLAN] = { 771 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 772 MLX5_EXPANSION_OUTER_IPV6), 773 .type = RTE_FLOW_ITEM_TYPE_VLAN, 774 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 775 }, 776 [MLX5_EXPANSION_OUTER_IPV4] = { 777 .next = MLX5_FLOW_EXPAND_RSS_NEXT 778 (MLX5_EXPANSION_OUTER_IPV4_UDP, 779 MLX5_EXPANSION_OUTER_IPV4_TCP, 780 MLX5_EXPANSION_OUTER_IPV4_ESP, 781 MLX5_EXPANSION_OUTER_IPV4_ICMP, 782 MLX5_EXPANSION_GRE, 783 MLX5_EXPANSION_NVGRE, 784 MLX5_EXPANSION_IPV4, 785 MLX5_EXPANSION_IPV6), 786 .type = RTE_FLOW_ITEM_TYPE_IPV4, 787 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 788 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 789 }, 790 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 791 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 792 MLX5_EXPANSION_VXLAN_GPE, 793 MLX5_EXPANSION_MPLS, 794 MLX5_EXPANSION_GENEVE, 795 MLX5_EXPANSION_GTP), 796 .type = RTE_FLOW_ITEM_TYPE_UDP, 797 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 798 }, 799 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 800 .type = RTE_FLOW_ITEM_TYPE_TCP, 801 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 802 }, 803 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 804 .type = RTE_FLOW_ITEM_TYPE_ESP, 805 .rss_types = RTE_ETH_RSS_ESP, 806 }, 807 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 808 .type = RTE_FLOW_ITEM_TYPE_ICMP, 809 }, 810 [MLX5_EXPANSION_OUTER_IPV6] = { 811 .next = MLX5_FLOW_EXPAND_RSS_NEXT 812 (MLX5_EXPANSION_OUTER_IPV6_UDP, 813 MLX5_EXPANSION_OUTER_IPV6_TCP, 814 MLX5_EXPANSION_OUTER_IPV6_ESP, 815 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 816 MLX5_EXPANSION_IPV4, 817 MLX5_EXPANSION_IPV6, 818 MLX5_EXPANSION_GRE, 819 MLX5_EXPANSION_NVGRE), 820 .type = RTE_FLOW_ITEM_TYPE_IPV6, 821 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 822 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 823 }, 824 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 825 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 826 MLX5_EXPANSION_VXLAN_GPE, 827 MLX5_EXPANSION_MPLS, 828 MLX5_EXPANSION_GENEVE, 829 MLX5_EXPANSION_GTP), 830 .type = RTE_FLOW_ITEM_TYPE_UDP, 831 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 832 }, 833 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 834 .type = RTE_FLOW_ITEM_TYPE_TCP, 835 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 836 }, 837 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 838 .type = RTE_FLOW_ITEM_TYPE_ESP, 839 .rss_types = RTE_ETH_RSS_ESP, 840 }, 841 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 842 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 843 }, 844 [MLX5_EXPANSION_VXLAN] = { 845 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 846 MLX5_EXPANSION_IPV4, 847 MLX5_EXPANSION_IPV6), 848 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 849 }, 850 [MLX5_EXPANSION_STD_VXLAN] = { 851 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 852 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 853 }, 854 [MLX5_EXPANSION_L3_VXLAN] = { 855 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 856 MLX5_EXPANSION_IPV6), 857 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 858 }, 859 [MLX5_EXPANSION_VXLAN_GPE] = { 860 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 861 MLX5_EXPANSION_IPV4, 862 MLX5_EXPANSION_IPV6), 863 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 864 }, 865 [MLX5_EXPANSION_GRE] = { 866 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 867 MLX5_EXPANSION_IPV4, 868 MLX5_EXPANSION_IPV6, 869 MLX5_EXPANSION_GRE_KEY, 870 MLX5_EXPANSION_MPLS), 871 .type = RTE_FLOW_ITEM_TYPE_GRE, 872 }, 873 [MLX5_EXPANSION_GRE_KEY] = { 874 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 875 MLX5_EXPANSION_IPV6, 876 MLX5_EXPANSION_MPLS), 877 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 878 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 879 }, 880 [MLX5_EXPANSION_NVGRE] = { 881 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 882 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 883 }, 884 [MLX5_EXPANSION_MPLS] = { 885 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 886 MLX5_EXPANSION_IPV6, 887 MLX5_EXPANSION_ETH), 888 .type = RTE_FLOW_ITEM_TYPE_MPLS, 889 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 890 }, 891 [MLX5_EXPANSION_ETH] = { 892 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 893 .type = RTE_FLOW_ITEM_TYPE_ETH, 894 }, 895 [MLX5_EXPANSION_VLAN] = { 896 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 897 MLX5_EXPANSION_IPV6), 898 .type = RTE_FLOW_ITEM_TYPE_VLAN, 899 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 900 }, 901 [MLX5_EXPANSION_IPV4] = { 902 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 903 MLX5_EXPANSION_IPV4_TCP, 904 MLX5_EXPANSION_IPV4_ESP, 905 MLX5_EXPANSION_IPV4_ICMP), 906 .type = RTE_FLOW_ITEM_TYPE_IPV4, 907 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 908 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 909 }, 910 [MLX5_EXPANSION_IPV4_UDP] = { 911 .type = RTE_FLOW_ITEM_TYPE_UDP, 912 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 913 }, 914 [MLX5_EXPANSION_IPV4_TCP] = { 915 .type = RTE_FLOW_ITEM_TYPE_TCP, 916 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 917 }, 918 [MLX5_EXPANSION_IPV4_ESP] = { 919 .type = RTE_FLOW_ITEM_TYPE_ESP, 920 .rss_types = RTE_ETH_RSS_ESP, 921 }, 922 [MLX5_EXPANSION_IPV4_ICMP] = { 923 .type = RTE_FLOW_ITEM_TYPE_ICMP, 924 }, 925 [MLX5_EXPANSION_IPV6] = { 926 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 927 MLX5_EXPANSION_IPV6_TCP, 928 MLX5_EXPANSION_IPV6_ESP, 929 MLX5_EXPANSION_IPV6_ICMP6, 930 MLX5_EXPANSION_IPV6_FRAG_EXT), 931 .type = RTE_FLOW_ITEM_TYPE_IPV6, 932 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 933 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 934 }, 935 [MLX5_EXPANSION_IPV6_UDP] = { 936 .type = RTE_FLOW_ITEM_TYPE_UDP, 937 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 938 }, 939 [MLX5_EXPANSION_IPV6_TCP] = { 940 .type = RTE_FLOW_ITEM_TYPE_TCP, 941 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 942 }, 943 [MLX5_EXPANSION_IPV6_ESP] = { 944 .type = RTE_FLOW_ITEM_TYPE_ESP, 945 .rss_types = RTE_ETH_RSS_ESP, 946 }, 947 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 948 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 949 }, 950 [MLX5_EXPANSION_IPV6_ICMP6] = { 951 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 952 }, 953 [MLX5_EXPANSION_GTP] = { 954 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 955 MLX5_EXPANSION_IPV6), 956 .type = RTE_FLOW_ITEM_TYPE_GTP, 957 }, 958 [MLX5_EXPANSION_GENEVE] = { 959 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 960 MLX5_EXPANSION_IPV4, 961 MLX5_EXPANSION_IPV6), 962 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 963 }, 964 }; 965 966 static struct rte_flow_action_handle * 967 mlx5_action_handle_create(struct rte_eth_dev *dev, 968 const struct rte_flow_indir_action_conf *conf, 969 const struct rte_flow_action *action, 970 struct rte_flow_error *error); 971 static int mlx5_action_handle_destroy 972 (struct rte_eth_dev *dev, 973 struct rte_flow_action_handle *handle, 974 struct rte_flow_error *error); 975 static int mlx5_action_handle_update 976 (struct rte_eth_dev *dev, 977 struct rte_flow_action_handle *handle, 978 const void *update, 979 struct rte_flow_error *error); 980 static int mlx5_action_handle_query 981 (struct rte_eth_dev *dev, 982 const struct rte_flow_action_handle *handle, 983 void *data, 984 struct rte_flow_error *error); 985 static int 986 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 987 struct rte_flow_tunnel *app_tunnel, 988 struct rte_flow_action **actions, 989 uint32_t *num_of_actions, 990 struct rte_flow_error *error); 991 static int 992 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 993 struct rte_flow_tunnel *app_tunnel, 994 struct rte_flow_item **items, 995 uint32_t *num_of_items, 996 struct rte_flow_error *error); 997 static int 998 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 999 struct rte_flow_item *pmd_items, 1000 uint32_t num_items, struct rte_flow_error *err); 1001 static int 1002 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 1003 struct rte_flow_action *pmd_actions, 1004 uint32_t num_actions, 1005 struct rte_flow_error *err); 1006 static int 1007 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 1008 struct rte_mbuf *m, 1009 struct rte_flow_restore_info *info, 1010 struct rte_flow_error *err); 1011 static struct rte_flow_item_flex_handle * 1012 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 1013 const struct rte_flow_item_flex_conf *conf, 1014 struct rte_flow_error *error); 1015 static int 1016 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 1017 const struct rte_flow_item_flex_handle *handle, 1018 struct rte_flow_error *error); 1019 static int 1020 mlx5_flow_info_get(struct rte_eth_dev *dev, 1021 struct rte_flow_port_info *port_info, 1022 struct rte_flow_queue_info *queue_info, 1023 struct rte_flow_error *error); 1024 static int 1025 mlx5_flow_port_configure(struct rte_eth_dev *dev, 1026 const struct rte_flow_port_attr *port_attr, 1027 uint16_t nb_queue, 1028 const struct rte_flow_queue_attr *queue_attr[], 1029 struct rte_flow_error *err); 1030 1031 static struct rte_flow_pattern_template * 1032 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 1033 const struct rte_flow_pattern_template_attr *attr, 1034 const struct rte_flow_item items[], 1035 struct rte_flow_error *error); 1036 1037 static int 1038 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1039 struct rte_flow_pattern_template *template, 1040 struct rte_flow_error *error); 1041 static struct rte_flow_actions_template * 1042 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1043 const struct rte_flow_actions_template_attr *attr, 1044 const struct rte_flow_action actions[], 1045 const struct rte_flow_action masks[], 1046 struct rte_flow_error *error); 1047 static int 1048 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1049 struct rte_flow_actions_template *template, 1050 struct rte_flow_error *error); 1051 1052 static struct rte_flow_template_table * 1053 mlx5_flow_table_create(struct rte_eth_dev *dev, 1054 const struct rte_flow_template_table_attr *attr, 1055 struct rte_flow_pattern_template *item_templates[], 1056 uint8_t nb_item_templates, 1057 struct rte_flow_actions_template *action_templates[], 1058 uint8_t nb_action_templates, 1059 struct rte_flow_error *error); 1060 static int 1061 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1062 struct rte_flow_template_table *table, 1063 struct rte_flow_error *error); 1064 static struct rte_flow * 1065 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1066 uint32_t queue, 1067 const struct rte_flow_op_attr *attr, 1068 struct rte_flow_template_table *table, 1069 const struct rte_flow_item items[], 1070 uint8_t pattern_template_index, 1071 const struct rte_flow_action actions[], 1072 uint8_t action_template_index, 1073 void *user_data, 1074 struct rte_flow_error *error); 1075 static struct rte_flow * 1076 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 1077 uint32_t queue, 1078 const struct rte_flow_op_attr *attr, 1079 struct rte_flow_template_table *table, 1080 uint32_t rule_index, 1081 const struct rte_flow_action actions[], 1082 uint8_t action_template_index, 1083 void *user_data, 1084 struct rte_flow_error *error); 1085 static int 1086 mlx5_flow_async_flow_update(struct rte_eth_dev *dev, 1087 uint32_t queue, 1088 const struct rte_flow_op_attr *attr, 1089 struct rte_flow *flow, 1090 const struct rte_flow_action actions[], 1091 uint8_t action_template_index, 1092 void *user_data, 1093 struct rte_flow_error *error); 1094 static int 1095 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1096 uint32_t queue, 1097 const struct rte_flow_op_attr *attr, 1098 struct rte_flow *flow, 1099 void *user_data, 1100 struct rte_flow_error *error); 1101 static int 1102 mlx5_flow_pull(struct rte_eth_dev *dev, 1103 uint32_t queue, 1104 struct rte_flow_op_result res[], 1105 uint16_t n_res, 1106 struct rte_flow_error *error); 1107 static int 1108 mlx5_flow_push(struct rte_eth_dev *dev, 1109 uint32_t queue, 1110 struct rte_flow_error *error); 1111 1112 static struct rte_flow_action_handle * 1113 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1114 const struct rte_flow_op_attr *attr, 1115 const struct rte_flow_indir_action_conf *conf, 1116 const struct rte_flow_action *action, 1117 void *user_data, 1118 struct rte_flow_error *error); 1119 1120 static int 1121 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1122 const struct rte_flow_op_attr *attr, 1123 struct rte_flow_action_handle *handle, 1124 const void *update, 1125 void *user_data, 1126 struct rte_flow_error *error); 1127 1128 static int 1129 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1130 const struct rte_flow_op_attr *attr, 1131 struct rte_flow_action_handle *handle, 1132 void *user_data, 1133 struct rte_flow_error *error); 1134 1135 static int 1136 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1137 const struct rte_flow_op_attr *attr, 1138 const struct rte_flow_action_handle *handle, 1139 void *data, 1140 void *user_data, 1141 struct rte_flow_error *error); 1142 static int 1143 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1144 struct rte_flow_action_handle *handle, 1145 const void *update, void *query, 1146 enum rte_flow_query_update_mode qu_mode, 1147 struct rte_flow_error *error); 1148 static int 1149 mlx5_flow_async_action_handle_query_update 1150 (struct rte_eth_dev *dev, uint32_t queue_id, 1151 const struct rte_flow_op_attr *op_attr, 1152 struct rte_flow_action_handle *action_handle, 1153 const void *update, void *query, 1154 enum rte_flow_query_update_mode qu_mode, 1155 void *user_data, struct rte_flow_error *error); 1156 1157 static struct rte_flow_action_list_handle * 1158 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 1159 const struct rte_flow_indir_action_conf *conf, 1160 const struct rte_flow_action *actions, 1161 struct rte_flow_error *error); 1162 1163 static int 1164 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 1165 struct rte_flow_action_list_handle *handle, 1166 struct rte_flow_error *error); 1167 1168 static struct rte_flow_action_list_handle * 1169 mlx5_flow_async_action_list_handle_create(struct rte_eth_dev *dev, uint32_t queue_id, 1170 const struct rte_flow_op_attr *attr, 1171 const struct 1172 rte_flow_indir_action_conf *conf, 1173 const struct rte_flow_action *actions, 1174 void *user_data, 1175 struct rte_flow_error *error); 1176 static int 1177 mlx5_flow_async_action_list_handle_destroy 1178 (struct rte_eth_dev *dev, uint32_t queue_id, 1179 const struct rte_flow_op_attr *op_attr, 1180 struct rte_flow_action_list_handle *action_handle, 1181 void *user_data, struct rte_flow_error *error); 1182 static int 1183 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 1184 const 1185 struct rte_flow_action_list_handle *handle, 1186 const void **update, void **query, 1187 enum rte_flow_query_update_mode mode, 1188 struct rte_flow_error *error); 1189 static int 1190 mlx5_flow_async_action_list_handle_query_update(struct rte_eth_dev *dev, 1191 uint32_t queue_id, 1192 const struct rte_flow_op_attr *attr, 1193 const struct 1194 rte_flow_action_list_handle *handle, 1195 const void **update, 1196 void **query, 1197 enum rte_flow_query_update_mode mode, 1198 void *user_data, 1199 struct rte_flow_error *error); 1200 static const struct rte_flow_ops mlx5_flow_ops = { 1201 .validate = mlx5_flow_validate, 1202 .create = mlx5_flow_create, 1203 .destroy = mlx5_flow_destroy, 1204 .flush = mlx5_flow_flush, 1205 .isolate = mlx5_flow_isolate, 1206 .query = mlx5_flow_query, 1207 .dev_dump = mlx5_flow_dev_dump, 1208 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1209 .get_aged_flows = mlx5_flow_get_aged_flows, 1210 .action_handle_create = mlx5_action_handle_create, 1211 .action_handle_destroy = mlx5_action_handle_destroy, 1212 .action_handle_update = mlx5_action_handle_update, 1213 .action_handle_query = mlx5_action_handle_query, 1214 .action_handle_query_update = mlx5_action_handle_query_update, 1215 .action_list_handle_create = mlx5_action_list_handle_create, 1216 .action_list_handle_destroy = mlx5_action_list_handle_destroy, 1217 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1218 .tunnel_match = mlx5_flow_tunnel_match, 1219 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1220 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1221 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1222 .flex_item_create = mlx5_flow_flex_item_create, 1223 .flex_item_release = mlx5_flow_flex_item_release, 1224 .info_get = mlx5_flow_info_get, 1225 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1226 .configure = mlx5_flow_port_configure, 1227 .pattern_template_create = mlx5_flow_pattern_template_create, 1228 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1229 .actions_template_create = mlx5_flow_actions_template_create, 1230 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1231 .template_table_create = mlx5_flow_table_create, 1232 .template_table_destroy = mlx5_flow_table_destroy, 1233 .async_create = mlx5_flow_async_flow_create, 1234 .async_create_by_index = mlx5_flow_async_flow_create_by_index, 1235 .async_destroy = mlx5_flow_async_flow_destroy, 1236 .pull = mlx5_flow_pull, 1237 .push = mlx5_flow_push, 1238 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1239 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1240 .async_action_handle_query_update = 1241 mlx5_flow_async_action_handle_query_update, 1242 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1243 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1244 .async_actions_update = mlx5_flow_async_flow_update, 1245 .async_action_list_handle_create = 1246 mlx5_flow_async_action_list_handle_create, 1247 .async_action_list_handle_destroy = 1248 mlx5_flow_async_action_list_handle_destroy, 1249 .action_list_handle_query_update = 1250 mlx5_flow_action_list_handle_query_update, 1251 .async_action_list_handle_query_update = 1252 mlx5_flow_async_action_list_handle_query_update, 1253 }; 1254 1255 /* Tunnel information. */ 1256 struct mlx5_flow_tunnel_info { 1257 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1258 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1259 }; 1260 1261 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1262 { 1263 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1264 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1265 }, 1266 { 1267 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1268 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1269 }, 1270 { 1271 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1272 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1273 }, 1274 { 1275 .tunnel = MLX5_FLOW_LAYER_GRE, 1276 .ptype = RTE_PTYPE_TUNNEL_GRE, 1277 }, 1278 { 1279 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1280 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1281 }, 1282 { 1283 .tunnel = MLX5_FLOW_LAYER_MPLS, 1284 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1285 }, 1286 { 1287 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1288 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1289 }, 1290 { 1291 .tunnel = MLX5_FLOW_LAYER_IPIP, 1292 .ptype = RTE_PTYPE_TUNNEL_IP, 1293 }, 1294 { 1295 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1296 .ptype = RTE_PTYPE_TUNNEL_IP, 1297 }, 1298 { 1299 .tunnel = MLX5_FLOW_LAYER_GTP, 1300 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1301 }, 1302 }; 1303 1304 1305 1306 /** 1307 * Translate tag ID to register. 1308 * 1309 * @param[in] dev 1310 * Pointer to the Ethernet device structure. 1311 * @param[in] feature 1312 * The feature that request the register. 1313 * @param[in] id 1314 * The request register ID. 1315 * @param[out] error 1316 * Error description in case of any. 1317 * 1318 * @return 1319 * The request register on success, a negative errno 1320 * value otherwise and rte_errno is set. 1321 */ 1322 int 1323 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1324 enum mlx5_feature_name feature, 1325 uint32_t id, 1326 struct rte_flow_error *error) 1327 { 1328 struct mlx5_priv *priv = dev->data->dev_private; 1329 struct mlx5_sh_config *config = &priv->sh->config; 1330 enum modify_reg start_reg; 1331 bool skip_mtr_reg = false; 1332 1333 switch (feature) { 1334 case MLX5_HAIRPIN_RX: 1335 return REG_B; 1336 case MLX5_HAIRPIN_TX: 1337 return REG_A; 1338 case MLX5_METADATA_RX: 1339 switch (config->dv_xmeta_en) { 1340 case MLX5_XMETA_MODE_LEGACY: 1341 return REG_B; 1342 case MLX5_XMETA_MODE_META16: 1343 return REG_C_0; 1344 case MLX5_XMETA_MODE_META32: 1345 return REG_C_1; 1346 case MLX5_XMETA_MODE_META32_HWS: 1347 return REG_C_1; 1348 } 1349 break; 1350 case MLX5_METADATA_TX: 1351 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1352 return REG_C_1; 1353 } else { 1354 return REG_A; 1355 } 1356 case MLX5_METADATA_FDB: 1357 switch (config->dv_xmeta_en) { 1358 case MLX5_XMETA_MODE_LEGACY: 1359 return REG_NON; 1360 case MLX5_XMETA_MODE_META16: 1361 return REG_C_0; 1362 case MLX5_XMETA_MODE_META32: 1363 return REG_C_1; 1364 case MLX5_XMETA_MODE_META32_HWS: 1365 return REG_C_1; 1366 } 1367 break; 1368 case MLX5_FLOW_MARK: 1369 switch (config->dv_xmeta_en) { 1370 case MLX5_XMETA_MODE_LEGACY: 1371 case MLX5_XMETA_MODE_META32_HWS: 1372 return REG_NON; 1373 case MLX5_XMETA_MODE_META16: 1374 return REG_C_1; 1375 case MLX5_XMETA_MODE_META32: 1376 return REG_C_0; 1377 } 1378 break; 1379 case MLX5_MTR_ID: 1380 /* 1381 * If meter color and meter id share one register, flow match 1382 * should use the meter color register for match. 1383 */ 1384 if (priv->mtr_reg_share) 1385 return priv->mtr_color_reg; 1386 else 1387 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1388 REG_C_3; 1389 case MLX5_MTR_COLOR: 1390 case MLX5_ASO_FLOW_HIT: 1391 case MLX5_ASO_CONNTRACK: 1392 case MLX5_SAMPLE_ID: 1393 /* All features use the same REG_C. */ 1394 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1395 return priv->mtr_color_reg; 1396 case MLX5_COPY_MARK: 1397 /* 1398 * Metadata COPY_MARK register using is in meter suffix sub 1399 * flow while with meter. It's safe to share the same register. 1400 */ 1401 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1402 case MLX5_APP_TAG: 1403 /* 1404 * If meter is enable, it will engage the register for color 1405 * match and flow match. If meter color match is not using the 1406 * REG_C_2, need to skip the REG_C_x be used by meter color 1407 * match. 1408 * If meter is disable, free to use all available registers. 1409 */ 1410 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1411 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1412 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1413 if (id > (uint32_t)(REG_C_7 - start_reg)) 1414 return rte_flow_error_set(error, EINVAL, 1415 RTE_FLOW_ERROR_TYPE_ITEM, 1416 NULL, "invalid tag id"); 1417 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1418 return rte_flow_error_set(error, ENOTSUP, 1419 RTE_FLOW_ERROR_TYPE_ITEM, 1420 NULL, "unsupported tag id"); 1421 /* 1422 * This case means meter is using the REG_C_x great than 2. 1423 * Take care not to conflict with meter color REG_C_x. 1424 * If the available index REG_C_y >= REG_C_x, skip the 1425 * color register. 1426 */ 1427 if (skip_mtr_reg && priv->sh->flow_mreg_c 1428 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1429 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1430 return rte_flow_error_set(error, EINVAL, 1431 RTE_FLOW_ERROR_TYPE_ITEM, 1432 NULL, "invalid tag id"); 1433 if (priv->sh->flow_mreg_c 1434 [id + 1 + start_reg - REG_C_0] != REG_NON) 1435 return priv->sh->flow_mreg_c 1436 [id + 1 + start_reg - REG_C_0]; 1437 return rte_flow_error_set(error, ENOTSUP, 1438 RTE_FLOW_ERROR_TYPE_ITEM, 1439 NULL, "unsupported tag id"); 1440 } 1441 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1442 } 1443 MLX5_ASSERT(false); 1444 return rte_flow_error_set(error, EINVAL, 1445 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1446 NULL, "invalid feature name"); 1447 } 1448 1449 /** 1450 * Check extensive flow metadata register support. 1451 * 1452 * @param dev 1453 * Pointer to rte_eth_dev structure. 1454 * 1455 * @return 1456 * True if device supports extensive flow metadata register, otherwise false. 1457 */ 1458 bool 1459 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1460 { 1461 struct mlx5_priv *priv = dev->data->dev_private; 1462 1463 /* 1464 * Having available reg_c can be regarded inclusively as supporting 1465 * extensive flow metadata register, which could mean, 1466 * - metadata register copy action by modify header. 1467 * - 16 modify header actions is supported. 1468 * - reg_c's are preserved across different domain (FDB and NIC) on 1469 * packet loopback by flow lookup miss. 1470 */ 1471 return priv->sh->flow_mreg_c[2] != REG_NON; 1472 } 1473 1474 /** 1475 * Get the lowest priority. 1476 * 1477 * @param[in] dev 1478 * Pointer to the Ethernet device structure. 1479 * @param[in] attributes 1480 * Pointer to device flow rule attributes. 1481 * 1482 * @return 1483 * The value of lowest priority of flow. 1484 */ 1485 uint32_t 1486 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1487 const struct rte_flow_attr *attr) 1488 { 1489 struct mlx5_priv *priv = dev->data->dev_private; 1490 1491 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1492 return priv->sh->flow_max_priority - 2; 1493 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1494 } 1495 1496 /** 1497 * Calculate matcher priority of the flow. 1498 * 1499 * @param[in] dev 1500 * Pointer to the Ethernet device structure. 1501 * @param[in] attr 1502 * Pointer to device flow rule attributes. 1503 * @param[in] subpriority 1504 * The priority based on the items. 1505 * @param[in] external 1506 * Flow is user flow. 1507 * @return 1508 * The matcher priority of the flow. 1509 */ 1510 uint16_t 1511 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1512 const struct rte_flow_attr *attr, 1513 uint32_t subpriority, bool external) 1514 { 1515 uint16_t priority = (uint16_t)attr->priority; 1516 struct mlx5_priv *priv = dev->data->dev_private; 1517 1518 /* NIC root rules */ 1519 if (!attr->group && !attr->transfer) { 1520 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1521 priority = priv->sh->flow_max_priority - 1; 1522 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1523 /* FDB root rules */ 1524 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1525 attr->group == 0 && 1526 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1527 return (priv->sh->flow_max_priority - 1) * 3; 1528 } 1529 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1530 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1531 return priority * 3 + subpriority; 1532 } 1533 1534 /** 1535 * Verify the @p item specifications (spec, last, mask) are compatible with the 1536 * NIC capabilities. 1537 * 1538 * @param[in] item 1539 * Item specification. 1540 * @param[in] mask 1541 * @p item->mask or flow default bit-masks. 1542 * @param[in] nic_mask 1543 * Bit-masks covering supported fields by the NIC to compare with user mask. 1544 * @param[in] size 1545 * Bit-masks size in bytes. 1546 * @param[in] range_accepted 1547 * True if range of values is accepted for specific fields, false otherwise. 1548 * @param[out] error 1549 * Pointer to error structure. 1550 * 1551 * @return 1552 * 0 on success, a negative errno value otherwise and rte_errno is set. 1553 */ 1554 int 1555 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1556 const uint8_t *mask, 1557 const uint8_t *nic_mask, 1558 unsigned int size, 1559 bool range_accepted, 1560 struct rte_flow_error *error) 1561 { 1562 unsigned int i; 1563 1564 MLX5_ASSERT(nic_mask); 1565 for (i = 0; i < size; ++i) 1566 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1567 return rte_flow_error_set(error, ENOTSUP, 1568 RTE_FLOW_ERROR_TYPE_ITEM, 1569 item, 1570 "mask enables non supported" 1571 " bits"); 1572 if (!item->spec && (item->mask || item->last)) 1573 return rte_flow_error_set(error, EINVAL, 1574 RTE_FLOW_ERROR_TYPE_ITEM, item, 1575 "mask/last without a spec is not" 1576 " supported"); 1577 if (item->spec && item->last && !range_accepted) { 1578 uint8_t spec[size]; 1579 uint8_t last[size]; 1580 unsigned int i; 1581 int ret; 1582 1583 for (i = 0; i < size; ++i) { 1584 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1585 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1586 } 1587 ret = memcmp(spec, last, size); 1588 if (ret != 0) 1589 return rte_flow_error_set(error, EINVAL, 1590 RTE_FLOW_ERROR_TYPE_ITEM, 1591 item, 1592 "range is not valid"); 1593 } 1594 return 0; 1595 } 1596 1597 /** 1598 * Adjust the hash fields according to the @p flow information. 1599 * 1600 * @param[in] dev_flow. 1601 * Pointer to the mlx5_flow. 1602 * @param[in] tunnel 1603 * 1 when the hash field is for a tunnel item. 1604 * @param[in] layer_types 1605 * RTE_ETH_RSS_* types. 1606 * @param[in] hash_fields 1607 * Item hash fields. 1608 * 1609 * @return 1610 * The hash fields that should be used. 1611 */ 1612 uint64_t 1613 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1614 int tunnel __rte_unused, uint64_t layer_types, 1615 uint64_t hash_fields) 1616 { 1617 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1618 int rss_request_inner = rss_desc->level >= 2; 1619 1620 /* Check RSS hash level for tunnel. */ 1621 if (tunnel && rss_request_inner) 1622 hash_fields |= IBV_RX_HASH_INNER; 1623 else if (tunnel || rss_request_inner) 1624 return 0; 1625 #endif 1626 /* Check if requested layer matches RSS hash fields. */ 1627 if (!(rss_desc->types & layer_types)) 1628 return 0; 1629 return hash_fields; 1630 } 1631 1632 /** 1633 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1634 * if several tunnel rules are used on this queue, the tunnel ptype will be 1635 * cleared. 1636 * 1637 * @param rxq_ctrl 1638 * Rx queue to update. 1639 */ 1640 static void 1641 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1642 { 1643 unsigned int i; 1644 uint32_t tunnel_ptype = 0; 1645 1646 /* Look up for the ptype to use. */ 1647 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1648 if (!rxq_ctrl->flow_tunnels_n[i]) 1649 continue; 1650 if (!tunnel_ptype) { 1651 tunnel_ptype = tunnels_info[i].ptype; 1652 } else { 1653 tunnel_ptype = 0; 1654 break; 1655 } 1656 } 1657 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1658 } 1659 1660 /** 1661 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1662 * flow. 1663 * 1664 * @param[in] dev 1665 * Pointer to the Ethernet device structure. 1666 * @param[in] dev_handle 1667 * Pointer to device flow handle structure. 1668 */ 1669 void 1670 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1671 struct mlx5_flow_handle *dev_handle) 1672 { 1673 struct mlx5_priv *priv = dev->data->dev_private; 1674 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1675 struct mlx5_ind_table_obj *ind_tbl = NULL; 1676 unsigned int i; 1677 1678 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1679 struct mlx5_hrxq *hrxq; 1680 1681 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1682 dev_handle->rix_hrxq); 1683 if (hrxq) 1684 ind_tbl = hrxq->ind_table; 1685 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1686 struct mlx5_shared_action_rss *shared_rss; 1687 1688 shared_rss = mlx5_ipool_get 1689 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1690 dev_handle->rix_srss); 1691 if (shared_rss) 1692 ind_tbl = shared_rss->ind_tbl; 1693 } 1694 if (!ind_tbl) 1695 return; 1696 for (i = 0; i != ind_tbl->queues_n; ++i) { 1697 int idx = ind_tbl->queues[i]; 1698 struct mlx5_rxq_ctrl *rxq_ctrl; 1699 1700 if (mlx5_is_external_rxq(dev, idx)) 1701 continue; 1702 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1703 MLX5_ASSERT(rxq_ctrl != NULL); 1704 if (rxq_ctrl == NULL) 1705 continue; 1706 /* 1707 * To support metadata register copy on Tx loopback, 1708 * this must be always enabled (metadata may arive 1709 * from other port - not from local flows only. 1710 */ 1711 if (tunnel) { 1712 unsigned int j; 1713 1714 /* Increase the counter matching the flow. */ 1715 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1716 if ((tunnels_info[j].tunnel & 1717 dev_handle->layers) == 1718 tunnels_info[j].tunnel) { 1719 rxq_ctrl->flow_tunnels_n[j]++; 1720 break; 1721 } 1722 } 1723 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1724 } 1725 } 1726 } 1727 1728 static void 1729 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1730 { 1731 struct mlx5_priv *priv = dev->data->dev_private; 1732 struct mlx5_rxq_ctrl *rxq_ctrl; 1733 uint16_t port_id; 1734 1735 if (priv->sh->shared_mark_enabled) 1736 return; 1737 if (priv->master || priv->representor) { 1738 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1739 struct mlx5_priv *opriv = 1740 rte_eth_devices[port_id].data->dev_private; 1741 1742 if (!opriv || 1743 opriv->sh != priv->sh || 1744 opriv->domain_id != priv->domain_id || 1745 opriv->mark_enabled) 1746 continue; 1747 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1748 rxq_ctrl->rxq.mark = 1; 1749 } 1750 opriv->mark_enabled = 1; 1751 } 1752 } else { 1753 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1754 rxq_ctrl->rxq.mark = 1; 1755 } 1756 priv->mark_enabled = 1; 1757 } 1758 priv->sh->shared_mark_enabled = 1; 1759 } 1760 1761 /** 1762 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1763 * 1764 * @param[in] dev 1765 * Pointer to the Ethernet device structure. 1766 * @param[in] flow 1767 * Pointer to flow structure. 1768 */ 1769 static void 1770 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1771 { 1772 struct mlx5_priv *priv = dev->data->dev_private; 1773 uint32_t handle_idx; 1774 struct mlx5_flow_handle *dev_handle; 1775 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1776 1777 MLX5_ASSERT(wks); 1778 if (wks->mark) 1779 flow_rxq_mark_flag_set(dev); 1780 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1781 handle_idx, dev_handle, next) 1782 flow_drv_rxq_flags_set(dev, dev_handle); 1783 } 1784 1785 /** 1786 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1787 * device flow if no other flow uses it with the same kind of request. 1788 * 1789 * @param dev 1790 * Pointer to Ethernet device. 1791 * @param[in] dev_handle 1792 * Pointer to the device flow handle structure. 1793 */ 1794 static void 1795 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1796 struct mlx5_flow_handle *dev_handle) 1797 { 1798 struct mlx5_priv *priv = dev->data->dev_private; 1799 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1800 struct mlx5_ind_table_obj *ind_tbl = NULL; 1801 unsigned int i; 1802 1803 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1804 struct mlx5_hrxq *hrxq; 1805 1806 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1807 dev_handle->rix_hrxq); 1808 if (hrxq) 1809 ind_tbl = hrxq->ind_table; 1810 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1811 struct mlx5_shared_action_rss *shared_rss; 1812 1813 shared_rss = mlx5_ipool_get 1814 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1815 dev_handle->rix_srss); 1816 if (shared_rss) 1817 ind_tbl = shared_rss->ind_tbl; 1818 } 1819 if (!ind_tbl) 1820 return; 1821 MLX5_ASSERT(dev->data->dev_started); 1822 for (i = 0; i != ind_tbl->queues_n; ++i) { 1823 int idx = ind_tbl->queues[i]; 1824 struct mlx5_rxq_ctrl *rxq_ctrl; 1825 1826 if (mlx5_is_external_rxq(dev, idx)) 1827 continue; 1828 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1829 MLX5_ASSERT(rxq_ctrl != NULL); 1830 if (rxq_ctrl == NULL) 1831 continue; 1832 if (tunnel) { 1833 unsigned int j; 1834 1835 /* Decrease the counter matching the flow. */ 1836 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1837 if ((tunnels_info[j].tunnel & 1838 dev_handle->layers) == 1839 tunnels_info[j].tunnel) { 1840 rxq_ctrl->flow_tunnels_n[j]--; 1841 break; 1842 } 1843 } 1844 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1845 } 1846 } 1847 } 1848 1849 /** 1850 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1851 * @p flow if no other flow uses it with the same kind of request. 1852 * 1853 * @param dev 1854 * Pointer to Ethernet device. 1855 * @param[in] flow 1856 * Pointer to the flow. 1857 */ 1858 static void 1859 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1860 { 1861 struct mlx5_priv *priv = dev->data->dev_private; 1862 uint32_t handle_idx; 1863 struct mlx5_flow_handle *dev_handle; 1864 1865 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1866 handle_idx, dev_handle, next) 1867 flow_drv_rxq_flags_trim(dev, dev_handle); 1868 } 1869 1870 /** 1871 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1872 * 1873 * @param dev 1874 * Pointer to Ethernet device. 1875 */ 1876 static void 1877 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1878 { 1879 struct mlx5_priv *priv = dev->data->dev_private; 1880 unsigned int i; 1881 1882 for (i = 0; i != priv->rxqs_n; ++i) { 1883 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1884 unsigned int j; 1885 1886 if (rxq == NULL || rxq->ctrl == NULL) 1887 continue; 1888 rxq->ctrl->rxq.mark = 0; 1889 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1890 rxq->ctrl->flow_tunnels_n[j] = 0; 1891 rxq->ctrl->rxq.tunnel = 0; 1892 } 1893 priv->mark_enabled = 0; 1894 priv->sh->shared_mark_enabled = 0; 1895 } 1896 1897 static uint64_t mlx5_restore_info_dynflag; 1898 1899 int 1900 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 1901 { 1902 struct mlx5_priv *priv = dev->data->dev_private; 1903 uint64_t supported = 0; 1904 1905 if (!is_tunnel_offload_active(dev)) { 1906 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 1907 supported |= RTE_ETH_RX_METADATA_USER_MARK; 1908 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) { 1909 DRV_LOG(DEBUG, 1910 "tunnel offload was not activated, consider setting dv_xmeta_en=%d", 1911 MLX5_XMETA_MODE_MISS_INFO); 1912 } 1913 } else { 1914 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 1915 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 && 1916 mlx5_restore_info_dynflag == 0) 1917 mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag(); 1918 } 1919 1920 if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) 1921 priv->tunnel_enabled = 1; 1922 else 1923 priv->tunnel_enabled = 0; 1924 1925 *features &= supported; 1926 return 0; 1927 } 1928 1929 /** 1930 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1931 * 1932 * @param[in] dev 1933 * Pointer to the Ethernet device structure. 1934 */ 1935 void 1936 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) 1937 { 1938 struct mlx5_priv *priv = dev->data->dev_private; 1939 uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID; 1940 unsigned int i; 1941 1942 if (priv->tunnel_enabled) 1943 mark_flag |= mlx5_restore_info_dynflag; 1944 1945 for (i = 0; i != priv->rxqs_n; ++i) { 1946 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1947 struct mlx5_rxq_data *data; 1948 1949 if (rxq == NULL || rxq->ctrl == NULL) 1950 continue; 1951 data = &rxq->ctrl->rxq; 1952 if (!rte_flow_dynf_metadata_avail()) { 1953 data->dynf_meta = 0; 1954 data->flow_meta_mask = 0; 1955 data->flow_meta_offset = -1; 1956 data->flow_meta_port_mask = 0; 1957 } else { 1958 data->dynf_meta = 1; 1959 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1960 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1961 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1962 } 1963 data->mark_flag = mark_flag; 1964 } 1965 } 1966 1967 /* 1968 * return a pointer to the desired action in the list of actions. 1969 * 1970 * @param[in] actions 1971 * The list of actions to search the action in. 1972 * @param[in] action 1973 * The action to find. 1974 * 1975 * @return 1976 * Pointer to the action in the list, if found. NULL otherwise. 1977 */ 1978 const struct rte_flow_action * 1979 mlx5_flow_find_action(const struct rte_flow_action *actions, 1980 enum rte_flow_action_type action) 1981 { 1982 if (actions == NULL) 1983 return NULL; 1984 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1985 if (actions->type == action) 1986 return actions; 1987 return NULL; 1988 } 1989 1990 /* 1991 * Validate the flag action. 1992 * 1993 * @param[in] action_flags 1994 * Bit-fields that holds the actions detected until now. 1995 * @param[in] attr 1996 * Attributes of flow that includes this action. 1997 * @param[out] error 1998 * Pointer to error structure. 1999 * 2000 * @return 2001 * 0 on success, a negative errno value otherwise and rte_errno is set. 2002 */ 2003 int 2004 mlx5_flow_validate_action_flag(uint64_t action_flags, 2005 const struct rte_flow_attr *attr, 2006 struct rte_flow_error *error) 2007 { 2008 if (action_flags & MLX5_FLOW_ACTION_MARK) 2009 return rte_flow_error_set(error, EINVAL, 2010 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2011 "can't mark and flag in same flow"); 2012 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2013 return rte_flow_error_set(error, EINVAL, 2014 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2015 "can't have 2 flag" 2016 " actions in same flow"); 2017 if (attr->egress) 2018 return rte_flow_error_set(error, ENOTSUP, 2019 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2020 "flag action not supported for " 2021 "egress"); 2022 return 0; 2023 } 2024 2025 /* 2026 * Validate the mark action. 2027 * 2028 * @param[in] action 2029 * Pointer to the queue action. 2030 * @param[in] action_flags 2031 * Bit-fields that holds the actions detected until now. 2032 * @param[in] attr 2033 * Attributes of flow that includes this action. 2034 * @param[out] error 2035 * Pointer to error structure. 2036 * 2037 * @return 2038 * 0 on success, a negative errno value otherwise and rte_errno is set. 2039 */ 2040 int 2041 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 2042 uint64_t action_flags, 2043 const struct rte_flow_attr *attr, 2044 struct rte_flow_error *error) 2045 { 2046 const struct rte_flow_action_mark *mark = action->conf; 2047 2048 if (!mark) 2049 return rte_flow_error_set(error, EINVAL, 2050 RTE_FLOW_ERROR_TYPE_ACTION, 2051 action, 2052 "configuration cannot be null"); 2053 if (mark->id >= MLX5_FLOW_MARK_MAX) 2054 return rte_flow_error_set(error, EINVAL, 2055 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2056 &mark->id, 2057 "mark id must in 0 <= id < " 2058 RTE_STR(MLX5_FLOW_MARK_MAX)); 2059 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2060 return rte_flow_error_set(error, EINVAL, 2061 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2062 "can't flag and mark in same flow"); 2063 if (action_flags & MLX5_FLOW_ACTION_MARK) 2064 return rte_flow_error_set(error, EINVAL, 2065 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2066 "can't have 2 mark actions in same" 2067 " flow"); 2068 if (attr->egress) 2069 return rte_flow_error_set(error, ENOTSUP, 2070 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2071 "mark action not supported for " 2072 "egress"); 2073 return 0; 2074 } 2075 2076 /* 2077 * Validate the drop action. 2078 * 2079 * @param[in] dev 2080 * Pointer to the Ethernet device structure. 2081 * @param[in] is_root 2082 * True if flow is validated for root table. False otherwise. 2083 * @param[in] attr 2084 * Attributes of flow that includes this action. 2085 * @param[out] error 2086 * Pointer to error structure. 2087 * 2088 * @return 2089 * 0 on success, a negative errno value otherwise and rte_errno is set. 2090 */ 2091 int 2092 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 2093 bool is_root, 2094 const struct rte_flow_attr *attr, 2095 struct rte_flow_error *error) 2096 { 2097 struct mlx5_priv *priv = dev->data->dev_private; 2098 2099 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 2100 return rte_flow_error_set(error, ENOTSUP, 2101 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2102 "drop action not supported for " 2103 "egress"); 2104 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 2105 !priv->sh->dr_root_drop_action_en) { 2106 return rte_flow_error_set(error, ENOTSUP, 2107 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 2108 "drop action not supported for " 2109 "egress and transfer on group 0"); 2110 } 2111 return 0; 2112 } 2113 2114 /* 2115 * Validate the queue action. 2116 * 2117 * @param[in] action 2118 * Pointer to the queue action. 2119 * @param[in] action_flags 2120 * Bit-fields that holds the actions detected until now. 2121 * @param[in] dev 2122 * Pointer to the Ethernet device structure. 2123 * @param[in] attr 2124 * Attributes of flow that includes this action. 2125 * @param[out] error 2126 * Pointer to error structure. 2127 * 2128 * @return 2129 * 0 on success, a negative errno value otherwise and rte_errno is set. 2130 */ 2131 int 2132 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 2133 uint64_t action_flags, 2134 struct rte_eth_dev *dev, 2135 const struct rte_flow_attr *attr, 2136 struct rte_flow_error *error) 2137 { 2138 struct mlx5_priv *priv = dev->data->dev_private; 2139 const struct rte_flow_action_queue *queue = action->conf; 2140 2141 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2142 return rte_flow_error_set(error, EINVAL, 2143 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2144 "can't have 2 fate actions in" 2145 " same flow"); 2146 if (attr->egress) 2147 return rte_flow_error_set(error, ENOTSUP, 2148 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2149 "queue action not supported for egress."); 2150 if (mlx5_is_external_rxq(dev, queue->index)) 2151 return 0; 2152 if (!priv->rxqs_n) 2153 return rte_flow_error_set(error, EINVAL, 2154 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2155 NULL, "No Rx queues configured"); 2156 if (queue->index >= priv->rxqs_n) 2157 return rte_flow_error_set(error, EINVAL, 2158 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2159 &queue->index, 2160 "queue index out of range"); 2161 if (mlx5_rxq_get(dev, queue->index) == NULL) 2162 return rte_flow_error_set(error, EINVAL, 2163 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2164 &queue->index, 2165 "queue is not configured"); 2166 return 0; 2167 } 2168 2169 /** 2170 * Validate queue numbers for device RSS. 2171 * 2172 * @param[in] dev 2173 * Configured device. 2174 * @param[in] queues 2175 * Array of queue numbers. 2176 * @param[in] queues_n 2177 * Size of the @p queues array. 2178 * @param[out] error 2179 * On error, filled with a textual error description. 2180 * @param[out] queue_idx 2181 * On error, filled with an offending queue index in @p queues array. 2182 * 2183 * @return 2184 * 0 on success, a negative errno code on error. 2185 */ 2186 static int 2187 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2188 const uint16_t *queues, uint32_t queues_n, 2189 const char **error, uint32_t *queue_idx) 2190 { 2191 const struct mlx5_priv *priv = dev->data->dev_private; 2192 bool is_hairpin = false; 2193 bool is_ext_rss = false; 2194 uint32_t i; 2195 2196 for (i = 0; i != queues_n; ++i) { 2197 struct mlx5_rxq_ctrl *rxq_ctrl; 2198 2199 if (mlx5_is_external_rxq(dev, queues[0])) { 2200 is_ext_rss = true; 2201 continue; 2202 } 2203 if (is_ext_rss) { 2204 *error = "Combining external and regular RSS queues is not supported"; 2205 *queue_idx = i; 2206 return -ENOTSUP; 2207 } 2208 if (queues[i] >= priv->rxqs_n) { 2209 *error = "queue index out of range"; 2210 *queue_idx = i; 2211 return -EINVAL; 2212 } 2213 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2214 if (rxq_ctrl == NULL) { 2215 *error = "queue is not configured"; 2216 *queue_idx = i; 2217 return -EINVAL; 2218 } 2219 if (i == 0 && rxq_ctrl->is_hairpin) 2220 is_hairpin = true; 2221 if (is_hairpin != rxq_ctrl->is_hairpin) { 2222 *error = "combining hairpin and regular RSS queues is not supported"; 2223 *queue_idx = i; 2224 return -ENOTSUP; 2225 } 2226 } 2227 return 0; 2228 } 2229 2230 /* 2231 * Validate the rss action. 2232 * 2233 * @param[in] dev 2234 * Pointer to the Ethernet device structure. 2235 * @param[in] action 2236 * Pointer to the queue action. 2237 * @param[out] error 2238 * Pointer to error structure. 2239 * 2240 * @return 2241 * 0 on success, a negative errno value otherwise and rte_errno is set. 2242 */ 2243 int 2244 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2245 const struct rte_flow_action *action, 2246 struct rte_flow_error *error) 2247 { 2248 struct mlx5_priv *priv = dev->data->dev_private; 2249 const struct rte_flow_action_rss *rss = action->conf; 2250 int ret; 2251 const char *message; 2252 uint32_t queue_idx; 2253 2254 if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 2255 DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT", 2256 dev->data->port_id); 2257 } else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2258 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2259 return rte_flow_error_set(error, ENOTSUP, 2260 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2261 &rss->func, 2262 "RSS hash function not supported"); 2263 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2264 if (rss->level > 2) 2265 #else 2266 if (rss->level > 1) 2267 #endif 2268 return rte_flow_error_set(error, ENOTSUP, 2269 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2270 &rss->level, 2271 "tunnel RSS is not supported"); 2272 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2273 if (rss->key_len == 0 && rss->key != NULL) 2274 return rte_flow_error_set(error, ENOTSUP, 2275 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2276 &rss->key_len, 2277 "RSS hash key length 0"); 2278 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2279 return rte_flow_error_set(error, ENOTSUP, 2280 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2281 &rss->key_len, 2282 "RSS hash key too small"); 2283 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2284 return rte_flow_error_set(error, ENOTSUP, 2285 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2286 &rss->key_len, 2287 "RSS hash key too large"); 2288 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2289 return rte_flow_error_set(error, ENOTSUP, 2290 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2291 &rss->queue_num, 2292 "number of queues too large"); 2293 if (rss->types & MLX5_RSS_HF_MASK) 2294 return rte_flow_error_set(error, ENOTSUP, 2295 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2296 &rss->types, 2297 "some RSS protocols are not" 2298 " supported"); 2299 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2300 !(rss->types & RTE_ETH_RSS_IP)) 2301 return rte_flow_error_set(error, EINVAL, 2302 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2303 "L3 partial RSS requested but L3 RSS" 2304 " type not specified"); 2305 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2306 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2307 return rte_flow_error_set(error, EINVAL, 2308 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2309 "L4 partial RSS requested but L4 RSS" 2310 " type not specified"); 2311 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2312 return rte_flow_error_set(error, EINVAL, 2313 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2314 NULL, "No Rx queues configured"); 2315 if (!rss->queue_num) 2316 return rte_flow_error_set(error, EINVAL, 2317 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2318 NULL, "No queues configured"); 2319 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2320 &message, &queue_idx); 2321 if (ret != 0) { 2322 return rte_flow_error_set(error, -ret, 2323 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2324 &rss->queue[queue_idx], message); 2325 } 2326 return 0; 2327 } 2328 2329 /* 2330 * Validate the rss action. 2331 * 2332 * @param[in] action 2333 * Pointer to the queue action. 2334 * @param[in] action_flags 2335 * Bit-fields that holds the actions detected until now. 2336 * @param[in] dev 2337 * Pointer to the Ethernet device structure. 2338 * @param[in] attr 2339 * Attributes of flow that includes this action. 2340 * @param[in] item_flags 2341 * Items that were detected. 2342 * @param[out] error 2343 * Pointer to error structure. 2344 * 2345 * @return 2346 * 0 on success, a negative errno value otherwise and rte_errno is set. 2347 */ 2348 int 2349 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2350 uint64_t action_flags, 2351 struct rte_eth_dev *dev, 2352 const struct rte_flow_attr *attr, 2353 uint64_t item_flags, 2354 struct rte_flow_error *error) 2355 { 2356 const struct rte_flow_action_rss *rss = action->conf; 2357 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2358 int ret; 2359 2360 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2361 return rte_flow_error_set(error, EINVAL, 2362 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2363 "can't have 2 fate actions" 2364 " in same flow"); 2365 ret = mlx5_validate_action_rss(dev, action, error); 2366 if (ret) 2367 return ret; 2368 if (attr->egress) 2369 return rte_flow_error_set(error, ENOTSUP, 2370 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2371 "rss action not supported for " 2372 "egress"); 2373 if (rss->level > 1 && !tunnel) 2374 return rte_flow_error_set(error, EINVAL, 2375 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2376 "inner RSS is not supported for " 2377 "non-tunnel flows"); 2378 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2379 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2380 return rte_flow_error_set(error, EINVAL, 2381 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2382 "RSS on eCPRI is not supported now"); 2383 } 2384 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2385 !(item_flags & 2386 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2387 rss->level > 1) 2388 return rte_flow_error_set(error, EINVAL, 2389 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2390 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2391 return 0; 2392 } 2393 2394 /* 2395 * Validate the default miss action. 2396 * 2397 * @param[in] action_flags 2398 * Bit-fields that holds the actions detected until now. 2399 * @param[out] error 2400 * Pointer to error structure. 2401 * 2402 * @return 2403 * 0 on success, a negative errno value otherwise and rte_errno is set. 2404 */ 2405 int 2406 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2407 const struct rte_flow_attr *attr, 2408 struct rte_flow_error *error) 2409 { 2410 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2411 return rte_flow_error_set(error, EINVAL, 2412 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2413 "can't have 2 fate actions in" 2414 " same flow"); 2415 if (attr->egress) 2416 return rte_flow_error_set(error, ENOTSUP, 2417 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2418 "default miss action not supported " 2419 "for egress"); 2420 if (attr->group) 2421 return rte_flow_error_set(error, ENOTSUP, 2422 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2423 "only group 0 is supported"); 2424 if (attr->transfer) 2425 return rte_flow_error_set(error, ENOTSUP, 2426 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2427 NULL, "transfer is not supported"); 2428 return 0; 2429 } 2430 2431 /* 2432 * Validate the count action. 2433 * 2434 * @param[in] dev 2435 * Pointer to the Ethernet device structure. 2436 * @param[in] attr 2437 * Attributes of flow that includes this action. 2438 * @param[out] error 2439 * Pointer to error structure. 2440 * 2441 * @return 2442 * 0 on success, a negative errno value otherwise and rte_errno is set. 2443 */ 2444 int 2445 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2446 const struct rte_flow_attr *attr, 2447 struct rte_flow_error *error) 2448 { 2449 if (attr->egress) 2450 return rte_flow_error_set(error, ENOTSUP, 2451 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2452 "count action not supported for " 2453 "egress"); 2454 return 0; 2455 } 2456 2457 /* 2458 * Validate the ASO CT action. 2459 * 2460 * @param[in] dev 2461 * Pointer to the Ethernet device structure. 2462 * @param[in] conntrack 2463 * Pointer to the CT action profile. 2464 * @param[out] error 2465 * Pointer to error structure. 2466 * 2467 * @return 2468 * 0 on success, a negative errno value otherwise and rte_errno is set. 2469 */ 2470 int 2471 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2472 const struct rte_flow_action_conntrack *conntrack, 2473 struct rte_flow_error *error) 2474 { 2475 RTE_SET_USED(dev); 2476 2477 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2478 return rte_flow_error_set(error, EINVAL, 2479 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2480 "Invalid CT state"); 2481 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2482 return rte_flow_error_set(error, EINVAL, 2483 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2484 "Invalid last TCP packet flag"); 2485 return 0; 2486 } 2487 2488 /** 2489 * Validate the level value for modify field action. 2490 * 2491 * @param[in] data 2492 * Pointer to the rte_flow_action_modify_data structure either src or dst. 2493 * @param[out] error 2494 * Pointer to error structure. 2495 * 2496 * @return 2497 * 0 on success, a negative errno value otherwise and rte_errno is set. 2498 */ 2499 int 2500 flow_validate_modify_field_level(const struct rte_flow_action_modify_data *data, 2501 struct rte_flow_error *error) 2502 { 2503 if (data->level == 0) 2504 return 0; 2505 if (data->field != RTE_FLOW_FIELD_TAG && 2506 data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) 2507 return rte_flow_error_set(error, ENOTSUP, 2508 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2509 "inner header fields modification is not supported"); 2510 if (data->tag_index != 0) 2511 return rte_flow_error_set(error, EINVAL, 2512 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2513 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2514 /* 2515 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2516 * 'tag_index' field. In old API, it was provided using 'level' field 2517 * and it is still supported for backwards compatibility. 2518 */ 2519 DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field."); 2520 return 0; 2521 } 2522 2523 /** 2524 * Validate ICMP6 item. 2525 * 2526 * @param[in] item 2527 * Item specification. 2528 * @param[in] item_flags 2529 * Bit-fields that holds the items detected until now. 2530 * @param[in] ext_vlan_sup 2531 * Whether extended VLAN features are supported or not. 2532 * @param[out] error 2533 * Pointer to error structure. 2534 * 2535 * @return 2536 * 0 on success, a negative errno value otherwise and rte_errno is set. 2537 */ 2538 int 2539 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2540 uint64_t item_flags, 2541 uint8_t target_protocol, 2542 struct rte_flow_error *error) 2543 { 2544 const struct rte_flow_item_icmp6 *mask = item->mask; 2545 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2546 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2547 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2548 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2549 MLX5_FLOW_LAYER_OUTER_L4; 2550 int ret; 2551 2552 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2553 return rte_flow_error_set(error, EINVAL, 2554 RTE_FLOW_ERROR_TYPE_ITEM, item, 2555 "protocol filtering not compatible" 2556 " with ICMP6 layer"); 2557 if (!(item_flags & l3m)) 2558 return rte_flow_error_set(error, EINVAL, 2559 RTE_FLOW_ERROR_TYPE_ITEM, item, 2560 "IPv6 is mandatory to filter on" 2561 " ICMP6"); 2562 if (item_flags & l4m) 2563 return rte_flow_error_set(error, EINVAL, 2564 RTE_FLOW_ERROR_TYPE_ITEM, item, 2565 "multiple L4 layers not supported"); 2566 if (!mask) 2567 mask = &rte_flow_item_icmp6_mask; 2568 ret = mlx5_flow_item_acceptable 2569 (item, (const uint8_t *)mask, 2570 (const uint8_t *)&rte_flow_item_icmp6_mask, 2571 sizeof(struct rte_flow_item_icmp6), 2572 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2573 if (ret < 0) 2574 return ret; 2575 return 0; 2576 } 2577 2578 /** 2579 * Validate ICMP6 echo request/reply item. 2580 * 2581 * @param[in] item 2582 * Item specification. 2583 * @param[in] item_flags 2584 * Bit-fields that holds the items detected until now. 2585 * @param[in] ext_vlan_sup 2586 * Whether extended VLAN features are supported or not. 2587 * @param[out] error 2588 * Pointer to error structure. 2589 * 2590 * @return 2591 * 0 on success, a negative errno value otherwise and rte_errno is set. 2592 */ 2593 int 2594 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2595 uint64_t item_flags, 2596 uint8_t target_protocol, 2597 struct rte_flow_error *error) 2598 { 2599 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2600 const struct rte_flow_item_icmp6_echo nic_mask = { 2601 .hdr.base.type = 0xff, 2602 .hdr.base.code = 0xff, 2603 .hdr.identifier = RTE_BE16(0xffff), 2604 .hdr.sequence = RTE_BE16(0xffff), 2605 }; 2606 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2607 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2608 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2609 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2610 MLX5_FLOW_LAYER_OUTER_L4; 2611 int ret; 2612 2613 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2614 return rte_flow_error_set(error, EINVAL, 2615 RTE_FLOW_ERROR_TYPE_ITEM, item, 2616 "protocol filtering not compatible" 2617 " with ICMP6 layer"); 2618 if (!(item_flags & l3m)) 2619 return rte_flow_error_set(error, EINVAL, 2620 RTE_FLOW_ERROR_TYPE_ITEM, item, 2621 "IPv6 is mandatory to filter on" 2622 " ICMP6"); 2623 if (item_flags & l4m) 2624 return rte_flow_error_set(error, EINVAL, 2625 RTE_FLOW_ERROR_TYPE_ITEM, item, 2626 "multiple L4 layers not supported"); 2627 if (!mask) 2628 mask = &nic_mask; 2629 ret = mlx5_flow_item_acceptable 2630 (item, (const uint8_t *)mask, 2631 (const uint8_t *)&nic_mask, 2632 sizeof(struct rte_flow_item_icmp6_echo), 2633 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2634 if (ret < 0) 2635 return ret; 2636 return 0; 2637 } 2638 2639 /** 2640 * Validate ICMP item. 2641 * 2642 * @param[in] item 2643 * Item specification. 2644 * @param[in] item_flags 2645 * Bit-fields that holds the items detected until now. 2646 * @param[out] error 2647 * Pointer to error structure. 2648 * 2649 * @return 2650 * 0 on success, a negative errno value otherwise and rte_errno is set. 2651 */ 2652 int 2653 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2654 uint64_t item_flags, 2655 uint8_t target_protocol, 2656 struct rte_flow_error *error) 2657 { 2658 const struct rte_flow_item_icmp *mask = item->mask; 2659 const struct rte_flow_item_icmp nic_mask = { 2660 .hdr.icmp_type = 0xff, 2661 .hdr.icmp_code = 0xff, 2662 .hdr.icmp_ident = RTE_BE16(0xffff), 2663 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2664 }; 2665 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2666 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2667 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2668 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2669 MLX5_FLOW_LAYER_OUTER_L4; 2670 int ret; 2671 2672 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2673 return rte_flow_error_set(error, EINVAL, 2674 RTE_FLOW_ERROR_TYPE_ITEM, item, 2675 "protocol filtering not compatible" 2676 " with ICMP layer"); 2677 if (!(item_flags & l3m)) 2678 return rte_flow_error_set(error, EINVAL, 2679 RTE_FLOW_ERROR_TYPE_ITEM, item, 2680 "IPv4 is mandatory to filter" 2681 " on ICMP"); 2682 if (item_flags & l4m) 2683 return rte_flow_error_set(error, EINVAL, 2684 RTE_FLOW_ERROR_TYPE_ITEM, item, 2685 "multiple L4 layers not supported"); 2686 if (!mask) 2687 mask = &nic_mask; 2688 ret = mlx5_flow_item_acceptable 2689 (item, (const uint8_t *)mask, 2690 (const uint8_t *)&nic_mask, 2691 sizeof(struct rte_flow_item_icmp), 2692 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2693 if (ret < 0) 2694 return ret; 2695 return 0; 2696 } 2697 2698 /** 2699 * Validate Ethernet item. 2700 * 2701 * @param[in] item 2702 * Item specification. 2703 * @param[in] item_flags 2704 * Bit-fields that holds the items detected until now. 2705 * @param[out] error 2706 * Pointer to error structure. 2707 * 2708 * @return 2709 * 0 on success, a negative errno value otherwise and rte_errno is set. 2710 */ 2711 int 2712 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2713 uint64_t item_flags, bool ext_vlan_sup, 2714 struct rte_flow_error *error) 2715 { 2716 const struct rte_flow_item_eth *mask = item->mask; 2717 const struct rte_flow_item_eth nic_mask = { 2718 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2719 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2720 .hdr.ether_type = RTE_BE16(0xffff), 2721 .has_vlan = ext_vlan_sup ? 1 : 0, 2722 }; 2723 int ret; 2724 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2725 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2726 MLX5_FLOW_LAYER_OUTER_L2; 2727 2728 if (item_flags & ethm) 2729 return rte_flow_error_set(error, ENOTSUP, 2730 RTE_FLOW_ERROR_TYPE_ITEM, item, 2731 "multiple L2 layers not supported"); 2732 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2733 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2734 return rte_flow_error_set(error, EINVAL, 2735 RTE_FLOW_ERROR_TYPE_ITEM, item, 2736 "L2 layer should not follow " 2737 "L3 layers"); 2738 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2739 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2740 return rte_flow_error_set(error, EINVAL, 2741 RTE_FLOW_ERROR_TYPE_ITEM, item, 2742 "L2 layer should not follow VLAN"); 2743 if (item_flags & MLX5_FLOW_LAYER_GTP) 2744 return rte_flow_error_set(error, EINVAL, 2745 RTE_FLOW_ERROR_TYPE_ITEM, item, 2746 "L2 layer should not follow GTP"); 2747 if (!mask) 2748 mask = &rte_flow_item_eth_mask; 2749 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2750 (const uint8_t *)&nic_mask, 2751 sizeof(struct rte_flow_item_eth), 2752 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2753 return ret; 2754 } 2755 2756 /** 2757 * Validate VLAN item. 2758 * 2759 * @param[in] item 2760 * Item specification. 2761 * @param[in] item_flags 2762 * Bit-fields that holds the items detected until now. 2763 * @param[in] dev 2764 * Ethernet device flow is being created on. 2765 * @param[out] error 2766 * Pointer to error structure. 2767 * 2768 * @return 2769 * 0 on success, a negative errno value otherwise and rte_errno is set. 2770 */ 2771 int 2772 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2773 uint64_t item_flags, 2774 struct rte_eth_dev *dev, 2775 struct rte_flow_error *error) 2776 { 2777 const struct rte_flow_item_vlan *spec = item->spec; 2778 const struct rte_flow_item_vlan *mask = item->mask; 2779 const struct rte_flow_item_vlan nic_mask = { 2780 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2781 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2782 }; 2783 uint16_t vlan_tag = 0; 2784 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2785 int ret; 2786 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2787 MLX5_FLOW_LAYER_INNER_L4) : 2788 (MLX5_FLOW_LAYER_OUTER_L3 | 2789 MLX5_FLOW_LAYER_OUTER_L4); 2790 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2791 MLX5_FLOW_LAYER_OUTER_VLAN; 2792 2793 if (item_flags & vlanm) 2794 return rte_flow_error_set(error, EINVAL, 2795 RTE_FLOW_ERROR_TYPE_ITEM, item, 2796 "multiple VLAN layers not supported"); 2797 else if ((item_flags & l34m) != 0) 2798 return rte_flow_error_set(error, EINVAL, 2799 RTE_FLOW_ERROR_TYPE_ITEM, item, 2800 "VLAN cannot follow L3/L4 layer"); 2801 if (!mask) 2802 mask = &rte_flow_item_vlan_mask; 2803 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2804 (const uint8_t *)&nic_mask, 2805 sizeof(struct rte_flow_item_vlan), 2806 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2807 if (ret) 2808 return ret; 2809 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2810 struct mlx5_priv *priv = dev->data->dev_private; 2811 2812 if (priv->vmwa_context) { 2813 /* 2814 * Non-NULL context means we have a virtual machine 2815 * and SR-IOV enabled, we have to create VLAN interface 2816 * to make hypervisor to setup E-Switch vport 2817 * context correctly. We avoid creating the multiple 2818 * VLAN interfaces, so we cannot support VLAN tag mask. 2819 */ 2820 return rte_flow_error_set(error, EINVAL, 2821 RTE_FLOW_ERROR_TYPE_ITEM, 2822 item, 2823 "VLAN tag mask is not" 2824 " supported in virtual" 2825 " environment"); 2826 } 2827 } 2828 if (spec) { 2829 vlan_tag = spec->hdr.vlan_tci; 2830 vlan_tag &= mask->hdr.vlan_tci; 2831 } 2832 /* 2833 * From verbs perspective an empty VLAN is equivalent 2834 * to a packet without VLAN layer. 2835 */ 2836 if (!vlan_tag) 2837 return rte_flow_error_set(error, EINVAL, 2838 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2839 item->spec, 2840 "VLAN cannot be empty"); 2841 return 0; 2842 } 2843 2844 /** 2845 * Validate IPV4 item. 2846 * 2847 * @param[in] item 2848 * Item specification. 2849 * @param[in] item_flags 2850 * Bit-fields that holds the items detected until now. 2851 * @param[in] last_item 2852 * Previous validated item in the pattern items. 2853 * @param[in] ether_type 2854 * Type in the ethernet layer header (including dot1q). 2855 * @param[in] acc_mask 2856 * Acceptable mask, if NULL default internal default mask 2857 * will be used to check whether item fields are supported. 2858 * @param[in] range_accepted 2859 * True if range of values is accepted for specific fields, false otherwise. 2860 * @param[out] error 2861 * Pointer to error structure. 2862 * 2863 * @return 2864 * 0 on success, a negative errno value otherwise and rte_errno is set. 2865 */ 2866 int 2867 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2868 uint64_t item_flags, 2869 uint64_t last_item, 2870 uint16_t ether_type, 2871 const struct rte_flow_item_ipv4 *acc_mask, 2872 bool range_accepted, 2873 struct rte_flow_error *error) 2874 { 2875 const struct rte_flow_item_ipv4 *mask = item->mask; 2876 const struct rte_flow_item_ipv4 *spec = item->spec; 2877 const struct rte_flow_item_ipv4 nic_mask = { 2878 .hdr = { 2879 .src_addr = RTE_BE32(0xffffffff), 2880 .dst_addr = RTE_BE32(0xffffffff), 2881 .type_of_service = 0xff, 2882 .next_proto_id = 0xff, 2883 }, 2884 }; 2885 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2886 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2887 MLX5_FLOW_LAYER_OUTER_L3; 2888 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2889 MLX5_FLOW_LAYER_OUTER_L4; 2890 int ret; 2891 uint8_t next_proto = 0xFF; 2892 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2893 MLX5_FLOW_LAYER_OUTER_VLAN | 2894 MLX5_FLOW_LAYER_INNER_VLAN); 2895 2896 if ((last_item & l2_vlan) && ether_type && 2897 ether_type != RTE_ETHER_TYPE_IPV4) 2898 return rte_flow_error_set(error, EINVAL, 2899 RTE_FLOW_ERROR_TYPE_ITEM, item, 2900 "IPv4 cannot follow L2/VLAN layer " 2901 "which ether type is not IPv4"); 2902 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2903 if (mask && spec) 2904 next_proto = mask->hdr.next_proto_id & 2905 spec->hdr.next_proto_id; 2906 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2907 return rte_flow_error_set(error, EINVAL, 2908 RTE_FLOW_ERROR_TYPE_ITEM, 2909 item, 2910 "multiple tunnel " 2911 "not supported"); 2912 } 2913 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2914 return rte_flow_error_set(error, EINVAL, 2915 RTE_FLOW_ERROR_TYPE_ITEM, item, 2916 "wrong tunnel type - IPv6 specified " 2917 "but IPv4 item provided"); 2918 if (item_flags & l3m) 2919 return rte_flow_error_set(error, ENOTSUP, 2920 RTE_FLOW_ERROR_TYPE_ITEM, item, 2921 "multiple L3 layers not supported"); 2922 else if (item_flags & l4m) 2923 return rte_flow_error_set(error, EINVAL, 2924 RTE_FLOW_ERROR_TYPE_ITEM, item, 2925 "L3 cannot follow an L4 layer."); 2926 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2927 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2928 return rte_flow_error_set(error, EINVAL, 2929 RTE_FLOW_ERROR_TYPE_ITEM, item, 2930 "L3 cannot follow an NVGRE layer."); 2931 if (!mask) 2932 mask = &rte_flow_item_ipv4_mask; 2933 else if (mask->hdr.next_proto_id != 0 && 2934 mask->hdr.next_proto_id != 0xff) 2935 return rte_flow_error_set(error, EINVAL, 2936 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2937 "partial mask is not supported" 2938 " for protocol"); 2939 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2940 acc_mask ? (const uint8_t *)acc_mask 2941 : (const uint8_t *)&nic_mask, 2942 sizeof(struct rte_flow_item_ipv4), 2943 range_accepted, error); 2944 if (ret < 0) 2945 return ret; 2946 return 0; 2947 } 2948 2949 /** 2950 * Validate IPV6 item. 2951 * 2952 * @param[in] item 2953 * Item specification. 2954 * @param[in] item_flags 2955 * Bit-fields that holds the items detected until now. 2956 * @param[in] last_item 2957 * Previous validated item in the pattern items. 2958 * @param[in] ether_type 2959 * Type in the ethernet layer header (including dot1q). 2960 * @param[in] acc_mask 2961 * Acceptable mask, if NULL default internal default mask 2962 * will be used to check whether item fields are supported. 2963 * @param[out] error 2964 * Pointer to error structure. 2965 * 2966 * @return 2967 * 0 on success, a negative errno value otherwise and rte_errno is set. 2968 */ 2969 int 2970 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2971 uint64_t item_flags, 2972 uint64_t last_item, 2973 uint16_t ether_type, 2974 const struct rte_flow_item_ipv6 *acc_mask, 2975 struct rte_flow_error *error) 2976 { 2977 const struct rte_flow_item_ipv6 *mask = item->mask; 2978 const struct rte_flow_item_ipv6 *spec = item->spec; 2979 const struct rte_flow_item_ipv6 nic_mask = { 2980 .hdr = { 2981 .src_addr = 2982 "\xff\xff\xff\xff\xff\xff\xff\xff" 2983 "\xff\xff\xff\xff\xff\xff\xff\xff", 2984 .dst_addr = 2985 "\xff\xff\xff\xff\xff\xff\xff\xff" 2986 "\xff\xff\xff\xff\xff\xff\xff\xff", 2987 .vtc_flow = RTE_BE32(0xffffffff), 2988 .proto = 0xff, 2989 }, 2990 }; 2991 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2992 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2993 MLX5_FLOW_LAYER_OUTER_L3; 2994 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2995 MLX5_FLOW_LAYER_OUTER_L4; 2996 int ret; 2997 uint8_t next_proto = 0xFF; 2998 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2999 MLX5_FLOW_LAYER_OUTER_VLAN | 3000 MLX5_FLOW_LAYER_INNER_VLAN); 3001 3002 if ((last_item & l2_vlan) && ether_type && 3003 ether_type != RTE_ETHER_TYPE_IPV6) 3004 return rte_flow_error_set(error, EINVAL, 3005 RTE_FLOW_ERROR_TYPE_ITEM, item, 3006 "IPv6 cannot follow L2/VLAN layer " 3007 "which ether type is not IPv6"); 3008 if (mask && mask->hdr.proto == UINT8_MAX && spec) 3009 next_proto = spec->hdr.proto; 3010 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 3011 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 3012 return rte_flow_error_set(error, EINVAL, 3013 RTE_FLOW_ERROR_TYPE_ITEM, 3014 item, 3015 "multiple tunnel " 3016 "not supported"); 3017 } 3018 if (next_proto == IPPROTO_HOPOPTS || 3019 next_proto == IPPROTO_ROUTING || 3020 next_proto == IPPROTO_FRAGMENT || 3021 next_proto == IPPROTO_ESP || 3022 next_proto == IPPROTO_AH || 3023 next_proto == IPPROTO_DSTOPTS) 3024 return rte_flow_error_set(error, EINVAL, 3025 RTE_FLOW_ERROR_TYPE_ITEM, item, 3026 "IPv6 proto (next header) should " 3027 "not be set as extension header"); 3028 if (item_flags & MLX5_FLOW_LAYER_IPIP) 3029 return rte_flow_error_set(error, EINVAL, 3030 RTE_FLOW_ERROR_TYPE_ITEM, item, 3031 "wrong tunnel type - IPv4 specified " 3032 "but IPv6 item provided"); 3033 if (item_flags & l3m) 3034 return rte_flow_error_set(error, ENOTSUP, 3035 RTE_FLOW_ERROR_TYPE_ITEM, item, 3036 "multiple L3 layers not supported"); 3037 else if (item_flags & l4m) 3038 return rte_flow_error_set(error, EINVAL, 3039 RTE_FLOW_ERROR_TYPE_ITEM, item, 3040 "L3 cannot follow an L4 layer."); 3041 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 3042 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 3043 return rte_flow_error_set(error, EINVAL, 3044 RTE_FLOW_ERROR_TYPE_ITEM, item, 3045 "L3 cannot follow an NVGRE layer."); 3046 if (!mask) 3047 mask = &rte_flow_item_ipv6_mask; 3048 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3049 acc_mask ? (const uint8_t *)acc_mask 3050 : (const uint8_t *)&nic_mask, 3051 sizeof(struct rte_flow_item_ipv6), 3052 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3053 if (ret < 0) 3054 return ret; 3055 return 0; 3056 } 3057 3058 /** 3059 * Validate UDP item. 3060 * 3061 * @param[in] item 3062 * Item specification. 3063 * @param[in] item_flags 3064 * Bit-fields that holds the items detected until now. 3065 * @param[in] target_protocol 3066 * The next protocol in the previous item. 3067 * @param[in] flow_mask 3068 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 3069 * @param[out] error 3070 * Pointer to error structure. 3071 * 3072 * @return 3073 * 0 on success, a negative errno value otherwise and rte_errno is set. 3074 */ 3075 int 3076 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 3077 uint64_t item_flags, 3078 uint8_t target_protocol, 3079 struct rte_flow_error *error) 3080 { 3081 const struct rte_flow_item_udp *mask = item->mask; 3082 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3083 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3084 MLX5_FLOW_LAYER_OUTER_L3; 3085 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3086 MLX5_FLOW_LAYER_OUTER_L4; 3087 int ret; 3088 3089 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 3090 return rte_flow_error_set(error, EINVAL, 3091 RTE_FLOW_ERROR_TYPE_ITEM, item, 3092 "protocol filtering not compatible" 3093 " with UDP layer"); 3094 if (!(item_flags & l3m)) 3095 return rte_flow_error_set(error, EINVAL, 3096 RTE_FLOW_ERROR_TYPE_ITEM, item, 3097 "L3 is mandatory to filter on L4"); 3098 if (item_flags & l4m) 3099 return rte_flow_error_set(error, EINVAL, 3100 RTE_FLOW_ERROR_TYPE_ITEM, item, 3101 "multiple L4 layers not supported"); 3102 if (!mask) 3103 mask = &rte_flow_item_udp_mask; 3104 ret = mlx5_flow_item_acceptable 3105 (item, (const uint8_t *)mask, 3106 (const uint8_t *)&rte_flow_item_udp_mask, 3107 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3108 error); 3109 if (ret < 0) 3110 return ret; 3111 return 0; 3112 } 3113 3114 /** 3115 * Validate TCP item. 3116 * 3117 * @param[in] item 3118 * Item specification. 3119 * @param[in] item_flags 3120 * Bit-fields that holds the items detected until now. 3121 * @param[in] target_protocol 3122 * The next protocol in the previous item. 3123 * @param[out] error 3124 * Pointer to error structure. 3125 * 3126 * @return 3127 * 0 on success, a negative errno value otherwise and rte_errno is set. 3128 */ 3129 int 3130 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 3131 uint64_t item_flags, 3132 uint8_t target_protocol, 3133 const struct rte_flow_item_tcp *flow_mask, 3134 struct rte_flow_error *error) 3135 { 3136 const struct rte_flow_item_tcp *mask = item->mask; 3137 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3138 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3139 MLX5_FLOW_LAYER_OUTER_L3; 3140 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3141 MLX5_FLOW_LAYER_OUTER_L4; 3142 int ret; 3143 3144 MLX5_ASSERT(flow_mask); 3145 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3146 return rte_flow_error_set(error, EINVAL, 3147 RTE_FLOW_ERROR_TYPE_ITEM, item, 3148 "protocol filtering not compatible" 3149 " with TCP layer"); 3150 if (!(item_flags & l3m)) 3151 return rte_flow_error_set(error, EINVAL, 3152 RTE_FLOW_ERROR_TYPE_ITEM, item, 3153 "L3 is mandatory to filter on L4"); 3154 if (item_flags & l4m) 3155 return rte_flow_error_set(error, EINVAL, 3156 RTE_FLOW_ERROR_TYPE_ITEM, item, 3157 "multiple L4 layers not supported"); 3158 if (!mask) 3159 mask = &rte_flow_item_tcp_mask; 3160 ret = mlx5_flow_item_acceptable 3161 (item, (const uint8_t *)mask, 3162 (const uint8_t *)flow_mask, 3163 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3164 error); 3165 if (ret < 0) 3166 return ret; 3167 return 0; 3168 } 3169 3170 /** 3171 * Validate VXLAN item. 3172 * 3173 * @param[in] dev 3174 * Pointer to the Ethernet device structure. 3175 * @param[in] udp_dport 3176 * UDP destination port 3177 * @param[in] item 3178 * Item specification. 3179 * @param[in] item_flags 3180 * Bit-fields that holds the items detected until now. 3181 * @param root 3182 * Whether action is on root table. 3183 * @param[out] error 3184 * Pointer to error structure. 3185 * 3186 * @return 3187 * 0 on success, a negative errno value otherwise and rte_errno is set. 3188 */ 3189 int 3190 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3191 uint16_t udp_dport, 3192 const struct rte_flow_item *item, 3193 uint64_t item_flags, 3194 bool root, 3195 struct rte_flow_error *error) 3196 { 3197 const struct rte_flow_item_vxlan *spec = item->spec; 3198 const struct rte_flow_item_vxlan *mask = item->mask; 3199 int ret; 3200 struct mlx5_priv *priv = dev->data->dev_private; 3201 union vni { 3202 uint32_t vlan_id; 3203 uint8_t vni[4]; 3204 } id = { .vlan_id = 0, }; 3205 const struct rte_flow_item_vxlan nic_mask = { 3206 .hdr.vni = "\xff\xff\xff", 3207 .hdr.rsvd1 = 0xff, 3208 }; 3209 const struct rte_flow_item_vxlan *valid_mask; 3210 3211 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3212 return rte_flow_error_set(error, ENOTSUP, 3213 RTE_FLOW_ERROR_TYPE_ITEM, item, 3214 "multiple tunnel layers not" 3215 " supported"); 3216 valid_mask = &rte_flow_item_vxlan_mask; 3217 /* 3218 * Verify only UDPv4 is present as defined in 3219 * https://tools.ietf.org/html/rfc7348 3220 */ 3221 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3222 return rte_flow_error_set(error, EINVAL, 3223 RTE_FLOW_ERROR_TYPE_ITEM, item, 3224 "no outer UDP layer found"); 3225 if (!mask) 3226 mask = &rte_flow_item_vxlan_mask; 3227 3228 if (priv->sh->steering_format_version != 3229 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3230 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3231 /* non-root table */ 3232 if (!root && priv->sh->misc5_cap) 3233 valid_mask = &nic_mask; 3234 /* Group zero in NIC domain */ 3235 if (!root && priv->sh->tunnel_header_0_1) 3236 valid_mask = &nic_mask; 3237 } 3238 ret = mlx5_flow_item_acceptable 3239 (item, (const uint8_t *)mask, 3240 (const uint8_t *)valid_mask, 3241 sizeof(struct rte_flow_item_vxlan), 3242 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3243 if (ret < 0) 3244 return ret; 3245 if (spec) { 3246 memcpy(&id.vni[1], spec->hdr.vni, 3); 3247 memcpy(&id.vni[1], mask->hdr.vni, 3); 3248 } 3249 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3250 return rte_flow_error_set(error, ENOTSUP, 3251 RTE_FLOW_ERROR_TYPE_ITEM, item, 3252 "VXLAN tunnel must be fully defined"); 3253 return 0; 3254 } 3255 3256 /** 3257 * Validate VXLAN_GPE item. 3258 * 3259 * @param[in] item 3260 * Item specification. 3261 * @param[in] item_flags 3262 * Bit-fields that holds the items detected until now. 3263 * @param[in] priv 3264 * Pointer to the private data structure. 3265 * @param[in] target_protocol 3266 * The next protocol in the previous item. 3267 * @param[out] error 3268 * Pointer to error structure. 3269 * 3270 * @return 3271 * 0 on success, a negative errno value otherwise and rte_errno is set. 3272 */ 3273 int 3274 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3275 uint64_t item_flags, 3276 struct rte_eth_dev *dev, 3277 struct rte_flow_error *error) 3278 { 3279 struct mlx5_priv *priv = dev->data->dev_private; 3280 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3281 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3282 int ret; 3283 union vni { 3284 uint32_t vlan_id; 3285 uint8_t vni[4]; 3286 } id = { .vlan_id = 0, }; 3287 3288 struct rte_flow_item_vxlan_gpe nic_mask = { 3289 .vni = "\xff\xff\xff", 3290 .protocol = 0xff, 3291 }; 3292 3293 if (!priv->sh->config.l3_vxlan_en) 3294 return rte_flow_error_set(error, ENOTSUP, 3295 RTE_FLOW_ERROR_TYPE_ITEM, item, 3296 "L3 VXLAN is not enabled by device" 3297 " parameter and/or not configured in" 3298 " firmware"); 3299 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3300 return rte_flow_error_set(error, ENOTSUP, 3301 RTE_FLOW_ERROR_TYPE_ITEM, item, 3302 "multiple tunnel layers not" 3303 " supported"); 3304 /* 3305 * Verify only UDPv4 is present as defined in 3306 * https://tools.ietf.org/html/rfc7348 3307 */ 3308 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3309 return rte_flow_error_set(error, EINVAL, 3310 RTE_FLOW_ERROR_TYPE_ITEM, item, 3311 "no outer UDP layer found"); 3312 if (!mask) 3313 mask = &rte_flow_item_vxlan_gpe_mask; 3314 ret = mlx5_flow_item_acceptable 3315 (item, (const uint8_t *)mask, 3316 (const uint8_t *)&nic_mask, 3317 sizeof(struct rte_flow_item_vxlan_gpe), 3318 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3319 if (ret < 0) 3320 return ret; 3321 if (spec) { 3322 memcpy(&id.vni[1], spec->hdr.vni, 3); 3323 memcpy(&id.vni[1], mask->hdr.vni, 3); 3324 } 3325 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3326 return rte_flow_error_set(error, ENOTSUP, 3327 RTE_FLOW_ERROR_TYPE_ITEM, item, 3328 "VXLAN-GPE tunnel must be fully" 3329 " defined"); 3330 return 0; 3331 } 3332 /** 3333 * Validate GRE Key item. 3334 * 3335 * @param[in] item 3336 * Item specification. 3337 * @param[in] item_flags 3338 * Bit flags to mark detected items. 3339 * @param[in] gre_item 3340 * Pointer to gre_item 3341 * @param[out] error 3342 * Pointer to error structure. 3343 * 3344 * @return 3345 * 0 on success, a negative errno value otherwise and rte_errno is set. 3346 */ 3347 int 3348 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3349 uint64_t item_flags, 3350 const struct rte_flow_item *gre_item, 3351 struct rte_flow_error *error) 3352 { 3353 const rte_be32_t *mask = item->mask; 3354 int ret = 0; 3355 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3356 const struct rte_flow_item_gre *gre_spec; 3357 const struct rte_flow_item_gre *gre_mask; 3358 3359 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3360 return rte_flow_error_set(error, ENOTSUP, 3361 RTE_FLOW_ERROR_TYPE_ITEM, item, 3362 "Multiple GRE key not support"); 3363 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3364 return rte_flow_error_set(error, ENOTSUP, 3365 RTE_FLOW_ERROR_TYPE_ITEM, item, 3366 "No preceding GRE header"); 3367 if (item_flags & MLX5_FLOW_LAYER_INNER) 3368 return rte_flow_error_set(error, ENOTSUP, 3369 RTE_FLOW_ERROR_TYPE_ITEM, item, 3370 "GRE key following a wrong item"); 3371 gre_mask = gre_item->mask; 3372 if (!gre_mask) 3373 gre_mask = &rte_flow_item_gre_mask; 3374 gre_spec = gre_item->spec; 3375 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3376 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3377 return rte_flow_error_set(error, EINVAL, 3378 RTE_FLOW_ERROR_TYPE_ITEM, item, 3379 "Key bit must be on"); 3380 3381 if (!mask) 3382 mask = &gre_key_default_mask; 3383 ret = mlx5_flow_item_acceptable 3384 (item, (const uint8_t *)mask, 3385 (const uint8_t *)&gre_key_default_mask, 3386 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3387 return ret; 3388 } 3389 3390 /** 3391 * Validate GRE optional item. 3392 * 3393 * @param[in] dev 3394 * Pointer to the Ethernet device structure. 3395 * @param[in] item 3396 * Item specification. 3397 * @param[in] item_flags 3398 * Bit flags to mark detected items. 3399 * @param[in] attr 3400 * Flow rule attributes. 3401 * @param[in] gre_item 3402 * Pointer to gre_item 3403 * @param[out] error 3404 * Pointer to error structure. 3405 * 3406 * @return 3407 * 0 on success, a negative errno value otherwise and rte_errno is set. 3408 */ 3409 int 3410 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3411 const struct rte_flow_item *item, 3412 uint64_t item_flags, 3413 const struct rte_flow_attr *attr, 3414 const struct rte_flow_item *gre_item, 3415 struct rte_flow_error *error) 3416 { 3417 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3418 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3419 const struct rte_flow_item_gre_opt *spec = item->spec; 3420 const struct rte_flow_item_gre_opt *mask = item->mask; 3421 struct mlx5_priv *priv = dev->data->dev_private; 3422 int ret = 0; 3423 struct rte_flow_item_gre_opt nic_mask = { 3424 .checksum_rsvd = { 3425 .checksum = RTE_BE16(UINT16_MAX), 3426 .reserved1 = 0x0, 3427 }, 3428 .key = { 3429 .key = RTE_BE32(UINT32_MAX), 3430 }, 3431 .sequence = { 3432 .sequence = RTE_BE32(UINT32_MAX), 3433 }, 3434 }; 3435 3436 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3437 return rte_flow_error_set(error, ENOTSUP, 3438 RTE_FLOW_ERROR_TYPE_ITEM, item, 3439 "No preceding GRE header"); 3440 if (item_flags & MLX5_FLOW_LAYER_INNER) 3441 return rte_flow_error_set(error, ENOTSUP, 3442 RTE_FLOW_ERROR_TYPE_ITEM, item, 3443 "GRE option following a wrong item"); 3444 if (!spec || !mask) 3445 return rte_flow_error_set(error, EINVAL, 3446 RTE_FLOW_ERROR_TYPE_ITEM, item, 3447 "At least one field gre_option(checksum/key/sequence) must be specified"); 3448 if (!gre_mask) 3449 gre_mask = &rte_flow_item_gre_mask; 3450 if (mask->checksum_rsvd.checksum) 3451 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3452 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3453 return rte_flow_error_set(error, EINVAL, 3454 RTE_FLOW_ERROR_TYPE_ITEM, 3455 item, 3456 "Checksum bit must be on"); 3457 if (mask->key.key) 3458 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3459 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3460 return rte_flow_error_set(error, EINVAL, 3461 RTE_FLOW_ERROR_TYPE_ITEM, 3462 item, "Key bit must be on"); 3463 if (mask->sequence.sequence) 3464 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3465 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3466 return rte_flow_error_set(error, EINVAL, 3467 RTE_FLOW_ERROR_TYPE_ITEM, 3468 item, 3469 "Sequence bit must be on"); 3470 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3471 if (priv->sh->steering_format_version == 3472 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3473 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3474 !priv->sh->misc5_cap) || 3475 (!(priv->sh->tunnel_header_0_1 && 3476 priv->sh->tunnel_header_2_3) && 3477 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3478 return rte_flow_error_set(error, EINVAL, 3479 RTE_FLOW_ERROR_TYPE_ITEM, 3480 item, 3481 "Checksum/Sequence not supported"); 3482 } 3483 ret = mlx5_flow_item_acceptable 3484 (item, (const uint8_t *)mask, 3485 (const uint8_t *)&nic_mask, 3486 sizeof(struct rte_flow_item_gre_opt), 3487 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3488 return ret; 3489 } 3490 3491 /** 3492 * Validate GRE item. 3493 * 3494 * @param[in] item 3495 * Item specification. 3496 * @param[in] item_flags 3497 * Bit flags to mark detected items. 3498 * @param[in] target_protocol 3499 * The next protocol in the previous item. 3500 * @param[out] error 3501 * Pointer to error structure. 3502 * 3503 * @return 3504 * 0 on success, a negative errno value otherwise and rte_errno is set. 3505 */ 3506 int 3507 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3508 uint64_t item_flags, 3509 uint8_t target_protocol, 3510 struct rte_flow_error *error) 3511 { 3512 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3513 const struct rte_flow_item_gre *mask = item->mask; 3514 int ret; 3515 const struct rte_flow_item_gre nic_mask = { 3516 .c_rsvd0_ver = RTE_BE16(0xB000), 3517 .protocol = RTE_BE16(UINT16_MAX), 3518 }; 3519 3520 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3521 return rte_flow_error_set(error, EINVAL, 3522 RTE_FLOW_ERROR_TYPE_ITEM, item, 3523 "protocol filtering not compatible" 3524 " with this GRE layer"); 3525 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3526 return rte_flow_error_set(error, ENOTSUP, 3527 RTE_FLOW_ERROR_TYPE_ITEM, item, 3528 "multiple tunnel layers not" 3529 " supported"); 3530 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3531 return rte_flow_error_set(error, ENOTSUP, 3532 RTE_FLOW_ERROR_TYPE_ITEM, item, 3533 "L3 Layer is missing"); 3534 if (!mask) 3535 mask = &rte_flow_item_gre_mask; 3536 ret = mlx5_flow_item_acceptable 3537 (item, (const uint8_t *)mask, 3538 (const uint8_t *)&nic_mask, 3539 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3540 error); 3541 if (ret < 0) 3542 return ret; 3543 #ifndef HAVE_MLX5DV_DR 3544 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3545 if (spec && (spec->protocol & mask->protocol)) 3546 return rte_flow_error_set(error, ENOTSUP, 3547 RTE_FLOW_ERROR_TYPE_ITEM, item, 3548 "without MPLS support the" 3549 " specification cannot be used for" 3550 " filtering"); 3551 #endif 3552 #endif 3553 return 0; 3554 } 3555 3556 /** 3557 * Validate Geneve item. 3558 * 3559 * @param[in] item 3560 * Item specification. 3561 * @param[in] itemFlags 3562 * Bit-fields that holds the items detected until now. 3563 * @param[in] enPriv 3564 * Pointer to the private data structure. 3565 * @param[out] error 3566 * Pointer to error structure. 3567 * 3568 * @return 3569 * 0 on success, a negative errno value otherwise and rte_errno is set. 3570 */ 3571 3572 int 3573 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3574 uint64_t item_flags, 3575 struct rte_eth_dev *dev, 3576 struct rte_flow_error *error) 3577 { 3578 struct mlx5_priv *priv = dev->data->dev_private; 3579 const struct rte_flow_item_geneve *spec = item->spec; 3580 const struct rte_flow_item_geneve *mask = item->mask; 3581 int ret; 3582 uint16_t gbhdr; 3583 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3584 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3585 const struct rte_flow_item_geneve nic_mask = { 3586 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3587 .vni = "\xff\xff\xff", 3588 .protocol = RTE_BE16(UINT16_MAX), 3589 }; 3590 3591 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3592 return rte_flow_error_set(error, ENOTSUP, 3593 RTE_FLOW_ERROR_TYPE_ITEM, item, 3594 "L3 Geneve is not enabled by device" 3595 " parameter and/or not configured in" 3596 " firmware"); 3597 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3598 return rte_flow_error_set(error, ENOTSUP, 3599 RTE_FLOW_ERROR_TYPE_ITEM, item, 3600 "multiple tunnel layers not" 3601 " supported"); 3602 /* 3603 * Verify only UDPv4 is present as defined in 3604 * https://tools.ietf.org/html/rfc7348 3605 */ 3606 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3607 return rte_flow_error_set(error, EINVAL, 3608 RTE_FLOW_ERROR_TYPE_ITEM, item, 3609 "no outer UDP layer found"); 3610 if (!mask) 3611 mask = &rte_flow_item_geneve_mask; 3612 ret = mlx5_flow_item_acceptable 3613 (item, (const uint8_t *)mask, 3614 (const uint8_t *)&nic_mask, 3615 sizeof(struct rte_flow_item_geneve), 3616 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3617 if (ret) 3618 return ret; 3619 if (spec) { 3620 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3621 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3622 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3623 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3624 return rte_flow_error_set(error, ENOTSUP, 3625 RTE_FLOW_ERROR_TYPE_ITEM, 3626 item, 3627 "Geneve protocol unsupported" 3628 " fields are being used"); 3629 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3630 return rte_flow_error_set 3631 (error, ENOTSUP, 3632 RTE_FLOW_ERROR_TYPE_ITEM, 3633 item, 3634 "Unsupported Geneve options length"); 3635 } 3636 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3637 return rte_flow_error_set 3638 (error, ENOTSUP, 3639 RTE_FLOW_ERROR_TYPE_ITEM, item, 3640 "Geneve tunnel must be fully defined"); 3641 return 0; 3642 } 3643 3644 /** 3645 * Validate Geneve TLV option item. 3646 * 3647 * @param[in] item 3648 * Item specification. 3649 * @param[in] last_item 3650 * Previous validated item in the pattern items. 3651 * @param[in] geneve_item 3652 * Previous GENEVE item specification. 3653 * @param[in] dev 3654 * Pointer to the rte_eth_dev structure. 3655 * @param[out] error 3656 * Pointer to error structure. 3657 * 3658 * @return 3659 * 0 on success, a negative errno value otherwise and rte_errno is set. 3660 */ 3661 int 3662 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3663 uint64_t last_item, 3664 const struct rte_flow_item *geneve_item, 3665 struct rte_eth_dev *dev, 3666 struct rte_flow_error *error) 3667 { 3668 struct mlx5_priv *priv = dev->data->dev_private; 3669 struct mlx5_dev_ctx_shared *sh = priv->sh; 3670 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3671 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3672 uint8_t data_max_supported = 3673 hca_attr->max_geneve_tlv_option_data_len * 4; 3674 const struct rte_flow_item_geneve *geneve_spec; 3675 const struct rte_flow_item_geneve *geneve_mask; 3676 const struct rte_flow_item_geneve_opt *spec = item->spec; 3677 const struct rte_flow_item_geneve_opt *mask = item->mask; 3678 unsigned int i; 3679 unsigned int data_len; 3680 uint8_t tlv_option_len; 3681 uint16_t optlen_m, optlen_v; 3682 const struct rte_flow_item_geneve_opt full_mask = { 3683 .option_class = RTE_BE16(0xffff), 3684 .option_type = 0xff, 3685 .option_len = 0x1f, 3686 }; 3687 3688 if (!mask) 3689 mask = &rte_flow_item_geneve_opt_mask; 3690 if (!spec) 3691 return rte_flow_error_set 3692 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3693 "Geneve TLV opt class/type/length must be specified"); 3694 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3695 return rte_flow_error_set 3696 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3697 "Geneve TLV opt length exceeds the limit (31)"); 3698 /* Check if class type and length masks are full. */ 3699 if (full_mask.option_class != mask->option_class || 3700 full_mask.option_type != mask->option_type || 3701 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3702 return rte_flow_error_set 3703 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3704 "Geneve TLV opt class/type/length masks must be full"); 3705 /* Check if length is supported */ 3706 if ((uint32_t)spec->option_len > 3707 hca_attr->max_geneve_tlv_option_data_len) 3708 return rte_flow_error_set 3709 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3710 "Geneve TLV opt length not supported"); 3711 if (hca_attr->max_geneve_tlv_options > 1) 3712 DRV_LOG(DEBUG, 3713 "max_geneve_tlv_options supports more than 1 option"); 3714 /* Check GENEVE item preceding. */ 3715 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3716 return rte_flow_error_set 3717 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3718 "Geneve opt item must be preceded with Geneve item"); 3719 geneve_spec = geneve_item->spec; 3720 geneve_mask = geneve_item->mask ? geneve_item->mask : 3721 &rte_flow_item_geneve_mask; 3722 /* Check if GENEVE TLV option size doesn't exceed option length */ 3723 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3724 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3725 tlv_option_len = spec->option_len & mask->option_len; 3726 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3727 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3728 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3729 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3730 if ((optlen_v & optlen_m) <= tlv_option_len) 3731 return rte_flow_error_set 3732 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3733 "GENEVE TLV option length exceeds optlen"); 3734 } 3735 /* Check if length is 0 or data is 0. */ 3736 if (spec->data == NULL || spec->option_len == 0) 3737 return rte_flow_error_set 3738 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3739 "Geneve TLV opt with zero data/length not supported"); 3740 /* Check not all data & mask are 0. */ 3741 data_len = spec->option_len * 4; 3742 if (mask->data == NULL) { 3743 for (i = 0; i < data_len; i++) 3744 if (spec->data[i]) 3745 break; 3746 if (i == data_len) 3747 return rte_flow_error_set(error, ENOTSUP, 3748 RTE_FLOW_ERROR_TYPE_ITEM, item, 3749 "Can't match on Geneve option data 0"); 3750 } else { 3751 for (i = 0; i < data_len; i++) 3752 if (spec->data[i] & mask->data[i]) 3753 break; 3754 if (i == data_len) 3755 return rte_flow_error_set(error, ENOTSUP, 3756 RTE_FLOW_ERROR_TYPE_ITEM, item, 3757 "Can't match on Geneve option data and mask 0"); 3758 /* Check data mask supported. */ 3759 for (i = data_max_supported; i < data_len ; i++) 3760 if (mask->data[i]) 3761 return rte_flow_error_set(error, ENOTSUP, 3762 RTE_FLOW_ERROR_TYPE_ITEM, item, 3763 "Data mask is of unsupported size"); 3764 } 3765 /* Check GENEVE option is supported in NIC. */ 3766 if (!hca_attr->geneve_tlv_opt) 3767 return rte_flow_error_set 3768 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3769 "Geneve TLV opt not supported"); 3770 /* Check if we already have geneve option with different type/class. */ 3771 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3772 geneve_opt_resource = sh->geneve_tlv_option_resource; 3773 if (geneve_opt_resource != NULL) 3774 if (geneve_opt_resource->option_class != spec->option_class || 3775 geneve_opt_resource->option_type != spec->option_type || 3776 geneve_opt_resource->length != spec->option_len) { 3777 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3778 return rte_flow_error_set(error, ENOTSUP, 3779 RTE_FLOW_ERROR_TYPE_ITEM, item, 3780 "Only one Geneve TLV option supported"); 3781 } 3782 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3783 return 0; 3784 } 3785 3786 /** 3787 * Validate MPLS item. 3788 * 3789 * @param[in] dev 3790 * Pointer to the rte_eth_dev structure. 3791 * @param[in] item 3792 * Item specification. 3793 * @param[in] item_flags 3794 * Bit-fields that holds the items detected until now. 3795 * @param[in] prev_layer 3796 * The protocol layer indicated in previous item. 3797 * @param[out] error 3798 * Pointer to error structure. 3799 * 3800 * @return 3801 * 0 on success, a negative errno value otherwise and rte_errno is set. 3802 */ 3803 int 3804 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3805 const struct rte_flow_item *item __rte_unused, 3806 uint64_t item_flags __rte_unused, 3807 uint64_t prev_layer __rte_unused, 3808 struct rte_flow_error *error) 3809 { 3810 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3811 const struct rte_flow_item_mpls *mask = item->mask; 3812 struct mlx5_priv *priv = dev->data->dev_private; 3813 int ret; 3814 3815 if (!priv->sh->dev_cap.mpls_en) 3816 return rte_flow_error_set(error, ENOTSUP, 3817 RTE_FLOW_ERROR_TYPE_ITEM, item, 3818 "MPLS not supported or" 3819 " disabled in firmware" 3820 " configuration."); 3821 /* MPLS over UDP, GRE is allowed */ 3822 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3823 MLX5_FLOW_LAYER_GRE | 3824 MLX5_FLOW_LAYER_GRE_KEY))) 3825 return rte_flow_error_set(error, EINVAL, 3826 RTE_FLOW_ERROR_TYPE_ITEM, item, 3827 "protocol filtering not compatible" 3828 " with MPLS layer"); 3829 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3830 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3831 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3832 return rte_flow_error_set(error, ENOTSUP, 3833 RTE_FLOW_ERROR_TYPE_ITEM, item, 3834 "multiple tunnel layers not" 3835 " supported"); 3836 if (!mask) 3837 mask = &rte_flow_item_mpls_mask; 3838 ret = mlx5_flow_item_acceptable 3839 (item, (const uint8_t *)mask, 3840 (const uint8_t *)&rte_flow_item_mpls_mask, 3841 sizeof(struct rte_flow_item_mpls), 3842 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3843 if (ret < 0) 3844 return ret; 3845 return 0; 3846 #else 3847 return rte_flow_error_set(error, ENOTSUP, 3848 RTE_FLOW_ERROR_TYPE_ITEM, item, 3849 "MPLS is not supported by Verbs, please" 3850 " update."); 3851 #endif 3852 } 3853 3854 /** 3855 * Validate NVGRE item. 3856 * 3857 * @param[in] item 3858 * Item specification. 3859 * @param[in] item_flags 3860 * Bit flags to mark detected items. 3861 * @param[in] target_protocol 3862 * The next protocol in the previous item. 3863 * @param[out] error 3864 * Pointer to error structure. 3865 * 3866 * @return 3867 * 0 on success, a negative errno value otherwise and rte_errno is set. 3868 */ 3869 int 3870 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3871 uint64_t item_flags, 3872 uint8_t target_protocol, 3873 struct rte_flow_error *error) 3874 { 3875 const struct rte_flow_item_nvgre *mask = item->mask; 3876 int ret; 3877 3878 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3879 return rte_flow_error_set(error, EINVAL, 3880 RTE_FLOW_ERROR_TYPE_ITEM, item, 3881 "protocol filtering not compatible" 3882 " with this GRE layer"); 3883 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3884 return rte_flow_error_set(error, ENOTSUP, 3885 RTE_FLOW_ERROR_TYPE_ITEM, item, 3886 "multiple tunnel layers not" 3887 " supported"); 3888 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3889 return rte_flow_error_set(error, ENOTSUP, 3890 RTE_FLOW_ERROR_TYPE_ITEM, item, 3891 "L3 Layer is missing"); 3892 if (!mask) 3893 mask = &rte_flow_item_nvgre_mask; 3894 ret = mlx5_flow_item_acceptable 3895 (item, (const uint8_t *)mask, 3896 (const uint8_t *)&rte_flow_item_nvgre_mask, 3897 sizeof(struct rte_flow_item_nvgre), 3898 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3899 if (ret < 0) 3900 return ret; 3901 return 0; 3902 } 3903 3904 /** 3905 * Validate eCPRI item. 3906 * 3907 * @param[in] item 3908 * Item specification. 3909 * @param[in] item_flags 3910 * Bit-fields that holds the items detected until now. 3911 * @param[in] last_item 3912 * Previous validated item in the pattern items. 3913 * @param[in] ether_type 3914 * Type in the ethernet layer header (including dot1q). 3915 * @param[in] acc_mask 3916 * Acceptable mask, if NULL default internal default mask 3917 * will be used to check whether item fields are supported. 3918 * @param[out] error 3919 * Pointer to error structure. 3920 * 3921 * @return 3922 * 0 on success, a negative errno value otherwise and rte_errno is set. 3923 */ 3924 int 3925 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3926 uint64_t item_flags, 3927 uint64_t last_item, 3928 uint16_t ether_type, 3929 const struct rte_flow_item_ecpri *acc_mask, 3930 struct rte_flow_error *error) 3931 { 3932 const struct rte_flow_item_ecpri *mask = item->mask; 3933 const struct rte_flow_item_ecpri nic_mask = { 3934 .hdr = { 3935 .common = { 3936 .u32 = 3937 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3938 .type = 0xFF, 3939 }).u32), 3940 }, 3941 .dummy[0] = 0xFFFFFFFF, 3942 }, 3943 }; 3944 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3945 MLX5_FLOW_LAYER_OUTER_VLAN); 3946 struct rte_flow_item_ecpri mask_lo; 3947 3948 if (!(last_item & outer_l2_vlan) && 3949 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3950 return rte_flow_error_set(error, EINVAL, 3951 RTE_FLOW_ERROR_TYPE_ITEM, item, 3952 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3953 if ((last_item & outer_l2_vlan) && ether_type && 3954 ether_type != RTE_ETHER_TYPE_ECPRI) 3955 return rte_flow_error_set(error, EINVAL, 3956 RTE_FLOW_ERROR_TYPE_ITEM, item, 3957 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3958 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3959 return rte_flow_error_set(error, EINVAL, 3960 RTE_FLOW_ERROR_TYPE_ITEM, item, 3961 "eCPRI with tunnel is not supported right now"); 3962 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3963 return rte_flow_error_set(error, ENOTSUP, 3964 RTE_FLOW_ERROR_TYPE_ITEM, item, 3965 "multiple L3 layers not supported"); 3966 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3967 return rte_flow_error_set(error, EINVAL, 3968 RTE_FLOW_ERROR_TYPE_ITEM, item, 3969 "eCPRI cannot coexist with a TCP layer"); 3970 /* In specification, eCPRI could be over UDP layer. */ 3971 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3972 return rte_flow_error_set(error, EINVAL, 3973 RTE_FLOW_ERROR_TYPE_ITEM, item, 3974 "eCPRI over UDP layer is not yet supported right now"); 3975 /* Mask for type field in common header could be zero. */ 3976 if (!mask) 3977 mask = &rte_flow_item_ecpri_mask; 3978 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3979 /* Input mask is in big-endian format. */ 3980 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3981 return rte_flow_error_set(error, EINVAL, 3982 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3983 "partial mask is not supported for protocol"); 3984 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3985 return rte_flow_error_set(error, EINVAL, 3986 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3987 "message header mask must be after a type mask"); 3988 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3989 acc_mask ? (const uint8_t *)acc_mask 3990 : (const uint8_t *)&nic_mask, 3991 sizeof(struct rte_flow_item_ecpri), 3992 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3993 } 3994 3995 /** 3996 * Validate the NSH item. 3997 * 3998 * @param[in] dev 3999 * Pointer to Ethernet device on which flow rule is being created on. 4000 * @param[out] error 4001 * Pointer to error structure. 4002 * 4003 * @return 4004 * 0 on success, a negative errno value otherwise and rte_errno is set. 4005 */ 4006 int 4007 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev, 4008 const struct rte_flow_item *item, 4009 struct rte_flow_error *error) 4010 { 4011 struct mlx5_priv *priv = dev->data->dev_private; 4012 4013 if (item->mask) { 4014 return rte_flow_error_set(error, ENOTSUP, 4015 RTE_FLOW_ERROR_TYPE_ITEM, item, 4016 "NSH fields matching is not supported"); 4017 } 4018 4019 if (!priv->sh->config.dv_flow_en) { 4020 return rte_flow_error_set(error, ENOTSUP, 4021 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4022 NULL, "NSH support requires DV flow interface"); 4023 } 4024 4025 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) { 4026 return rte_flow_error_set(error, ENOTSUP, 4027 RTE_FLOW_ERROR_TYPE_ITEM, item, 4028 "Current FW does not support matching on NSH"); 4029 } 4030 4031 return 0; 4032 } 4033 4034 static int 4035 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 4036 const struct rte_flow_attr *attr __rte_unused, 4037 const struct rte_flow_item items[] __rte_unused, 4038 const struct rte_flow_action actions[] __rte_unused, 4039 bool external __rte_unused, 4040 int hairpin __rte_unused, 4041 struct rte_flow_error *error) 4042 { 4043 return rte_flow_error_set(error, ENOTSUP, 4044 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4045 } 4046 4047 static struct mlx5_flow * 4048 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 4049 const struct rte_flow_attr *attr __rte_unused, 4050 const struct rte_flow_item items[] __rte_unused, 4051 const struct rte_flow_action actions[] __rte_unused, 4052 struct rte_flow_error *error) 4053 { 4054 rte_flow_error_set(error, ENOTSUP, 4055 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4056 return NULL; 4057 } 4058 4059 static int 4060 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 4061 struct mlx5_flow *dev_flow __rte_unused, 4062 const struct rte_flow_attr *attr __rte_unused, 4063 const struct rte_flow_item items[] __rte_unused, 4064 const struct rte_flow_action actions[] __rte_unused, 4065 struct rte_flow_error *error) 4066 { 4067 return rte_flow_error_set(error, ENOTSUP, 4068 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4069 } 4070 4071 static int 4072 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 4073 struct rte_flow *flow __rte_unused, 4074 struct rte_flow_error *error) 4075 { 4076 return rte_flow_error_set(error, ENOTSUP, 4077 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4078 } 4079 4080 static void 4081 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 4082 struct rte_flow *flow __rte_unused) 4083 { 4084 } 4085 4086 static void 4087 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 4088 struct rte_flow *flow __rte_unused) 4089 { 4090 } 4091 4092 static int 4093 flow_null_query(struct rte_eth_dev *dev __rte_unused, 4094 struct rte_flow *flow __rte_unused, 4095 const struct rte_flow_action *actions __rte_unused, 4096 void *data __rte_unused, 4097 struct rte_flow_error *error) 4098 { 4099 return rte_flow_error_set(error, ENOTSUP, 4100 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4101 } 4102 4103 static int 4104 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 4105 uint32_t domains __rte_unused, 4106 uint32_t flags __rte_unused) 4107 { 4108 return 0; 4109 } 4110 4111 int 4112 flow_null_get_aged_flows(struct rte_eth_dev *dev, 4113 void **context __rte_unused, 4114 uint32_t nb_contexts __rte_unused, 4115 struct rte_flow_error *error __rte_unused) 4116 { 4117 DRV_LOG(ERR, "port %u get aged flows is not supported.", 4118 dev->data->port_id); 4119 return -ENOTSUP; 4120 } 4121 4122 uint32_t 4123 flow_null_counter_allocate(struct rte_eth_dev *dev) 4124 { 4125 DRV_LOG(ERR, "port %u counter allocate is not supported.", 4126 dev->data->port_id); 4127 return 0; 4128 } 4129 4130 void 4131 flow_null_counter_free(struct rte_eth_dev *dev, 4132 uint32_t counter __rte_unused) 4133 { 4134 DRV_LOG(ERR, "port %u counter free is not supported.", 4135 dev->data->port_id); 4136 } 4137 4138 int 4139 flow_null_counter_query(struct rte_eth_dev *dev, 4140 uint32_t counter __rte_unused, 4141 bool clear __rte_unused, 4142 uint64_t *pkts __rte_unused, 4143 uint64_t *bytes __rte_unused, 4144 void **action __rte_unused) 4145 { 4146 DRV_LOG(ERR, "port %u counter query is not supported.", 4147 dev->data->port_id); 4148 return -ENOTSUP; 4149 } 4150 4151 /* Void driver to protect from null pointer reference. */ 4152 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 4153 .validate = flow_null_validate, 4154 .prepare = flow_null_prepare, 4155 .translate = flow_null_translate, 4156 .apply = flow_null_apply, 4157 .remove = flow_null_remove, 4158 .destroy = flow_null_destroy, 4159 .query = flow_null_query, 4160 .sync_domain = flow_null_sync_domain, 4161 .get_aged_flows = flow_null_get_aged_flows, 4162 .counter_alloc = flow_null_counter_allocate, 4163 .counter_free = flow_null_counter_free, 4164 .counter_query = flow_null_counter_query 4165 }; 4166 4167 /** 4168 * Select flow driver type according to flow attributes and device 4169 * configuration. 4170 * 4171 * @param[in] dev 4172 * Pointer to the dev structure. 4173 * @param[in] attr 4174 * Pointer to the flow attributes. 4175 * 4176 * @return 4177 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4178 */ 4179 static enum mlx5_flow_drv_type 4180 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4181 { 4182 struct mlx5_priv *priv = dev->data->dev_private; 4183 /* The OS can determine first a specific flow type (DV, VERBS) */ 4184 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4185 4186 if (type != MLX5_FLOW_TYPE_MAX) 4187 return type; 4188 /* 4189 * Currently when dv_flow_en == 2, only HW steering engine is 4190 * supported. New engines can also be chosen here if ready. 4191 */ 4192 if (priv->sh->config.dv_flow_en == 2) 4193 return MLX5_FLOW_TYPE_HW; 4194 if (!attr) 4195 return MLX5_FLOW_TYPE_MIN; 4196 /* If no OS specific type - continue with DV/VERBS selection */ 4197 if (attr->transfer && priv->sh->config.dv_esw_en) 4198 type = MLX5_FLOW_TYPE_DV; 4199 if (!attr->transfer) 4200 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4201 MLX5_FLOW_TYPE_VERBS; 4202 return type; 4203 } 4204 4205 #define flow_get_drv_ops(type) flow_drv_ops[type] 4206 4207 /** 4208 * Flow driver validation API. This abstracts calling driver specific functions. 4209 * The type of flow driver is determined according to flow attributes. 4210 * 4211 * @param[in] dev 4212 * Pointer to the dev structure. 4213 * @param[in] attr 4214 * Pointer to the flow attributes. 4215 * @param[in] items 4216 * Pointer to the list of items. 4217 * @param[in] actions 4218 * Pointer to the list of actions. 4219 * @param[in] external 4220 * This flow rule is created by request external to PMD. 4221 * @param[in] hairpin 4222 * Number of hairpin TX actions, 0 means classic flow. 4223 * @param[out] error 4224 * Pointer to the error structure. 4225 * 4226 * @return 4227 * 0 on success, a negative errno value otherwise and rte_errno is set. 4228 */ 4229 static inline int 4230 flow_drv_validate(struct rte_eth_dev *dev, 4231 const struct rte_flow_attr *attr, 4232 const struct rte_flow_item items[], 4233 const struct rte_flow_action actions[], 4234 bool external, int hairpin, struct rte_flow_error *error) 4235 { 4236 const struct mlx5_flow_driver_ops *fops; 4237 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4238 4239 fops = flow_get_drv_ops(type); 4240 return fops->validate(dev, attr, items, actions, external, 4241 hairpin, error); 4242 } 4243 4244 /** 4245 * Flow driver preparation API. This abstracts calling driver specific 4246 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4247 * calculates the size of memory required for device flow, allocates the memory, 4248 * initializes the device flow and returns the pointer. 4249 * 4250 * @note 4251 * This function initializes device flow structure such as dv or verbs in 4252 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4253 * rest. For example, adding returning device flow to flow->dev_flow list and 4254 * setting backward reference to the flow should be done out of this function. 4255 * layers field is not filled either. 4256 * 4257 * @param[in] dev 4258 * Pointer to the dev structure. 4259 * @param[in] attr 4260 * Pointer to the flow attributes. 4261 * @param[in] items 4262 * Pointer to the list of items. 4263 * @param[in] actions 4264 * Pointer to the list of actions. 4265 * @param[in] flow_idx 4266 * This memory pool index to the flow. 4267 * @param[out] error 4268 * Pointer to the error structure. 4269 * 4270 * @return 4271 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4272 */ 4273 static inline struct mlx5_flow * 4274 flow_drv_prepare(struct rte_eth_dev *dev, 4275 const struct rte_flow *flow, 4276 const struct rte_flow_attr *attr, 4277 const struct rte_flow_item items[], 4278 const struct rte_flow_action actions[], 4279 uint32_t flow_idx, 4280 struct rte_flow_error *error) 4281 { 4282 const struct mlx5_flow_driver_ops *fops; 4283 enum mlx5_flow_drv_type type = flow->drv_type; 4284 struct mlx5_flow *mlx5_flow = NULL; 4285 4286 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4287 fops = flow_get_drv_ops(type); 4288 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4289 if (mlx5_flow) 4290 mlx5_flow->flow_idx = flow_idx; 4291 return mlx5_flow; 4292 } 4293 4294 /** 4295 * Flow driver translation API. This abstracts calling driver specific 4296 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4297 * translates a generic flow into a driver flow. flow_drv_prepare() must 4298 * precede. 4299 * 4300 * @note 4301 * dev_flow->layers could be filled as a result of parsing during translation 4302 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4303 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4304 * flow->actions could be overwritten even though all the expanded dev_flows 4305 * have the same actions. 4306 * 4307 * @param[in] dev 4308 * Pointer to the rte dev structure. 4309 * @param[in, out] dev_flow 4310 * Pointer to the mlx5 flow. 4311 * @param[in] attr 4312 * Pointer to the flow attributes. 4313 * @param[in] items 4314 * Pointer to the list of items. 4315 * @param[in] actions 4316 * Pointer to the list of actions. 4317 * @param[out] error 4318 * Pointer to the error structure. 4319 * 4320 * @return 4321 * 0 on success, a negative errno value otherwise and rte_errno is set. 4322 */ 4323 static inline int 4324 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4325 const struct rte_flow_attr *attr, 4326 const struct rte_flow_item items[], 4327 const struct rte_flow_action actions[], 4328 struct rte_flow_error *error) 4329 { 4330 const struct mlx5_flow_driver_ops *fops; 4331 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4332 4333 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4334 fops = flow_get_drv_ops(type); 4335 return fops->translate(dev, dev_flow, attr, items, actions, error); 4336 } 4337 4338 /** 4339 * Flow driver apply API. This abstracts calling driver specific functions. 4340 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4341 * translated driver flows on to device. flow_drv_translate() must precede. 4342 * 4343 * @param[in] dev 4344 * Pointer to Ethernet device structure. 4345 * @param[in, out] flow 4346 * Pointer to flow structure. 4347 * @param[out] error 4348 * Pointer to error structure. 4349 * 4350 * @return 4351 * 0 on success, a negative errno value otherwise and rte_errno is set. 4352 */ 4353 static inline int 4354 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4355 struct rte_flow_error *error) 4356 { 4357 const struct mlx5_flow_driver_ops *fops; 4358 enum mlx5_flow_drv_type type = flow->drv_type; 4359 4360 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4361 fops = flow_get_drv_ops(type); 4362 return fops->apply(dev, flow, error); 4363 } 4364 4365 /** 4366 * Flow driver destroy API. This abstracts calling driver specific functions. 4367 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4368 * on device and releases resources of the flow. 4369 * 4370 * @param[in] dev 4371 * Pointer to Ethernet device. 4372 * @param[in, out] flow 4373 * Pointer to flow structure. 4374 */ 4375 static inline void 4376 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4377 { 4378 const struct mlx5_flow_driver_ops *fops; 4379 enum mlx5_flow_drv_type type = flow->drv_type; 4380 4381 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4382 fops = flow_get_drv_ops(type); 4383 fops->destroy(dev, flow); 4384 } 4385 4386 /** 4387 * Flow driver find RSS policy tbl API. This abstracts calling driver 4388 * specific functions. Parent flow (rte_flow) should have driver 4389 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4390 * 4391 * @param[in] dev 4392 * Pointer to Ethernet device. 4393 * @param[in, out] flow 4394 * Pointer to flow structure. 4395 * @param[in] policy 4396 * Pointer to meter policy table. 4397 * @param[in] rss_desc 4398 * Pointer to rss_desc 4399 */ 4400 static struct mlx5_flow_meter_sub_policy * 4401 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4402 struct rte_flow *flow, 4403 struct mlx5_flow_meter_policy *policy, 4404 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4405 { 4406 const struct mlx5_flow_driver_ops *fops; 4407 enum mlx5_flow_drv_type type = flow->drv_type; 4408 4409 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4410 fops = flow_get_drv_ops(type); 4411 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4412 } 4413 4414 /** 4415 * Flow driver color tag rule API. This abstracts calling driver 4416 * specific functions. Parent flow (rte_flow) should have driver 4417 * type (drv_type). It will create the color tag rules in hierarchy meter. 4418 * 4419 * @param[in] dev 4420 * Pointer to Ethernet device. 4421 * @param[in, out] flow 4422 * Pointer to flow structure. 4423 * @param[in] fm 4424 * Pointer to flow meter structure. 4425 * @param[in] src_port 4426 * The src port this extra rule should use. 4427 * @param[in] item 4428 * The src port id match item. 4429 * @param[out] error 4430 * Pointer to error structure. 4431 */ 4432 static int 4433 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4434 struct rte_flow *flow, 4435 struct mlx5_flow_meter_info *fm, 4436 int32_t src_port, 4437 const struct rte_flow_item *item, 4438 struct rte_flow_error *error) 4439 { 4440 const struct mlx5_flow_driver_ops *fops; 4441 enum mlx5_flow_drv_type type = flow->drv_type; 4442 4443 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4444 fops = flow_get_drv_ops(type); 4445 return fops->meter_hierarchy_rule_create(dev, fm, 4446 src_port, item, error); 4447 } 4448 4449 /** 4450 * Get RSS action from the action list. 4451 * 4452 * @param[in] dev 4453 * Pointer to Ethernet device. 4454 * @param[in] actions 4455 * Pointer to the list of actions. 4456 * @param[in] flow 4457 * Parent flow structure pointer. 4458 * 4459 * @return 4460 * Pointer to the RSS action if exist, else return NULL. 4461 */ 4462 static const struct rte_flow_action_rss* 4463 flow_get_rss_action(struct rte_eth_dev *dev, 4464 const struct rte_flow_action actions[]) 4465 { 4466 struct mlx5_priv *priv = dev->data->dev_private; 4467 const struct rte_flow_action_rss *rss = NULL; 4468 struct mlx5_meter_policy_action_container *acg; 4469 struct mlx5_meter_policy_action_container *acy; 4470 4471 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4472 switch (actions->type) { 4473 case RTE_FLOW_ACTION_TYPE_RSS: 4474 rss = actions->conf; 4475 break; 4476 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4477 { 4478 const struct rte_flow_action_sample *sample = 4479 actions->conf; 4480 const struct rte_flow_action *act = sample->actions; 4481 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4482 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4483 rss = act->conf; 4484 break; 4485 } 4486 case RTE_FLOW_ACTION_TYPE_METER: 4487 { 4488 uint32_t mtr_idx; 4489 struct mlx5_flow_meter_info *fm; 4490 struct mlx5_flow_meter_policy *policy; 4491 const struct rte_flow_action_meter *mtr = actions->conf; 4492 4493 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4494 if (fm && !fm->def_policy) { 4495 policy = mlx5_flow_meter_policy_find(dev, 4496 fm->policy_id, NULL); 4497 MLX5_ASSERT(policy); 4498 if (policy->is_hierarchy) { 4499 policy = 4500 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4501 policy); 4502 if (!policy) 4503 return NULL; 4504 } 4505 if (policy->is_rss) { 4506 acg = 4507 &policy->act_cnt[RTE_COLOR_GREEN]; 4508 acy = 4509 &policy->act_cnt[RTE_COLOR_YELLOW]; 4510 if (acg->fate_action == 4511 MLX5_FLOW_FATE_SHARED_RSS) 4512 rss = acg->rss->conf; 4513 else if (acy->fate_action == 4514 MLX5_FLOW_FATE_SHARED_RSS) 4515 rss = acy->rss->conf; 4516 } 4517 } 4518 break; 4519 } 4520 default: 4521 break; 4522 } 4523 } 4524 return rss; 4525 } 4526 4527 /** 4528 * Get ASO age action by index. 4529 * 4530 * @param[in] dev 4531 * Pointer to the Ethernet device structure. 4532 * @param[in] age_idx 4533 * Index to the ASO age action. 4534 * 4535 * @return 4536 * The specified ASO age action. 4537 */ 4538 struct mlx5_aso_age_action* 4539 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4540 { 4541 uint16_t pool_idx = age_idx & UINT16_MAX; 4542 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4543 struct mlx5_priv *priv = dev->data->dev_private; 4544 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4545 struct mlx5_aso_age_pool *pool; 4546 4547 rte_rwlock_read_lock(&mng->resize_rwl); 4548 pool = mng->pools[pool_idx]; 4549 rte_rwlock_read_unlock(&mng->resize_rwl); 4550 return &pool->actions[offset - 1]; 4551 } 4552 4553 /* maps indirect action to translated direct in some actions array */ 4554 struct mlx5_translated_action_handle { 4555 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4556 int index; /**< Index in related array of rte_flow_action. */ 4557 }; 4558 4559 /** 4560 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4561 * direct action if translation possible. 4562 * This functionality used to run same execution path for both direct and 4563 * indirect actions on flow create. All necessary preparations for indirect 4564 * action handling should be performed on *handle* actions list returned 4565 * from this call. 4566 * 4567 * @param[in] dev 4568 * Pointer to Ethernet device. 4569 * @param[in] actions 4570 * List of actions to translate. 4571 * @param[out] handle 4572 * List to store translated indirect action object handles. 4573 * @param[in, out] indir_n 4574 * Size of *handle* array. On return should be updated with number of 4575 * indirect actions retrieved from the *actions* list. 4576 * @param[out] translated_actions 4577 * List of actions where all indirect actions were translated to direct 4578 * if possible. NULL if no translation took place. 4579 * @param[out] error 4580 * Pointer to the error structure. 4581 * 4582 * @return 4583 * 0 on success, a negative errno value otherwise and rte_errno is set. 4584 */ 4585 static int 4586 flow_action_handles_translate(struct rte_eth_dev *dev, 4587 const struct rte_flow_action actions[], 4588 struct mlx5_translated_action_handle *handle, 4589 int *indir_n, 4590 struct rte_flow_action **translated_actions, 4591 struct rte_flow_error *error) 4592 { 4593 struct mlx5_priv *priv = dev->data->dev_private; 4594 struct rte_flow_action *translated = NULL; 4595 size_t actions_size; 4596 int n; 4597 int copied_n = 0; 4598 struct mlx5_translated_action_handle *handle_end = NULL; 4599 4600 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4601 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4602 continue; 4603 if (copied_n == *indir_n) { 4604 return rte_flow_error_set 4605 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4606 NULL, "too many shared actions"); 4607 } 4608 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4609 sizeof(actions[n].conf)); 4610 handle[copied_n].index = n; 4611 copied_n++; 4612 } 4613 n++; 4614 *indir_n = copied_n; 4615 if (!copied_n) 4616 return 0; 4617 actions_size = sizeof(struct rte_flow_action) * n; 4618 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4619 if (!translated) { 4620 rte_errno = ENOMEM; 4621 return -ENOMEM; 4622 } 4623 memcpy(translated, actions, actions_size); 4624 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4625 struct mlx5_shared_action_rss *shared_rss; 4626 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4627 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4628 uint32_t idx = act_idx & 4629 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4630 4631 switch (type) { 4632 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4633 shared_rss = mlx5_ipool_get 4634 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4635 translated[handle->index].type = 4636 RTE_FLOW_ACTION_TYPE_RSS; 4637 translated[handle->index].conf = 4638 &shared_rss->origin; 4639 break; 4640 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4641 translated[handle->index].type = 4642 (enum rte_flow_action_type) 4643 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4644 translated[handle->index].conf = (void *)(uintptr_t)idx; 4645 break; 4646 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4647 translated[handle->index].type = 4648 (enum rte_flow_action_type) 4649 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4650 translated[handle->index].conf = (void *)(uintptr_t)idx; 4651 break; 4652 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4653 if (priv->sh->flow_hit_aso_en) { 4654 translated[handle->index].type = 4655 (enum rte_flow_action_type) 4656 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4657 translated[handle->index].conf = 4658 (void *)(uintptr_t)idx; 4659 break; 4660 } 4661 /* Fall-through */ 4662 case MLX5_INDIRECT_ACTION_TYPE_CT: 4663 if (priv->sh->ct_aso_en) { 4664 translated[handle->index].type = 4665 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4666 translated[handle->index].conf = 4667 (void *)(uintptr_t)idx; 4668 break; 4669 } 4670 /* Fall-through */ 4671 default: 4672 mlx5_free(translated); 4673 return rte_flow_error_set 4674 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4675 NULL, "invalid indirect action type"); 4676 } 4677 } 4678 *translated_actions = translated; 4679 return 0; 4680 } 4681 4682 /** 4683 * Get Shared RSS action from the action list. 4684 * 4685 * @param[in] dev 4686 * Pointer to Ethernet device. 4687 * @param[in] shared 4688 * Pointer to the list of actions. 4689 * @param[in] shared_n 4690 * Actions list length. 4691 * 4692 * @return 4693 * The MLX5 RSS action ID if exists, otherwise return 0. 4694 */ 4695 static uint32_t 4696 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4697 struct mlx5_translated_action_handle *handle, 4698 int shared_n) 4699 { 4700 struct mlx5_translated_action_handle *handle_end; 4701 struct mlx5_priv *priv = dev->data->dev_private; 4702 struct mlx5_shared_action_rss *shared_rss; 4703 4704 4705 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4706 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4707 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4708 uint32_t idx = act_idx & 4709 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4710 switch (type) { 4711 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4712 shared_rss = mlx5_ipool_get 4713 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4714 idx); 4715 __atomic_fetch_add(&shared_rss->refcnt, 1, 4716 __ATOMIC_RELAXED); 4717 return idx; 4718 default: 4719 break; 4720 } 4721 } 4722 return 0; 4723 } 4724 4725 static unsigned int 4726 find_graph_root(uint32_t rss_level) 4727 { 4728 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4729 MLX5_EXPANSION_ROOT_OUTER; 4730 } 4731 4732 /** 4733 * Get layer flags from the prefix flow. 4734 * 4735 * Some flows may be split to several subflows, the prefix subflow gets the 4736 * match items and the suffix sub flow gets the actions. 4737 * Some actions need the user defined match item flags to get the detail for 4738 * the action. 4739 * This function helps the suffix flow to get the item layer flags from prefix 4740 * subflow. 4741 * 4742 * @param[in] dev_flow 4743 * Pointer the created prefix subflow. 4744 * 4745 * @return 4746 * The layers get from prefix subflow. 4747 */ 4748 static inline uint64_t 4749 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4750 { 4751 uint64_t layers = 0; 4752 4753 /* 4754 * Layers bits could be localization, but usually the compiler will 4755 * help to do the optimization work for source code. 4756 * If no decap actions, use the layers directly. 4757 */ 4758 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4759 return dev_flow->handle->layers; 4760 /* Convert L3 layers with decap action. */ 4761 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4762 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4763 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4764 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4765 /* Convert L4 layers with decap action. */ 4766 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4767 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4768 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4769 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4770 return layers; 4771 } 4772 4773 /** 4774 * Get metadata split action information. 4775 * 4776 * @param[in] actions 4777 * Pointer to the list of actions. 4778 * @param[out] qrss 4779 * Pointer to the return pointer. 4780 * @param[out] qrss_type 4781 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4782 * if no QUEUE/RSS is found. 4783 * @param[out] encap_idx 4784 * Pointer to the index of the encap action if exists, otherwise the last 4785 * action index. 4786 * 4787 * @return 4788 * Total number of actions. 4789 */ 4790 static int 4791 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4792 const struct rte_flow_action **qrss, 4793 int *encap_idx) 4794 { 4795 const struct rte_flow_action_raw_encap *raw_encap; 4796 int actions_n = 0; 4797 int raw_decap_idx = -1; 4798 4799 *encap_idx = -1; 4800 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4801 switch (actions->type) { 4802 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4803 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4804 *encap_idx = actions_n; 4805 break; 4806 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4807 raw_decap_idx = actions_n; 4808 break; 4809 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4810 raw_encap = actions->conf; 4811 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4812 *encap_idx = raw_decap_idx != -1 ? 4813 raw_decap_idx : actions_n; 4814 break; 4815 case RTE_FLOW_ACTION_TYPE_QUEUE: 4816 case RTE_FLOW_ACTION_TYPE_RSS: 4817 *qrss = actions; 4818 break; 4819 default: 4820 break; 4821 } 4822 actions_n++; 4823 } 4824 if (*encap_idx == -1) 4825 *encap_idx = actions_n; 4826 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4827 return actions_n + 1; 4828 } 4829 4830 /** 4831 * Check if the action will change packet. 4832 * 4833 * @param dev 4834 * Pointer to Ethernet device. 4835 * @param[in] type 4836 * action type. 4837 * 4838 * @return 4839 * true if action will change packet, false otherwise. 4840 */ 4841 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4842 enum rte_flow_action_type type) 4843 { 4844 struct mlx5_priv *priv = dev->data->dev_private; 4845 4846 switch (type) { 4847 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4848 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4849 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4850 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4851 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4852 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4853 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4854 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4855 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4856 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4857 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4858 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4859 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4860 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4861 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4862 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4863 case RTE_FLOW_ACTION_TYPE_SET_META: 4864 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4865 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4866 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4867 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4868 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4869 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4870 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4871 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4872 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4873 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4874 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4875 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4876 return true; 4877 case RTE_FLOW_ACTION_TYPE_FLAG: 4878 case RTE_FLOW_ACTION_TYPE_MARK: 4879 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4880 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4881 return true; 4882 else 4883 return false; 4884 default: 4885 return false; 4886 } 4887 } 4888 4889 /** 4890 * Check meter action from the action list. 4891 * 4892 * @param dev 4893 * Pointer to Ethernet device. 4894 * @param[in] actions 4895 * Pointer to the list of actions. 4896 * @param[out] has_mtr 4897 * Pointer to the meter exist flag. 4898 * @param[out] has_modify 4899 * Pointer to the flag showing there's packet change action. 4900 * @param[out] meter_id 4901 * Pointer to the meter id. 4902 * 4903 * @return 4904 * Total number of actions. 4905 */ 4906 static int 4907 flow_check_meter_action(struct rte_eth_dev *dev, 4908 const struct rte_flow_action actions[], 4909 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4910 { 4911 const struct rte_flow_action_meter *mtr = NULL; 4912 int actions_n = 0; 4913 4914 MLX5_ASSERT(has_mtr); 4915 *has_mtr = false; 4916 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4917 switch (actions->type) { 4918 case RTE_FLOW_ACTION_TYPE_METER: 4919 mtr = actions->conf; 4920 *meter_id = mtr->mtr_id; 4921 *has_mtr = true; 4922 break; 4923 default: 4924 break; 4925 } 4926 if (!*has_mtr) 4927 *has_modify |= flow_check_modify_action_type(dev, 4928 actions->type); 4929 actions_n++; 4930 } 4931 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4932 return actions_n + 1; 4933 } 4934 4935 /** 4936 * Check if the flow should be split due to hairpin. 4937 * The reason for the split is that in current HW we can't 4938 * support encap and push-vlan on Rx, so if a flow contains 4939 * these actions we move it to Tx. 4940 * 4941 * @param dev 4942 * Pointer to Ethernet device. 4943 * @param[in] attr 4944 * Flow rule attributes. 4945 * @param[in] actions 4946 * Associated actions (list terminated by the END action). 4947 * 4948 * @return 4949 * > 0 the number of actions and the flow should be split, 4950 * 0 when no split required. 4951 */ 4952 static int 4953 flow_check_hairpin_split(struct rte_eth_dev *dev, 4954 const struct rte_flow_attr *attr, 4955 const struct rte_flow_action actions[]) 4956 { 4957 int queue_action = 0; 4958 int action_n = 0; 4959 int split = 0; 4960 int push_vlan = 0; 4961 const struct rte_flow_action_queue *queue; 4962 const struct rte_flow_action_rss *rss; 4963 const struct rte_flow_action_raw_encap *raw_encap; 4964 const struct rte_eth_hairpin_conf *conf; 4965 4966 if (!attr->ingress) 4967 return 0; 4968 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4969 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4970 push_vlan = 1; 4971 switch (actions->type) { 4972 case RTE_FLOW_ACTION_TYPE_QUEUE: 4973 queue = actions->conf; 4974 if (queue == NULL) 4975 return 0; 4976 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4977 if (conf == NULL || conf->tx_explicit != 0) 4978 return 0; 4979 queue_action = 1; 4980 action_n++; 4981 break; 4982 case RTE_FLOW_ACTION_TYPE_RSS: 4983 rss = actions->conf; 4984 if (rss == NULL || rss->queue_num == 0) 4985 return 0; 4986 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4987 if (conf == NULL || conf->tx_explicit != 0) 4988 return 0; 4989 queue_action = 1; 4990 action_n++; 4991 break; 4992 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4993 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4994 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4995 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4996 split++; 4997 action_n++; 4998 break; 4999 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5000 if (push_vlan) 5001 split++; 5002 action_n++; 5003 break; 5004 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5005 raw_encap = actions->conf; 5006 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5007 split++; 5008 action_n++; 5009 break; 5010 default: 5011 action_n++; 5012 break; 5013 } 5014 } 5015 if (split && queue_action) 5016 return action_n; 5017 return 0; 5018 } 5019 5020 /* Declare flow create/destroy prototype in advance. */ 5021 static uint32_t 5022 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 5023 const struct rte_flow_attr *attr, 5024 const struct rte_flow_item items[], 5025 const struct rte_flow_action actions[], 5026 bool external, struct rte_flow_error *error); 5027 5028 static void 5029 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 5030 uint32_t flow_idx); 5031 5032 int 5033 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 5034 struct mlx5_list_entry *entry, void *cb_ctx) 5035 { 5036 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 5037 struct mlx5_flow_mreg_copy_resource *mcp_res = 5038 container_of(entry, typeof(*mcp_res), hlist_ent); 5039 5040 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 5041 } 5042 5043 struct mlx5_list_entry * 5044 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 5045 { 5046 struct rte_eth_dev *dev = tool_ctx; 5047 struct mlx5_priv *priv = dev->data->dev_private; 5048 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 5049 struct mlx5_flow_mreg_copy_resource *mcp_res; 5050 struct rte_flow_error *error = ctx->error; 5051 uint32_t idx = 0; 5052 int ret; 5053 uint32_t mark_id = *(uint32_t *)(ctx->data); 5054 struct rte_flow_attr attr = { 5055 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5056 .ingress = 1, 5057 }; 5058 struct mlx5_rte_flow_item_tag tag_spec = { 5059 .data = mark_id, 5060 }; 5061 struct rte_flow_item items[] = { 5062 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 5063 }; 5064 struct rte_flow_action_mark ftag = { 5065 .id = mark_id, 5066 }; 5067 struct mlx5_flow_action_copy_mreg cp_mreg = { 5068 .dst = REG_B, 5069 .src = REG_NON, 5070 }; 5071 struct rte_flow_action_jump jump = { 5072 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5073 }; 5074 struct rte_flow_action actions[] = { 5075 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 5076 }; 5077 5078 /* Fill the register fields in the flow. */ 5079 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 5080 if (ret < 0) 5081 return NULL; 5082 tag_spec.id = ret; 5083 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5084 if (ret < 0) 5085 return NULL; 5086 cp_mreg.src = ret; 5087 /* Provide the full width of FLAG specific value. */ 5088 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 5089 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 5090 /* Build a new flow. */ 5091 if (mark_id != MLX5_DEFAULT_COPY_ID) { 5092 items[0] = (struct rte_flow_item){ 5093 .type = (enum rte_flow_item_type) 5094 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5095 .spec = &tag_spec, 5096 }; 5097 items[1] = (struct rte_flow_item){ 5098 .type = RTE_FLOW_ITEM_TYPE_END, 5099 }; 5100 actions[0] = (struct rte_flow_action){ 5101 .type = (enum rte_flow_action_type) 5102 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 5103 .conf = &ftag, 5104 }; 5105 actions[1] = (struct rte_flow_action){ 5106 .type = (enum rte_flow_action_type) 5107 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5108 .conf = &cp_mreg, 5109 }; 5110 actions[2] = (struct rte_flow_action){ 5111 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5112 .conf = &jump, 5113 }; 5114 actions[3] = (struct rte_flow_action){ 5115 .type = RTE_FLOW_ACTION_TYPE_END, 5116 }; 5117 } else { 5118 /* Default rule, wildcard match. */ 5119 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 5120 items[0] = (struct rte_flow_item){ 5121 .type = RTE_FLOW_ITEM_TYPE_END, 5122 }; 5123 actions[0] = (struct rte_flow_action){ 5124 .type = (enum rte_flow_action_type) 5125 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5126 .conf = &cp_mreg, 5127 }; 5128 actions[1] = (struct rte_flow_action){ 5129 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5130 .conf = &jump, 5131 }; 5132 actions[2] = (struct rte_flow_action){ 5133 .type = RTE_FLOW_ACTION_TYPE_END, 5134 }; 5135 } 5136 /* Build a new entry. */ 5137 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5138 if (!mcp_res) { 5139 rte_errno = ENOMEM; 5140 return NULL; 5141 } 5142 mcp_res->idx = idx; 5143 mcp_res->mark_id = mark_id; 5144 /* 5145 * The copy Flows are not included in any list. There 5146 * ones are referenced from other Flows and can not 5147 * be applied, removed, deleted in arbitrary order 5148 * by list traversing. 5149 */ 5150 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 5151 &attr, items, actions, false, error); 5152 if (!mcp_res->rix_flow) { 5153 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 5154 return NULL; 5155 } 5156 return &mcp_res->hlist_ent; 5157 } 5158 5159 struct mlx5_list_entry * 5160 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5161 void *cb_ctx __rte_unused) 5162 { 5163 struct rte_eth_dev *dev = tool_ctx; 5164 struct mlx5_priv *priv = dev->data->dev_private; 5165 struct mlx5_flow_mreg_copy_resource *mcp_res; 5166 uint32_t idx = 0; 5167 5168 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5169 if (!mcp_res) { 5170 rte_errno = ENOMEM; 5171 return NULL; 5172 } 5173 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 5174 mcp_res->idx = idx; 5175 return &mcp_res->hlist_ent; 5176 } 5177 5178 void 5179 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5180 { 5181 struct mlx5_flow_mreg_copy_resource *mcp_res = 5182 container_of(entry, typeof(*mcp_res), hlist_ent); 5183 struct rte_eth_dev *dev = tool_ctx; 5184 struct mlx5_priv *priv = dev->data->dev_private; 5185 5186 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5187 } 5188 5189 /** 5190 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5191 * 5192 * As mark_id is unique, if there's already a registered flow for the mark_id, 5193 * return by increasing the reference counter of the resource. Otherwise, create 5194 * the resource (mcp_res) and flow. 5195 * 5196 * Flow looks like, 5197 * - If ingress port is ANY and reg_c[1] is mark_id, 5198 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5199 * 5200 * For default flow (zero mark_id), flow is like, 5201 * - If ingress port is ANY, 5202 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5203 * 5204 * @param dev 5205 * Pointer to Ethernet device. 5206 * @param mark_id 5207 * ID of MARK action, zero means default flow for META. 5208 * @param[out] error 5209 * Perform verbose error reporting if not NULL. 5210 * 5211 * @return 5212 * Associated resource on success, NULL otherwise and rte_errno is set. 5213 */ 5214 static struct mlx5_flow_mreg_copy_resource * 5215 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5216 struct rte_flow_error *error) 5217 { 5218 struct mlx5_priv *priv = dev->data->dev_private; 5219 struct mlx5_list_entry *entry; 5220 struct mlx5_flow_cb_ctx ctx = { 5221 .dev = dev, 5222 .error = error, 5223 .data = &mark_id, 5224 }; 5225 5226 /* Check if already registered. */ 5227 MLX5_ASSERT(priv->mreg_cp_tbl); 5228 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5229 if (!entry) 5230 return NULL; 5231 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5232 hlist_ent); 5233 } 5234 5235 void 5236 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5237 { 5238 struct mlx5_flow_mreg_copy_resource *mcp_res = 5239 container_of(entry, typeof(*mcp_res), hlist_ent); 5240 struct rte_eth_dev *dev = tool_ctx; 5241 struct mlx5_priv *priv = dev->data->dev_private; 5242 5243 MLX5_ASSERT(mcp_res->rix_flow); 5244 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5245 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5246 } 5247 5248 /** 5249 * Release flow in RX_CP_TBL. 5250 * 5251 * @param dev 5252 * Pointer to Ethernet device. 5253 * @flow 5254 * Parent flow for wich copying is provided. 5255 */ 5256 static void 5257 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5258 struct rte_flow *flow) 5259 { 5260 struct mlx5_flow_mreg_copy_resource *mcp_res; 5261 struct mlx5_priv *priv = dev->data->dev_private; 5262 5263 if (!flow->rix_mreg_copy) 5264 return; 5265 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5266 flow->rix_mreg_copy); 5267 if (!mcp_res || !priv->mreg_cp_tbl) 5268 return; 5269 MLX5_ASSERT(mcp_res->rix_flow); 5270 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5271 flow->rix_mreg_copy = 0; 5272 } 5273 5274 /** 5275 * Remove the default copy action from RX_CP_TBL. 5276 * 5277 * This functions is called in the mlx5_dev_start(). No thread safe 5278 * is guaranteed. 5279 * 5280 * @param dev 5281 * Pointer to Ethernet device. 5282 */ 5283 static void 5284 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5285 { 5286 struct mlx5_list_entry *entry; 5287 struct mlx5_priv *priv = dev->data->dev_private; 5288 struct mlx5_flow_cb_ctx ctx; 5289 uint32_t mark_id; 5290 5291 /* Check if default flow is registered. */ 5292 if (!priv->mreg_cp_tbl) 5293 return; 5294 mark_id = MLX5_DEFAULT_COPY_ID; 5295 ctx.data = &mark_id; 5296 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5297 if (!entry) 5298 return; 5299 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5300 } 5301 5302 /** 5303 * Add the default copy action in RX_CP_TBL. 5304 * 5305 * This functions is called in the mlx5_dev_start(). No thread safe 5306 * is guaranteed. 5307 * 5308 * @param dev 5309 * Pointer to Ethernet device. 5310 * @param[out] error 5311 * Perform verbose error reporting if not NULL. 5312 * 5313 * @return 5314 * 0 for success, negative value otherwise and rte_errno is set. 5315 */ 5316 static int 5317 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5318 struct rte_flow_error *error) 5319 { 5320 struct mlx5_priv *priv = dev->data->dev_private; 5321 struct mlx5_flow_mreg_copy_resource *mcp_res; 5322 struct mlx5_flow_cb_ctx ctx; 5323 uint32_t mark_id; 5324 5325 /* Check whether extensive metadata feature is engaged. */ 5326 if (!priv->sh->config.dv_flow_en || 5327 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5328 !mlx5_flow_ext_mreg_supported(dev) || 5329 !priv->sh->dv_regc0_mask) 5330 return 0; 5331 /* 5332 * Add default mreg copy flow may be called multiple time, but 5333 * only be called once in stop. Avoid register it twice. 5334 */ 5335 mark_id = MLX5_DEFAULT_COPY_ID; 5336 ctx.data = &mark_id; 5337 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5338 return 0; 5339 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5340 if (!mcp_res) 5341 return -rte_errno; 5342 return 0; 5343 } 5344 5345 /** 5346 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5347 * 5348 * All the flow having Q/RSS action should be split by 5349 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5350 * performs the following, 5351 * - CQE->flow_tag := reg_c[1] (MARK) 5352 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5353 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5354 * but there should be a flow per each MARK ID set by MARK action. 5355 * 5356 * For the aforementioned reason, if there's a MARK action in flow's action 5357 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5358 * the MARK ID to CQE's flow_tag like, 5359 * - If reg_c[1] is mark_id, 5360 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5361 * 5362 * For SET_META action which stores value in reg_c[0], as the destination is 5363 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5364 * MARK ID means the default flow. The default flow looks like, 5365 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5366 * 5367 * @param dev 5368 * Pointer to Ethernet device. 5369 * @param flow 5370 * Pointer to flow structure. 5371 * @param[in] actions 5372 * Pointer to the list of actions. 5373 * @param[out] error 5374 * Perform verbose error reporting if not NULL. 5375 * 5376 * @return 5377 * 0 on success, negative value otherwise and rte_errno is set. 5378 */ 5379 static int 5380 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5381 struct rte_flow *flow, 5382 const struct rte_flow_action *actions, 5383 struct rte_flow_error *error) 5384 { 5385 struct mlx5_priv *priv = dev->data->dev_private; 5386 struct mlx5_sh_config *config = &priv->sh->config; 5387 struct mlx5_flow_mreg_copy_resource *mcp_res; 5388 const struct rte_flow_action_mark *mark; 5389 5390 /* Check whether extensive metadata feature is engaged. */ 5391 if (!config->dv_flow_en || 5392 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5393 !mlx5_flow_ext_mreg_supported(dev) || 5394 !priv->sh->dv_regc0_mask) 5395 return 0; 5396 /* Find MARK action. */ 5397 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5398 switch (actions->type) { 5399 case RTE_FLOW_ACTION_TYPE_FLAG: 5400 mcp_res = flow_mreg_add_copy_action 5401 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5402 if (!mcp_res) 5403 return -rte_errno; 5404 flow->rix_mreg_copy = mcp_res->idx; 5405 return 0; 5406 case RTE_FLOW_ACTION_TYPE_MARK: 5407 mark = (const struct rte_flow_action_mark *) 5408 actions->conf; 5409 mcp_res = 5410 flow_mreg_add_copy_action(dev, mark->id, error); 5411 if (!mcp_res) 5412 return -rte_errno; 5413 flow->rix_mreg_copy = mcp_res->idx; 5414 return 0; 5415 default: 5416 break; 5417 } 5418 } 5419 return 0; 5420 } 5421 5422 #define MLX5_MAX_SPLIT_ACTIONS 24 5423 #define MLX5_MAX_SPLIT_ITEMS 24 5424 5425 /** 5426 * Split the hairpin flow. 5427 * Since HW can't support encap and push-vlan on Rx, we move these 5428 * actions to Tx. 5429 * If the count action is after the encap then we also 5430 * move the count action. in this case the count will also measure 5431 * the outer bytes. 5432 * 5433 * @param dev 5434 * Pointer to Ethernet device. 5435 * @param[in] actions 5436 * Associated actions (list terminated by the END action). 5437 * @param[out] actions_rx 5438 * Rx flow actions. 5439 * @param[out] actions_tx 5440 * Tx flow actions.. 5441 * @param[out] pattern_tx 5442 * The pattern items for the Tx flow. 5443 * @param[out] flow_id 5444 * The flow ID connected to this flow. 5445 * 5446 * @return 5447 * 0 on success. 5448 */ 5449 static int 5450 flow_hairpin_split(struct rte_eth_dev *dev, 5451 const struct rte_flow_action actions[], 5452 struct rte_flow_action actions_rx[], 5453 struct rte_flow_action actions_tx[], 5454 struct rte_flow_item pattern_tx[], 5455 uint32_t flow_id) 5456 { 5457 const struct rte_flow_action_raw_encap *raw_encap; 5458 const struct rte_flow_action_raw_decap *raw_decap; 5459 struct mlx5_rte_flow_action_set_tag *set_tag; 5460 struct rte_flow_action *tag_action; 5461 struct mlx5_rte_flow_item_tag *tag_item; 5462 struct rte_flow_item *item; 5463 char *addr; 5464 int push_vlan = 0; 5465 int encap = 0; 5466 5467 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5468 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5469 push_vlan = 1; 5470 switch (actions->type) { 5471 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5472 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5473 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5474 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5475 rte_memcpy(actions_tx, actions, 5476 sizeof(struct rte_flow_action)); 5477 actions_tx++; 5478 break; 5479 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5480 if (push_vlan) { 5481 rte_memcpy(actions_tx, actions, 5482 sizeof(struct rte_flow_action)); 5483 actions_tx++; 5484 } else { 5485 rte_memcpy(actions_rx, actions, 5486 sizeof(struct rte_flow_action)); 5487 actions_rx++; 5488 } 5489 break; 5490 case RTE_FLOW_ACTION_TYPE_COUNT: 5491 if (encap) { 5492 rte_memcpy(actions_tx, actions, 5493 sizeof(struct rte_flow_action)); 5494 actions_tx++; 5495 } else { 5496 rte_memcpy(actions_rx, actions, 5497 sizeof(struct rte_flow_action)); 5498 actions_rx++; 5499 } 5500 break; 5501 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5502 raw_encap = actions->conf; 5503 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5504 memcpy(actions_tx, actions, 5505 sizeof(struct rte_flow_action)); 5506 actions_tx++; 5507 encap = 1; 5508 } else { 5509 rte_memcpy(actions_rx, actions, 5510 sizeof(struct rte_flow_action)); 5511 actions_rx++; 5512 } 5513 break; 5514 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5515 raw_decap = actions->conf; 5516 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5517 memcpy(actions_tx, actions, 5518 sizeof(struct rte_flow_action)); 5519 actions_tx++; 5520 } else { 5521 rte_memcpy(actions_rx, actions, 5522 sizeof(struct rte_flow_action)); 5523 actions_rx++; 5524 } 5525 break; 5526 default: 5527 rte_memcpy(actions_rx, actions, 5528 sizeof(struct rte_flow_action)); 5529 actions_rx++; 5530 break; 5531 } 5532 } 5533 /* Add set meta action and end action for the Rx flow. */ 5534 tag_action = actions_rx; 5535 tag_action->type = (enum rte_flow_action_type) 5536 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5537 actions_rx++; 5538 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5539 actions_rx++; 5540 set_tag = (void *)actions_rx; 5541 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5542 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5543 .data = flow_id, 5544 }; 5545 MLX5_ASSERT(set_tag->id > REG_NON); 5546 tag_action->conf = set_tag; 5547 /* Create Tx item list. */ 5548 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5549 addr = (void *)&pattern_tx[2]; 5550 item = pattern_tx; 5551 item->type = (enum rte_flow_item_type) 5552 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5553 tag_item = (void *)addr; 5554 tag_item->data = flow_id; 5555 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5556 MLX5_ASSERT(set_tag->id > REG_NON); 5557 item->spec = tag_item; 5558 addr += sizeof(struct mlx5_rte_flow_item_tag); 5559 tag_item = (void *)addr; 5560 tag_item->data = UINT32_MAX; 5561 tag_item->id = UINT16_MAX; 5562 item->mask = tag_item; 5563 item->last = NULL; 5564 item++; 5565 item->type = RTE_FLOW_ITEM_TYPE_END; 5566 return 0; 5567 } 5568 5569 /** 5570 * The last stage of splitting chain, just creates the subflow 5571 * without any modification. 5572 * 5573 * @param[in] dev 5574 * Pointer to Ethernet device. 5575 * @param[in] flow 5576 * Parent flow structure pointer. 5577 * @param[in, out] sub_flow 5578 * Pointer to return the created subflow, may be NULL. 5579 * @param[in] attr 5580 * Flow rule attributes. 5581 * @param[in] items 5582 * Pattern specification (list terminated by the END pattern item). 5583 * @param[in] actions 5584 * Associated actions (list terminated by the END action). 5585 * @param[in] flow_split_info 5586 * Pointer to flow split info structure. 5587 * @param[out] error 5588 * Perform verbose error reporting if not NULL. 5589 * @return 5590 * 0 on success, negative value otherwise 5591 */ 5592 static int 5593 flow_create_split_inner(struct rte_eth_dev *dev, 5594 struct rte_flow *flow, 5595 struct mlx5_flow **sub_flow, 5596 const struct rte_flow_attr *attr, 5597 const struct rte_flow_item items[], 5598 const struct rte_flow_action actions[], 5599 struct mlx5_flow_split_info *flow_split_info, 5600 struct rte_flow_error *error) 5601 { 5602 struct mlx5_flow *dev_flow; 5603 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5604 5605 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5606 flow_split_info->flow_idx, error); 5607 if (!dev_flow) 5608 return -rte_errno; 5609 dev_flow->flow = flow; 5610 dev_flow->external = flow_split_info->external; 5611 dev_flow->skip_scale = flow_split_info->skip_scale; 5612 /* Subflow object was created, we must include one in the list. */ 5613 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5614 dev_flow->handle, next); 5615 /* 5616 * If dev_flow is as one of the suffix flow, some actions in suffix 5617 * flow may need some user defined item layer flags, and pass the 5618 * Metadata rxq mark flag to suffix flow as well. 5619 */ 5620 if (flow_split_info->prefix_layers) 5621 dev_flow->handle->layers = flow_split_info->prefix_layers; 5622 if (flow_split_info->prefix_mark) { 5623 MLX5_ASSERT(wks); 5624 wks->mark = 1; 5625 } 5626 if (sub_flow) 5627 *sub_flow = dev_flow; 5628 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5629 dev_flow->dv.table_id = flow_split_info->table_id; 5630 #endif 5631 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5632 } 5633 5634 /** 5635 * Get the sub policy of a meter. 5636 * 5637 * @param[in] dev 5638 * Pointer to Ethernet device. 5639 * @param[in] flow 5640 * Parent flow structure pointer. 5641 * @param wks 5642 * Pointer to thread flow work space. 5643 * @param[in] attr 5644 * Flow rule attributes. 5645 * @param[in] items 5646 * Pattern specification (list terminated by the END pattern item). 5647 * @param[out] error 5648 * Perform verbose error reporting if not NULL. 5649 * 5650 * @return 5651 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5652 */ 5653 static struct mlx5_flow_meter_sub_policy * 5654 get_meter_sub_policy(struct rte_eth_dev *dev, 5655 struct rte_flow *flow, 5656 struct mlx5_flow_workspace *wks, 5657 const struct rte_flow_attr *attr, 5658 const struct rte_flow_item items[], 5659 struct rte_flow_error *error) 5660 { 5661 struct mlx5_flow_meter_policy *policy; 5662 struct mlx5_flow_meter_policy *final_policy; 5663 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5664 5665 policy = wks->policy; 5666 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5667 if (final_policy->is_rss || final_policy->is_queue) { 5668 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5669 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5670 uint32_t i; 5671 5672 /* 5673 * This is a tmp dev_flow, 5674 * no need to register any matcher for it in translate. 5675 */ 5676 wks->skip_matcher_reg = 1; 5677 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5678 struct mlx5_flow dev_flow = {0}; 5679 struct mlx5_flow_handle dev_handle = { {0} }; 5680 uint8_t fate = final_policy->act_cnt[i].fate_action; 5681 5682 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5683 const struct rte_flow_action_rss *rss_act = 5684 final_policy->act_cnt[i].rss->conf; 5685 struct rte_flow_action rss_actions[2] = { 5686 [0] = { 5687 .type = RTE_FLOW_ACTION_TYPE_RSS, 5688 .conf = rss_act, 5689 }, 5690 [1] = { 5691 .type = RTE_FLOW_ACTION_TYPE_END, 5692 .conf = NULL, 5693 } 5694 }; 5695 5696 dev_flow.handle = &dev_handle; 5697 dev_flow.ingress = attr->ingress; 5698 dev_flow.flow = flow; 5699 dev_flow.external = 0; 5700 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5701 dev_flow.dv.transfer = attr->transfer; 5702 #endif 5703 /** 5704 * Translate RSS action to get rss hash fields. 5705 */ 5706 if (flow_drv_translate(dev, &dev_flow, attr, 5707 items, rss_actions, error)) 5708 goto exit; 5709 rss_desc_v[i] = wks->rss_desc; 5710 rss_desc_v[i].symmetric_hash_function = 5711 dev_flow.symmetric_hash_function; 5712 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5713 rss_desc_v[i].hash_fields = 5714 dev_flow.hash_fields; 5715 rss_desc_v[i].queue_num = 5716 rss_desc_v[i].hash_fields ? 5717 rss_desc_v[i].queue_num : 1; 5718 rss_desc_v[i].tunnel = 5719 !!(dev_flow.handle->layers & 5720 MLX5_FLOW_LAYER_TUNNEL); 5721 /* Use the RSS queues in the containers. */ 5722 rss_desc_v[i].queue = 5723 (uint16_t *)(uintptr_t)rss_act->queue; 5724 rss_desc[i] = &rss_desc_v[i]; 5725 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5726 /* This is queue action. */ 5727 rss_desc_v[i] = wks->rss_desc; 5728 rss_desc_v[i].key_len = 0; 5729 rss_desc_v[i].hash_fields = 0; 5730 rss_desc_v[i].queue = 5731 &final_policy->act_cnt[i].queue; 5732 rss_desc_v[i].queue_num = 1; 5733 rss_desc[i] = &rss_desc_v[i]; 5734 } else { 5735 rss_desc[i] = NULL; 5736 } 5737 } 5738 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5739 flow, policy, rss_desc); 5740 } else { 5741 enum mlx5_meter_domain mtr_domain = 5742 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5743 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5744 MLX5_MTR_DOMAIN_INGRESS); 5745 sub_policy = policy->sub_policys[mtr_domain][0]; 5746 } 5747 if (!sub_policy) 5748 rte_flow_error_set(error, EINVAL, 5749 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5750 "Failed to get meter sub-policy."); 5751 exit: 5752 return sub_policy; 5753 } 5754 5755 /** 5756 * Split the meter flow. 5757 * 5758 * As meter flow will split to three sub flow, other than meter 5759 * action, the other actions make sense to only meter accepts 5760 * the packet. If it need to be dropped, no other additional 5761 * actions should be take. 5762 * 5763 * One kind of special action which decapsulates the L3 tunnel 5764 * header will be in the prefix sub flow, as not to take the 5765 * L3 tunnel header into account. 5766 * 5767 * @param[in] dev 5768 * Pointer to Ethernet device. 5769 * @param[in] flow 5770 * Parent flow structure pointer. 5771 * @param wks 5772 * Pointer to thread flow work space. 5773 * @param[in] attr 5774 * Flow rule attributes. 5775 * @param[in] items 5776 * Pattern specification (list terminated by the END pattern item). 5777 * @param[out] sfx_items 5778 * Suffix flow match items (list terminated by the END pattern item). 5779 * @param[in] actions 5780 * Associated actions (list terminated by the END action). 5781 * @param[out] actions_sfx 5782 * Suffix flow actions. 5783 * @param[out] actions_pre 5784 * Prefix flow actions. 5785 * @param[out] mtr_flow_id 5786 * Pointer to meter flow id. 5787 * @param[out] error 5788 * Perform verbose error reporting if not NULL. 5789 * 5790 * @return 5791 * 0 on success, a negative errno value otherwise and rte_errno is set. 5792 */ 5793 static int 5794 flow_meter_split_prep(struct rte_eth_dev *dev, 5795 struct rte_flow *flow, 5796 struct mlx5_flow_workspace *wks, 5797 const struct rte_flow_attr *attr, 5798 const struct rte_flow_item items[], 5799 struct rte_flow_item sfx_items[], 5800 const struct rte_flow_action actions[], 5801 struct rte_flow_action actions_sfx[], 5802 struct rte_flow_action actions_pre[], 5803 uint32_t *mtr_flow_id, 5804 struct rte_flow_error *error) 5805 { 5806 struct mlx5_priv *priv = dev->data->dev_private; 5807 struct mlx5_flow_meter_info *fm = wks->fm; 5808 struct rte_flow_action *tag_action = NULL; 5809 struct rte_flow_item *tag_item; 5810 struct mlx5_rte_flow_action_set_tag *set_tag; 5811 const struct rte_flow_action_raw_encap *raw_encap; 5812 const struct rte_flow_action_raw_decap *raw_decap; 5813 struct mlx5_rte_flow_item_tag *tag_item_spec; 5814 struct mlx5_rte_flow_item_tag *tag_item_mask; 5815 uint32_t tag_id = 0; 5816 struct rte_flow_item *vlan_item_dst = NULL; 5817 const struct rte_flow_item *vlan_item_src = NULL; 5818 const struct rte_flow_item *orig_items = items; 5819 struct rte_flow_action *hw_mtr_action; 5820 struct rte_flow_action *action_pre_head = NULL; 5821 uint16_t flow_src_port = priv->representor_id; 5822 bool mtr_first; 5823 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5824 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5825 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5826 uint32_t flow_id = 0; 5827 uint32_t flow_id_reversed = 0; 5828 uint8_t flow_id_bits = 0; 5829 bool after_meter = false; 5830 int shift; 5831 5832 /* Prepare the suffix subflow items. */ 5833 tag_item = sfx_items++; 5834 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5835 int item_type = items->type; 5836 5837 switch (item_type) { 5838 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5839 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5840 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5841 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5842 return -rte_errno; 5843 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5844 flow_src_port != priv->representor_id) { 5845 if (flow_drv_mtr_hierarchy_rule_create(dev, 5846 flow, fm, 5847 flow_src_port, 5848 items, 5849 error)) 5850 return -rte_errno; 5851 } 5852 memcpy(sfx_items, items, sizeof(*sfx_items)); 5853 sfx_items++; 5854 break; 5855 case RTE_FLOW_ITEM_TYPE_VLAN: 5856 /* Determine if copy vlan item below. */ 5857 vlan_item_src = items; 5858 vlan_item_dst = sfx_items++; 5859 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5860 break; 5861 default: 5862 break; 5863 } 5864 } 5865 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5866 sfx_items++; 5867 mtr_first = priv->sh->meter_aso_en && 5868 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5869 /* For ASO meter, meter must be before tag in TX direction. */ 5870 if (mtr_first) { 5871 action_pre_head = actions_pre++; 5872 /* Leave space for tag action. */ 5873 tag_action = actions_pre++; 5874 } 5875 /* Prepare the actions for prefix and suffix flow. */ 5876 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5877 struct rte_flow_action *action_cur = NULL; 5878 5879 switch (actions->type) { 5880 case RTE_FLOW_ACTION_TYPE_METER: 5881 if (mtr_first) { 5882 action_cur = action_pre_head; 5883 } else { 5884 /* Leave space for tag action. */ 5885 tag_action = actions_pre++; 5886 action_cur = actions_pre++; 5887 } 5888 after_meter = true; 5889 break; 5890 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5891 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5892 action_cur = actions_pre++; 5893 break; 5894 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5895 raw_encap = actions->conf; 5896 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5897 action_cur = actions_pre++; 5898 break; 5899 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5900 raw_decap = actions->conf; 5901 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5902 action_cur = actions_pre++; 5903 break; 5904 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5905 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5906 if (vlan_item_dst && vlan_item_src) { 5907 memcpy(vlan_item_dst, vlan_item_src, 5908 sizeof(*vlan_item_dst)); 5909 /* 5910 * Convert to internal match item, it is used 5911 * for vlan push and set vid. 5912 */ 5913 vlan_item_dst->type = (enum rte_flow_item_type) 5914 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5915 } 5916 break; 5917 case RTE_FLOW_ACTION_TYPE_COUNT: 5918 if (fm->def_policy) 5919 action_cur = after_meter ? 5920 actions_sfx++ : actions_pre++; 5921 break; 5922 default: 5923 break; 5924 } 5925 if (!action_cur) 5926 action_cur = (fm->def_policy) ? 5927 actions_sfx++ : actions_pre++; 5928 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5929 } 5930 /* Add end action to the actions. */ 5931 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5932 if (priv->sh->meter_aso_en) { 5933 /** 5934 * For ASO meter, need to add an extra jump action explicitly, 5935 * to jump from meter to policer table. 5936 */ 5937 struct mlx5_flow_meter_sub_policy *sub_policy; 5938 struct mlx5_flow_tbl_data_entry *tbl_data; 5939 5940 if (!fm->def_policy) { 5941 sub_policy = get_meter_sub_policy(dev, flow, wks, 5942 attr, orig_items, 5943 error); 5944 if (!sub_policy) 5945 return -rte_errno; 5946 } else { 5947 enum mlx5_meter_domain mtr_domain = 5948 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5949 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5950 MLX5_MTR_DOMAIN_INGRESS); 5951 5952 sub_policy = 5953 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5954 } 5955 tbl_data = container_of(sub_policy->tbl_rsc, 5956 struct mlx5_flow_tbl_data_entry, tbl); 5957 hw_mtr_action = actions_pre++; 5958 hw_mtr_action->type = (enum rte_flow_action_type) 5959 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5960 hw_mtr_action->conf = tbl_data->jump.action; 5961 } 5962 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5963 actions_pre++; 5964 if (!tag_action) 5965 return rte_flow_error_set(error, ENOMEM, 5966 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5967 NULL, "No tag action space."); 5968 if (!mtr_flow_id) { 5969 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5970 goto exit; 5971 } 5972 /* Only default-policy Meter creates mtr flow id. */ 5973 if (fm->def_policy) { 5974 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5975 if (!tag_id) 5976 return rte_flow_error_set(error, ENOMEM, 5977 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5978 "Failed to allocate meter flow id."); 5979 flow_id = tag_id - 1; 5980 flow_id_bits = (!flow_id) ? 1 : 5981 (MLX5_REG_BITS - rte_clz32(flow_id)); 5982 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5983 mtr_reg_bits) { 5984 mlx5_ipool_free(fm->flow_ipool, tag_id); 5985 return rte_flow_error_set(error, EINVAL, 5986 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5987 "Meter flow id exceeds max limit."); 5988 } 5989 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5990 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5991 } 5992 /* Build tag actions and items for meter_id/meter flow_id. */ 5993 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5994 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5995 tag_item_mask = tag_item_spec + 1; 5996 /* Both flow_id and meter_id share the same register. */ 5997 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5998 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5999 0, error), 6000 .offset = mtr_id_offset, 6001 .length = mtr_reg_bits, 6002 .data = flow->meter, 6003 }; 6004 /* 6005 * The color Reg bits used by flow_id are growing from 6006 * msb to lsb, so must do bit reverse for flow_id val in RegC. 6007 */ 6008 for (shift = 0; shift < flow_id_bits; shift++) 6009 flow_id_reversed = (flow_id_reversed << 1) | 6010 ((flow_id >> shift) & 0x1); 6011 set_tag->data |= 6012 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 6013 tag_item_spec->id = set_tag->id; 6014 tag_item_spec->data = set_tag->data << mtr_id_offset; 6015 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 6016 tag_action->type = (enum rte_flow_action_type) 6017 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 6018 tag_action->conf = set_tag; 6019 tag_item->type = (enum rte_flow_item_type) 6020 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 6021 tag_item->spec = tag_item_spec; 6022 tag_item->last = NULL; 6023 tag_item->mask = tag_item_mask; 6024 exit: 6025 if (mtr_flow_id) 6026 *mtr_flow_id = tag_id; 6027 return 0; 6028 } 6029 6030 /** 6031 * Split action list having QUEUE/RSS for metadata register copy. 6032 * 6033 * Once Q/RSS action is detected in user's action list, the flow action 6034 * should be split in order to copy metadata registers, which will happen in 6035 * RX_CP_TBL like, 6036 * - CQE->flow_tag := reg_c[1] (MARK) 6037 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 6038 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 6039 * This is because the last action of each flow must be a terminal action 6040 * (QUEUE, RSS or DROP). 6041 * 6042 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 6043 * stored and kept in the mlx5_flow structure per each sub_flow. 6044 * 6045 * The Q/RSS action is replaced with, 6046 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 6047 * And the following JUMP action is added at the end, 6048 * - JUMP, to RX_CP_TBL. 6049 * 6050 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 6051 * flow_create_split_metadata() routine. The flow will look like, 6052 * - If flow ID matches (reg_c[2]), perform Q/RSS. 6053 * 6054 * @param dev 6055 * Pointer to Ethernet device. 6056 * @param[out] split_actions 6057 * Pointer to store split actions to jump to CP_TBL. 6058 * @param[in] actions 6059 * Pointer to the list of original flow actions. 6060 * @param[in] qrss 6061 * Pointer to the Q/RSS action. 6062 * @param[in] actions_n 6063 * Number of original actions. 6064 * @param[in] mtr_sfx 6065 * Check if it is in meter suffix table. 6066 * @param[out] error 6067 * Perform verbose error reporting if not NULL. 6068 * 6069 * @return 6070 * non-zero unique flow_id on success, otherwise 0 and 6071 * error/rte_error are set. 6072 */ 6073 static uint32_t 6074 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 6075 struct rte_flow_action *split_actions, 6076 const struct rte_flow_action *actions, 6077 const struct rte_flow_action *qrss, 6078 int actions_n, int mtr_sfx, 6079 struct rte_flow_error *error) 6080 { 6081 struct mlx5_priv *priv = dev->data->dev_private; 6082 struct mlx5_rte_flow_action_set_tag *set_tag; 6083 struct rte_flow_action_jump *jump; 6084 const int qrss_idx = qrss - actions; 6085 uint32_t flow_id = 0; 6086 int ret = 0; 6087 6088 /* 6089 * Given actions will be split 6090 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 6091 * - Add jump to mreg CP_TBL. 6092 * As a result, there will be one more action. 6093 */ 6094 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 6095 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 6096 ++actions_n; 6097 set_tag = (void *)(split_actions + actions_n); 6098 /* 6099 * If we are not the meter suffix flow, add the tag action. 6100 * Since meter suffix flow already has the tag added. 6101 */ 6102 if (!mtr_sfx) { 6103 /* 6104 * Allocate the new subflow ID. This one is unique within 6105 * device and not shared with representors. Otherwise, 6106 * we would have to resolve multi-thread access synch 6107 * issue. Each flow on the shared device is appended 6108 * with source vport identifier, so the resulting 6109 * flows will be unique in the shared (by master and 6110 * representors) domain even if they have coinciding 6111 * IDs. 6112 */ 6113 mlx5_ipool_malloc(priv->sh->ipool 6114 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 6115 if (!flow_id) 6116 return rte_flow_error_set(error, ENOMEM, 6117 RTE_FLOW_ERROR_TYPE_ACTION, 6118 NULL, "can't allocate id " 6119 "for split Q/RSS subflow"); 6120 /* Internal SET_TAG action to set flow ID. */ 6121 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 6122 .data = flow_id, 6123 }; 6124 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 6125 if (ret < 0) 6126 return ret; 6127 set_tag->id = ret; 6128 /* Construct new actions array. */ 6129 /* Replace QUEUE/RSS action. */ 6130 split_actions[qrss_idx] = (struct rte_flow_action){ 6131 .type = (enum rte_flow_action_type) 6132 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6133 .conf = set_tag, 6134 }; 6135 } else { 6136 /* 6137 * If we are the suffix flow of meter, tag already exist. 6138 * Set the QUEUE/RSS action to void. 6139 */ 6140 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 6141 } 6142 /* JUMP action to jump to mreg copy table (CP_TBL). */ 6143 jump = (void *)(set_tag + 1); 6144 *jump = (struct rte_flow_action_jump){ 6145 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6146 }; 6147 split_actions[actions_n - 2] = (struct rte_flow_action){ 6148 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6149 .conf = jump, 6150 }; 6151 split_actions[actions_n - 1] = (struct rte_flow_action){ 6152 .type = RTE_FLOW_ACTION_TYPE_END, 6153 }; 6154 return flow_id; 6155 } 6156 6157 /** 6158 * Extend the given action list for Tx metadata copy. 6159 * 6160 * Copy the given action list to the ext_actions and add flow metadata register 6161 * copy action in order to copy reg_a set by WQE to reg_c[0]. 6162 * 6163 * @param[out] ext_actions 6164 * Pointer to the extended action list. 6165 * @param[in] actions 6166 * Pointer to the list of actions. 6167 * @param[in] actions_n 6168 * Number of actions in the list. 6169 * @param[out] error 6170 * Perform verbose error reporting if not NULL. 6171 * @param[in] encap_idx 6172 * The encap action index. 6173 * 6174 * @return 6175 * 0 on success, negative value otherwise 6176 */ 6177 static int 6178 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6179 struct rte_flow_action *ext_actions, 6180 const struct rte_flow_action *actions, 6181 int actions_n, struct rte_flow_error *error, 6182 int encap_idx) 6183 { 6184 struct mlx5_flow_action_copy_mreg *cp_mreg = 6185 (struct mlx5_flow_action_copy_mreg *) 6186 (ext_actions + actions_n + 1); 6187 int ret; 6188 6189 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6190 if (ret < 0) 6191 return ret; 6192 cp_mreg->dst = ret; 6193 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6194 if (ret < 0) 6195 return ret; 6196 cp_mreg->src = ret; 6197 if (encap_idx != 0) 6198 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6199 if (encap_idx == actions_n - 1) { 6200 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6201 .type = (enum rte_flow_action_type) 6202 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6203 .conf = cp_mreg, 6204 }; 6205 ext_actions[actions_n] = (struct rte_flow_action){ 6206 .type = RTE_FLOW_ACTION_TYPE_END, 6207 }; 6208 } else { 6209 ext_actions[encap_idx] = (struct rte_flow_action){ 6210 .type = (enum rte_flow_action_type) 6211 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6212 .conf = cp_mreg, 6213 }; 6214 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6215 sizeof(*ext_actions) * (actions_n - encap_idx)); 6216 } 6217 return 0; 6218 } 6219 6220 /** 6221 * Check the match action from the action list. 6222 * 6223 * @param[in] actions 6224 * Pointer to the list of actions. 6225 * @param[in] attr 6226 * Flow rule attributes. 6227 * @param[in] action 6228 * The action to be check if exist. 6229 * @param[out] match_action_pos 6230 * Pointer to the position of the matched action if exists, otherwise is -1. 6231 * @param[out] qrss_action_pos 6232 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6233 * @param[out] modify_after_mirror 6234 * Pointer to the flag of modify action after FDB mirroring. 6235 * 6236 * @return 6237 * > 0 the total number of actions. 6238 * 0 if not found match action in action list. 6239 */ 6240 static int 6241 flow_check_match_action(const struct rte_flow_action actions[], 6242 const struct rte_flow_attr *attr, 6243 enum rte_flow_action_type action, 6244 int *match_action_pos, int *qrss_action_pos, 6245 int *modify_after_mirror) 6246 { 6247 const struct rte_flow_action_sample *sample; 6248 const struct rte_flow_action_raw_decap *decap; 6249 const struct rte_flow_action *action_cur = NULL; 6250 int actions_n = 0; 6251 uint32_t ratio = 0; 6252 int sub_type = 0; 6253 int flag = 0; 6254 int fdb_mirror = 0; 6255 6256 *match_action_pos = -1; 6257 *qrss_action_pos = -1; 6258 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6259 if (actions->type == action) { 6260 flag = 1; 6261 *match_action_pos = actions_n; 6262 } 6263 switch (actions->type) { 6264 case RTE_FLOW_ACTION_TYPE_QUEUE: 6265 case RTE_FLOW_ACTION_TYPE_RSS: 6266 *qrss_action_pos = actions_n; 6267 break; 6268 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6269 sample = actions->conf; 6270 ratio = sample->ratio; 6271 sub_type = ((const struct rte_flow_action *) 6272 (sample->actions))->type; 6273 if (ratio == 1 && attr->transfer && 6274 sub_type != RTE_FLOW_ACTION_TYPE_END) 6275 fdb_mirror = 1; 6276 break; 6277 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6278 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6279 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6280 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6281 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6282 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6283 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6284 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6285 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6286 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6287 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6288 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6289 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6290 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6291 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6292 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6293 case RTE_FLOW_ACTION_TYPE_FLAG: 6294 case RTE_FLOW_ACTION_TYPE_MARK: 6295 case RTE_FLOW_ACTION_TYPE_SET_META: 6296 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6297 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6298 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6299 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6300 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6301 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6302 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6303 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6304 case RTE_FLOW_ACTION_TYPE_METER: 6305 if (fdb_mirror) 6306 *modify_after_mirror = 1; 6307 break; 6308 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6309 decap = actions->conf; 6310 action_cur = actions; 6311 while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID) 6312 ; 6313 if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6314 const struct rte_flow_action_raw_encap *encap = 6315 action_cur->conf; 6316 if (decap->size <= 6317 MLX5_ENCAPSULATION_DECISION_SIZE && 6318 encap->size > 6319 MLX5_ENCAPSULATION_DECISION_SIZE) 6320 /* L3 encap. */ 6321 break; 6322 } 6323 if (fdb_mirror) 6324 *modify_after_mirror = 1; 6325 break; 6326 default: 6327 break; 6328 } 6329 actions_n++; 6330 } 6331 if (flag && fdb_mirror && !*modify_after_mirror) { 6332 /* FDB mirroring uses the destination array to implement 6333 * instead of FLOW_SAMPLER object. 6334 */ 6335 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6336 flag = 0; 6337 } 6338 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6339 return flag ? actions_n + 1 : 0; 6340 } 6341 6342 #define SAMPLE_SUFFIX_ITEM 3 6343 6344 /** 6345 * Split the sample flow. 6346 * 6347 * As sample flow will split to two sub flow, sample flow with 6348 * sample action, the other actions will move to new suffix flow. 6349 * 6350 * Also add unique tag id with tag action in the sample flow, 6351 * the same tag id will be as match in the suffix flow. 6352 * 6353 * @param dev 6354 * Pointer to Ethernet device. 6355 * @param[in] add_tag 6356 * Add extra tag action flag. 6357 * @param[out] sfx_items 6358 * Suffix flow match items (list terminated by the END pattern item). 6359 * @param[in] actions 6360 * Associated actions (list terminated by the END action). 6361 * @param[out] actions_sfx 6362 * Suffix flow actions. 6363 * @param[out] actions_pre 6364 * Prefix flow actions. 6365 * @param[in] actions_n 6366 * The total number of actions. 6367 * @param[in] sample_action_pos 6368 * The sample action position. 6369 * @param[in] qrss_action_pos 6370 * The Queue/RSS action position. 6371 * @param[in] jump_table 6372 * Add extra jump action flag. 6373 * @param[out] error 6374 * Perform verbose error reporting if not NULL. 6375 * 6376 * @return 6377 * 0 on success, or unique flow_id, a negative errno value 6378 * otherwise and rte_errno is set. 6379 */ 6380 static int 6381 flow_sample_split_prep(struct rte_eth_dev *dev, 6382 int add_tag, 6383 const struct rte_flow_item items[], 6384 struct rte_flow_item sfx_items[], 6385 const struct rte_flow_action actions[], 6386 struct rte_flow_action actions_sfx[], 6387 struct rte_flow_action actions_pre[], 6388 int actions_n, 6389 int sample_action_pos, 6390 int qrss_action_pos, 6391 int jump_table, 6392 struct rte_flow_error *error) 6393 { 6394 struct mlx5_priv *priv = dev->data->dev_private; 6395 struct mlx5_rte_flow_action_set_tag *set_tag; 6396 struct mlx5_rte_flow_item_tag *tag_spec; 6397 struct mlx5_rte_flow_item_tag *tag_mask; 6398 struct rte_flow_action_jump *jump_action; 6399 uint32_t tag_id = 0; 6400 int append_index = 0; 6401 int set_tag_idx = -1; 6402 int index; 6403 int ret; 6404 6405 if (sample_action_pos < 0) 6406 return rte_flow_error_set(error, EINVAL, 6407 RTE_FLOW_ERROR_TYPE_ACTION, 6408 NULL, "invalid position of sample " 6409 "action in list"); 6410 /* Prepare the actions for prefix and suffix flow. */ 6411 if (add_tag) { 6412 /* Update the new added tag action index preceding 6413 * the PUSH_VLAN or ENCAP action. 6414 */ 6415 const struct rte_flow_action_raw_encap *raw_encap; 6416 const struct rte_flow_action *action = actions; 6417 int encap_idx; 6418 int action_idx = 0; 6419 int raw_decap_idx = -1; 6420 int push_vlan_idx = -1; 6421 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6422 switch (action->type) { 6423 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6424 raw_decap_idx = action_idx; 6425 break; 6426 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6427 raw_encap = action->conf; 6428 if (raw_encap->size > 6429 MLX5_ENCAPSULATION_DECISION_SIZE) { 6430 encap_idx = raw_decap_idx != -1 ? 6431 raw_decap_idx : action_idx; 6432 if (encap_idx < sample_action_pos && 6433 push_vlan_idx == -1) 6434 set_tag_idx = encap_idx; 6435 } 6436 break; 6437 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6438 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6439 encap_idx = action_idx; 6440 if (encap_idx < sample_action_pos && 6441 push_vlan_idx == -1) 6442 set_tag_idx = encap_idx; 6443 break; 6444 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6445 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6446 if (action_idx < sample_action_pos && 6447 push_vlan_idx == -1) { 6448 set_tag_idx = action_idx; 6449 push_vlan_idx = action_idx; 6450 } 6451 break; 6452 default: 6453 break; 6454 } 6455 action_idx++; 6456 } 6457 } 6458 /* Prepare the actions for prefix and suffix flow. */ 6459 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6460 index = qrss_action_pos; 6461 /* Put the preceding the Queue/RSS action into prefix flow. */ 6462 if (index != 0) 6463 memcpy(actions_pre, actions, 6464 sizeof(struct rte_flow_action) * index); 6465 /* Put others preceding the sample action into prefix flow. */ 6466 if (sample_action_pos > index + 1) 6467 memcpy(actions_pre + index, actions + index + 1, 6468 sizeof(struct rte_flow_action) * 6469 (sample_action_pos - index - 1)); 6470 index = sample_action_pos - 1; 6471 /* Put Queue/RSS action into Suffix flow. */ 6472 memcpy(actions_sfx, actions + qrss_action_pos, 6473 sizeof(struct rte_flow_action)); 6474 actions_sfx++; 6475 } else if (add_tag && set_tag_idx >= 0) { 6476 if (set_tag_idx > 0) 6477 memcpy(actions_pre, actions, 6478 sizeof(struct rte_flow_action) * set_tag_idx); 6479 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6480 sizeof(struct rte_flow_action) * 6481 (sample_action_pos - set_tag_idx)); 6482 index = sample_action_pos; 6483 } else { 6484 index = sample_action_pos; 6485 if (index != 0) 6486 memcpy(actions_pre, actions, 6487 sizeof(struct rte_flow_action) * index); 6488 } 6489 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6490 * For CX6DX and above, metadata registers Cx preserve their value, 6491 * add an extra tag action for NIC-RX and E-Switch Domain. 6492 */ 6493 if (add_tag) { 6494 /* Prepare the prefix tag action. */ 6495 append_index++; 6496 set_tag = (void *)(actions_pre + actions_n + append_index); 6497 /* Trust VF/SF on CX5 not supported meter so that the reserved 6498 * metadata regC is REG_NON, back to use application tag 6499 * index 0. 6500 */ 6501 if (unlikely(priv->mtr_color_reg == REG_NON)) 6502 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6503 else 6504 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6505 if (ret < 0) 6506 return ret; 6507 mlx5_ipool_malloc(priv->sh->ipool 6508 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6509 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6510 .id = ret, 6511 .data = tag_id, 6512 }; 6513 /* Prepare the suffix subflow items. */ 6514 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6515 tag_spec->data = tag_id; 6516 tag_spec->id = set_tag->id; 6517 tag_mask = tag_spec + 1; 6518 tag_mask->data = UINT32_MAX; 6519 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6520 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6521 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6522 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6523 memcpy(sfx_items, items, sizeof(*sfx_items)); 6524 sfx_items++; 6525 break; 6526 } 6527 } 6528 sfx_items[0] = (struct rte_flow_item){ 6529 .type = (enum rte_flow_item_type) 6530 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6531 .spec = tag_spec, 6532 .last = NULL, 6533 .mask = tag_mask, 6534 }; 6535 sfx_items[1] = (struct rte_flow_item){ 6536 .type = (enum rte_flow_item_type) 6537 RTE_FLOW_ITEM_TYPE_END, 6538 }; 6539 /* Prepare the tag action in prefix subflow. */ 6540 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6541 actions_pre[set_tag_idx] = 6542 (struct rte_flow_action){ 6543 .type = (enum rte_flow_action_type) 6544 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6545 .conf = set_tag, 6546 }; 6547 /* Update next sample position due to add one tag action */ 6548 index += 1; 6549 } 6550 /* Copy the sample action into prefix flow. */ 6551 memcpy(actions_pre + index, actions + sample_action_pos, 6552 sizeof(struct rte_flow_action)); 6553 index += 1; 6554 /* For the modify action after the sample action in E-Switch mirroring, 6555 * Add the extra jump action in prefix subflow and jump into the next 6556 * table, then do the modify action in the new table. 6557 */ 6558 if (jump_table) { 6559 /* Prepare the prefix jump action. */ 6560 append_index++; 6561 jump_action = (void *)(actions_pre + actions_n + append_index); 6562 jump_action->group = jump_table; 6563 actions_pre[index++] = 6564 (struct rte_flow_action){ 6565 .type = (enum rte_flow_action_type) 6566 RTE_FLOW_ACTION_TYPE_JUMP, 6567 .conf = jump_action, 6568 }; 6569 } 6570 actions_pre[index] = (struct rte_flow_action){ 6571 .type = (enum rte_flow_action_type) 6572 RTE_FLOW_ACTION_TYPE_END, 6573 }; 6574 /* Put the actions after sample into Suffix flow. */ 6575 memcpy(actions_sfx, actions + sample_action_pos + 1, 6576 sizeof(struct rte_flow_action) * 6577 (actions_n - sample_action_pos - 1)); 6578 return tag_id; 6579 } 6580 6581 /** 6582 * The splitting for metadata feature. 6583 * 6584 * - Q/RSS action on NIC Rx should be split in order to pass by 6585 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6586 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6587 * 6588 * - All the actions on NIC Tx should have a mreg copy action to 6589 * copy reg_a from WQE to reg_c[0]. 6590 * 6591 * @param dev 6592 * Pointer to Ethernet device. 6593 * @param[in] flow 6594 * Parent flow structure pointer. 6595 * @param[in] attr 6596 * Flow rule attributes. 6597 * @param[in] items 6598 * Pattern specification (list terminated by the END pattern item). 6599 * @param[in] actions 6600 * Associated actions (list terminated by the END action). 6601 * @param[in] flow_split_info 6602 * Pointer to flow split info structure. 6603 * @param[out] error 6604 * Perform verbose error reporting if not NULL. 6605 * @return 6606 * 0 on success, negative value otherwise 6607 */ 6608 static int 6609 flow_create_split_metadata(struct rte_eth_dev *dev, 6610 struct rte_flow *flow, 6611 const struct rte_flow_attr *attr, 6612 const struct rte_flow_item items[], 6613 const struct rte_flow_action actions[], 6614 struct mlx5_flow_split_info *flow_split_info, 6615 struct rte_flow_error *error) 6616 { 6617 struct mlx5_priv *priv = dev->data->dev_private; 6618 struct mlx5_sh_config *config = &priv->sh->config; 6619 const struct rte_flow_action *qrss = NULL; 6620 struct rte_flow_action *ext_actions = NULL; 6621 struct mlx5_flow *dev_flow = NULL; 6622 uint32_t qrss_id = 0; 6623 int mtr_sfx = 0; 6624 size_t act_size; 6625 int actions_n; 6626 int encap_idx; 6627 int ret; 6628 6629 /* Check whether extensive metadata feature is engaged. */ 6630 if (!config->dv_flow_en || 6631 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6632 !mlx5_flow_ext_mreg_supported(dev)) 6633 return flow_create_split_inner(dev, flow, NULL, attr, items, 6634 actions, flow_split_info, error); 6635 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6636 &encap_idx); 6637 if (qrss) { 6638 /* Exclude hairpin flows from splitting. */ 6639 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6640 const struct rte_flow_action_queue *queue; 6641 6642 queue = qrss->conf; 6643 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6644 qrss = NULL; 6645 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6646 const struct rte_flow_action_rss *rss; 6647 6648 rss = qrss->conf; 6649 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6650 qrss = NULL; 6651 } 6652 } 6653 if (qrss) { 6654 /* Check if it is in meter suffix table. */ 6655 mtr_sfx = attr->group == 6656 ((attr->transfer && priv->fdb_def_rule) ? 6657 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6658 MLX5_FLOW_TABLE_LEVEL_METER); 6659 /* 6660 * Q/RSS action on NIC Rx should be split in order to pass by 6661 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6662 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6663 */ 6664 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6665 sizeof(struct rte_flow_action_set_tag) + 6666 sizeof(struct rte_flow_action_jump); 6667 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6668 SOCKET_ID_ANY); 6669 if (!ext_actions) 6670 return rte_flow_error_set(error, ENOMEM, 6671 RTE_FLOW_ERROR_TYPE_ACTION, 6672 NULL, "no memory to split " 6673 "metadata flow"); 6674 /* 6675 * Create the new actions list with removed Q/RSS action 6676 * and appended set tag and jump to register copy table 6677 * (RX_CP_TBL). We should preallocate unique tag ID here 6678 * in advance, because it is needed for set tag action. 6679 */ 6680 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6681 qrss, actions_n, 6682 mtr_sfx, error); 6683 if (!mtr_sfx && !qrss_id) { 6684 ret = -rte_errno; 6685 goto exit; 6686 } 6687 } else if (attr->egress) { 6688 /* 6689 * All the actions on NIC Tx should have a metadata register 6690 * copy action to copy reg_a from WQE to reg_c[meta] 6691 */ 6692 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6693 sizeof(struct mlx5_flow_action_copy_mreg); 6694 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6695 SOCKET_ID_ANY); 6696 if (!ext_actions) 6697 return rte_flow_error_set(error, ENOMEM, 6698 RTE_FLOW_ERROR_TYPE_ACTION, 6699 NULL, "no memory to split " 6700 "metadata flow"); 6701 /* Create the action list appended with copy register. */ 6702 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6703 actions_n, error, encap_idx); 6704 if (ret < 0) 6705 goto exit; 6706 } 6707 /* Add the unmodified original or prefix subflow. */ 6708 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6709 items, ext_actions ? ext_actions : 6710 actions, flow_split_info, error); 6711 if (ret < 0) 6712 goto exit; 6713 MLX5_ASSERT(dev_flow); 6714 if (qrss) { 6715 const struct rte_flow_attr q_attr = { 6716 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6717 .ingress = 1, 6718 }; 6719 /* Internal PMD action to set register. */ 6720 struct mlx5_rte_flow_item_tag q_tag_spec = { 6721 .data = qrss_id, 6722 .id = REG_NON, 6723 }; 6724 struct rte_flow_item q_items[] = { 6725 { 6726 .type = (enum rte_flow_item_type) 6727 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6728 .spec = &q_tag_spec, 6729 .last = NULL, 6730 .mask = NULL, 6731 }, 6732 { 6733 .type = RTE_FLOW_ITEM_TYPE_END, 6734 }, 6735 }; 6736 struct rte_flow_action q_actions[] = { 6737 { 6738 .type = qrss->type, 6739 .conf = qrss->conf, 6740 }, 6741 { 6742 .type = RTE_FLOW_ACTION_TYPE_END, 6743 }, 6744 }; 6745 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6746 6747 /* 6748 * Configure the tag item only if there is no meter subflow. 6749 * Since tag is already marked in the meter suffix subflow 6750 * we can just use the meter suffix items as is. 6751 */ 6752 if (qrss_id) { 6753 /* Not meter subflow. */ 6754 MLX5_ASSERT(!mtr_sfx); 6755 /* 6756 * Put unique id in prefix flow due to it is destroyed 6757 * after suffix flow and id will be freed after there 6758 * is no actual flows with this id and identifier 6759 * reallocation becomes possible (for example, for 6760 * other flows in other threads). 6761 */ 6762 dev_flow->handle->split_flow_id = qrss_id; 6763 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6764 error); 6765 if (ret < 0) 6766 goto exit; 6767 q_tag_spec.id = ret; 6768 } 6769 dev_flow = NULL; 6770 /* Add suffix subflow to execute Q/RSS. */ 6771 flow_split_info->prefix_layers = layers; 6772 flow_split_info->prefix_mark = 0; 6773 flow_split_info->table_id = 0; 6774 ret = flow_create_split_inner(dev, flow, &dev_flow, 6775 &q_attr, mtr_sfx ? items : 6776 q_items, q_actions, 6777 flow_split_info, error); 6778 if (ret < 0) 6779 goto exit; 6780 /* qrss ID should be freed if failed. */ 6781 qrss_id = 0; 6782 MLX5_ASSERT(dev_flow); 6783 } 6784 6785 exit: 6786 /* 6787 * We do not destroy the partially created sub_flows in case of error. 6788 * These ones are included into parent flow list and will be destroyed 6789 * by flow_drv_destroy. 6790 */ 6791 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6792 qrss_id); 6793 mlx5_free(ext_actions); 6794 return ret; 6795 } 6796 6797 /** 6798 * Create meter internal drop flow with the original pattern. 6799 * 6800 * @param dev 6801 * Pointer to Ethernet device. 6802 * @param[in] flow 6803 * Parent flow structure pointer. 6804 * @param[in] attr 6805 * Flow rule attributes. 6806 * @param[in] items 6807 * Pattern specification (list terminated by the END pattern item). 6808 * @param[in] flow_split_info 6809 * Pointer to flow split info structure. 6810 * @param[in] fm 6811 * Pointer to flow meter structure. 6812 * @param[out] error 6813 * Perform verbose error reporting if not NULL. 6814 * @return 6815 * 0 on success, negative value otherwise 6816 */ 6817 static uint32_t 6818 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6819 struct rte_flow *flow, 6820 const struct rte_flow_attr *attr, 6821 const struct rte_flow_item items[], 6822 struct mlx5_flow_split_info *flow_split_info, 6823 struct mlx5_flow_meter_info *fm, 6824 struct rte_flow_error *error) 6825 { 6826 struct mlx5_flow *dev_flow = NULL; 6827 struct rte_flow_attr drop_attr = *attr; 6828 struct rte_flow_action drop_actions[3]; 6829 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6830 6831 MLX5_ASSERT(fm->drop_cnt); 6832 drop_actions[0].type = 6833 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6834 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6835 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6836 drop_actions[1].conf = NULL; 6837 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6838 drop_actions[2].conf = NULL; 6839 drop_split_info.external = false; 6840 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6841 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6842 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6843 return flow_create_split_inner(dev, flow, &dev_flow, 6844 &drop_attr, items, drop_actions, 6845 &drop_split_info, error); 6846 } 6847 6848 /** 6849 * The splitting for meter feature. 6850 * 6851 * - The meter flow will be split to two flows as prefix and 6852 * suffix flow. The packets make sense only it pass the prefix 6853 * meter action. 6854 * 6855 * - Reg_C_5 is used for the packet to match betweend prefix and 6856 * suffix flow. 6857 * 6858 * @param dev 6859 * Pointer to Ethernet device. 6860 * @param[in] flow 6861 * Parent flow structure pointer. 6862 * @param[in] attr 6863 * Flow rule attributes. 6864 * @param[in] items 6865 * Pattern specification (list terminated by the END pattern item). 6866 * @param[in] actions 6867 * Associated actions (list terminated by the END action). 6868 * @param[in] flow_split_info 6869 * Pointer to flow split info structure. 6870 * @param[out] error 6871 * Perform verbose error reporting if not NULL. 6872 * @return 6873 * 0 on success, negative value otherwise 6874 */ 6875 static int 6876 flow_create_split_meter(struct rte_eth_dev *dev, 6877 struct rte_flow *flow, 6878 const struct rte_flow_attr *attr, 6879 const struct rte_flow_item items[], 6880 const struct rte_flow_action actions[], 6881 struct mlx5_flow_split_info *flow_split_info, 6882 struct rte_flow_error *error) 6883 { 6884 struct mlx5_priv *priv = dev->data->dev_private; 6885 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6886 struct rte_flow_action *sfx_actions = NULL; 6887 struct rte_flow_action *pre_actions = NULL; 6888 struct rte_flow_item *sfx_items = NULL; 6889 struct mlx5_flow *dev_flow = NULL; 6890 struct rte_flow_attr sfx_attr = *attr; 6891 struct mlx5_flow_meter_info *fm = NULL; 6892 uint8_t skip_scale_restore; 6893 bool has_mtr = false; 6894 bool has_modify = false; 6895 bool set_mtr_reg = true; 6896 bool is_mtr_hierarchy = false; 6897 uint32_t meter_id = 0; 6898 uint32_t mtr_idx = 0; 6899 uint32_t mtr_flow_id = 0; 6900 size_t act_size; 6901 size_t item_size; 6902 int actions_n = 0; 6903 int ret = 0; 6904 6905 if (priv->mtr_en) 6906 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6907 &has_modify, &meter_id); 6908 if (has_mtr) { 6909 if (flow->meter) { 6910 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6911 if (!fm) 6912 return rte_flow_error_set(error, EINVAL, 6913 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6914 NULL, "Meter not found."); 6915 } else { 6916 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6917 if (!fm) 6918 return rte_flow_error_set(error, EINVAL, 6919 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6920 NULL, "Meter not found."); 6921 ret = mlx5_flow_meter_attach(priv, fm, 6922 &sfx_attr, error); 6923 if (ret) 6924 return -rte_errno; 6925 flow->meter = mtr_idx; 6926 } 6927 MLX5_ASSERT(wks); 6928 wks->fm = fm; 6929 if (!fm->def_policy) { 6930 wks->policy = mlx5_flow_meter_policy_find(dev, 6931 fm->policy_id, 6932 NULL); 6933 MLX5_ASSERT(wks->policy); 6934 if (wks->policy->mark) 6935 wks->mark = 1; 6936 if (wks->policy->is_hierarchy) { 6937 wks->final_policy = 6938 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6939 wks->policy); 6940 if (!wks->final_policy) 6941 return rte_flow_error_set(error, 6942 EINVAL, 6943 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6944 "Failed to find terminal policy of hierarchy."); 6945 is_mtr_hierarchy = true; 6946 } 6947 } 6948 /* 6949 * If it isn't default-policy Meter, and 6950 * 1. Not meter hierarchy and there's no action in flow to change 6951 * packet (modify/encap/decap etc.), OR 6952 * 2. No drop count needed for this meter. 6953 * Then no need to use regC to save meter id anymore. 6954 */ 6955 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6956 set_mtr_reg = false; 6957 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6958 #define METER_PREFIX_ACTION 7 6959 act_size = (sizeof(struct rte_flow_action) * 6960 (actions_n + METER_PREFIX_ACTION)) + 6961 sizeof(struct mlx5_rte_flow_action_set_tag); 6962 /* Suffix items: tag, vlan, port id, end. */ 6963 #define METER_SUFFIX_ITEM 4 6964 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6965 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6966 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6967 0, SOCKET_ID_ANY); 6968 if (!sfx_actions) 6969 return rte_flow_error_set(error, ENOMEM, 6970 RTE_FLOW_ERROR_TYPE_ACTION, 6971 NULL, "no memory to split " 6972 "meter flow"); 6973 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6974 act_size); 6975 /* There's no suffix flow for meter of non-default policy. */ 6976 if (!fm->def_policy) 6977 pre_actions = sfx_actions + 1; 6978 else 6979 pre_actions = sfx_actions + actions_n; 6980 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6981 items, sfx_items, actions, 6982 sfx_actions, pre_actions, 6983 (set_mtr_reg ? &mtr_flow_id : NULL), 6984 error); 6985 if (ret) { 6986 ret = -rte_errno; 6987 goto exit; 6988 } 6989 /* Add the prefix subflow. */ 6990 skip_scale_restore = flow_split_info->skip_scale; 6991 flow_split_info->skip_scale |= 6992 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6993 ret = flow_create_split_inner(dev, flow, &dev_flow, 6994 attr, items, pre_actions, 6995 flow_split_info, error); 6996 flow_split_info->skip_scale = skip_scale_restore; 6997 if (ret) { 6998 if (mtr_flow_id) 6999 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 7000 ret = -rte_errno; 7001 goto exit; 7002 } 7003 if (mtr_flow_id) { 7004 dev_flow->handle->split_flow_id = mtr_flow_id; 7005 dev_flow->handle->is_meter_flow_id = 1; 7006 } 7007 if (!fm->def_policy) { 7008 if (!set_mtr_reg && fm->drop_cnt) 7009 ret = 7010 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 7011 &sfx_attr, items, 7012 flow_split_info, 7013 fm, error); 7014 goto exit; 7015 } 7016 /* Setting the sfx group atrr. */ 7017 sfx_attr.group = sfx_attr.transfer ? 7018 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 7019 MLX5_FLOW_TABLE_LEVEL_METER; 7020 flow_split_info->prefix_layers = 7021 flow_get_prefix_layer_flags(dev_flow); 7022 flow_split_info->prefix_mark |= wks->mark; 7023 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 7024 } 7025 /* Add the prefix subflow. */ 7026 ret = flow_create_split_metadata(dev, flow, 7027 &sfx_attr, sfx_items ? 7028 sfx_items : items, 7029 sfx_actions ? sfx_actions : actions, 7030 flow_split_info, error); 7031 exit: 7032 if (sfx_actions) 7033 mlx5_free(sfx_actions); 7034 return ret; 7035 } 7036 7037 /** 7038 * The splitting for sample feature. 7039 * 7040 * Once Sample action is detected in the action list, the flow actions should 7041 * be split into prefix sub flow and suffix sub flow. 7042 * 7043 * The original items remain in the prefix sub flow, all actions preceding the 7044 * sample action and the sample action itself will be copied to the prefix 7045 * sub flow, the actions following the sample action will be copied to the 7046 * suffix sub flow, Queue action always be located in the suffix sub flow. 7047 * 7048 * In order to make the packet from prefix sub flow matches with suffix sub 7049 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 7050 * flow uses tag item with the unique flow id. 7051 * 7052 * @param dev 7053 * Pointer to Ethernet device. 7054 * @param[in] flow 7055 * Parent flow structure pointer. 7056 * @param[in] attr 7057 * Flow rule attributes. 7058 * @param[in] items 7059 * Pattern specification (list terminated by the END pattern item). 7060 * @param[in] actions 7061 * Associated actions (list terminated by the END action). 7062 * @param[in] flow_split_info 7063 * Pointer to flow split info structure. 7064 * @param[out] error 7065 * Perform verbose error reporting if not NULL. 7066 * @return 7067 * 0 on success, negative value otherwise 7068 */ 7069 static int 7070 flow_create_split_sample(struct rte_eth_dev *dev, 7071 struct rte_flow *flow, 7072 const struct rte_flow_attr *attr, 7073 const struct rte_flow_item items[], 7074 const struct rte_flow_action actions[], 7075 struct mlx5_flow_split_info *flow_split_info, 7076 struct rte_flow_error *error) 7077 { 7078 struct mlx5_priv *priv = dev->data->dev_private; 7079 struct rte_flow_action *sfx_actions = NULL; 7080 struct rte_flow_action *pre_actions = NULL; 7081 struct rte_flow_item *sfx_items = NULL; 7082 struct mlx5_flow *dev_flow = NULL; 7083 struct rte_flow_attr sfx_attr = *attr; 7084 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7085 struct mlx5_flow_dv_sample_resource *sample_res; 7086 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 7087 struct mlx5_flow_tbl_resource *sfx_tbl; 7088 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 7089 #endif 7090 size_t act_size; 7091 size_t item_size; 7092 uint32_t fdb_tx = 0; 7093 int32_t tag_id = 0; 7094 int actions_n = 0; 7095 int sample_action_pos; 7096 int qrss_action_pos; 7097 int add_tag = 0; 7098 int modify_after_mirror = 0; 7099 uint16_t jump_table = 0; 7100 const uint32_t next_ft_step = 1; 7101 int ret = 0; 7102 struct mlx5_priv *item_port_priv = NULL; 7103 const struct rte_flow_item *item; 7104 7105 if (priv->sampler_en) 7106 actions_n = flow_check_match_action(actions, attr, 7107 RTE_FLOW_ACTION_TYPE_SAMPLE, 7108 &sample_action_pos, &qrss_action_pos, 7109 &modify_after_mirror); 7110 if (actions_n) { 7111 /* The prefix actions must includes sample, tag, end. */ 7112 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 7113 + sizeof(struct mlx5_rte_flow_action_set_tag); 7114 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 7115 sizeof(struct mlx5_rte_flow_item_tag) * 2; 7116 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 7117 item_size), 0, SOCKET_ID_ANY); 7118 if (!sfx_actions) 7119 return rte_flow_error_set(error, ENOMEM, 7120 RTE_FLOW_ERROR_TYPE_ACTION, 7121 NULL, "no memory to split " 7122 "sample flow"); 7123 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 7124 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 7125 const struct rte_flow_item_port_id *spec; 7126 7127 spec = (const struct rte_flow_item_port_id *)item->spec; 7128 if (spec) 7129 item_port_priv = 7130 mlx5_port_to_eswitch_info(spec->id, true); 7131 break; 7132 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 7133 const struct rte_flow_item_ethdev *spec; 7134 7135 spec = (const struct rte_flow_item_ethdev *)item->spec; 7136 if (spec) 7137 item_port_priv = 7138 mlx5_port_to_eswitch_info(spec->port_id, true); 7139 break; 7140 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 7141 const struct rte_flow_item_ethdev *spec; 7142 7143 spec = (const struct rte_flow_item_ethdev *)item->spec; 7144 if (spec) 7145 item_port_priv = 7146 mlx5_port_to_eswitch_info(spec->port_id, true); 7147 break; 7148 } 7149 } 7150 /* The representor_id is UINT16_MAX for uplink. */ 7151 fdb_tx = (attr->transfer && 7152 flow_source_vport_representor(priv, item_port_priv)); 7153 /* 7154 * When reg_c_preserve is set, metadata registers Cx preserve 7155 * their value even through packet duplication. 7156 */ 7157 add_tag = (!fdb_tx || 7158 priv->sh->cdev->config.hca_attr.reg_c_preserve); 7159 if (add_tag) 7160 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 7161 + act_size); 7162 if (modify_after_mirror) 7163 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 7164 next_ft_step; 7165 pre_actions = sfx_actions + actions_n; 7166 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 7167 actions, sfx_actions, 7168 pre_actions, actions_n, 7169 sample_action_pos, 7170 qrss_action_pos, jump_table, 7171 error); 7172 if (tag_id < 0 || (add_tag && !tag_id)) { 7173 ret = -rte_errno; 7174 goto exit; 7175 } 7176 if (modify_after_mirror) 7177 flow_split_info->skip_scale = 7178 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 7179 /* Add the prefix subflow. */ 7180 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7181 items, pre_actions, 7182 flow_split_info, error); 7183 if (ret) { 7184 ret = -rte_errno; 7185 goto exit; 7186 } 7187 dev_flow->handle->split_flow_id = tag_id; 7188 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7189 if (!modify_after_mirror) { 7190 /* Set the sfx group attr. */ 7191 sample_res = (struct mlx5_flow_dv_sample_resource *) 7192 dev_flow->dv.sample_res; 7193 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7194 sample_res->normal_path_tbl; 7195 sfx_tbl_data = container_of(sfx_tbl, 7196 struct mlx5_flow_tbl_data_entry, 7197 tbl); 7198 sfx_attr.group = sfx_attr.transfer ? 7199 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7200 } else { 7201 MLX5_ASSERT(attr->transfer); 7202 sfx_attr.group = jump_table; 7203 } 7204 flow_split_info->prefix_layers = 7205 flow_get_prefix_layer_flags(dev_flow); 7206 MLX5_ASSERT(wks); 7207 flow_split_info->prefix_mark |= wks->mark; 7208 /* Suffix group level already be scaled with factor, set 7209 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7210 * again in translation. 7211 */ 7212 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7213 #endif 7214 } 7215 /* Add the suffix subflow. */ 7216 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7217 sfx_items ? sfx_items : items, 7218 sfx_actions ? sfx_actions : actions, 7219 flow_split_info, error); 7220 exit: 7221 if (sfx_actions) 7222 mlx5_free(sfx_actions); 7223 return ret; 7224 } 7225 7226 /** 7227 * Split the flow to subflow set. The splitters might be linked 7228 * in the chain, like this: 7229 * flow_create_split_outer() calls: 7230 * flow_create_split_meter() calls: 7231 * flow_create_split_metadata(meter_subflow_0) calls: 7232 * flow_create_split_inner(metadata_subflow_0) 7233 * flow_create_split_inner(metadata_subflow_1) 7234 * flow_create_split_inner(metadata_subflow_2) 7235 * flow_create_split_metadata(meter_subflow_1) calls: 7236 * flow_create_split_inner(metadata_subflow_0) 7237 * flow_create_split_inner(metadata_subflow_1) 7238 * flow_create_split_inner(metadata_subflow_2) 7239 * 7240 * This provide flexible way to add new levels of flow splitting. 7241 * The all of successfully created subflows are included to the 7242 * parent flow dev_flow list. 7243 * 7244 * @param dev 7245 * Pointer to Ethernet device. 7246 * @param[in] flow 7247 * Parent flow structure pointer. 7248 * @param[in] attr 7249 * Flow rule attributes. 7250 * @param[in] items 7251 * Pattern specification (list terminated by the END pattern item). 7252 * @param[in] actions 7253 * Associated actions (list terminated by the END action). 7254 * @param[in] flow_split_info 7255 * Pointer to flow split info structure. 7256 * @param[out] error 7257 * Perform verbose error reporting if not NULL. 7258 * @return 7259 * 0 on success, negative value otherwise 7260 */ 7261 static int 7262 flow_create_split_outer(struct rte_eth_dev *dev, 7263 struct rte_flow *flow, 7264 const struct rte_flow_attr *attr, 7265 const struct rte_flow_item items[], 7266 const struct rte_flow_action actions[], 7267 struct mlx5_flow_split_info *flow_split_info, 7268 struct rte_flow_error *error) 7269 { 7270 int ret; 7271 7272 ret = flow_create_split_sample(dev, flow, attr, items, 7273 actions, flow_split_info, error); 7274 MLX5_ASSERT(ret <= 0); 7275 return ret; 7276 } 7277 7278 static inline struct mlx5_flow_tunnel * 7279 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7280 { 7281 struct mlx5_flow_tunnel *tunnel; 7282 7283 #pragma GCC diagnostic push 7284 #pragma GCC diagnostic ignored "-Wcast-qual" 7285 tunnel = (typeof(tunnel))flow->tunnel; 7286 #pragma GCC diagnostic pop 7287 7288 return tunnel; 7289 } 7290 7291 /** 7292 * Create a flow and add it to @p list. 7293 * 7294 * @param dev 7295 * Pointer to Ethernet device. 7296 * @param list 7297 * Pointer to a TAILQ flow list. If this parameter NULL, 7298 * no list insertion occurred, flow is just created, 7299 * this is caller's responsibility to track the 7300 * created flow. 7301 * @param[in] attr 7302 * Flow rule attributes. 7303 * @param[in] items 7304 * Pattern specification (list terminated by the END pattern item). 7305 * @param[in] actions 7306 * Associated actions (list terminated by the END action). 7307 * @param[in] external 7308 * This flow rule is created by request external to PMD. 7309 * @param[out] error 7310 * Perform verbose error reporting if not NULL. 7311 * 7312 * @return 7313 * A flow index on success, 0 otherwise and rte_errno is set. 7314 */ 7315 static uint32_t 7316 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7317 const struct rte_flow_attr *attr, 7318 const struct rte_flow_item items[], 7319 const struct rte_flow_action original_actions[], 7320 bool external, struct rte_flow_error *error) 7321 { 7322 struct mlx5_priv *priv = dev->data->dev_private; 7323 struct rte_flow *flow = NULL; 7324 struct mlx5_flow *dev_flow; 7325 const struct rte_flow_action_rss *rss = NULL; 7326 struct mlx5_translated_action_handle 7327 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7328 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7329 union { 7330 struct mlx5_flow_expand_rss buf; 7331 uint8_t buffer[8192]; 7332 } expand_buffer; 7333 union { 7334 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7335 uint8_t buffer[2048]; 7336 } actions_rx; 7337 union { 7338 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7339 uint8_t buffer[2048]; 7340 } actions_hairpin_tx; 7341 union { 7342 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7343 uint8_t buffer[2048]; 7344 } items_tx; 7345 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7346 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7347 struct mlx5_flow_rss_desc *rss_desc; 7348 const struct rte_flow_action *p_actions_rx; 7349 uint32_t i; 7350 uint32_t idx = 0; 7351 int hairpin_flow; 7352 struct rte_flow_attr attr_tx = { .priority = 0 }; 7353 const struct rte_flow_action *actions; 7354 struct rte_flow_action *translated_actions = NULL; 7355 struct mlx5_flow_tunnel *tunnel; 7356 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7357 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7358 struct mlx5_flow_split_info flow_split_info = { 7359 .external = !!external, 7360 .skip_scale = 0, 7361 .flow_idx = 0, 7362 .prefix_mark = 0, 7363 .prefix_layers = 0, 7364 .table_id = 0 7365 }; 7366 int ret; 7367 7368 MLX5_ASSERT(wks); 7369 rss_desc = &wks->rss_desc; 7370 ret = flow_action_handles_translate(dev, original_actions, 7371 indir_actions, 7372 &indir_actions_n, 7373 &translated_actions, error); 7374 if (ret < 0) { 7375 MLX5_ASSERT(translated_actions == NULL); 7376 return 0; 7377 } 7378 actions = translated_actions ? translated_actions : original_actions; 7379 p_actions_rx = actions; 7380 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7381 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7382 external, hairpin_flow, error); 7383 if (ret < 0) 7384 goto error_before_hairpin_split; 7385 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7386 if (!flow) { 7387 rte_errno = ENOMEM; 7388 goto error_before_hairpin_split; 7389 } 7390 if (hairpin_flow > 0) { 7391 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7392 rte_errno = EINVAL; 7393 goto error_before_hairpin_split; 7394 } 7395 flow_hairpin_split(dev, actions, actions_rx.actions, 7396 actions_hairpin_tx.actions, items_tx.items, 7397 idx); 7398 p_actions_rx = actions_rx.actions; 7399 } 7400 flow_split_info.flow_idx = idx; 7401 flow->drv_type = flow_get_drv_type(dev, attr); 7402 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7403 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7404 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7405 /* RSS Action only works on NIC RX domain */ 7406 if (attr->ingress) 7407 rss = flow_get_rss_action(dev, p_actions_rx); 7408 if (rss) { 7409 MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); 7410 rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func); 7411 /* 7412 * The following information is required by 7413 * mlx5_flow_hashfields_adjust() in advance. 7414 */ 7415 rss_desc->level = rss->level; 7416 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7417 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7418 } 7419 flow->dev_handles = 0; 7420 if (rss && rss->types) { 7421 unsigned int graph_root; 7422 7423 graph_root = find_graph_root(rss->level); 7424 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7425 items, rss->types, 7426 mlx5_support_expansion, graph_root); 7427 MLX5_ASSERT(ret > 0 && 7428 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7429 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7430 for (i = 0; i < buf->entries; ++i) 7431 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7432 } 7433 } else { 7434 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7435 sizeof(expand_buffer.buffer), 7436 items, sq_specs); 7437 if (ret) { 7438 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7439 NULL, "not enough memory for rte_flow"); 7440 goto error; 7441 } 7442 if (buf->entries == 0) { 7443 buf->entries = 1; 7444 buf->entry[0].pattern = (void *)(uintptr_t)items; 7445 } 7446 } 7447 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7448 indir_actions_n); 7449 for (i = 0; i < buf->entries; ++i) { 7450 /* Initialize flow split data. */ 7451 flow_split_info.prefix_layers = 0; 7452 flow_split_info.prefix_mark = 0; 7453 flow_split_info.skip_scale = 0; 7454 /* 7455 * The splitter may create multiple dev_flows, 7456 * depending on configuration. In the simplest 7457 * case it just creates unmodified original flow. 7458 */ 7459 ret = flow_create_split_outer(dev, flow, attr, 7460 buf->entry[i].pattern, 7461 p_actions_rx, &flow_split_info, 7462 error); 7463 if (ret < 0) 7464 goto error; 7465 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7466 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7467 p_actions_rx, 7468 idx, 7469 wks->flows[0].tunnel, 7470 &default_miss_ctx, 7471 error); 7472 if (ret < 0) { 7473 mlx5_free(default_miss_ctx.queue); 7474 goto error; 7475 } 7476 } 7477 } 7478 /* Create the tx flow. */ 7479 if (hairpin_flow) { 7480 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7481 attr_tx.ingress = 0; 7482 attr_tx.egress = 1; 7483 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7484 actions_hairpin_tx.actions, 7485 idx, error); 7486 if (!dev_flow) 7487 goto error; 7488 dev_flow->flow = flow; 7489 dev_flow->external = 0; 7490 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7491 dev_flow->handle, next); 7492 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7493 items_tx.items, 7494 actions_hairpin_tx.actions, error); 7495 if (ret < 0) 7496 goto error; 7497 } 7498 /* 7499 * Update the metadata register copy table. If extensive 7500 * metadata feature is enabled and registers are supported 7501 * we might create the extra rte_flow for each unique 7502 * MARK/FLAG action ID. 7503 * 7504 * The table is updated for ingress and transfer flows only, because 7505 * the egress Flows belong to the different device and 7506 * copy table should be updated in peer NIC Rx domain. 7507 */ 7508 if ((attr->ingress || attr->transfer) && 7509 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7510 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7511 if (ret) 7512 goto error; 7513 } 7514 /* 7515 * If the flow is external (from application) OR device is started, 7516 * OR mreg discover, then apply immediately. 7517 */ 7518 if (external || dev->data->dev_started || 7519 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7520 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7521 ret = flow_drv_apply(dev, flow, error); 7522 if (ret < 0) 7523 goto error; 7524 } 7525 flow->type = type; 7526 flow_rxq_flags_set(dev, flow); 7527 rte_free(translated_actions); 7528 tunnel = flow_tunnel_from_rule(wks->flows); 7529 if (tunnel) { 7530 flow->tunnel = 1; 7531 flow->tunnel_id = tunnel->tunnel_id; 7532 __atomic_fetch_add(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7533 mlx5_free(default_miss_ctx.queue); 7534 } 7535 mlx5_flow_pop_thread_workspace(); 7536 return idx; 7537 error: 7538 MLX5_ASSERT(flow); 7539 ret = rte_errno; /* Save rte_errno before cleanup. */ 7540 flow_mreg_del_copy_action(dev, flow); 7541 flow_drv_destroy(dev, flow); 7542 if (rss_desc->shared_rss) 7543 __atomic_fetch_sub(&((struct mlx5_shared_action_rss *) 7544 mlx5_ipool_get 7545 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7546 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7547 mlx5_ipool_free(priv->flows[type], idx); 7548 rte_errno = ret; /* Restore rte_errno. */ 7549 ret = rte_errno; 7550 rte_errno = ret; 7551 error_before_hairpin_split: 7552 mlx5_flow_pop_thread_workspace(); 7553 rte_free(translated_actions); 7554 return 0; 7555 } 7556 7557 /** 7558 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7559 * incoming packets to table 1. 7560 * 7561 * Other flow rules, requested for group n, will be created in 7562 * e-switch table n+1. 7563 * Jump action to e-switch group n will be created to group n+1. 7564 * 7565 * Used when working in switchdev mode, to utilise advantages of table 1 7566 * and above. 7567 * 7568 * @param dev 7569 * Pointer to Ethernet device. 7570 * 7571 * @return 7572 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7573 */ 7574 struct rte_flow * 7575 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7576 { 7577 const struct rte_flow_attr attr = { 7578 .group = 0, 7579 .priority = 0, 7580 .ingress = 0, 7581 .egress = 0, 7582 .transfer = 1, 7583 }; 7584 const struct rte_flow_item pattern = { 7585 .type = RTE_FLOW_ITEM_TYPE_END, 7586 }; 7587 struct rte_flow_action_jump jump = { 7588 .group = 1, 7589 }; 7590 const struct rte_flow_action actions[] = { 7591 { 7592 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7593 .conf = &jump, 7594 }, 7595 { 7596 .type = RTE_FLOW_ACTION_TYPE_END, 7597 }, 7598 }; 7599 struct rte_flow_error error; 7600 7601 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7602 &attr, &pattern, 7603 actions, false, &error); 7604 } 7605 7606 /** 7607 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7608 * and sq number, directs all packets to peer vport. 7609 * 7610 * @param dev 7611 * Pointer to Ethernet device. 7612 * @param sq_num 7613 * SQ number. 7614 * 7615 * @return 7616 * Flow ID on success, 0 otherwise and rte_errno is set. 7617 */ 7618 uint32_t 7619 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7620 { 7621 struct rte_flow_attr attr = { 7622 .group = 0, 7623 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7624 .ingress = 0, 7625 .egress = 0, 7626 .transfer = 1, 7627 }; 7628 struct rte_flow_item_port_id port_spec = { 7629 .id = MLX5_PORT_ESW_MGR, 7630 }; 7631 struct mlx5_rte_flow_item_sq sq_spec = { 7632 .queue = sq_num, 7633 }; 7634 struct rte_flow_item pattern[] = { 7635 { 7636 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7637 .spec = &port_spec, 7638 }, 7639 { 7640 .type = (enum rte_flow_item_type) 7641 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7642 .spec = &sq_spec, 7643 }, 7644 { 7645 .type = RTE_FLOW_ITEM_TYPE_END, 7646 }, 7647 }; 7648 struct rte_flow_action_jump jump = { 7649 .group = 1, 7650 }; 7651 struct rte_flow_action_port_id port = { 7652 .id = dev->data->port_id, 7653 }; 7654 struct rte_flow_action actions[] = { 7655 { 7656 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7657 .conf = &jump, 7658 }, 7659 { 7660 .type = RTE_FLOW_ACTION_TYPE_END, 7661 }, 7662 }; 7663 struct rte_flow_error error; 7664 7665 /* 7666 * Creates group 0, highest priority jump flow. 7667 * Matches txq to bypass kernel packets. 7668 */ 7669 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7670 false, &error) == 0) 7671 return 0; 7672 /* Create group 1, lowest priority redirect flow for txq. */ 7673 attr.group = 1; 7674 actions[0].conf = &port; 7675 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7676 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7677 actions, false, &error); 7678 } 7679 7680 /** 7681 * Validate a flow supported by the NIC. 7682 * 7683 * @see rte_flow_validate() 7684 * @see rte_flow_ops 7685 */ 7686 int 7687 mlx5_flow_validate(struct rte_eth_dev *dev, 7688 const struct rte_flow_attr *attr, 7689 const struct rte_flow_item items[], 7690 const struct rte_flow_action original_actions[], 7691 struct rte_flow_error *error) 7692 { 7693 int hairpin_flow; 7694 struct mlx5_translated_action_handle 7695 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7696 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7697 const struct rte_flow_action *actions; 7698 struct rte_flow_action *translated_actions = NULL; 7699 int ret = flow_action_handles_translate(dev, original_actions, 7700 indir_actions, 7701 &indir_actions_n, 7702 &translated_actions, error); 7703 7704 if (ret) 7705 return ret; 7706 actions = translated_actions ? translated_actions : original_actions; 7707 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7708 ret = flow_drv_validate(dev, attr, items, actions, 7709 true, hairpin_flow, error); 7710 rte_free(translated_actions); 7711 return ret; 7712 } 7713 7714 static int 7715 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7716 const struct rte_flow_attr *attr, 7717 const uint32_t orig_prio, 7718 const struct rte_flow_item *items, 7719 const struct rte_flow_action *actions, 7720 uint32_t flow_idx) 7721 { 7722 struct mlx5_priv *priv = dev->data->dev_private; 7723 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7724 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7725 struct rte_flow_error error; 7726 int len, ret; 7727 7728 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7729 if (!flow_info) { 7730 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7731 return -1; 7732 } 7733 flow_info->orig_prio = orig_prio; 7734 flow_info->attr = *attr; 7735 /* Standby mode rule awlays saves it in low priority entry. */ 7736 flow_info->flow_idx_low_prio = flow_idx; 7737 7738 /* Store matching items. */ 7739 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7740 if (ret <= 0) { 7741 DRV_LOG(ERR, "Can't get items length."); 7742 goto end; 7743 } 7744 len = RTE_ALIGN(ret, 16); 7745 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7746 if (!flow_info->items) { 7747 DRV_LOG(ERR, "No enough memory for items caching."); 7748 goto end; 7749 } 7750 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7751 if (ret <= 0) { 7752 DRV_LOG(ERR, "Can't duplicate items."); 7753 goto end; 7754 } 7755 7756 /* Store flow actions. */ 7757 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7758 if (ret <= 0) { 7759 DRV_LOG(ERR, "Can't get actions length."); 7760 goto end; 7761 } 7762 len = RTE_ALIGN(ret, 16); 7763 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7764 if (!flow_info->actions) { 7765 DRV_LOG(ERR, "No enough memory for actions caching."); 7766 goto end; 7767 } 7768 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7769 if (ret <= 0) { 7770 DRV_LOG(ERR, "Can't duplicate actions."); 7771 goto end; 7772 } 7773 7774 /* Insert to the list end. */ 7775 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7776 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7777 } else { 7778 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7779 while (LIST_NEXT(tmp_info, next)) 7780 tmp_info = LIST_NEXT(tmp_info, next); 7781 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7782 } 7783 return 0; 7784 end: 7785 if (flow_info->items) 7786 mlx5_free(flow_info->items); 7787 if (flow_info->actions) 7788 mlx5_free(flow_info->actions); 7789 mlx5_free(flow_info); 7790 return -1; 7791 } 7792 7793 static int 7794 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7795 { 7796 struct mlx5_priv *priv = dev->data->dev_private; 7797 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7798 struct mlx5_dv_flow_info *flow_info; 7799 struct rte_flow_attr attr; 7800 struct rte_flow_error error; 7801 struct rte_flow *high, *low; 7802 7803 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7804 while (flow_info) { 7805 /* DUP flow may have the same priority. */ 7806 if (flow_info->orig_prio != flow_info->attr.priority) { 7807 attr = flow_info->attr; 7808 if (orig_prio) 7809 attr.priority = flow_info->orig_prio; 7810 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7811 &attr, flow_info->items, flow_info->actions, 7812 true, &error); 7813 if (!flow_info->flow_idx_high_prio) { 7814 DRV_LOG(ERR, "Priority toggle failed internally."); 7815 goto err; 7816 } 7817 } 7818 flow_info = LIST_NEXT(flow_info, next); 7819 } 7820 /* Delete the low priority rules and swap the flow handle. */ 7821 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7822 while (flow_info) { 7823 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7824 if (flow_info->orig_prio != flow_info->attr.priority) { 7825 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7826 flow_info->flow_idx_high_prio); 7827 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7828 flow_info->flow_idx_low_prio); 7829 if (high && low) { 7830 RTE_SWAP(*low, *high); 7831 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7832 flow_info->flow_idx_low_prio); 7833 flow_info->flow_idx_high_prio = 0; 7834 } 7835 } 7836 flow_info = LIST_NEXT(flow_info, next); 7837 } 7838 return 0; 7839 err: 7840 /* Destroy preceding successful high priority rules. */ 7841 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7842 while (flow_info) { 7843 if (flow_info->orig_prio != flow_info->attr.priority) { 7844 if (flow_info->flow_idx_high_prio) 7845 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7846 flow_info->flow_idx_high_prio); 7847 else 7848 break; 7849 flow_info->flow_idx_high_prio = 0; 7850 } 7851 flow_info = LIST_NEXT(flow_info, next); 7852 } 7853 return -1; 7854 } 7855 7856 /** 7857 * Set the mode of the flow engine of a process to active or standby during live migration. 7858 * 7859 * @param[in] mode 7860 * MLX5 flow engine mode, @see `enum mlx5_flow_engine_mode`. 7861 * @param[in] flags 7862 * Flow engine mode specific flags. 7863 * 7864 * @return 7865 * Negative value on error, positive on success. 7866 */ 7867 int 7868 rte_pmd_mlx5_flow_engine_set_mode(enum mlx5_flow_engine_mode mode, uint32_t flags) 7869 { 7870 struct mlx5_priv *priv; 7871 struct mlx5_flow_engine_mode_info *mode_info; 7872 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7873 uint16_t port, port_id; 7874 uint16_t toggle_num = 0; 7875 struct rte_eth_dev *dev; 7876 enum mlx5_flow_engine_mode orig_mode; 7877 uint32_t orig_flags; 7878 bool need_toggle = false; 7879 7880 /* Check if flags combinations are supported. */ 7881 if (flags && flags != MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7882 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7883 return -1; 7884 } 7885 MLX5_ETH_FOREACH_DEV(port, NULL) { 7886 dev = &rte_eth_devices[port]; 7887 priv = dev->data->dev_private; 7888 mode_info = &priv->mode_info; 7889 /* No mode change. Assume all devices hold the same mode. */ 7890 if (mode_info->mode == mode) { 7891 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7892 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7893 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7894 port, mode_info->mode_flag); 7895 orig_mode = mode_info->mode; 7896 orig_flags = mode_info->mode_flag; 7897 goto err; 7898 } 7899 mode_info->mode_flag = flags; 7900 toggle_num++; 7901 continue; 7902 } 7903 /* Active -> standby. */ 7904 if (mode == MLX5_FLOW_ENGINE_MODE_STANDBY) { 7905 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7906 DRV_LOG(ERR, "Cached rule existed"); 7907 orig_mode = mode_info->mode; 7908 orig_flags = mode_info->mode_flag; 7909 goto err; 7910 } 7911 mode_info->mode_flag = flags; 7912 mode_info->mode = mode; 7913 toggle_num++; 7914 /* Standby -> active. */ 7915 } else if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7916 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7917 DRV_LOG(INFO, "No cached rule existed"); 7918 } else { 7919 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7920 orig_mode = mode_info->mode; 7921 orig_flags = mode_info->mode_flag; 7922 need_toggle = true; 7923 goto err; 7924 } 7925 } 7926 toggle_num++; 7927 } 7928 } 7929 if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7930 /* Clear cache flow rules. */ 7931 MLX5_ETH_FOREACH_DEV(port, NULL) { 7932 priv = rte_eth_devices[port].data->dev_private; 7933 mode_info = &priv->mode_info; 7934 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7935 while (flow_info) { 7936 tmp_info = LIST_NEXT(flow_info, next); 7937 LIST_REMOVE(flow_info, next); 7938 mlx5_free(flow_info->actions); 7939 mlx5_free(flow_info->items); 7940 mlx5_free(flow_info); 7941 flow_info = tmp_info; 7942 } 7943 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7944 } 7945 } 7946 return toggle_num; 7947 err: 7948 /* Rollback all preceding successful ports. */ 7949 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7950 if (port_id == port) 7951 break; 7952 priv = rte_eth_devices[port_id].data->dev_private; 7953 mode_info = &priv->mode_info; 7954 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7955 mlx5_flow_cache_flow_toggle(dev, false)) 7956 return -EPERM; 7957 mode_info->mode = orig_mode; 7958 mode_info->mode_flag = orig_flags; 7959 } 7960 return -EINVAL; 7961 } 7962 /** 7963 * Create a flow. 7964 * 7965 * @see rte_flow_create() 7966 * @see rte_flow_ops 7967 */ 7968 struct rte_flow * 7969 mlx5_flow_create(struct rte_eth_dev *dev, 7970 const struct rte_flow_attr *attr, 7971 const struct rte_flow_item items[], 7972 const struct rte_flow_action actions[], 7973 struct rte_flow_error *error) 7974 { 7975 struct mlx5_priv *priv = dev->data->dev_private; 7976 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7977 uint32_t prio = attr->priority; 7978 uint32_t flow_idx; 7979 7980 if (priv->sh->config.dv_flow_en == 2) { 7981 rte_flow_error_set(error, ENOTSUP, 7982 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7983 NULL, 7984 "Flow non-Q creation not supported"); 7985 return NULL; 7986 } 7987 /* 7988 * If the device is not started yet, it is not allowed to created a 7989 * flow from application. PMD default flows and traffic control flows 7990 * are not affected. 7991 */ 7992 if (unlikely(!dev->data->dev_started)) { 7993 DRV_LOG(DEBUG, "port %u is not started when " 7994 "inserting a flow", dev->data->port_id); 7995 rte_flow_error_set(error, ENODEV, 7996 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7997 NULL, 7998 "port not started"); 7999 return NULL; 8000 } 8001 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 8002 if (attr->transfer || 8003 (attr->ingress && 8004 !(priv->mode_info.mode_flag & MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 8005 new_attr->priority += 1; 8006 } 8007 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 8008 if (!flow_idx) 8009 return NULL; 8010 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 8011 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 8012 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8013 flow_idx = 0; 8014 } 8015 } 8016 return (void *)(uintptr_t)flow_idx; 8017 } 8018 8019 /** 8020 * Destroy a flow in a list. 8021 * 8022 * @param dev 8023 * Pointer to Ethernet device. 8024 * @param[in] flow_idx 8025 * Index of flow to destroy. 8026 */ 8027 static void 8028 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 8029 uint32_t flow_idx) 8030 { 8031 struct mlx5_priv *priv = dev->data->dev_private; 8032 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 8033 8034 if (!flow) 8035 return; 8036 MLX5_ASSERT(flow->type == type); 8037 /* 8038 * Update RX queue flags only if port is started, otherwise it is 8039 * already clean. 8040 */ 8041 if (dev->data->dev_started) 8042 flow_rxq_flags_trim(dev, flow); 8043 flow_drv_destroy(dev, flow); 8044 if (flow->tunnel) { 8045 struct mlx5_flow_tunnel *tunnel; 8046 8047 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 8048 RTE_VERIFY(tunnel); 8049 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 8050 mlx5_flow_tunnel_free(dev, tunnel); 8051 } 8052 flow_mreg_del_copy_action(dev, flow); 8053 mlx5_ipool_free(priv->flows[type], flow_idx); 8054 } 8055 8056 /** 8057 * Destroy all flows. 8058 * 8059 * @param dev 8060 * Pointer to Ethernet device. 8061 * @param type 8062 * Flow type to be flushed. 8063 * @param active 8064 * If flushing is called actively. 8065 */ 8066 void 8067 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 8068 bool active) 8069 { 8070 struct mlx5_priv *priv = dev->data->dev_private; 8071 uint32_t num_flushed = 0, fidx = 1; 8072 struct rte_flow *flow; 8073 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8074 struct mlx5_dv_flow_info *flow_info; 8075 8076 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8077 if (priv->sh->config.dv_flow_en == 2 && 8078 type == MLX5_FLOW_TYPE_GEN) { 8079 flow_hw_q_flow_flush(dev, NULL); 8080 return; 8081 } 8082 #endif 8083 8084 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 8085 flow_list_destroy(dev, type, fidx); 8086 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 8087 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8088 while (flow_info) { 8089 /* Romove the cache flow info. */ 8090 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 8091 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8092 LIST_REMOVE(flow_info, next); 8093 mlx5_free(flow_info->items); 8094 mlx5_free(flow_info->actions); 8095 mlx5_free(flow_info); 8096 break; 8097 } 8098 flow_info = LIST_NEXT(flow_info, next); 8099 } 8100 } 8101 num_flushed++; 8102 } 8103 if (active) { 8104 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 8105 dev->data->port_id, num_flushed); 8106 } 8107 } 8108 8109 /** 8110 * Stop all default actions for flows. 8111 * 8112 * @param dev 8113 * Pointer to Ethernet device. 8114 */ 8115 void 8116 mlx5_flow_stop_default(struct rte_eth_dev *dev) 8117 { 8118 flow_mreg_del_default_copy_action(dev); 8119 flow_rxq_flags_clear(dev); 8120 } 8121 8122 /** 8123 * Set rxq flag. 8124 * 8125 * @param[in] dev 8126 * Pointer to the rte_eth_dev structure. 8127 * @param[in] enable 8128 * Flag to enable or not. 8129 */ 8130 void 8131 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 8132 { 8133 struct mlx5_priv *priv = dev->data->dev_private; 8134 unsigned int i; 8135 8136 if ((!priv->mark_enabled && !enable) || 8137 (priv->mark_enabled && enable)) 8138 return; 8139 for (i = 0; i < priv->rxqs_n; ++i) { 8140 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 8141 8142 /* With RXQ start/stop feature, RXQ might be stopped. */ 8143 if (!rxq_ctrl) 8144 continue; 8145 rxq_ctrl->rxq.mark = enable; 8146 } 8147 priv->mark_enabled = enable; 8148 } 8149 8150 /** 8151 * Start all default actions for flows. 8152 * 8153 * @param dev 8154 * Pointer to Ethernet device. 8155 * @return 8156 * 0 on success, a negative errno value otherwise and rte_errno is set. 8157 */ 8158 int 8159 mlx5_flow_start_default(struct rte_eth_dev *dev) 8160 { 8161 struct rte_flow_error error; 8162 8163 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8164 return flow_mreg_add_default_copy_action(dev, &error); 8165 } 8166 8167 /** 8168 * Release key of thread specific flow workspace data. 8169 */ 8170 void 8171 flow_release_workspace(void *data) 8172 { 8173 struct mlx5_flow_workspace *wks = data; 8174 struct mlx5_flow_workspace *next; 8175 8176 while (wks) { 8177 next = wks->next; 8178 free(wks); 8179 wks = next; 8180 } 8181 } 8182 8183 static struct mlx5_flow_workspace *gc_head; 8184 static rte_spinlock_t mlx5_flow_workspace_lock = RTE_SPINLOCK_INITIALIZER; 8185 8186 static void 8187 mlx5_flow_workspace_gc_add(struct mlx5_flow_workspace *ws) 8188 { 8189 rte_spinlock_lock(&mlx5_flow_workspace_lock); 8190 ws->gc = gc_head; 8191 gc_head = ws; 8192 rte_spinlock_unlock(&mlx5_flow_workspace_lock); 8193 } 8194 8195 void 8196 mlx5_flow_workspace_gc_release(void) 8197 { 8198 while (gc_head) { 8199 struct mlx5_flow_workspace *wks = gc_head; 8200 8201 gc_head = wks->gc; 8202 flow_release_workspace(wks); 8203 } 8204 } 8205 8206 /** 8207 * Get thread specific current flow workspace. 8208 * 8209 * @return pointer to thread specific flow workspace data, NULL on error. 8210 */ 8211 struct mlx5_flow_workspace* 8212 mlx5_flow_get_thread_workspace(void) 8213 { 8214 struct mlx5_flow_workspace *data; 8215 8216 data = mlx5_flow_os_get_specific_workspace(); 8217 MLX5_ASSERT(data && data->inuse); 8218 if (!data || !data->inuse) 8219 DRV_LOG(ERR, "flow workspace not initialized."); 8220 return data; 8221 } 8222 8223 /** 8224 * Allocate and init new flow workspace. 8225 * 8226 * @return pointer to flow workspace data, NULL on error. 8227 */ 8228 static struct mlx5_flow_workspace* 8229 flow_alloc_thread_workspace(void) 8230 { 8231 size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); 8232 size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; 8233 struct mlx5_flow_workspace *data = calloc(1, data_size + 8234 rss_queue_array_size); 8235 8236 if (!data) { 8237 DRV_LOG(ERR, "Failed to allocate flow workspace memory."); 8238 return NULL; 8239 } 8240 data->rss_desc.queue = RTE_PTR_ADD(data, data_size); 8241 return data; 8242 } 8243 8244 /** 8245 * Get new thread specific flow workspace. 8246 * 8247 * If current workspace inuse, create new one and set as current. 8248 * 8249 * @return pointer to thread specific flow workspace data, NULL on error. 8250 */ 8251 struct mlx5_flow_workspace* 8252 mlx5_flow_push_thread_workspace(void) 8253 { 8254 struct mlx5_flow_workspace *curr; 8255 struct mlx5_flow_workspace *data; 8256 8257 curr = mlx5_flow_os_get_specific_workspace(); 8258 if (!curr) { 8259 data = flow_alloc_thread_workspace(); 8260 if (!data) 8261 return NULL; 8262 mlx5_flow_workspace_gc_add(data); 8263 } else if (!curr->inuse) { 8264 data = curr; 8265 } else if (curr->next) { 8266 data = curr->next; 8267 } else { 8268 data = flow_alloc_thread_workspace(); 8269 if (!data) 8270 return NULL; 8271 curr->next = data; 8272 data->prev = curr; 8273 } 8274 data->inuse = 1; 8275 data->flow_idx = 0; 8276 /* Set as current workspace */ 8277 if (mlx5_flow_os_set_specific_workspace(data)) 8278 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8279 return data; 8280 } 8281 8282 /** 8283 * Close current thread specific flow workspace. 8284 * 8285 * If previous workspace available, set it as current. 8286 * 8287 * @return pointer to thread specific flow workspace data, NULL on error. 8288 */ 8289 void 8290 mlx5_flow_pop_thread_workspace(void) 8291 { 8292 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8293 8294 if (!data) 8295 return; 8296 if (!data->inuse) { 8297 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8298 return; 8299 } 8300 data->inuse = 0; 8301 if (!data->prev) 8302 return; 8303 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8304 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8305 } 8306 8307 /** 8308 * Verify the flow list is empty 8309 * 8310 * @param dev 8311 * Pointer to Ethernet device. 8312 * 8313 * @return the number of flows not released. 8314 */ 8315 int 8316 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8317 { 8318 struct mlx5_priv *priv = dev->data->dev_private; 8319 struct rte_flow *flow; 8320 uint32_t idx = 0; 8321 int ret = 0, i; 8322 8323 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8324 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8325 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8326 dev->data->port_id, (void *)flow); 8327 ret++; 8328 } 8329 } 8330 return ret; 8331 } 8332 8333 /** 8334 * Enable default hairpin egress flow. 8335 * 8336 * @param dev 8337 * Pointer to Ethernet device. 8338 * @param sq_num 8339 * The SQ hw number. 8340 * 8341 * @return 8342 * 0 on success, a negative errno value otherwise and rte_errno is set. 8343 */ 8344 int 8345 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8346 uint32_t sq_num) 8347 { 8348 const struct rte_flow_attr attr = { 8349 .egress = 1, 8350 .priority = 0, 8351 }; 8352 struct mlx5_rte_flow_item_sq queue_spec = { 8353 .queue = sq_num, 8354 }; 8355 struct mlx5_rte_flow_item_sq queue_mask = { 8356 .queue = UINT32_MAX, 8357 }; 8358 struct rte_flow_item items[] = { 8359 { 8360 .type = (enum rte_flow_item_type) 8361 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8362 .spec = &queue_spec, 8363 .last = NULL, 8364 .mask = &queue_mask, 8365 }, 8366 { 8367 .type = RTE_FLOW_ITEM_TYPE_END, 8368 }, 8369 }; 8370 struct rte_flow_action_jump jump = { 8371 .group = MLX5_HAIRPIN_TX_TABLE, 8372 }; 8373 struct rte_flow_action actions[2]; 8374 uint32_t flow_idx; 8375 struct rte_flow_error error; 8376 8377 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8378 actions[0].conf = &jump; 8379 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8380 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8381 &attr, items, actions, false, &error); 8382 if (!flow_idx) { 8383 DRV_LOG(DEBUG, 8384 "Failed to create ctrl flow: rte_errno(%d)," 8385 " type(%d), message(%s)", 8386 rte_errno, error.type, 8387 error.message ? error.message : " (no stated reason)"); 8388 return -rte_errno; 8389 } 8390 return 0; 8391 } 8392 8393 /** 8394 * Enable a control flow configured from the control plane. 8395 * 8396 * @param dev 8397 * Pointer to Ethernet device. 8398 * @param eth_spec 8399 * An Ethernet flow spec to apply. 8400 * @param eth_mask 8401 * An Ethernet flow mask to apply. 8402 * @param vlan_spec 8403 * A VLAN flow spec to apply. 8404 * @param vlan_mask 8405 * A VLAN flow mask to apply. 8406 * 8407 * @return 8408 * 0 on success, a negative errno value otherwise and rte_errno is set. 8409 */ 8410 int 8411 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8412 struct rte_flow_item_eth *eth_spec, 8413 struct rte_flow_item_eth *eth_mask, 8414 struct rte_flow_item_vlan *vlan_spec, 8415 struct rte_flow_item_vlan *vlan_mask) 8416 { 8417 struct mlx5_priv *priv = dev->data->dev_private; 8418 const struct rte_flow_attr attr = { 8419 .ingress = 1, 8420 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8421 }; 8422 struct rte_flow_item items[] = { 8423 { 8424 .type = RTE_FLOW_ITEM_TYPE_ETH, 8425 .spec = eth_spec, 8426 .last = NULL, 8427 .mask = eth_mask, 8428 }, 8429 { 8430 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8431 RTE_FLOW_ITEM_TYPE_END, 8432 .spec = vlan_spec, 8433 .last = NULL, 8434 .mask = vlan_mask, 8435 }, 8436 { 8437 .type = RTE_FLOW_ITEM_TYPE_END, 8438 }, 8439 }; 8440 uint16_t queue[priv->reta_idx_n]; 8441 struct rte_flow_action_rss action_rss = { 8442 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8443 .level = 0, 8444 .types = priv->rss_conf.rss_hf, 8445 .key_len = priv->rss_conf.rss_key_len, 8446 .queue_num = priv->reta_idx_n, 8447 .key = priv->rss_conf.rss_key, 8448 .queue = queue, 8449 }; 8450 struct rte_flow_action actions[] = { 8451 { 8452 .type = RTE_FLOW_ACTION_TYPE_RSS, 8453 .conf = &action_rss, 8454 }, 8455 { 8456 .type = RTE_FLOW_ACTION_TYPE_END, 8457 }, 8458 }; 8459 uint32_t flow_idx; 8460 struct rte_flow_error error; 8461 unsigned int i; 8462 8463 if (!priv->reta_idx_n || !priv->rxqs_n) { 8464 return 0; 8465 } 8466 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8467 action_rss.types = 0; 8468 for (i = 0; i != priv->reta_idx_n; ++i) 8469 queue[i] = (*priv->reta_idx)[i]; 8470 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8471 &attr, items, actions, false, &error); 8472 if (!flow_idx) 8473 return -rte_errno; 8474 return 0; 8475 } 8476 8477 /** 8478 * Enable a flow control configured from the control plane. 8479 * 8480 * @param dev 8481 * Pointer to Ethernet device. 8482 * @param eth_spec 8483 * An Ethernet flow spec to apply. 8484 * @param eth_mask 8485 * An Ethernet flow mask to apply. 8486 * 8487 * @return 8488 * 0 on success, a negative errno value otherwise and rte_errno is set. 8489 */ 8490 int 8491 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8492 struct rte_flow_item_eth *eth_spec, 8493 struct rte_flow_item_eth *eth_mask) 8494 { 8495 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8496 } 8497 8498 /** 8499 * Create default miss flow rule matching lacp traffic 8500 * 8501 * @param dev 8502 * Pointer to Ethernet device. 8503 * @param eth_spec 8504 * An Ethernet flow spec to apply. 8505 * 8506 * @return 8507 * 0 on success, a negative errno value otherwise and rte_errno is set. 8508 */ 8509 int 8510 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8511 { 8512 /* 8513 * The LACP matching is done by only using ether type since using 8514 * a multicast dst mac causes kernel to give low priority to this flow. 8515 */ 8516 static const struct rte_flow_item_eth lacp_spec = { 8517 .hdr.ether_type = RTE_BE16(0x8809), 8518 }; 8519 static const struct rte_flow_item_eth lacp_mask = { 8520 .hdr.ether_type = 0xffff, 8521 }; 8522 const struct rte_flow_attr attr = { 8523 .ingress = 1, 8524 }; 8525 struct rte_flow_item items[] = { 8526 { 8527 .type = RTE_FLOW_ITEM_TYPE_ETH, 8528 .spec = &lacp_spec, 8529 .mask = &lacp_mask, 8530 }, 8531 { 8532 .type = RTE_FLOW_ITEM_TYPE_END, 8533 }, 8534 }; 8535 struct rte_flow_action actions[] = { 8536 { 8537 .type = (enum rte_flow_action_type) 8538 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8539 }, 8540 { 8541 .type = RTE_FLOW_ACTION_TYPE_END, 8542 }, 8543 }; 8544 struct rte_flow_error error; 8545 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8546 &attr, items, actions, 8547 false, &error); 8548 8549 if (!flow_idx) 8550 return -rte_errno; 8551 return 0; 8552 } 8553 8554 /** 8555 * Destroy a flow. 8556 * 8557 * @see rte_flow_destroy() 8558 * @see rte_flow_ops 8559 */ 8560 int 8561 mlx5_flow_destroy(struct rte_eth_dev *dev, 8562 struct rte_flow *flow, 8563 struct rte_flow_error *error __rte_unused) 8564 { 8565 struct mlx5_priv *priv = dev->data->dev_private; 8566 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8567 struct mlx5_dv_flow_info *flow_info; 8568 8569 if (priv->sh->config.dv_flow_en == 2) 8570 return rte_flow_error_set(error, ENOTSUP, 8571 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8572 NULL, 8573 "Flow non-Q destruction not supported"); 8574 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8575 (uintptr_t)(void *)flow); 8576 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8577 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8578 while (flow_info) { 8579 /* Romove the cache flow info. */ 8580 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8581 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8582 LIST_REMOVE(flow_info, next); 8583 mlx5_free(flow_info->items); 8584 mlx5_free(flow_info->actions); 8585 mlx5_free(flow_info); 8586 break; 8587 } 8588 flow_info = LIST_NEXT(flow_info, next); 8589 } 8590 } 8591 return 0; 8592 } 8593 8594 /** 8595 * Destroy all flows. 8596 * 8597 * @see rte_flow_flush() 8598 * @see rte_flow_ops 8599 */ 8600 int 8601 mlx5_flow_flush(struct rte_eth_dev *dev, 8602 struct rte_flow_error *error __rte_unused) 8603 { 8604 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8605 return 0; 8606 } 8607 8608 /** 8609 * Isolated mode. 8610 * 8611 * @see rte_flow_isolate() 8612 * @see rte_flow_ops 8613 */ 8614 int 8615 mlx5_flow_isolate(struct rte_eth_dev *dev, 8616 int enable, 8617 struct rte_flow_error *error) 8618 { 8619 struct mlx5_priv *priv = dev->data->dev_private; 8620 8621 if (dev->data->dev_started) { 8622 rte_flow_error_set(error, EBUSY, 8623 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8624 NULL, 8625 "port must be stopped first"); 8626 return -rte_errno; 8627 } 8628 if (!enable && !priv->sh->config.repr_matching) 8629 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8630 "isolated mode cannot be disabled when " 8631 "representor matching is disabled"); 8632 priv->isolated = !!enable; 8633 if (enable) 8634 dev->dev_ops = &mlx5_dev_ops_isolate; 8635 else 8636 dev->dev_ops = &mlx5_dev_ops; 8637 8638 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8639 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8640 8641 return 0; 8642 } 8643 8644 /** 8645 * Query a flow. 8646 * 8647 * @see rte_flow_query() 8648 * @see rte_flow_ops 8649 */ 8650 static int 8651 flow_drv_query(struct rte_eth_dev *dev, 8652 struct rte_flow *eflow, 8653 const struct rte_flow_action *actions, 8654 void *data, 8655 struct rte_flow_error *error) 8656 { 8657 struct mlx5_priv *priv = dev->data->dev_private; 8658 const struct mlx5_flow_driver_ops *fops; 8659 struct rte_flow *flow = NULL; 8660 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8661 8662 if (priv->sh->config.dv_flow_en == 2) { 8663 #ifdef HAVE_MLX5_HWS_SUPPORT 8664 flow = eflow; 8665 ftype = MLX5_FLOW_TYPE_HW; 8666 #endif 8667 } else { 8668 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8669 (uintptr_t)(void *)eflow); 8670 } 8671 if (!flow) { 8672 return rte_flow_error_set(error, ENOENT, 8673 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8674 NULL, 8675 "invalid flow handle"); 8676 } 8677 if (ftype == MLX5_FLOW_TYPE_MIN) 8678 ftype = flow->drv_type; 8679 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8680 fops = flow_get_drv_ops(ftype); 8681 8682 return fops->query(dev, flow, actions, data, error); 8683 } 8684 8685 /** 8686 * Query a flow. 8687 * 8688 * @see rte_flow_query() 8689 * @see rte_flow_ops 8690 */ 8691 int 8692 mlx5_flow_query(struct rte_eth_dev *dev, 8693 struct rte_flow *flow, 8694 const struct rte_flow_action *actions, 8695 void *data, 8696 struct rte_flow_error *error) 8697 { 8698 int ret; 8699 8700 ret = flow_drv_query(dev, flow, actions, data, 8701 error); 8702 if (ret < 0) 8703 return ret; 8704 return 0; 8705 } 8706 8707 /** 8708 * Get rte_flow callbacks. 8709 * 8710 * @param dev 8711 * Pointer to Ethernet device structure. 8712 * @param ops 8713 * Pointer to operation-specific structure. 8714 * 8715 * @return 0 8716 */ 8717 int 8718 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8719 const struct rte_flow_ops **ops) 8720 { 8721 *ops = &mlx5_flow_ops; 8722 return 0; 8723 } 8724 8725 /** 8726 * Validate meter policy actions. 8727 * Dispatcher for action type specific validation. 8728 * 8729 * @param[in] dev 8730 * Pointer to the Ethernet device structure. 8731 * @param[in] action 8732 * The meter policy action object to validate. 8733 * @param[in] attr 8734 * Attributes of flow to determine steering domain. 8735 * @param[out] is_rss 8736 * Is RSS or not. 8737 * @param[out] domain_bitmap 8738 * Domain bitmap. 8739 * @param[out] is_def_policy 8740 * Is default policy or not. 8741 * @param[out] error 8742 * Perform verbose error reporting if not NULL. Initialized in case of 8743 * error only. 8744 * 8745 * @return 8746 * 0 on success, otherwise negative errno value. 8747 */ 8748 int 8749 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8750 const struct rte_flow_action *actions[RTE_COLORS], 8751 struct rte_flow_attr *attr, 8752 bool *is_rss, 8753 uint8_t *domain_bitmap, 8754 uint8_t *policy_mode, 8755 struct rte_mtr_error *error) 8756 { 8757 const struct mlx5_flow_driver_ops *fops; 8758 8759 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8760 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8761 domain_bitmap, policy_mode, error); 8762 } 8763 8764 /** 8765 * Destroy the meter table set. 8766 * 8767 * @param[in] dev 8768 * Pointer to Ethernet device. 8769 * @param[in] mtr_policy 8770 * Meter policy struct. 8771 */ 8772 void 8773 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8774 struct mlx5_flow_meter_policy *mtr_policy) 8775 { 8776 const struct mlx5_flow_driver_ops *fops; 8777 8778 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8779 fops->destroy_mtr_acts(dev, mtr_policy); 8780 } 8781 8782 /** 8783 * Create policy action, lock free, 8784 * (mutex should be acquired by caller). 8785 * Dispatcher for action type specific call. 8786 * 8787 * @param[in] dev 8788 * Pointer to the Ethernet device structure. 8789 * @param[in] mtr_policy 8790 * Meter policy struct. 8791 * @param[in] action 8792 * Action specification used to create meter actions. 8793 * @param[in] attr 8794 * Flow rule attributes. 8795 * @param[out] error 8796 * Perform verbose error reporting if not NULL. Initialized in case of 8797 * error only. 8798 * 8799 * @return 8800 * 0 on success, otherwise negative errno value. 8801 */ 8802 int 8803 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8804 struct mlx5_flow_meter_policy *mtr_policy, 8805 const struct rte_flow_action *actions[RTE_COLORS], 8806 struct rte_flow_attr *attr, 8807 struct rte_mtr_error *error) 8808 { 8809 const struct mlx5_flow_driver_ops *fops; 8810 8811 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8812 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8813 } 8814 8815 /** 8816 * Create policy rules, lock free, 8817 * (mutex should be acquired by caller). 8818 * Dispatcher for action type specific call. 8819 * 8820 * @param[in] dev 8821 * Pointer to the Ethernet device structure. 8822 * @param[in] mtr_policy 8823 * Meter policy struct. 8824 * 8825 * @return 8826 * 0 on success, -1 otherwise. 8827 */ 8828 int 8829 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8830 struct mlx5_flow_meter_policy *mtr_policy) 8831 { 8832 const struct mlx5_flow_driver_ops *fops; 8833 8834 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8835 return fops->create_policy_rules(dev, mtr_policy); 8836 } 8837 8838 /** 8839 * Destroy policy rules, lock free, 8840 * (mutex should be acquired by caller). 8841 * Dispatcher for action type specific call. 8842 * 8843 * @param[in] dev 8844 * Pointer to the Ethernet device structure. 8845 * @param[in] mtr_policy 8846 * Meter policy struct. 8847 */ 8848 void 8849 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8850 struct mlx5_flow_meter_policy *mtr_policy) 8851 { 8852 const struct mlx5_flow_driver_ops *fops; 8853 8854 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8855 fops->destroy_policy_rules(dev, mtr_policy); 8856 } 8857 8858 /** 8859 * Destroy the default policy table set. 8860 * 8861 * @param[in] dev 8862 * Pointer to Ethernet device. 8863 */ 8864 void 8865 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8866 { 8867 const struct mlx5_flow_driver_ops *fops; 8868 8869 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8870 fops->destroy_def_policy(dev); 8871 } 8872 8873 /** 8874 * Destroy the default policy table set. 8875 * 8876 * @param[in] dev 8877 * Pointer to Ethernet device. 8878 * 8879 * @return 8880 * 0 on success, -1 otherwise. 8881 */ 8882 int 8883 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8884 { 8885 const struct mlx5_flow_driver_ops *fops; 8886 8887 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8888 return fops->create_def_policy(dev); 8889 } 8890 8891 /** 8892 * Create the needed meter and suffix tables. 8893 * 8894 * @param[in] dev 8895 * Pointer to Ethernet device. 8896 * 8897 * @return 8898 * 0 on success, -1 otherwise. 8899 */ 8900 int 8901 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8902 struct mlx5_flow_meter_info *fm, 8903 uint32_t mtr_idx, 8904 uint8_t domain_bitmap) 8905 { 8906 const struct mlx5_flow_driver_ops *fops; 8907 8908 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8909 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8910 } 8911 8912 /** 8913 * Destroy the meter table set. 8914 * 8915 * @param[in] dev 8916 * Pointer to Ethernet device. 8917 * @param[in] tbl 8918 * Pointer to the meter table set. 8919 */ 8920 void 8921 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8922 struct mlx5_flow_meter_info *fm) 8923 { 8924 const struct mlx5_flow_driver_ops *fops; 8925 8926 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8927 fops->destroy_mtr_tbls(dev, fm); 8928 } 8929 8930 /** 8931 * Destroy the global meter drop table. 8932 * 8933 * @param[in] dev 8934 * Pointer to Ethernet device. 8935 */ 8936 void 8937 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8938 { 8939 const struct mlx5_flow_driver_ops *fops; 8940 8941 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8942 fops->destroy_mtr_drop_tbls(dev); 8943 } 8944 8945 /** 8946 * Destroy the sub policy table with RX queue. 8947 * 8948 * @param[in] dev 8949 * Pointer to Ethernet device. 8950 * @param[in] mtr_policy 8951 * Pointer to meter policy table. 8952 */ 8953 void 8954 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8955 struct mlx5_flow_meter_policy *mtr_policy) 8956 { 8957 const struct mlx5_flow_driver_ops *fops; 8958 8959 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8960 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8961 } 8962 8963 /** 8964 * Allocate the needed aso flow meter id. 8965 * 8966 * @param[in] dev 8967 * Pointer to Ethernet device. 8968 * 8969 * @return 8970 * Index to aso flow meter on success, NULL otherwise. 8971 */ 8972 uint32_t 8973 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8974 { 8975 const struct mlx5_flow_driver_ops *fops; 8976 8977 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8978 return fops->create_meter(dev); 8979 } 8980 8981 /** 8982 * Free the aso flow meter id. 8983 * 8984 * @param[in] dev 8985 * Pointer to Ethernet device. 8986 * @param[in] mtr_idx 8987 * Index to aso flow meter to be free. 8988 * 8989 * @return 8990 * 0 on success. 8991 */ 8992 void 8993 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8994 { 8995 const struct mlx5_flow_driver_ops *fops; 8996 8997 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8998 fops->free_meter(dev, mtr_idx); 8999 } 9000 9001 /** 9002 * Allocate a counter. 9003 * 9004 * @param[in] dev 9005 * Pointer to Ethernet device structure. 9006 * 9007 * @return 9008 * Index to allocated counter on success, 0 otherwise. 9009 */ 9010 uint32_t 9011 mlx5_counter_alloc(struct rte_eth_dev *dev) 9012 { 9013 struct rte_flow_attr attr = { .transfer = 0 }; 9014 9015 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 9016 (dev); 9017 } 9018 9019 /** 9020 * Free a counter. 9021 * 9022 * @param[in] dev 9023 * Pointer to Ethernet device structure. 9024 * @param[in] cnt 9025 * Index to counter to be free. 9026 */ 9027 void 9028 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 9029 { 9030 struct rte_flow_attr attr = { .transfer = 0 }; 9031 9032 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 9033 } 9034 9035 /** 9036 * Query counter statistics. 9037 * 9038 * @param[in] dev 9039 * Pointer to Ethernet device structure. 9040 * @param[in] cnt 9041 * Index to counter to query. 9042 * @param[in] clear 9043 * Set to clear counter statistics. 9044 * @param[out] pkts 9045 * The counter hits packets number to save. 9046 * @param[out] bytes 9047 * The counter hits bytes number to save. 9048 * 9049 * @return 9050 * 0 on success, a negative errno value otherwise. 9051 */ 9052 int 9053 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 9054 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 9055 { 9056 struct rte_flow_attr attr = { .transfer = 0 }; 9057 9058 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 9059 (dev, cnt, clear, pkts, bytes, action); 9060 } 9061 9062 /** 9063 * Get information about HWS pre-configurable resources. 9064 * 9065 * @param[in] dev 9066 * Pointer to the rte_eth_dev structure. 9067 * @param[out] port_info 9068 * Pointer to port information. 9069 * @param[out] queue_info 9070 * Pointer to queue information. 9071 * @param[out] error 9072 * Pointer to error structure. 9073 * 9074 * @return 9075 * 0 on success, a negative errno value otherwise and rte_errno is set. 9076 */ 9077 static int 9078 mlx5_flow_info_get(struct rte_eth_dev *dev, 9079 struct rte_flow_port_info *port_info, 9080 struct rte_flow_queue_info *queue_info, 9081 struct rte_flow_error *error) 9082 { 9083 const struct mlx5_flow_driver_ops *fops; 9084 struct rte_flow_attr attr = {0}; 9085 9086 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9087 return rte_flow_error_set(error, ENOTSUP, 9088 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9089 NULL, 9090 "info get with incorrect steering mode"); 9091 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9092 return fops->info_get(dev, port_info, queue_info, error); 9093 } 9094 9095 /** 9096 * Configure port HWS resources. 9097 * 9098 * @param[in] dev 9099 * Pointer to the rte_eth_dev structure. 9100 * @param[in] port_attr 9101 * Port configuration attributes. 9102 * @param[in] nb_queue 9103 * Number of queue. 9104 * @param[in] queue_attr 9105 * Array that holds attributes for each flow queue. 9106 * @param[out] error 9107 * Pointer to error structure. 9108 * 9109 * @return 9110 * 0 on success, a negative errno value otherwise and rte_errno is set. 9111 */ 9112 static int 9113 mlx5_flow_port_configure(struct rte_eth_dev *dev, 9114 const struct rte_flow_port_attr *port_attr, 9115 uint16_t nb_queue, 9116 const struct rte_flow_queue_attr *queue_attr[], 9117 struct rte_flow_error *error) 9118 { 9119 const struct mlx5_flow_driver_ops *fops; 9120 struct rte_flow_attr attr = {0}; 9121 9122 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9123 return rte_flow_error_set(error, ENOTSUP, 9124 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9125 NULL, 9126 "port configure with incorrect steering mode"); 9127 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9128 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 9129 } 9130 9131 /** 9132 * Validate item template. 9133 * 9134 * @param[in] dev 9135 * Pointer to the rte_eth_dev structure. 9136 * @param[in] attr 9137 * Pointer to the item template attributes. 9138 * @param[in] items 9139 * The template item pattern. 9140 * @param[out] error 9141 * Pointer to error structure. 9142 * 9143 * @return 9144 * 0 on success, a negative errno value otherwise and rte_errno is set. 9145 */ 9146 int 9147 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 9148 const struct rte_flow_pattern_template_attr *attr, 9149 const struct rte_flow_item items[], 9150 struct rte_flow_error *error) 9151 { 9152 const struct mlx5_flow_driver_ops *fops; 9153 struct rte_flow_attr fattr = {0}; 9154 9155 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9156 rte_flow_error_set(error, ENOTSUP, 9157 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9158 "pattern validate with incorrect steering mode"); 9159 return -ENOTSUP; 9160 } 9161 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9162 return fops->pattern_validate(dev, attr, items, error); 9163 } 9164 9165 /** 9166 * Create flow item template. 9167 * 9168 * @param[in] dev 9169 * Pointer to the rte_eth_dev structure. 9170 * @param[in] attr 9171 * Pointer to the item template attributes. 9172 * @param[in] items 9173 * The template item pattern. 9174 * @param[out] error 9175 * Pointer to error structure. 9176 * 9177 * @return 9178 * 0 on success, a negative errno value otherwise and rte_errno is set. 9179 */ 9180 static struct rte_flow_pattern_template * 9181 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9182 const struct rte_flow_pattern_template_attr *attr, 9183 const struct rte_flow_item items[], 9184 struct rte_flow_error *error) 9185 { 9186 const struct mlx5_flow_driver_ops *fops; 9187 struct rte_flow_attr fattr = {0}; 9188 9189 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9190 rte_flow_error_set(error, ENOTSUP, 9191 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9192 NULL, 9193 "pattern create with incorrect steering mode"); 9194 return NULL; 9195 } 9196 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9197 return fops->pattern_template_create(dev, attr, items, error); 9198 } 9199 9200 /** 9201 * Destroy flow item template. 9202 * 9203 * @param[in] dev 9204 * Pointer to the rte_eth_dev structure. 9205 * @param[in] template 9206 * Pointer to the item template to be destroyed. 9207 * @param[out] error 9208 * Pointer to error structure. 9209 * 9210 * @return 9211 * 0 on success, a negative errno value otherwise and rte_errno is set. 9212 */ 9213 static int 9214 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9215 struct rte_flow_pattern_template *template, 9216 struct rte_flow_error *error) 9217 { 9218 const struct mlx5_flow_driver_ops *fops; 9219 struct rte_flow_attr attr = {0}; 9220 9221 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9222 return rte_flow_error_set(error, ENOTSUP, 9223 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9224 NULL, 9225 "pattern destroy with incorrect steering mode"); 9226 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9227 return fops->pattern_template_destroy(dev, template, error); 9228 } 9229 9230 /** 9231 * Validate flow actions template. 9232 * 9233 * @param[in] dev 9234 * Pointer to the rte_eth_dev structure. 9235 * @param[in] attr 9236 * Pointer to the action template attributes. 9237 * @param[in] actions 9238 * Associated actions (list terminated by the END action). 9239 * @param[in] masks 9240 * List of actions that marks which of the action's member is constant. 9241 * @param[out] error 9242 * Pointer to error structure. 9243 * 9244 * @return 9245 * 0 on success, a negative errno value otherwise and rte_errno is set. 9246 */ 9247 int 9248 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9249 const struct rte_flow_actions_template_attr *attr, 9250 const struct rte_flow_action actions[], 9251 const struct rte_flow_action masks[], 9252 struct rte_flow_error *error) 9253 { 9254 const struct mlx5_flow_driver_ops *fops; 9255 struct rte_flow_attr fattr = {0}; 9256 9257 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9258 rte_flow_error_set(error, ENOTSUP, 9259 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9260 "actions validate with incorrect steering mode"); 9261 return -ENOTSUP; 9262 } 9263 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9264 return fops->actions_validate(dev, attr, actions, masks, error); 9265 } 9266 9267 /** 9268 * Create flow item template. 9269 * 9270 * @param[in] dev 9271 * Pointer to the rte_eth_dev structure. 9272 * @param[in] attr 9273 * Pointer to the action template attributes. 9274 * @param[in] actions 9275 * Associated actions (list terminated by the END action). 9276 * @param[in] masks 9277 * List of actions that marks which of the action's member is constant. 9278 * @param[out] error 9279 * Pointer to error structure. 9280 * 9281 * @return 9282 * 0 on success, a negative errno value otherwise and rte_errno is set. 9283 */ 9284 static struct rte_flow_actions_template * 9285 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9286 const struct rte_flow_actions_template_attr *attr, 9287 const struct rte_flow_action actions[], 9288 const struct rte_flow_action masks[], 9289 struct rte_flow_error *error) 9290 { 9291 const struct mlx5_flow_driver_ops *fops; 9292 struct rte_flow_attr fattr = {0}; 9293 9294 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9295 rte_flow_error_set(error, ENOTSUP, 9296 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9297 NULL, 9298 "action create with incorrect steering mode"); 9299 return NULL; 9300 } 9301 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9302 return fops->actions_template_create(dev, attr, actions, masks, error); 9303 } 9304 9305 /** 9306 * Destroy flow action template. 9307 * 9308 * @param[in] dev 9309 * Pointer to the rte_eth_dev structure. 9310 * @param[in] template 9311 * Pointer to the action template to be destroyed. 9312 * @param[out] error 9313 * Pointer to error structure. 9314 * 9315 * @return 9316 * 0 on success, a negative errno value otherwise and rte_errno is set. 9317 */ 9318 static int 9319 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9320 struct rte_flow_actions_template *template, 9321 struct rte_flow_error *error) 9322 { 9323 const struct mlx5_flow_driver_ops *fops; 9324 struct rte_flow_attr attr = {0}; 9325 9326 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9327 return rte_flow_error_set(error, ENOTSUP, 9328 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9329 NULL, 9330 "action destroy with incorrect steering mode"); 9331 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9332 return fops->actions_template_destroy(dev, template, error); 9333 } 9334 9335 /** 9336 * Create flow table. 9337 * 9338 * @param[in] dev 9339 * Pointer to the rte_eth_dev structure. 9340 * @param[in] attr 9341 * Pointer to the table attributes. 9342 * @param[in] item_templates 9343 * Item template array to be binded to the table. 9344 * @param[in] nb_item_templates 9345 * Number of item template. 9346 * @param[in] action_templates 9347 * Action template array to be binded to the table. 9348 * @param[in] nb_action_templates 9349 * Number of action template. 9350 * @param[out] error 9351 * Pointer to error structure. 9352 * 9353 * @return 9354 * Table on success, NULL otherwise and rte_errno is set. 9355 */ 9356 static struct rte_flow_template_table * 9357 mlx5_flow_table_create(struct rte_eth_dev *dev, 9358 const struct rte_flow_template_table_attr *attr, 9359 struct rte_flow_pattern_template *item_templates[], 9360 uint8_t nb_item_templates, 9361 struct rte_flow_actions_template *action_templates[], 9362 uint8_t nb_action_templates, 9363 struct rte_flow_error *error) 9364 { 9365 const struct mlx5_flow_driver_ops *fops; 9366 struct rte_flow_attr fattr = {0}; 9367 9368 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9369 rte_flow_error_set(error, ENOTSUP, 9370 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9371 NULL, 9372 "table create with incorrect steering mode"); 9373 return NULL; 9374 } 9375 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9376 return fops->template_table_create(dev, 9377 attr, 9378 item_templates, 9379 nb_item_templates, 9380 action_templates, 9381 nb_action_templates, 9382 error); 9383 } 9384 9385 /** 9386 * PMD destroy flow table. 9387 * 9388 * @param[in] dev 9389 * Pointer to the rte_eth_dev structure. 9390 * @param[in] table 9391 * Pointer to the table to be destroyed. 9392 * @param[out] error 9393 * Pointer to error structure. 9394 * 9395 * @return 9396 * 0 on success, a negative errno value otherwise and rte_errno is set. 9397 */ 9398 static int 9399 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9400 struct rte_flow_template_table *table, 9401 struct rte_flow_error *error) 9402 { 9403 const struct mlx5_flow_driver_ops *fops; 9404 struct rte_flow_attr attr = {0}; 9405 9406 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9407 return rte_flow_error_set(error, ENOTSUP, 9408 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9409 NULL, 9410 "table destroy with incorrect steering mode"); 9411 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9412 return fops->template_table_destroy(dev, table, error); 9413 } 9414 9415 /** 9416 * Enqueue flow creation. 9417 * 9418 * @param[in] dev 9419 * Pointer to the rte_eth_dev structure. 9420 * @param[in] queue_id 9421 * The queue to create the flow. 9422 * @param[in] attr 9423 * Pointer to the flow operation attributes. 9424 * @param[in] items 9425 * Items with flow spec value. 9426 * @param[in] pattern_template_index 9427 * The item pattern flow follows from the table. 9428 * @param[in] actions 9429 * Action with flow spec value. 9430 * @param[in] action_template_index 9431 * The action pattern flow follows from the table. 9432 * @param[in] user_data 9433 * Pointer to the user_data. 9434 * @param[out] error 9435 * Pointer to error structure. 9436 * 9437 * @return 9438 * Flow pointer on success, NULL otherwise and rte_errno is set. 9439 */ 9440 static struct rte_flow * 9441 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 9442 uint32_t queue_id, 9443 const struct rte_flow_op_attr *attr, 9444 struct rte_flow_template_table *table, 9445 const struct rte_flow_item items[], 9446 uint8_t pattern_template_index, 9447 const struct rte_flow_action actions[], 9448 uint8_t action_template_index, 9449 void *user_data, 9450 struct rte_flow_error *error) 9451 { 9452 const struct mlx5_flow_driver_ops *fops; 9453 struct rte_flow_attr fattr = {0}; 9454 9455 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9456 rte_flow_error_set(error, ENOTSUP, 9457 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9458 NULL, 9459 "flow_q create with incorrect steering mode"); 9460 return NULL; 9461 } 9462 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9463 return fops->async_flow_create(dev, queue_id, attr, table, 9464 items, pattern_template_index, 9465 actions, action_template_index, 9466 user_data, error); 9467 } 9468 9469 /** 9470 * Enqueue flow creation by index. 9471 * 9472 * @param[in] dev 9473 * Pointer to the rte_eth_dev structure. 9474 * @param[in] queue_id 9475 * The queue to create the flow. 9476 * @param[in] attr 9477 * Pointer to the flow operation attributes. 9478 * @param[in] rule_index 9479 * The item pattern flow follows from the table. 9480 * @param[in] actions 9481 * Action with flow spec value. 9482 * @param[in] action_template_index 9483 * The action pattern flow follows from the table. 9484 * @param[in] user_data 9485 * Pointer to the user_data. 9486 * @param[out] error 9487 * Pointer to error structure. 9488 * 9489 * @return 9490 * Flow pointer on success, NULL otherwise and rte_errno is set. 9491 */ 9492 static struct rte_flow * 9493 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 9494 uint32_t queue_id, 9495 const struct rte_flow_op_attr *attr, 9496 struct rte_flow_template_table *table, 9497 uint32_t rule_index, 9498 const struct rte_flow_action actions[], 9499 uint8_t action_template_index, 9500 void *user_data, 9501 struct rte_flow_error *error) 9502 { 9503 const struct mlx5_flow_driver_ops *fops; 9504 struct rte_flow_attr fattr = {0}; 9505 9506 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9507 rte_flow_error_set(error, ENOTSUP, 9508 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9509 NULL, 9510 "flow_q create with incorrect steering mode"); 9511 return NULL; 9512 } 9513 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9514 return fops->async_flow_create_by_index(dev, queue_id, attr, table, 9515 rule_index, actions, action_template_index, 9516 user_data, error); 9517 } 9518 9519 /** 9520 * Enqueue flow update. 9521 * 9522 * @param[in] dev 9523 * Pointer to the rte_eth_dev structure. 9524 * @param[in] queue 9525 * The queue to destroy the flow. 9526 * @param[in] attr 9527 * Pointer to the flow operation attributes. 9528 * @param[in] flow 9529 * Pointer to the flow to be destroyed. 9530 * @param[in] actions 9531 * Action with flow spec value. 9532 * @param[in] action_template_index 9533 * The action pattern flow follows from the table. 9534 * @param[in] user_data 9535 * Pointer to the user_data. 9536 * @param[out] error 9537 * Pointer to error structure. 9538 * 9539 * @return 9540 * 0 on success, negative value otherwise and rte_errno is set. 9541 */ 9542 static int 9543 mlx5_flow_async_flow_update(struct rte_eth_dev *dev, 9544 uint32_t queue, 9545 const struct rte_flow_op_attr *attr, 9546 struct rte_flow *flow, 9547 const struct rte_flow_action actions[], 9548 uint8_t action_template_index, 9549 void *user_data, 9550 struct rte_flow_error *error) 9551 { 9552 const struct mlx5_flow_driver_ops *fops; 9553 struct rte_flow_attr fattr = {0}; 9554 9555 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9556 return rte_flow_error_set(error, ENOTSUP, 9557 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9558 NULL, 9559 "flow_q update with incorrect steering mode"); 9560 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9561 return fops->async_flow_update(dev, queue, attr, flow, 9562 actions, action_template_index, user_data, error); 9563 } 9564 9565 /** 9566 * Enqueue flow destruction. 9567 * 9568 * @param[in] dev 9569 * Pointer to the rte_eth_dev structure. 9570 * @param[in] queue 9571 * The queue to destroy the flow. 9572 * @param[in] attr 9573 * Pointer to the flow operation attributes. 9574 * @param[in] flow 9575 * Pointer to the flow to be destroyed. 9576 * @param[in] user_data 9577 * Pointer to the user_data. 9578 * @param[out] error 9579 * Pointer to error structure. 9580 * 9581 * @return 9582 * 0 on success, negative value otherwise and rte_errno is set. 9583 */ 9584 static int 9585 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 9586 uint32_t queue, 9587 const struct rte_flow_op_attr *attr, 9588 struct rte_flow *flow, 9589 void *user_data, 9590 struct rte_flow_error *error) 9591 { 9592 const struct mlx5_flow_driver_ops *fops; 9593 struct rte_flow_attr fattr = {0}; 9594 9595 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9596 return rte_flow_error_set(error, ENOTSUP, 9597 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9598 NULL, 9599 "flow_q destroy with incorrect steering mode"); 9600 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9601 return fops->async_flow_destroy(dev, queue, attr, flow, 9602 user_data, error); 9603 } 9604 9605 /** 9606 * Pull the enqueued flows. 9607 * 9608 * @param[in] dev 9609 * Pointer to the rte_eth_dev structure. 9610 * @param[in] queue 9611 * The queue to pull the result. 9612 * @param[in/out] res 9613 * Array to save the results. 9614 * @param[in] n_res 9615 * Available result with the array. 9616 * @param[out] error 9617 * Pointer to error structure. 9618 * 9619 * @return 9620 * Result number on success, negative value otherwise and rte_errno is set. 9621 */ 9622 static int 9623 mlx5_flow_pull(struct rte_eth_dev *dev, 9624 uint32_t queue, 9625 struct rte_flow_op_result res[], 9626 uint16_t n_res, 9627 struct rte_flow_error *error) 9628 { 9629 const struct mlx5_flow_driver_ops *fops; 9630 struct rte_flow_attr attr = {0}; 9631 9632 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9633 return rte_flow_error_set(error, ENOTSUP, 9634 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9635 NULL, 9636 "flow_q pull with incorrect steering mode"); 9637 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9638 return fops->pull(dev, queue, res, n_res, error); 9639 } 9640 9641 /** 9642 * Push the enqueued flows. 9643 * 9644 * @param[in] dev 9645 * Pointer to the rte_eth_dev structure. 9646 * @param[in] queue 9647 * The queue to push the flows. 9648 * @param[out] error 9649 * Pointer to error structure. 9650 * 9651 * @return 9652 * 0 on success, negative value otherwise and rte_errno is set. 9653 */ 9654 static int 9655 mlx5_flow_push(struct rte_eth_dev *dev, 9656 uint32_t queue, 9657 struct rte_flow_error *error) 9658 { 9659 const struct mlx5_flow_driver_ops *fops; 9660 struct rte_flow_attr attr = {0}; 9661 9662 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9663 return rte_flow_error_set(error, ENOTSUP, 9664 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9665 NULL, 9666 "flow_q push with incorrect steering mode"); 9667 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9668 return fops->push(dev, queue, error); 9669 } 9670 9671 /** 9672 * Create shared action. 9673 * 9674 * @param[in] dev 9675 * Pointer to the rte_eth_dev structure. 9676 * @param[in] queue 9677 * Which queue to be used.. 9678 * @param[in] attr 9679 * Operation attribute. 9680 * @param[in] conf 9681 * Indirect action configuration. 9682 * @param[in] action 9683 * rte_flow action detail. 9684 * @param[in] user_data 9685 * Pointer to the user_data. 9686 * @param[out] error 9687 * Pointer to error structure. 9688 * 9689 * @return 9690 * Action handle on success, NULL otherwise and rte_errno is set. 9691 */ 9692 static struct rte_flow_action_handle * 9693 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 9694 const struct rte_flow_op_attr *attr, 9695 const struct rte_flow_indir_action_conf *conf, 9696 const struct rte_flow_action *action, 9697 void *user_data, 9698 struct rte_flow_error *error) 9699 { 9700 const struct mlx5_flow_driver_ops *fops = 9701 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9702 9703 return fops->async_action_create(dev, queue, attr, conf, action, 9704 user_data, error); 9705 } 9706 9707 /** 9708 * Update shared action. 9709 * 9710 * @param[in] dev 9711 * Pointer to the rte_eth_dev structure. 9712 * @param[in] queue 9713 * Which queue to be used.. 9714 * @param[in] attr 9715 * Operation attribute. 9716 * @param[in] handle 9717 * Action handle to be updated. 9718 * @param[in] update 9719 * Update value. 9720 * @param[in] user_data 9721 * Pointer to the user_data. 9722 * @param[out] error 9723 * Pointer to error structure. 9724 * 9725 * @return 9726 * 0 on success, negative value otherwise and rte_errno is set. 9727 */ 9728 static int 9729 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 9730 const struct rte_flow_op_attr *attr, 9731 struct rte_flow_action_handle *handle, 9732 const void *update, 9733 void *user_data, 9734 struct rte_flow_error *error) 9735 { 9736 const struct mlx5_flow_driver_ops *fops = 9737 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9738 9739 return fops->async_action_update(dev, queue, attr, handle, 9740 update, user_data, error); 9741 } 9742 9743 static int 9744 mlx5_flow_async_action_handle_query_update 9745 (struct rte_eth_dev *dev, uint32_t queue_id, 9746 const struct rte_flow_op_attr *op_attr, 9747 struct rte_flow_action_handle *action_handle, 9748 const void *update, void *query, 9749 enum rte_flow_query_update_mode qu_mode, 9750 void *user_data, struct rte_flow_error *error) 9751 { 9752 const struct mlx5_flow_driver_ops *fops = 9753 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9754 9755 if (!fops || !fops->async_action_query_update) 9756 return rte_flow_error_set(error, ENOTSUP, 9757 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 9758 "async query_update not supported"); 9759 return fops->async_action_query_update 9760 (dev, queue_id, op_attr, action_handle, 9761 update, query, qu_mode, user_data, error); 9762 } 9763 9764 /** 9765 * Query shared action. 9766 * 9767 * @param[in] dev 9768 * Pointer to the rte_eth_dev structure. 9769 * @param[in] queue 9770 * Which queue to be used.. 9771 * @param[in] attr 9772 * Operation attribute. 9773 * @param[in] handle 9774 * Action handle to be updated. 9775 * @param[in] data 9776 * Pointer query result data. 9777 * @param[in] user_data 9778 * Pointer to the user_data. 9779 * @param[out] error 9780 * Pointer to error structure. 9781 * 9782 * @return 9783 * 0 on success, negative value otherwise and rte_errno is set. 9784 */ 9785 static int 9786 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9787 const struct rte_flow_op_attr *attr, 9788 const struct rte_flow_action_handle *handle, 9789 void *data, 9790 void *user_data, 9791 struct rte_flow_error *error) 9792 { 9793 const struct mlx5_flow_driver_ops *fops = 9794 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9795 9796 return fops->async_action_query(dev, queue, attr, handle, 9797 data, user_data, error); 9798 } 9799 9800 /** 9801 * Destroy shared action. 9802 * 9803 * @param[in] dev 9804 * Pointer to the rte_eth_dev structure. 9805 * @param[in] queue 9806 * Which queue to be used.. 9807 * @param[in] attr 9808 * Operation attribute. 9809 * @param[in] handle 9810 * Action handle to be destroyed. 9811 * @param[in] user_data 9812 * Pointer to the user_data. 9813 * @param[out] error 9814 * Pointer to error structure. 9815 * 9816 * @return 9817 * 0 on success, negative value otherwise and rte_errno is set. 9818 */ 9819 static int 9820 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9821 const struct rte_flow_op_attr *attr, 9822 struct rte_flow_action_handle *handle, 9823 void *user_data, 9824 struct rte_flow_error *error) 9825 { 9826 const struct mlx5_flow_driver_ops *fops = 9827 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9828 9829 return fops->async_action_destroy(dev, queue, attr, handle, 9830 user_data, error); 9831 } 9832 9833 /** 9834 * Allocate a new memory for the counter values wrapped by all the needed 9835 * management. 9836 * 9837 * @param[in] sh 9838 * Pointer to mlx5_dev_ctx_shared object. 9839 * 9840 * @return 9841 * 0 on success, a negative errno value otherwise. 9842 */ 9843 static int 9844 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9845 { 9846 struct mlx5_counter_stats_mem_mng *mem_mng; 9847 volatile struct flow_counter_stats *raw_data; 9848 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9849 int size = (sizeof(struct flow_counter_stats) * 9850 MLX5_COUNTERS_PER_POOL + 9851 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9852 sizeof(struct mlx5_counter_stats_mem_mng); 9853 size_t pgsize = rte_mem_page_size(); 9854 uint8_t *mem; 9855 int ret; 9856 int i; 9857 9858 if (pgsize == (size_t)-1) { 9859 DRV_LOG(ERR, "Failed to get mem page size"); 9860 rte_errno = ENOMEM; 9861 return -ENOMEM; 9862 } 9863 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9864 if (!mem) { 9865 rte_errno = ENOMEM; 9866 return -ENOMEM; 9867 } 9868 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9869 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9870 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9871 sh->cdev->pdn, mem, size, 9872 &mem_mng->wm); 9873 if (ret) { 9874 rte_errno = errno; 9875 mlx5_free(mem); 9876 return -rte_errno; 9877 } 9878 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9879 raw_data = (volatile struct flow_counter_stats *)mem; 9880 for (i = 0; i < raws_n; ++i) { 9881 mem_mng->raws[i].mem_mng = mem_mng; 9882 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9883 } 9884 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9885 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9886 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9887 next); 9888 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9889 sh->sws_cmng.mem_mng = mem_mng; 9890 return 0; 9891 } 9892 9893 /** 9894 * Set the statistic memory to the new counter pool. 9895 * 9896 * @param[in] sh 9897 * Pointer to mlx5_dev_ctx_shared object. 9898 * @param[in] pool 9899 * Pointer to the pool to set the statistic memory. 9900 * 9901 * @return 9902 * 0 on success, a negative errno value otherwise. 9903 */ 9904 static int 9905 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9906 struct mlx5_flow_counter_pool *pool) 9907 { 9908 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9909 /* Resize statistic memory once used out. */ 9910 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9911 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9912 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9913 return -1; 9914 } 9915 rte_spinlock_lock(&pool->sl); 9916 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9917 rte_spinlock_unlock(&pool->sl); 9918 pool->raw_hw = NULL; 9919 return 0; 9920 } 9921 9922 #define MLX5_POOL_QUERY_FREQ_US 1000000 9923 9924 /** 9925 * Set the periodic procedure for triggering asynchronous batch queries for all 9926 * the counter pools. 9927 * 9928 * @param[in] sh 9929 * Pointer to mlx5_dev_ctx_shared object. 9930 */ 9931 void 9932 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9933 { 9934 uint32_t pools_n, us; 9935 9936 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9937 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9938 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9939 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9940 sh->sws_cmng.query_thread_on = 0; 9941 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9942 } else { 9943 sh->sws_cmng.query_thread_on = 1; 9944 } 9945 } 9946 9947 /** 9948 * The periodic procedure for triggering asynchronous batch queries for all the 9949 * counter pools. This function is probably called by the host thread. 9950 * 9951 * @param[in] arg 9952 * The parameter for the alarm process. 9953 */ 9954 void 9955 mlx5_flow_query_alarm(void *arg) 9956 { 9957 struct mlx5_dev_ctx_shared *sh = arg; 9958 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9959 uint16_t pool_index = cmng->pool_index; 9960 struct mlx5_flow_counter_pool *pool; 9961 uint16_t n_valid; 9962 int ret; 9963 9964 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9965 goto set_alarm; 9966 rte_spinlock_lock(&cmng->pool_update_sl); 9967 pool = cmng->pools[pool_index]; 9968 n_valid = cmng->n_valid; 9969 rte_spinlock_unlock(&cmng->pool_update_sl); 9970 /* Set the statistic memory to the new created pool. */ 9971 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9972 goto set_alarm; 9973 if (pool->raw_hw) 9974 /* There is a pool query in progress. */ 9975 goto set_alarm; 9976 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9977 if (!pool->raw_hw) 9978 /* No free counter statistics raw memory. */ 9979 goto set_alarm; 9980 /* 9981 * Identify the counters released between query trigger and query 9982 * handle more efficiently. The counter released in this gap period 9983 * should wait for a new round of query as the new arrived packets 9984 * will not be taken into account. 9985 */ 9986 pool->query_gen++; 9987 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9988 MLX5_COUNTERS_PER_POOL, 9989 NULL, NULL, 9990 pool->raw_hw->mem_mng->wm.lkey, 9991 (void *)(uintptr_t) 9992 pool->raw_hw->data, 9993 sh->devx_comp, 9994 (uint64_t)(uintptr_t)pool); 9995 if (ret) { 9996 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9997 " %d", pool->min_dcs->id); 9998 pool->raw_hw = NULL; 9999 goto set_alarm; 10000 } 10001 LIST_REMOVE(pool->raw_hw, next); 10002 cmng->pending_queries++; 10003 pool_index++; 10004 if (pool_index >= n_valid) 10005 pool_index = 0; 10006 set_alarm: 10007 cmng->pool_index = pool_index; 10008 mlx5_set_query_alarm(sh); 10009 } 10010 10011 /** 10012 * Check and callback event for new aged flow in the counter pool 10013 * 10014 * @param[in] sh 10015 * Pointer to mlx5_dev_ctx_shared object. 10016 * @param[in] pool 10017 * Pointer to Current counter pool. 10018 */ 10019 static void 10020 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 10021 struct mlx5_flow_counter_pool *pool) 10022 { 10023 struct mlx5_priv *priv; 10024 struct mlx5_flow_counter *cnt; 10025 struct mlx5_age_info *age_info; 10026 struct mlx5_age_param *age_param; 10027 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 10028 struct mlx5_counter_stats_raw *prev = pool->raw; 10029 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 10030 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 10031 uint16_t expected = AGE_CANDIDATE; 10032 uint32_t i; 10033 10034 pool->time_of_last_age_check = curr_time; 10035 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 10036 cnt = MLX5_POOL_GET_CNT(pool, i); 10037 age_param = MLX5_CNT_TO_AGE(cnt); 10038 if (__atomic_load_n(&age_param->state, 10039 __ATOMIC_RELAXED) != AGE_CANDIDATE) 10040 continue; 10041 if (cur->data[i].hits != prev->data[i].hits) { 10042 __atomic_store_n(&age_param->sec_since_last_hit, 0, 10043 __ATOMIC_RELAXED); 10044 continue; 10045 } 10046 if (__atomic_fetch_add(&age_param->sec_since_last_hit, 10047 time_delta, 10048 __ATOMIC_RELAXED) + time_delta <= age_param->timeout) 10049 continue; 10050 /** 10051 * Hold the lock first, or if between the 10052 * state AGE_TMOUT and tailq operation the 10053 * release happened, the release procedure 10054 * may delete a non-existent tailq node. 10055 */ 10056 priv = rte_eth_devices[age_param->port_id].data->dev_private; 10057 age_info = GET_PORT_AGE_INFO(priv); 10058 rte_spinlock_lock(&age_info->aged_sl); 10059 if (__atomic_compare_exchange_n(&age_param->state, &expected, 10060 AGE_TMOUT, false, 10061 __ATOMIC_RELAXED, 10062 __ATOMIC_RELAXED)) { 10063 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 10064 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 10065 } 10066 rte_spinlock_unlock(&age_info->aged_sl); 10067 } 10068 mlx5_age_event_prepare(sh); 10069 } 10070 10071 /** 10072 * Handler for the HW respond about ready values from an asynchronous batch 10073 * query. This function is probably called by the host thread. 10074 * 10075 * @param[in] sh 10076 * The pointer to the shared device context. 10077 * @param[in] async_id 10078 * The Devx async ID. 10079 * @param[in] status 10080 * The status of the completion. 10081 */ 10082 void 10083 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 10084 uint64_t async_id, int status) 10085 { 10086 struct mlx5_flow_counter_pool *pool = 10087 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 10088 struct mlx5_counter_stats_raw *raw_to_free; 10089 uint8_t query_gen = pool->query_gen ^ 1; 10090 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 10091 enum mlx5_counter_type cnt_type = 10092 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 10093 MLX5_COUNTER_TYPE_ORIGIN; 10094 10095 if (unlikely(status)) { 10096 raw_to_free = pool->raw_hw; 10097 } else { 10098 raw_to_free = pool->raw; 10099 if (pool->is_aged) 10100 mlx5_flow_aging_check(sh, pool); 10101 rte_spinlock_lock(&pool->sl); 10102 pool->raw = pool->raw_hw; 10103 rte_spinlock_unlock(&pool->sl); 10104 /* Be sure the new raw counters data is updated in memory. */ 10105 rte_io_wmb(); 10106 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 10107 rte_spinlock_lock(&cmng->csl[cnt_type]); 10108 TAILQ_CONCAT(&cmng->counters[cnt_type], 10109 &pool->counters[query_gen], next); 10110 rte_spinlock_unlock(&cmng->csl[cnt_type]); 10111 } 10112 } 10113 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 10114 pool->raw_hw = NULL; 10115 sh->sws_cmng.pending_queries--; 10116 } 10117 10118 static int 10119 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 10120 const struct flow_grp_info *grp_info, 10121 struct rte_flow_error *error) 10122 { 10123 if (grp_info->transfer && grp_info->external && 10124 grp_info->fdb_def_rule) { 10125 if (group == UINT32_MAX) 10126 return rte_flow_error_set 10127 (error, EINVAL, 10128 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10129 NULL, 10130 "group index not supported"); 10131 *table = group + 1; 10132 } else { 10133 *table = group; 10134 } 10135 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 10136 return 0; 10137 } 10138 10139 /** 10140 * Translate the rte_flow group index to HW table value. 10141 * 10142 * If tunnel offload is disabled, all group ids converted to flow table 10143 * id using the standard method. 10144 * If tunnel offload is enabled, group id can be converted using the 10145 * standard or tunnel conversion method. Group conversion method 10146 * selection depends on flags in `grp_info` parameter: 10147 * - Internal (grp_info.external == 0) groups conversion uses the 10148 * standard method. 10149 * - Group ids in JUMP action converted with the tunnel conversion. 10150 * - Group id in rule attribute conversion depends on a rule type and 10151 * group id value: 10152 * ** non zero group attributes converted with the tunnel method 10153 * ** zero group attribute in non-tunnel rule is converted using the 10154 * standard method - there's only one root table 10155 * ** zero group attribute in steer tunnel rule is converted with the 10156 * standard method - single root table 10157 * ** zero group attribute in match tunnel rule is a special OvS 10158 * case: that value is used for portability reasons. That group 10159 * id is converted with the tunnel conversion method. 10160 * 10161 * @param[in] dev 10162 * Port device 10163 * @param[in] tunnel 10164 * PMD tunnel offload object 10165 * @param[in] group 10166 * rte_flow group index value. 10167 * @param[out] table 10168 * HW table value. 10169 * @param[in] grp_info 10170 * flags used for conversion 10171 * @param[out] error 10172 * Pointer to error structure. 10173 * 10174 * @return 10175 * 0 on success, a negative errno value otherwise and rte_errno is set. 10176 */ 10177 int 10178 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 10179 const struct mlx5_flow_tunnel *tunnel, 10180 uint32_t group, uint32_t *table, 10181 const struct flow_grp_info *grp_info, 10182 struct rte_flow_error *error) 10183 { 10184 int ret; 10185 bool standard_translation; 10186 10187 if (!grp_info->skip_scale && grp_info->external && 10188 group < MLX5_MAX_TABLES_EXTERNAL) 10189 group *= MLX5_FLOW_TABLE_FACTOR; 10190 if (is_tunnel_offload_active(dev)) { 10191 standard_translation = !grp_info->external || 10192 grp_info->std_tbl_fix; 10193 } else { 10194 standard_translation = true; 10195 } 10196 DRV_LOG(DEBUG, 10197 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 10198 dev->data->port_id, group, grp_info->transfer, 10199 grp_info->external, grp_info->fdb_def_rule, 10200 standard_translation ? "STANDARD" : "TUNNEL"); 10201 if (standard_translation) 10202 ret = flow_group_to_table(dev->data->port_id, group, table, 10203 grp_info, error); 10204 else 10205 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 10206 table, error); 10207 10208 return ret; 10209 } 10210 10211 /** 10212 * Discover availability of metadata reg_c's. 10213 * 10214 * Iteratively use test flows to check availability. 10215 * 10216 * @param[in] dev 10217 * Pointer to the Ethernet device structure. 10218 * 10219 * @return 10220 * 0 on success, a negative errno value otherwise and rte_errno is set. 10221 */ 10222 int 10223 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 10224 { 10225 struct mlx5_priv *priv = dev->data->dev_private; 10226 enum modify_reg idx; 10227 int n = 0; 10228 10229 /* reg_c[0] and reg_c[1] are reserved. */ 10230 priv->sh->flow_mreg_c[n++] = REG_C_0; 10231 priv->sh->flow_mreg_c[n++] = REG_C_1; 10232 /* Discover availability of other reg_c's. */ 10233 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 10234 struct rte_flow_attr attr = { 10235 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 10236 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 10237 .ingress = 1, 10238 }; 10239 struct rte_flow_item items[] = { 10240 [0] = { 10241 .type = RTE_FLOW_ITEM_TYPE_END, 10242 }, 10243 }; 10244 struct rte_flow_action actions[] = { 10245 [0] = { 10246 .type = (enum rte_flow_action_type) 10247 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 10248 .conf = &(struct mlx5_flow_action_copy_mreg){ 10249 .src = REG_C_1, 10250 .dst = idx, 10251 }, 10252 }, 10253 [1] = { 10254 .type = RTE_FLOW_ACTION_TYPE_JUMP, 10255 .conf = &(struct rte_flow_action_jump){ 10256 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 10257 }, 10258 }, 10259 [2] = { 10260 .type = RTE_FLOW_ACTION_TYPE_END, 10261 }, 10262 }; 10263 uint32_t flow_idx; 10264 struct rte_flow *flow; 10265 struct rte_flow_error error; 10266 10267 if (!priv->sh->config.dv_flow_en) 10268 break; 10269 /* Create internal flow, validation skips copy action. */ 10270 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 10271 items, actions, false, &error); 10272 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10273 flow_idx); 10274 if (!flow) 10275 continue; 10276 priv->sh->flow_mreg_c[n++] = idx; 10277 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 10278 } 10279 for (; n < MLX5_MREG_C_NUM; ++n) 10280 priv->sh->flow_mreg_c[n] = REG_NON; 10281 priv->sh->metadata_regc_check_flag = 1; 10282 return 0; 10283 } 10284 10285 int 10286 save_dump_file(const uint8_t *data, uint32_t size, 10287 uint32_t type, uint64_t id, void *arg, FILE *file) 10288 { 10289 char line[BUF_SIZE]; 10290 uint32_t out = 0; 10291 uint32_t k; 10292 uint32_t actions_num; 10293 struct rte_flow_query_count *count; 10294 10295 memset(line, 0, BUF_SIZE); 10296 switch (type) { 10297 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 10298 actions_num = *(uint32_t *)(arg); 10299 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 10300 type, id, actions_num); 10301 break; 10302 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 10303 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 10304 type, id); 10305 break; 10306 case DR_DUMP_REC_TYPE_PMD_COUNTER: 10307 count = (struct rte_flow_query_count *)arg; 10308 fprintf(file, 10309 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 10310 type, id, count->hits, count->bytes); 10311 return 0; 10312 default: 10313 return -1; 10314 } 10315 10316 for (k = 0; k < size; k++) { 10317 /* Make sure we do not overrun the line buffer length. */ 10318 if (out >= BUF_SIZE - 4) { 10319 line[out] = '\0'; 10320 break; 10321 } 10322 out += snprintf(line + out, BUF_SIZE - out, "%02x", 10323 (data[k]) & 0xff); 10324 } 10325 fprintf(file, "%s\n", line); 10326 return 0; 10327 } 10328 10329 int 10330 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 10331 struct rte_flow_query_count *count, struct rte_flow_error *error) 10332 { 10333 struct rte_flow_action action[2]; 10334 enum mlx5_flow_drv_type ftype; 10335 const struct mlx5_flow_driver_ops *fops; 10336 10337 if (!flow) { 10338 return rte_flow_error_set(error, ENOENT, 10339 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10340 NULL, 10341 "invalid flow handle"); 10342 } 10343 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 10344 action[1].type = RTE_FLOW_ACTION_TYPE_END; 10345 if (flow->counter) { 10346 memset(count, 0, sizeof(struct rte_flow_query_count)); 10347 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 10348 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 10349 ftype < MLX5_FLOW_TYPE_MAX); 10350 fops = flow_get_drv_ops(ftype); 10351 return fops->query(dev, flow, action, count, error); 10352 } 10353 return -1; 10354 } 10355 10356 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10357 /** 10358 * Dump flow ipool data to file 10359 * 10360 * @param[in] dev 10361 * The pointer to Ethernet device. 10362 * @param[in] file 10363 * A pointer to a file for output. 10364 * @param[out] error 10365 * Perform verbose error reporting if not NULL. PMDs initialize this 10366 * structure in case of error only. 10367 * @return 10368 * 0 on success, a negative value otherwise. 10369 */ 10370 int 10371 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 10372 struct rte_flow *flow, FILE *file, 10373 struct rte_flow_error *error) 10374 { 10375 struct mlx5_priv *priv = dev->data->dev_private; 10376 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10377 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10378 uint32_t handle_idx; 10379 struct mlx5_flow_handle *dh; 10380 struct rte_flow_query_count count; 10381 uint32_t actions_num; 10382 const uint8_t *data; 10383 size_t size; 10384 uint64_t id; 10385 uint32_t type; 10386 void *action = NULL; 10387 10388 if (!flow) { 10389 return rte_flow_error_set(error, ENOENT, 10390 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10391 NULL, 10392 "invalid flow handle"); 10393 } 10394 handle_idx = flow->dev_handles; 10395 /* query counter */ 10396 if (flow->counter && 10397 (!mlx5_counter_query(dev, flow->counter, false, 10398 &count.hits, &count.bytes, &action)) && action) { 10399 id = (uint64_t)(uintptr_t)action; 10400 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10401 save_dump_file(NULL, 0, type, 10402 id, (void *)&count, file); 10403 } 10404 10405 while (handle_idx) { 10406 dh = mlx5_ipool_get(priv->sh->ipool 10407 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 10408 if (!dh) 10409 continue; 10410 handle_idx = dh->next.next; 10411 10412 /* Get modify_hdr and encap_decap buf from ipools. */ 10413 encap_decap = NULL; 10414 modify_hdr = dh->dvh.modify_hdr; 10415 10416 if (dh->dvh.rix_encap_decap) { 10417 encap_decap = mlx5_ipool_get(priv->sh->ipool 10418 [MLX5_IPOOL_DECAP_ENCAP], 10419 dh->dvh.rix_encap_decap); 10420 } 10421 if (modify_hdr) { 10422 data = (const uint8_t *)modify_hdr->actions; 10423 size = (size_t)(modify_hdr->actions_num) * 8; 10424 id = (uint64_t)(uintptr_t)modify_hdr->action; 10425 actions_num = modify_hdr->actions_num; 10426 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10427 save_dump_file(data, size, type, id, 10428 (void *)(&actions_num), file); 10429 } 10430 if (encap_decap) { 10431 data = encap_decap->buf; 10432 size = encap_decap->size; 10433 id = (uint64_t)(uintptr_t)encap_decap->action; 10434 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10435 save_dump_file(data, size, type, 10436 id, NULL, file); 10437 } 10438 } 10439 return 0; 10440 } 10441 10442 /** 10443 * Dump all flow's encap_decap/modify_hdr/counter data to file 10444 * 10445 * @param[in] dev 10446 * The pointer to Ethernet device. 10447 * @param[in] file 10448 * A pointer to a file for output. 10449 * @param[out] error 10450 * Perform verbose error reporting if not NULL. PMDs initialize this 10451 * structure in case of error only. 10452 * @return 10453 * 0 on success, a negative value otherwise. 10454 */ 10455 static int 10456 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 10457 FILE *file, struct rte_flow_error *error __rte_unused) 10458 { 10459 struct mlx5_priv *priv = dev->data->dev_private; 10460 struct mlx5_dev_ctx_shared *sh = priv->sh; 10461 struct mlx5_hlist *h; 10462 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10463 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10464 struct rte_flow_query_count count; 10465 uint32_t actions_num; 10466 const uint8_t *data; 10467 size_t size; 10468 uint64_t id; 10469 uint32_t type; 10470 uint32_t i; 10471 uint32_t j; 10472 struct mlx5_list_inconst *l_inconst; 10473 struct mlx5_list_entry *e; 10474 int lcore_index; 10475 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 10476 uint32_t max; 10477 void *action; 10478 10479 /* encap_decap hlist is lcore_share, get global core cache. */ 10480 i = MLX5_LIST_GLOBAL; 10481 h = sh->encaps_decaps; 10482 if (h) { 10483 for (j = 0; j <= h->mask; j++) { 10484 l_inconst = &h->buckets[j].l; 10485 if (!l_inconst || !l_inconst->cache[i]) 10486 continue; 10487 10488 e = LIST_FIRST(&l_inconst->cache[i]->h); 10489 while (e) { 10490 encap_decap = 10491 (struct mlx5_flow_dv_encap_decap_resource *)e; 10492 data = encap_decap->buf; 10493 size = encap_decap->size; 10494 id = (uint64_t)(uintptr_t)encap_decap->action; 10495 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10496 save_dump_file(data, size, type, 10497 id, NULL, file); 10498 e = LIST_NEXT(e, next); 10499 } 10500 } 10501 } 10502 10503 /* get modify_hdr */ 10504 h = sh->modify_cmds; 10505 if (h) { 10506 lcore_index = rte_lcore_index(rte_lcore_id()); 10507 if (unlikely(lcore_index == -1)) { 10508 lcore_index = MLX5_LIST_NLCORE; 10509 rte_spinlock_lock(&h->l_const.lcore_lock); 10510 } 10511 i = lcore_index; 10512 10513 if (lcore_index == MLX5_LIST_NLCORE) { 10514 for (i = 0; i <= (uint32_t)lcore_index; i++) { 10515 for (j = 0; j <= h->mask; j++) { 10516 l_inconst = &h->buckets[j].l; 10517 if (!l_inconst || !l_inconst->cache[i]) 10518 continue; 10519 10520 e = LIST_FIRST(&l_inconst->cache[i]->h); 10521 while (e) { 10522 modify_hdr = 10523 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10524 data = (const uint8_t *)modify_hdr->actions; 10525 size = (size_t)(modify_hdr->actions_num) * 8; 10526 actions_num = modify_hdr->actions_num; 10527 id = (uint64_t)(uintptr_t)modify_hdr->action; 10528 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10529 save_dump_file(data, size, type, id, 10530 (void *)(&actions_num), file); 10531 e = LIST_NEXT(e, next); 10532 } 10533 } 10534 } 10535 } else { 10536 for (j = 0; j <= h->mask; j++) { 10537 l_inconst = &h->buckets[j].l; 10538 if (!l_inconst || !l_inconst->cache[i]) 10539 continue; 10540 10541 e = LIST_FIRST(&l_inconst->cache[i]->h); 10542 while (e) { 10543 modify_hdr = 10544 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10545 data = (const uint8_t *)modify_hdr->actions; 10546 size = (size_t)(modify_hdr->actions_num) * 8; 10547 actions_num = modify_hdr->actions_num; 10548 id = (uint64_t)(uintptr_t)modify_hdr->action; 10549 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10550 save_dump_file(data, size, type, id, 10551 (void *)(&actions_num), file); 10552 e = LIST_NEXT(e, next); 10553 } 10554 } 10555 } 10556 10557 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10558 rte_spinlock_unlock(&h->l_const.lcore_lock); 10559 } 10560 10561 /* get counter */ 10562 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10563 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10564 for (j = 1; j <= max; j++) { 10565 action = NULL; 10566 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10567 &count.bytes, &action)) && action) { 10568 id = (uint64_t)(uintptr_t)action; 10569 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10570 save_dump_file(NULL, 0, type, 10571 id, (void *)&count, file); 10572 } 10573 } 10574 return 0; 10575 } 10576 #endif 10577 10578 /** 10579 * Dump flow raw hw data to file 10580 * 10581 * @param[in] dev 10582 * The pointer to Ethernet device. 10583 * @param[in] file 10584 * A pointer to a file for output. 10585 * @param[out] error 10586 * Perform verbose error reporting if not NULL. PMDs initialize this 10587 * structure in case of error only. 10588 * @return 10589 * 0 on success, a negative value otherwise. 10590 */ 10591 int 10592 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10593 FILE *file, 10594 struct rte_flow_error *error __rte_unused) 10595 { 10596 struct mlx5_priv *priv = dev->data->dev_private; 10597 struct mlx5_dev_ctx_shared *sh = priv->sh; 10598 uint32_t handle_idx; 10599 int ret; 10600 struct mlx5_flow_handle *dh; 10601 struct rte_flow *flow; 10602 10603 if (!sh->config.dv_flow_en) { 10604 if (fputs("device dv flow disabled\n", file) <= 0) 10605 return -errno; 10606 return -ENOTSUP; 10607 } 10608 10609 /* dump all */ 10610 if (!flow_idx) { 10611 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10612 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10613 return -EINVAL; 10614 10615 if (sh->config.dv_flow_en == 2) 10616 return mlx5dr_debug_dump(priv->dr_ctx, file); 10617 #endif 10618 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10619 sh->rx_domain, 10620 sh->tx_domain, file); 10621 } 10622 /* dump one */ 10623 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10624 (uintptr_t)(void *)flow_idx); 10625 if (!flow) 10626 return -EINVAL; 10627 10628 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10629 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10630 #endif 10631 handle_idx = flow->dev_handles; 10632 while (handle_idx) { 10633 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10634 handle_idx); 10635 if (!dh) 10636 return -ENOENT; 10637 if (dh->drv_flow) { 10638 if (sh->config.dv_flow_en == 2) 10639 return -ENOTSUP; 10640 10641 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10642 file); 10643 if (ret) 10644 return -ENOENT; 10645 } 10646 handle_idx = dh->next.next; 10647 } 10648 return 0; 10649 } 10650 10651 /** 10652 * Get aged-out flows. 10653 * 10654 * @param[in] dev 10655 * Pointer to the Ethernet device structure. 10656 * @param[in] context 10657 * The address of an array of pointers to the aged-out flows contexts. 10658 * @param[in] nb_countexts 10659 * The length of context array pointers. 10660 * @param[out] error 10661 * Perform verbose error reporting if not NULL. Initialized in case of 10662 * error only. 10663 * 10664 * @return 10665 * how many contexts get in success, otherwise negative errno value. 10666 * if nb_contexts is 0, return the amount of all aged contexts. 10667 * if nb_contexts is not 0 , return the amount of aged flows reported 10668 * in the context array. 10669 */ 10670 int 10671 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10672 uint32_t nb_contexts, struct rte_flow_error *error) 10673 { 10674 struct rte_flow_attr attr = { .transfer = 0 }; 10675 10676 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10677 (dev, contexts, nb_contexts, error); 10678 } 10679 10680 /** 10681 * Get aged-out flows per HWS queue. 10682 * 10683 * @param[in] dev 10684 * Pointer to the Ethernet device structure. 10685 * @param[in] queue_id 10686 * Flow queue to query. 10687 * @param[in] context 10688 * The address of an array of pointers to the aged-out flows contexts. 10689 * @param[in] nb_countexts 10690 * The length of context array pointers. 10691 * @param[out] error 10692 * Perform verbose error reporting if not NULL. Initialized in case of 10693 * error only. 10694 * 10695 * @return 10696 * how many contexts get in success, otherwise negative errno value. 10697 * if nb_contexts is 0, return the amount of all aged contexts. 10698 * if nb_contexts is not 0 , return the amount of aged flows reported 10699 * in the context array. 10700 */ 10701 int 10702 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10703 void **contexts, uint32_t nb_contexts, 10704 struct rte_flow_error *error) 10705 { 10706 const struct mlx5_flow_driver_ops *fops; 10707 struct rte_flow_attr attr = { 0 }; 10708 10709 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10710 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10711 return fops->get_q_aged_flows(dev, queue_id, contexts, 10712 nb_contexts, error); 10713 } 10714 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10715 dev->data->port_id, queue_id); 10716 return rte_flow_error_set(error, ENOTSUP, 10717 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10718 "get Q aged flows with incorrect steering mode"); 10719 } 10720 10721 /* Wrapper for driver action_validate op callback */ 10722 static int 10723 flow_drv_action_validate(struct rte_eth_dev *dev, 10724 const struct rte_flow_indir_action_conf *conf, 10725 const struct rte_flow_action *action, 10726 const struct mlx5_flow_driver_ops *fops, 10727 struct rte_flow_error *error) 10728 { 10729 static const char err_msg[] = "indirect action validation unsupported"; 10730 10731 if (!fops->action_validate) { 10732 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10733 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10734 NULL, err_msg); 10735 return -rte_errno; 10736 } 10737 return fops->action_validate(dev, conf, action, error); 10738 } 10739 10740 /** 10741 * Destroys the shared action by handle. 10742 * 10743 * @param dev 10744 * Pointer to Ethernet device structure. 10745 * @param[in] handle 10746 * Handle for the indirect action object to be destroyed. 10747 * @param[out] error 10748 * Perform verbose error reporting if not NULL. PMDs initialize this 10749 * structure in case of error only. 10750 * 10751 * @return 10752 * 0 on success, a negative errno value otherwise and rte_errno is set. 10753 * 10754 * @note: wrapper for driver action_create op callback. 10755 */ 10756 static int 10757 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10758 struct rte_flow_action_handle *handle, 10759 struct rte_flow_error *error) 10760 { 10761 static const char err_msg[] = "indirect action destruction unsupported"; 10762 struct rte_flow_attr attr = { .transfer = 0 }; 10763 const struct mlx5_flow_driver_ops *fops = 10764 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10765 10766 if (!fops->action_destroy) { 10767 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10768 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10769 NULL, err_msg); 10770 return -rte_errno; 10771 } 10772 return fops->action_destroy(dev, handle, error); 10773 } 10774 10775 /* Wrapper for driver action_destroy op callback */ 10776 static int 10777 flow_drv_action_update(struct rte_eth_dev *dev, 10778 struct rte_flow_action_handle *handle, 10779 const void *update, 10780 const struct mlx5_flow_driver_ops *fops, 10781 struct rte_flow_error *error) 10782 { 10783 static const char err_msg[] = "indirect action update unsupported"; 10784 10785 if (!fops->action_update) { 10786 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10787 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10788 NULL, err_msg); 10789 return -rte_errno; 10790 } 10791 return fops->action_update(dev, handle, update, error); 10792 } 10793 10794 /* Wrapper for driver action_destroy op callback */ 10795 static int 10796 flow_drv_action_query(struct rte_eth_dev *dev, 10797 const struct rte_flow_action_handle *handle, 10798 void *data, 10799 const struct mlx5_flow_driver_ops *fops, 10800 struct rte_flow_error *error) 10801 { 10802 static const char err_msg[] = "indirect action query unsupported"; 10803 10804 if (!fops->action_query) { 10805 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10806 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10807 NULL, err_msg); 10808 return -rte_errno; 10809 } 10810 return fops->action_query(dev, handle, data, error); 10811 } 10812 10813 /** 10814 * Create indirect action for reuse in multiple flow rules. 10815 * 10816 * @param dev 10817 * Pointer to Ethernet device structure. 10818 * @param conf 10819 * Pointer to indirect action object configuration. 10820 * @param[in] action 10821 * Action configuration for indirect action object creation. 10822 * @param[out] error 10823 * Perform verbose error reporting if not NULL. PMDs initialize this 10824 * structure in case of error only. 10825 * @return 10826 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10827 */ 10828 static struct rte_flow_action_handle * 10829 mlx5_action_handle_create(struct rte_eth_dev *dev, 10830 const struct rte_flow_indir_action_conf *conf, 10831 const struct rte_flow_action *action, 10832 struct rte_flow_error *error) 10833 { 10834 static const char err_msg[] = "indirect action creation unsupported"; 10835 struct rte_flow_attr attr = { .transfer = 0 }; 10836 const struct mlx5_flow_driver_ops *fops = 10837 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10838 10839 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10840 return NULL; 10841 if (!fops->action_create) { 10842 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10843 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10844 NULL, err_msg); 10845 return NULL; 10846 } 10847 return fops->action_create(dev, conf, action, error); 10848 } 10849 10850 /** 10851 * Updates inplace the indirect action configuration pointed by *handle* 10852 * with the configuration provided as *update* argument. 10853 * The update of the indirect action configuration effects all flow rules 10854 * reusing the action via handle. 10855 * 10856 * @param dev 10857 * Pointer to Ethernet device structure. 10858 * @param[in] handle 10859 * Handle for the indirect action to be updated. 10860 * @param[in] update 10861 * Action specification used to modify the action pointed by handle. 10862 * *update* could be of same type with the action pointed by the *handle* 10863 * handle argument, or some other structures like a wrapper, depending on 10864 * the indirect action type. 10865 * @param[out] error 10866 * Perform verbose error reporting if not NULL. PMDs initialize this 10867 * structure in case of error only. 10868 * 10869 * @return 10870 * 0 on success, a negative errno value otherwise and rte_errno is set. 10871 */ 10872 static int 10873 mlx5_action_handle_update(struct rte_eth_dev *dev, 10874 struct rte_flow_action_handle *handle, 10875 const void *update, 10876 struct rte_flow_error *error) 10877 { 10878 struct rte_flow_attr attr = { .transfer = 0 }; 10879 const struct mlx5_flow_driver_ops *fops = 10880 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10881 int ret; 10882 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 10883 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 10884 10885 switch (type) { 10886 case MLX5_INDIRECT_ACTION_TYPE_CT: 10887 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 10888 ret = 0; 10889 break; 10890 default: 10891 ret = flow_drv_action_validate(dev, NULL, 10892 (const struct rte_flow_action *)update, 10893 fops, error); 10894 } 10895 if (ret) 10896 return ret; 10897 return flow_drv_action_update(dev, handle, update, fops, 10898 error); 10899 } 10900 10901 /** 10902 * Query the indirect action by handle. 10903 * 10904 * This function allows retrieving action-specific data such as counters. 10905 * Data is gathered by special action which may be present/referenced in 10906 * more than one flow rule definition. 10907 * 10908 * see @RTE_FLOW_ACTION_TYPE_COUNT 10909 * 10910 * @param dev 10911 * Pointer to Ethernet device structure. 10912 * @param[in] handle 10913 * Handle for the indirect action to query. 10914 * @param[in, out] data 10915 * Pointer to storage for the associated query data type. 10916 * @param[out] error 10917 * Perform verbose error reporting if not NULL. PMDs initialize this 10918 * structure in case of error only. 10919 * 10920 * @return 10921 * 0 on success, a negative errno value otherwise and rte_errno is set. 10922 */ 10923 static int 10924 mlx5_action_handle_query(struct rte_eth_dev *dev, 10925 const struct rte_flow_action_handle *handle, 10926 void *data, 10927 struct rte_flow_error *error) 10928 { 10929 struct rte_flow_attr attr = { .transfer = 0 }; 10930 const struct mlx5_flow_driver_ops *fops = 10931 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10932 10933 return flow_drv_action_query(dev, handle, data, fops, error); 10934 } 10935 10936 static int 10937 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10938 struct rte_flow_action_handle *handle, 10939 const void *update, void *query, 10940 enum rte_flow_query_update_mode qu_mode, 10941 struct rte_flow_error *error) 10942 { 10943 struct rte_flow_attr attr = { .transfer = 0 }; 10944 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10945 const struct mlx5_flow_driver_ops *fops; 10946 10947 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10948 return rte_flow_error_set(error, ENOTSUP, 10949 RTE_FLOW_ERROR_TYPE_ACTION, 10950 NULL, "invalid driver type"); 10951 fops = flow_get_drv_ops(drv_type); 10952 if (!fops || !fops->action_query_update) 10953 return rte_flow_error_set(error, ENOTSUP, 10954 RTE_FLOW_ERROR_TYPE_ACTION, 10955 NULL, "no query_update handler"); 10956 return fops->action_query_update(dev, handle, update, 10957 query, qu_mode, error); 10958 } 10959 10960 10961 #define MLX5_DRV_FOPS_OR_ERR(dev, fops, drv_cb, ret) \ 10962 { \ 10963 struct rte_flow_attr attr = { .transfer = 0 }; \ 10964 enum mlx5_flow_drv_type drv_type = flow_get_drv_type((dev), &attr); \ 10965 if (drv_type == MLX5_FLOW_TYPE_MIN || \ 10966 drv_type == MLX5_FLOW_TYPE_MAX) { \ 10967 rte_flow_error_set(error, ENOTSUP, \ 10968 RTE_FLOW_ERROR_TYPE_ACTION, \ 10969 NULL, "invalid driver type"); \ 10970 return ret; \ 10971 } \ 10972 (fops) = flow_get_drv_ops(drv_type); \ 10973 if (!(fops) || !(fops)->drv_cb) { \ 10974 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, \ 10975 NULL, "no action_list handler"); \ 10976 return ret; \ 10977 } \ 10978 } 10979 10980 static struct rte_flow_action_list_handle * 10981 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 10982 const struct rte_flow_indir_action_conf *conf, 10983 const struct rte_flow_action *actions, 10984 struct rte_flow_error *error) 10985 { 10986 const struct mlx5_flow_driver_ops *fops; 10987 10988 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_create, NULL); 10989 return fops->action_list_handle_create(dev, conf, actions, error); 10990 } 10991 10992 static int 10993 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 10994 struct rte_flow_action_list_handle *handle, 10995 struct rte_flow_error *error) 10996 { 10997 const struct mlx5_flow_driver_ops *fops; 10998 10999 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_destroy, ENOTSUP); 11000 return fops->action_list_handle_destroy(dev, handle, error); 11001 } 11002 11003 static struct rte_flow_action_list_handle * 11004 mlx5_flow_async_action_list_handle_create(struct rte_eth_dev *dev, 11005 uint32_t queue_id, 11006 const struct 11007 rte_flow_op_attr *op_attr, 11008 const struct 11009 rte_flow_indir_action_conf *conf, 11010 const struct rte_flow_action *actions, 11011 void *user_data, 11012 struct rte_flow_error *error) 11013 { 11014 const struct mlx5_flow_driver_ops *fops; 11015 11016 MLX5_DRV_FOPS_OR_ERR(dev, fops, async_action_list_handle_create, NULL); 11017 return fops->async_action_list_handle_create(dev, queue_id, op_attr, 11018 conf, actions, user_data, 11019 error); 11020 } 11021 11022 static int 11023 mlx5_flow_async_action_list_handle_destroy 11024 (struct rte_eth_dev *dev, uint32_t queue_id, 11025 const struct rte_flow_op_attr *op_attr, 11026 struct rte_flow_action_list_handle *action_handle, 11027 void *user_data, struct rte_flow_error *error) 11028 { 11029 const struct mlx5_flow_driver_ops *fops; 11030 11031 MLX5_DRV_FOPS_OR_ERR(dev, fops, 11032 async_action_list_handle_destroy, ENOTSUP); 11033 return fops->async_action_list_handle_destroy(dev, queue_id, op_attr, 11034 action_handle, user_data, 11035 error); 11036 } 11037 11038 static int 11039 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 11040 const 11041 struct rte_flow_action_list_handle *handle, 11042 const void **update, void **query, 11043 enum rte_flow_query_update_mode mode, 11044 struct rte_flow_error *error) 11045 { 11046 const struct mlx5_flow_driver_ops *fops; 11047 11048 MLX5_DRV_FOPS_OR_ERR(dev, fops, 11049 action_list_handle_query_update, ENOTSUP); 11050 return fops->action_list_handle_query_update(dev, handle, update, query, 11051 mode, error); 11052 } 11053 11054 static int 11055 mlx5_flow_async_action_list_handle_query_update(struct rte_eth_dev *dev, 11056 uint32_t queue_id, 11057 const 11058 struct rte_flow_op_attr *op_attr, 11059 const struct 11060 rte_flow_action_list_handle *handle, 11061 const void **update, 11062 void **query, 11063 enum 11064 rte_flow_query_update_mode mode, 11065 void *user_data, 11066 struct rte_flow_error *error) 11067 { 11068 const struct mlx5_flow_driver_ops *fops; 11069 11070 MLX5_DRV_FOPS_OR_ERR(dev, fops, 11071 async_action_list_handle_query_update, ENOTSUP); 11072 return fops->async_action_list_handle_query_update(dev, queue_id, op_attr, 11073 handle, update, 11074 query, mode, 11075 user_data, error); 11076 } 11077 11078 11079 /** 11080 * Destroy all indirect actions (shared RSS). 11081 * 11082 * @param dev 11083 * Pointer to Ethernet device. 11084 * 11085 * @return 11086 * 0 on success, a negative errno value otherwise and rte_errno is set. 11087 */ 11088 int 11089 mlx5_action_handle_flush(struct rte_eth_dev *dev) 11090 { 11091 struct rte_flow_error error; 11092 struct mlx5_priv *priv = dev->data->dev_private; 11093 struct mlx5_shared_action_rss *shared_rss; 11094 int ret = 0; 11095 uint32_t idx; 11096 11097 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 11098 priv->rss_shared_actions, idx, shared_rss, next) { 11099 ret |= mlx5_action_handle_destroy(dev, 11100 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 11101 } 11102 return ret; 11103 } 11104 11105 /** 11106 * Validate existing indirect actions against current device configuration 11107 * and attach them to device resources. 11108 * 11109 * @param dev 11110 * Pointer to Ethernet device. 11111 * 11112 * @return 11113 * 0 on success, a negative errno value otherwise and rte_errno is set. 11114 */ 11115 int 11116 mlx5_action_handle_attach(struct rte_eth_dev *dev) 11117 { 11118 struct mlx5_priv *priv = dev->data->dev_private; 11119 int ret = 0; 11120 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 11121 11122 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11123 const char *message; 11124 uint32_t queue_idx; 11125 11126 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 11127 ind_tbl->queues_n, 11128 &message, &queue_idx); 11129 if (ret != 0) { 11130 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 11131 dev->data->port_id, ind_tbl->queues[queue_idx], 11132 message); 11133 break; 11134 } 11135 } 11136 if (ret != 0) 11137 return ret; 11138 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11139 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 11140 if (ret != 0) { 11141 DRV_LOG(ERR, "Port %u could not attach " 11142 "indirection table obj %p", 11143 dev->data->port_id, (void *)ind_tbl); 11144 goto error; 11145 } 11146 } 11147 11148 return 0; 11149 error: 11150 ind_tbl_last = ind_tbl; 11151 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11152 if (ind_tbl == ind_tbl_last) 11153 break; 11154 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 11155 DRV_LOG(CRIT, "Port %u could not detach " 11156 "indirection table obj %p on rollback", 11157 dev->data->port_id, (void *)ind_tbl); 11158 } 11159 return ret; 11160 } 11161 11162 /** 11163 * Detach indirect actions of the device from its resources. 11164 * 11165 * @param dev 11166 * Pointer to Ethernet device. 11167 * 11168 * @return 11169 * 0 on success, a negative errno value otherwise and rte_errno is set. 11170 */ 11171 int 11172 mlx5_action_handle_detach(struct rte_eth_dev *dev) 11173 { 11174 struct mlx5_priv *priv = dev->data->dev_private; 11175 int ret = 0; 11176 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 11177 11178 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11179 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 11180 if (ret != 0) { 11181 DRV_LOG(ERR, "Port %u could not detach " 11182 "indirection table obj %p", 11183 dev->data->port_id, (void *)ind_tbl); 11184 goto error; 11185 } 11186 } 11187 return 0; 11188 error: 11189 ind_tbl_last = ind_tbl; 11190 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11191 if (ind_tbl == ind_tbl_last) 11192 break; 11193 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 11194 DRV_LOG(CRIT, "Port %u could not attach " 11195 "indirection table obj %p on rollback", 11196 dev->data->port_id, (void *)ind_tbl); 11197 } 11198 return ret; 11199 } 11200 11201 #ifndef HAVE_MLX5DV_DR 11202 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 11203 #else 11204 #define MLX5_DOMAIN_SYNC_FLOW \ 11205 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 11206 #endif 11207 11208 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 11209 { 11210 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 11211 const struct mlx5_flow_driver_ops *fops; 11212 int ret; 11213 struct rte_flow_attr attr = { .transfer = 0 }; 11214 11215 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11216 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 11217 if (ret > 0) 11218 ret = -ret; 11219 return ret; 11220 } 11221 11222 const struct mlx5_flow_tunnel * 11223 mlx5_get_tof(const struct rte_flow_item *item, 11224 const struct rte_flow_action *action, 11225 enum mlx5_tof_rule_type *rule_type) 11226 { 11227 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11228 if (item->type == (typeof(item->type)) 11229 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 11230 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 11231 return flow_items_to_tunnel(item); 11232 } 11233 } 11234 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 11235 if (action->type == (typeof(action->type)) 11236 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 11237 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 11238 return flow_actions_to_tunnel(action); 11239 } 11240 } 11241 return NULL; 11242 } 11243 11244 /** 11245 * tunnel offload functionality is defined for DV environment only 11246 */ 11247 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 11248 __extension__ 11249 union tunnel_offload_mark { 11250 uint32_t val; 11251 struct { 11252 uint32_t app_reserve:8; 11253 uint32_t table_id:15; 11254 uint32_t transfer:1; 11255 uint32_t _unused_:8; 11256 }; 11257 }; 11258 11259 static bool 11260 mlx5_access_tunnel_offload_db 11261 (struct rte_eth_dev *dev, 11262 bool (*match)(struct rte_eth_dev *, 11263 struct mlx5_flow_tunnel *, const void *), 11264 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11265 void (*miss)(struct rte_eth_dev *, void *), 11266 void *ctx, bool lock_op); 11267 11268 static int 11269 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 11270 struct rte_flow *flow, 11271 const struct rte_flow_attr *attr, 11272 const struct rte_flow_action *app_actions, 11273 uint32_t flow_idx, 11274 const struct mlx5_flow_tunnel *tunnel, 11275 struct tunnel_default_miss_ctx *ctx, 11276 struct rte_flow_error *error) 11277 { 11278 struct mlx5_priv *priv = dev->data->dev_private; 11279 struct mlx5_flow *dev_flow; 11280 struct rte_flow_attr miss_attr = *attr; 11281 const struct rte_flow_item miss_items[2] = { 11282 { 11283 .type = RTE_FLOW_ITEM_TYPE_ETH, 11284 .spec = NULL, 11285 .last = NULL, 11286 .mask = NULL 11287 }, 11288 { 11289 .type = RTE_FLOW_ITEM_TYPE_END, 11290 .spec = NULL, 11291 .last = NULL, 11292 .mask = NULL 11293 } 11294 }; 11295 union tunnel_offload_mark mark_id; 11296 struct rte_flow_action_mark miss_mark; 11297 struct rte_flow_action miss_actions[3] = { 11298 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 11299 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 11300 }; 11301 const struct rte_flow_action_jump *jump_data; 11302 uint32_t i, flow_table = 0; /* prevent compilation warning */ 11303 struct flow_grp_info grp_info = { 11304 .external = 1, 11305 .transfer = attr->transfer, 11306 .fdb_def_rule = !!priv->fdb_def_rule, 11307 .std_tbl_fix = 0, 11308 }; 11309 int ret; 11310 11311 if (!attr->transfer) { 11312 uint32_t q_size; 11313 11314 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 11315 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 11316 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 11317 0, SOCKET_ID_ANY); 11318 if (!ctx->queue) 11319 return rte_flow_error_set 11320 (error, ENOMEM, 11321 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11322 NULL, "invalid default miss RSS"); 11323 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 11324 ctx->action_rss.level = 0, 11325 ctx->action_rss.types = priv->rss_conf.rss_hf, 11326 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 11327 ctx->action_rss.queue_num = priv->reta_idx_n, 11328 ctx->action_rss.key = priv->rss_conf.rss_key, 11329 ctx->action_rss.queue = ctx->queue; 11330 if (!priv->reta_idx_n || !priv->rxqs_n) 11331 return rte_flow_error_set 11332 (error, EINVAL, 11333 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11334 NULL, "invalid port configuration"); 11335 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 11336 ctx->action_rss.types = 0; 11337 for (i = 0; i != priv->reta_idx_n; ++i) 11338 ctx->queue[i] = (*priv->reta_idx)[i]; 11339 } else { 11340 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 11341 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 11342 } 11343 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 11344 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 11345 jump_data = app_actions->conf; 11346 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 11347 miss_attr.group = jump_data->group; 11348 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 11349 &flow_table, &grp_info, error); 11350 if (ret) 11351 return rte_flow_error_set(error, EINVAL, 11352 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11353 NULL, "invalid tunnel id"); 11354 mark_id.app_reserve = 0; 11355 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 11356 mark_id.transfer = !!attr->transfer; 11357 mark_id._unused_ = 0; 11358 miss_mark.id = mark_id.val; 11359 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 11360 miss_items, miss_actions, flow_idx, error); 11361 if (!dev_flow) 11362 return -rte_errno; 11363 dev_flow->flow = flow; 11364 dev_flow->external = true; 11365 dev_flow->tunnel = tunnel; 11366 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 11367 /* Subflow object was created, we must include one in the list. */ 11368 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 11369 dev_flow->handle, next); 11370 DRV_LOG(DEBUG, 11371 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 11372 dev->data->port_id, tunnel->app_tunnel.type, 11373 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 11374 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 11375 miss_actions, error); 11376 if (!ret) 11377 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 11378 error); 11379 11380 return ret; 11381 } 11382 11383 static const struct mlx5_flow_tbl_data_entry * 11384 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 11385 { 11386 struct mlx5_priv *priv = dev->data->dev_private; 11387 struct mlx5_dev_ctx_shared *sh = priv->sh; 11388 struct mlx5_list_entry *he; 11389 union tunnel_offload_mark mbits = { .val = mark }; 11390 union mlx5_flow_tbl_key table_key = { 11391 { 11392 .level = tunnel_id_to_flow_tbl(mbits.table_id), 11393 .id = 0, 11394 .reserved = 0, 11395 .dummy = 0, 11396 .is_fdb = !!mbits.transfer, 11397 .is_egress = 0, 11398 } 11399 }; 11400 struct mlx5_flow_cb_ctx ctx = { 11401 .data = &table_key.v64, 11402 }; 11403 11404 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 11405 return he ? 11406 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 11407 } 11408 11409 static void 11410 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 11411 struct mlx5_list_entry *entry) 11412 { 11413 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11414 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11415 11416 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11417 tunnel_flow_tbl_to_id(tte->flow_table)); 11418 mlx5_free(tte); 11419 } 11420 11421 static int 11422 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 11423 struct mlx5_list_entry *entry, void *cb_ctx) 11424 { 11425 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11426 union tunnel_tbl_key tbl = { 11427 .val = *(uint64_t *)(ctx->data), 11428 }; 11429 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11430 11431 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 11432 } 11433 11434 static struct mlx5_list_entry * 11435 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 11436 { 11437 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11438 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11439 struct tunnel_tbl_entry *tte; 11440 union tunnel_tbl_key tbl = { 11441 .val = *(uint64_t *)(ctx->data), 11442 }; 11443 11444 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 11445 sizeof(*tte), 0, 11446 SOCKET_ID_ANY); 11447 if (!tte) 11448 goto err; 11449 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11450 &tte->flow_table); 11451 if (tte->flow_table >= MLX5_MAX_TABLES) { 11452 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 11453 tte->flow_table); 11454 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11455 tte->flow_table); 11456 goto err; 11457 } else if (!tte->flow_table) { 11458 goto err; 11459 } 11460 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 11461 tte->tunnel_id = tbl.tunnel_id; 11462 tte->group = tbl.group; 11463 return &tte->hash; 11464 err: 11465 if (tte) 11466 mlx5_free(tte); 11467 return NULL; 11468 } 11469 11470 static struct mlx5_list_entry * 11471 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 11472 struct mlx5_list_entry *oentry, 11473 void *cb_ctx __rte_unused) 11474 { 11475 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 11476 0, SOCKET_ID_ANY); 11477 11478 if (!tte) 11479 return NULL; 11480 memcpy(tte, oentry, sizeof(*tte)); 11481 return &tte->hash; 11482 } 11483 11484 static void 11485 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 11486 struct mlx5_list_entry *entry) 11487 { 11488 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11489 11490 mlx5_free(tte); 11491 } 11492 11493 static uint32_t 11494 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 11495 const struct mlx5_flow_tunnel *tunnel, 11496 uint32_t group, uint32_t *table, 11497 struct rte_flow_error *error) 11498 { 11499 struct mlx5_list_entry *he; 11500 struct tunnel_tbl_entry *tte; 11501 union tunnel_tbl_key key = { 11502 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 11503 .group = group 11504 }; 11505 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11506 struct mlx5_hlist *group_hash; 11507 struct mlx5_flow_cb_ctx ctx = { 11508 .data = &key.val, 11509 }; 11510 11511 group_hash = tunnel ? tunnel->groups : thub->groups; 11512 he = mlx5_hlist_register(group_hash, key.val, &ctx); 11513 if (!he) 11514 return rte_flow_error_set(error, EINVAL, 11515 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 11516 NULL, 11517 "tunnel group index not supported"); 11518 tte = container_of(he, typeof(*tte), hash); 11519 *table = tte->flow_table; 11520 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 11521 dev->data->port_id, key.tunnel_id, group, *table); 11522 return 0; 11523 } 11524 11525 static void 11526 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 11527 struct mlx5_flow_tunnel *tunnel) 11528 { 11529 struct mlx5_priv *priv = dev->data->dev_private; 11530 struct mlx5_indexed_pool *ipool; 11531 11532 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11533 dev->data->port_id, tunnel->tunnel_id); 11534 LIST_REMOVE(tunnel, chain); 11535 mlx5_hlist_destroy(tunnel->groups); 11536 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11537 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11538 } 11539 11540 static bool 11541 mlx5_access_tunnel_offload_db 11542 (struct rte_eth_dev *dev, 11543 bool (*match)(struct rte_eth_dev *, 11544 struct mlx5_flow_tunnel *, const void *), 11545 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11546 void (*miss)(struct rte_eth_dev *, void *), 11547 void *ctx, bool lock_op) 11548 { 11549 bool verdict = false; 11550 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11551 struct mlx5_flow_tunnel *tunnel; 11552 11553 rte_spinlock_lock(&thub->sl); 11554 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11555 verdict = match(dev, tunnel, (const void *)ctx); 11556 if (verdict) 11557 break; 11558 } 11559 if (!lock_op) 11560 rte_spinlock_unlock(&thub->sl); 11561 if (verdict && hit) 11562 hit(dev, tunnel, ctx); 11563 if (!verdict && miss) 11564 miss(dev, ctx); 11565 if (lock_op) 11566 rte_spinlock_unlock(&thub->sl); 11567 11568 return verdict; 11569 } 11570 11571 struct tunnel_db_find_tunnel_id_ctx { 11572 uint32_t tunnel_id; 11573 struct mlx5_flow_tunnel *tunnel; 11574 }; 11575 11576 static bool 11577 find_tunnel_id_match(struct rte_eth_dev *dev, 11578 struct mlx5_flow_tunnel *tunnel, const void *x) 11579 { 11580 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11581 11582 RTE_SET_USED(dev); 11583 return tunnel->tunnel_id == ctx->tunnel_id; 11584 } 11585 11586 static void 11587 find_tunnel_id_hit(struct rte_eth_dev *dev, 11588 struct mlx5_flow_tunnel *tunnel, void *x) 11589 { 11590 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11591 RTE_SET_USED(dev); 11592 ctx->tunnel = tunnel; 11593 } 11594 11595 static struct mlx5_flow_tunnel * 11596 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11597 { 11598 struct tunnel_db_find_tunnel_id_ctx ctx = { 11599 .tunnel_id = id, 11600 }; 11601 11602 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11603 find_tunnel_id_hit, NULL, &ctx, true); 11604 11605 return ctx.tunnel; 11606 } 11607 11608 static struct mlx5_flow_tunnel * 11609 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11610 const struct rte_flow_tunnel *app_tunnel) 11611 { 11612 struct mlx5_priv *priv = dev->data->dev_private; 11613 struct mlx5_indexed_pool *ipool; 11614 struct mlx5_flow_tunnel *tunnel; 11615 uint32_t id; 11616 11617 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11618 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11619 if (!tunnel) 11620 return NULL; 11621 if (id >= MLX5_MAX_TUNNELS) { 11622 mlx5_ipool_free(ipool, id); 11623 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11624 return NULL; 11625 } 11626 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11627 priv->sh, 11628 mlx5_flow_tunnel_grp2tbl_create_cb, 11629 mlx5_flow_tunnel_grp2tbl_match_cb, 11630 mlx5_flow_tunnel_grp2tbl_remove_cb, 11631 mlx5_flow_tunnel_grp2tbl_clone_cb, 11632 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11633 if (!tunnel->groups) { 11634 mlx5_ipool_free(ipool, id); 11635 return NULL; 11636 } 11637 /* initiate new PMD tunnel */ 11638 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11639 tunnel->tunnel_id = id; 11640 tunnel->action.type = (typeof(tunnel->action.type)) 11641 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11642 tunnel->action.conf = tunnel; 11643 tunnel->item.type = (typeof(tunnel->item.type)) 11644 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11645 tunnel->item.spec = tunnel; 11646 tunnel->item.last = NULL; 11647 tunnel->item.mask = NULL; 11648 11649 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11650 dev->data->port_id, tunnel->tunnel_id); 11651 11652 return tunnel; 11653 } 11654 11655 struct tunnel_db_get_tunnel_ctx { 11656 const struct rte_flow_tunnel *app_tunnel; 11657 struct mlx5_flow_tunnel *tunnel; 11658 }; 11659 11660 static bool get_tunnel_match(struct rte_eth_dev *dev, 11661 struct mlx5_flow_tunnel *tunnel, const void *x) 11662 { 11663 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11664 11665 RTE_SET_USED(dev); 11666 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11667 sizeof(*ctx->app_tunnel)); 11668 } 11669 11670 static void get_tunnel_hit(struct rte_eth_dev *dev, 11671 struct mlx5_flow_tunnel *tunnel, void *x) 11672 { 11673 /* called under tunnel spinlock protection */ 11674 struct tunnel_db_get_tunnel_ctx *ctx = x; 11675 11676 RTE_SET_USED(dev); 11677 tunnel->refctn++; 11678 ctx->tunnel = tunnel; 11679 } 11680 11681 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11682 { 11683 /* called under tunnel spinlock protection */ 11684 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11685 struct tunnel_db_get_tunnel_ctx *ctx = x; 11686 11687 rte_spinlock_unlock(&thub->sl); 11688 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11689 rte_spinlock_lock(&thub->sl); 11690 if (ctx->tunnel) { 11691 ctx->tunnel->refctn = 1; 11692 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11693 } 11694 } 11695 11696 11697 static int 11698 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11699 const struct rte_flow_tunnel *app_tunnel, 11700 struct mlx5_flow_tunnel **tunnel) 11701 { 11702 struct tunnel_db_get_tunnel_ctx ctx = { 11703 .app_tunnel = app_tunnel, 11704 }; 11705 11706 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11707 get_tunnel_miss, &ctx, true); 11708 *tunnel = ctx.tunnel; 11709 return ctx.tunnel ? 0 : -ENOMEM; 11710 } 11711 11712 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11713 { 11714 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11715 11716 if (!thub) 11717 return; 11718 if (!LIST_EMPTY(&thub->tunnels)) 11719 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11720 mlx5_hlist_destroy(thub->groups); 11721 mlx5_free(thub); 11722 } 11723 11724 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11725 { 11726 int err; 11727 struct mlx5_flow_tunnel_hub *thub; 11728 11729 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11730 0, SOCKET_ID_ANY); 11731 if (!thub) 11732 return -ENOMEM; 11733 LIST_INIT(&thub->tunnels); 11734 rte_spinlock_init(&thub->sl); 11735 thub->groups = mlx5_hlist_create("flow groups", 64, 11736 false, true, sh, 11737 mlx5_flow_tunnel_grp2tbl_create_cb, 11738 mlx5_flow_tunnel_grp2tbl_match_cb, 11739 mlx5_flow_tunnel_grp2tbl_remove_cb, 11740 mlx5_flow_tunnel_grp2tbl_clone_cb, 11741 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11742 if (!thub->groups) { 11743 err = -rte_errno; 11744 goto err; 11745 } 11746 sh->tunnel_hub = thub; 11747 11748 return 0; 11749 11750 err: 11751 if (thub->groups) 11752 mlx5_hlist_destroy(thub->groups); 11753 if (thub) 11754 mlx5_free(thub); 11755 return err; 11756 } 11757 11758 static inline int 11759 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11760 struct rte_flow_tunnel *tunnel, 11761 struct rte_flow_error *error) 11762 { 11763 struct mlx5_priv *priv = dev->data->dev_private; 11764 11765 if (!priv->sh->config.dv_flow_en) 11766 return rte_flow_error_set(error, ENOTSUP, 11767 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11768 "flow DV interface is off"); 11769 if (!is_tunnel_offload_active(dev)) 11770 return rte_flow_error_set(error, ENOTSUP, 11771 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11772 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11773 if (!tunnel) 11774 return rte_flow_error_set(error, EINVAL, 11775 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11776 "no application tunnel"); 11777 switch (tunnel->type) { 11778 default: 11779 return rte_flow_error_set(error, EINVAL, 11780 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11781 "unsupported tunnel type"); 11782 case RTE_FLOW_ITEM_TYPE_VXLAN: 11783 case RTE_FLOW_ITEM_TYPE_GRE: 11784 case RTE_FLOW_ITEM_TYPE_NVGRE: 11785 case RTE_FLOW_ITEM_TYPE_GENEVE: 11786 break; 11787 } 11788 return 0; 11789 } 11790 11791 static int 11792 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11793 struct rte_flow_tunnel *app_tunnel, 11794 struct rte_flow_action **actions, 11795 uint32_t *num_of_actions, 11796 struct rte_flow_error *error) 11797 { 11798 struct mlx5_flow_tunnel *tunnel; 11799 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11800 11801 if (ret) 11802 return ret; 11803 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11804 if (ret < 0) { 11805 return rte_flow_error_set(error, ret, 11806 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11807 "failed to initialize pmd tunnel"); 11808 } 11809 *actions = &tunnel->action; 11810 *num_of_actions = 1; 11811 return 0; 11812 } 11813 11814 static int 11815 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11816 struct rte_flow_tunnel *app_tunnel, 11817 struct rte_flow_item **items, 11818 uint32_t *num_of_items, 11819 struct rte_flow_error *error) 11820 { 11821 struct mlx5_flow_tunnel *tunnel; 11822 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11823 11824 if (ret) 11825 return ret; 11826 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11827 if (ret < 0) { 11828 return rte_flow_error_set(error, ret, 11829 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11830 "failed to initialize pmd tunnel"); 11831 } 11832 *items = &tunnel->item; 11833 *num_of_items = 1; 11834 return 0; 11835 } 11836 11837 struct tunnel_db_element_release_ctx { 11838 struct rte_flow_item *items; 11839 struct rte_flow_action *actions; 11840 uint32_t num_elements; 11841 struct rte_flow_error *error; 11842 int ret; 11843 }; 11844 11845 static bool 11846 tunnel_element_release_match(struct rte_eth_dev *dev, 11847 struct mlx5_flow_tunnel *tunnel, const void *x) 11848 { 11849 const struct tunnel_db_element_release_ctx *ctx = x; 11850 11851 RTE_SET_USED(dev); 11852 if (ctx->num_elements != 1) 11853 return false; 11854 else if (ctx->items) 11855 return ctx->items == &tunnel->item; 11856 else if (ctx->actions) 11857 return ctx->actions == &tunnel->action; 11858 11859 return false; 11860 } 11861 11862 static void 11863 tunnel_element_release_hit(struct rte_eth_dev *dev, 11864 struct mlx5_flow_tunnel *tunnel, void *x) 11865 { 11866 struct tunnel_db_element_release_ctx *ctx = x; 11867 ctx->ret = 0; 11868 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 11869 mlx5_flow_tunnel_free(dev, tunnel); 11870 } 11871 11872 static void 11873 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11874 { 11875 struct tunnel_db_element_release_ctx *ctx = x; 11876 RTE_SET_USED(dev); 11877 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11878 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11879 "invalid argument"); 11880 } 11881 11882 static int 11883 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11884 struct rte_flow_item *pmd_items, 11885 uint32_t num_items, struct rte_flow_error *err) 11886 { 11887 struct tunnel_db_element_release_ctx ctx = { 11888 .items = pmd_items, 11889 .actions = NULL, 11890 .num_elements = num_items, 11891 .error = err, 11892 }; 11893 11894 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11895 tunnel_element_release_hit, 11896 tunnel_element_release_miss, &ctx, false); 11897 11898 return ctx.ret; 11899 } 11900 11901 static int 11902 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11903 struct rte_flow_action *pmd_actions, 11904 uint32_t num_actions, struct rte_flow_error *err) 11905 { 11906 struct tunnel_db_element_release_ctx ctx = { 11907 .items = NULL, 11908 .actions = pmd_actions, 11909 .num_elements = num_actions, 11910 .error = err, 11911 }; 11912 11913 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11914 tunnel_element_release_hit, 11915 tunnel_element_release_miss, &ctx, false); 11916 11917 return ctx.ret; 11918 } 11919 11920 static int 11921 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11922 struct rte_mbuf *m, 11923 struct rte_flow_restore_info *info, 11924 struct rte_flow_error *err) 11925 { 11926 uint64_t ol_flags = m->ol_flags; 11927 const struct mlx5_flow_tbl_data_entry *tble; 11928 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11929 struct mlx5_priv *priv = dev->data->dev_private; 11930 11931 if (priv->tunnel_enabled == 0) 11932 goto err; 11933 if ((ol_flags & mask) != mask) 11934 goto err; 11935 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11936 if (!tble) { 11937 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11938 dev->data->port_id, m->hash.fdir.hi); 11939 goto err; 11940 } 11941 MLX5_ASSERT(tble->tunnel); 11942 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11943 info->group_id = tble->group_id; 11944 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11945 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11946 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11947 11948 return 0; 11949 11950 err: 11951 return rte_flow_error_set(err, EINVAL, 11952 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11953 "failed to get restore info"); 11954 } 11955 11956 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11957 static int 11958 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11959 __rte_unused struct rte_flow_tunnel *app_tunnel, 11960 __rte_unused struct rte_flow_action **actions, 11961 __rte_unused uint32_t *num_of_actions, 11962 __rte_unused struct rte_flow_error *error) 11963 { 11964 return -ENOTSUP; 11965 } 11966 11967 static int 11968 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11969 __rte_unused struct rte_flow_tunnel *app_tunnel, 11970 __rte_unused struct rte_flow_item **items, 11971 __rte_unused uint32_t *num_of_items, 11972 __rte_unused struct rte_flow_error *error) 11973 { 11974 return -ENOTSUP; 11975 } 11976 11977 static int 11978 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11979 __rte_unused struct rte_flow_item *pmd_items, 11980 __rte_unused uint32_t num_items, 11981 __rte_unused struct rte_flow_error *err) 11982 { 11983 return -ENOTSUP; 11984 } 11985 11986 static int 11987 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11988 __rte_unused struct rte_flow_action *pmd_action, 11989 __rte_unused uint32_t num_actions, 11990 __rte_unused struct rte_flow_error *err) 11991 { 11992 return -ENOTSUP; 11993 } 11994 11995 static int 11996 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11997 __rte_unused struct rte_mbuf *m, 11998 __rte_unused struct rte_flow_restore_info *i, 11999 __rte_unused struct rte_flow_error *err) 12000 { 12001 return -ENOTSUP; 12002 } 12003 12004 static int 12005 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 12006 __rte_unused struct rte_flow *flow, 12007 __rte_unused const struct rte_flow_attr *attr, 12008 __rte_unused const struct rte_flow_action *actions, 12009 __rte_unused uint32_t flow_idx, 12010 __rte_unused const struct mlx5_flow_tunnel *tunnel, 12011 __rte_unused struct tunnel_default_miss_ctx *ctx, 12012 __rte_unused struct rte_flow_error *error) 12013 { 12014 return -ENOTSUP; 12015 } 12016 12017 static struct mlx5_flow_tunnel * 12018 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 12019 __rte_unused uint32_t id) 12020 { 12021 return NULL; 12022 } 12023 12024 static void 12025 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 12026 __rte_unused struct mlx5_flow_tunnel *tunnel) 12027 { 12028 } 12029 12030 static uint32_t 12031 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 12032 __rte_unused const struct mlx5_flow_tunnel *t, 12033 __rte_unused uint32_t group, 12034 __rte_unused uint32_t *table, 12035 struct rte_flow_error *error) 12036 { 12037 return rte_flow_error_set(error, ENOTSUP, 12038 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 12039 "tunnel offload requires DV support"); 12040 } 12041 12042 void 12043 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 12044 __rte_unused uint16_t port_id) 12045 { 12046 } 12047 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 12048 12049 /* Flex flow item API */ 12050 static struct rte_flow_item_flex_handle * 12051 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 12052 const struct rte_flow_item_flex_conf *conf, 12053 struct rte_flow_error *error) 12054 { 12055 static const char err_msg[] = "flex item creation unsupported"; 12056 struct mlx5_priv *priv = dev->data->dev_private; 12057 struct rte_flow_attr attr = { .transfer = 0 }; 12058 const struct mlx5_flow_driver_ops *fops = 12059 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 12060 12061 if (!priv->pci_dev) { 12062 rte_flow_error_set(error, ENOTSUP, 12063 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 12064 "create flex item on PF only"); 12065 return NULL; 12066 } 12067 switch (priv->pci_dev->id.device_id) { 12068 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 12069 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 12070 break; 12071 default: 12072 rte_flow_error_set(error, ENOTSUP, 12073 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 12074 "flex item available on BlueField ports only"); 12075 return NULL; 12076 } 12077 if (!fops->item_create) { 12078 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 12079 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 12080 NULL, err_msg); 12081 return NULL; 12082 } 12083 return fops->item_create(dev, conf, error); 12084 } 12085 12086 static int 12087 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 12088 const struct rte_flow_item_flex_handle *handle, 12089 struct rte_flow_error *error) 12090 { 12091 static const char err_msg[] = "flex item release unsupported"; 12092 struct rte_flow_attr attr = { .transfer = 0 }; 12093 const struct mlx5_flow_driver_ops *fops = 12094 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 12095 12096 if (!fops->item_release) { 12097 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 12098 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 12099 NULL, err_msg); 12100 return -rte_errno; 12101 } 12102 return fops->item_release(dev, handle, error); 12103 } 12104 12105 static void 12106 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 12107 { 12108 int ret; 12109 struct rte_flow_error error; 12110 12111 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 12112 char *item_name; 12113 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 12114 sizeof(item_name), 12115 (void *)(uintptr_t)item->type, &error); 12116 if (ret > 0) 12117 printf("%s ", item_name); 12118 else 12119 printf("%d\n", (int)item->type); 12120 } 12121 printf("END\n"); 12122 } 12123 12124 static int 12125 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 12126 { 12127 const struct rte_flow_item_udp *spec = udp_item->spec; 12128 const struct rte_flow_item_udp *mask = udp_item->mask; 12129 uint16_t udp_dport = 0; 12130 12131 if (spec != NULL) { 12132 if (!mask) 12133 mask = &rte_flow_item_udp_mask; 12134 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 12135 mask->hdr.dst_port); 12136 } 12137 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 12138 } 12139 12140 static const struct mlx5_flow_expand_node * 12141 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 12142 unsigned int item_idx, 12143 const struct mlx5_flow_expand_node graph[], 12144 const struct mlx5_flow_expand_node *node) 12145 { 12146 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 12147 12148 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 12149 node != NULL && 12150 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 12151 /* 12152 * The expansion node is VXLAN and it is also the last 12153 * expandable item in the pattern, so need to continue 12154 * expansion of the inner tunnel. 12155 */ 12156 MLX5_ASSERT(item_idx > 0); 12157 prev_item = pattern + item_idx - 1; 12158 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 12159 if (mlx5_flow_is_std_vxlan_port(prev_item)) 12160 return &graph[MLX5_EXPANSION_STD_VXLAN]; 12161 return &graph[MLX5_EXPANSION_L3_VXLAN]; 12162 } 12163 return node; 12164 } 12165 12166 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 12167 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 12168 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 12169 }; 12170 12171 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 12172 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 12173 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 12174 { 9, 10, 11 }, { 12, 13, 14 }, 12175 }; 12176 12177 /** 12178 * Discover the number of available flow priorities. 12179 * 12180 * @param dev 12181 * Ethernet device. 12182 * 12183 * @return 12184 * On success, number of available flow priorities. 12185 * On failure, a negative errno-style code and rte_errno is set. 12186 */ 12187 int 12188 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 12189 { 12190 static const uint16_t vprio[] = {8, 16}; 12191 const struct mlx5_priv *priv = dev->data->dev_private; 12192 const struct mlx5_flow_driver_ops *fops; 12193 enum mlx5_flow_drv_type type; 12194 int ret; 12195 12196 type = mlx5_flow_os_get_type(); 12197 if (type == MLX5_FLOW_TYPE_MAX) { 12198 type = MLX5_FLOW_TYPE_VERBS; 12199 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 12200 type = MLX5_FLOW_TYPE_DV; 12201 } 12202 fops = flow_get_drv_ops(type); 12203 if (fops->discover_priorities == NULL) { 12204 DRV_LOG(ERR, "Priority discovery not supported"); 12205 rte_errno = ENOTSUP; 12206 return -rte_errno; 12207 } 12208 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 12209 if (ret < 0) 12210 return ret; 12211 switch (ret) { 12212 case 8: 12213 ret = RTE_DIM(priority_map_3); 12214 break; 12215 case 16: 12216 ret = RTE_DIM(priority_map_5); 12217 break; 12218 default: 12219 rte_errno = ENOTSUP; 12220 DRV_LOG(ERR, 12221 "port %u maximum priority: %d expected 8/16", 12222 dev->data->port_id, ret); 12223 return -rte_errno; 12224 } 12225 DRV_LOG(INFO, "port %u supported flow priorities:" 12226 " 0-%d for ingress or egress root table," 12227 " 0-%d for non-root table or transfer root table.", 12228 dev->data->port_id, ret - 2, 12229 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 12230 return ret; 12231 } 12232 12233 /** 12234 * Adjust flow priority based on the highest layer and the request priority. 12235 * 12236 * @param[in] dev 12237 * Pointer to the Ethernet device structure. 12238 * @param[in] priority 12239 * The rule base priority. 12240 * @param[in] subpriority 12241 * The priority based on the items. 12242 * 12243 * @return 12244 * The new priority. 12245 */ 12246 uint32_t 12247 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 12248 uint32_t subpriority) 12249 { 12250 uint32_t res = 0; 12251 struct mlx5_priv *priv = dev->data->dev_private; 12252 12253 switch (priv->sh->flow_max_priority) { 12254 case RTE_DIM(priority_map_3): 12255 res = priority_map_3[priority][subpriority]; 12256 break; 12257 case RTE_DIM(priority_map_5): 12258 res = priority_map_5[priority][subpriority]; 12259 break; 12260 } 12261 return res; 12262 } 12263 12264 /** 12265 * Get the priority for sending traffic to kernel table. 12266 * 12267 * @param[in] dev 12268 * Pointer to the Ethernet device structure. 12269 * 12270 * @return 12271 * On success: the value of priority for sending traffic to kernel table 12272 * On failure: -1 12273 */ 12274 uint32_t 12275 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 12276 { 12277 struct mlx5_priv *priv = dev->data->dev_private; 12278 uint32_t res; 12279 12280 switch (priv->sh->flow_max_priority) { 12281 case RTE_DIM(priority_map_5): 12282 res = 15; 12283 break; 12284 case RTE_DIM(priority_map_3): 12285 res = 7; 12286 break; 12287 default: 12288 DRV_LOG(ERR, 12289 "port %u maximum priority: %d expected 8/16", 12290 dev->data->port_id, priv->sh->flow_max_priority); 12291 res = (uint32_t)-1; 12292 } 12293 return res; 12294 } 12295 12296 /** 12297 * Get the E-Switch Manager vport id. 12298 * 12299 * @param[in] dev 12300 * Pointer to the Ethernet device structure. 12301 * 12302 * @return 12303 * The vport id. 12304 */ 12305 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 12306 { 12307 struct mlx5_priv *priv = dev->data->dev_private; 12308 struct mlx5_common_device *cdev = priv->sh->cdev; 12309 12310 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 12311 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 12312 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 12313 12314 if (priv->pci_dev == NULL) 12315 return 0; 12316 switch (priv->pci_dev->id.device_id) { 12317 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 12318 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 12319 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 12320 /* 12321 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 12322 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 12323 * ECPF vport ID is selected and not the host port (0) in any BF case. 12324 */ 12325 return (int16_t)MLX5_ECPF_VPORT_ID; 12326 default: 12327 return MLX5_PF_VPORT_ID; 12328 } 12329 } 12330 12331 /** 12332 * Parse item to get the vport id. 12333 * 12334 * @param[in] dev 12335 * Pointer to the Ethernet device structure. 12336 * @param[in] item 12337 * The src port id match item. 12338 * @param[out] vport_id 12339 * Pointer to put the vport id. 12340 * @param[out] all_ports 12341 * Indicate if the item matches all ports. 12342 * @param[out] error 12343 * Pointer to error structure. 12344 * 12345 * @return 12346 * 0 on success, a negative errno value otherwise and rte_errno is set. 12347 */ 12348 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 12349 const struct rte_flow_item *item, 12350 uint16_t *vport_id, 12351 bool *all_ports, 12352 struct rte_flow_error *error) 12353 { 12354 struct mlx5_priv *port_priv; 12355 const struct rte_flow_item_port_id *pid_v = NULL; 12356 const struct rte_flow_item_ethdev *dev_v = NULL; 12357 uint32_t esw_mgr_port; 12358 uint32_t src_port; 12359 12360 if (all_ports) 12361 *all_ports = false; 12362 switch (item->type) { 12363 case RTE_FLOW_ITEM_TYPE_PORT_ID: 12364 pid_v = item->spec; 12365 if (!pid_v) 12366 return 0; 12367 src_port = pid_v->id; 12368 esw_mgr_port = MLX5_PORT_ESW_MGR; 12369 break; 12370 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 12371 dev_v = item->spec; 12372 if (!dev_v) { 12373 if (all_ports) 12374 *all_ports = true; 12375 return 0; 12376 } 12377 src_port = dev_v->port_id; 12378 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12379 break; 12380 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 12381 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12382 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12383 break; 12384 default: 12385 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12386 NULL, "Incorrect item type."); 12387 } 12388 if (src_port == esw_mgr_port) { 12389 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 12390 } else { 12391 port_priv = mlx5_port_to_eswitch_info(src_port, false); 12392 if (!port_priv) 12393 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12394 NULL, "Failed to get port info."); 12395 *vport_id = port_priv->representor_id; 12396 } 12397 12398 return 0; 12399 } 12400 12401 int 12402 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 12403 uint16_t *proxy_port_id, 12404 struct rte_flow_error *error) 12405 { 12406 const struct mlx5_priv *priv = dev->data->dev_private; 12407 uint16_t port_id; 12408 12409 if (!priv->sh->config.dv_esw_en) 12410 return rte_flow_error_set(error, EINVAL, 12411 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12412 NULL, 12413 "unable to provide a proxy port" 12414 " without E-Switch configured"); 12415 if (!priv->master && !priv->representor) 12416 return rte_flow_error_set(error, EINVAL, 12417 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12418 NULL, 12419 "unable to provide a proxy port" 12420 " for port which is not a master" 12421 " or a representor port"); 12422 if (priv->master) { 12423 *proxy_port_id = dev->data->port_id; 12424 return 0; 12425 } 12426 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 12427 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 12428 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 12429 12430 if (port_priv->master && 12431 port_priv->domain_id == priv->domain_id) { 12432 *proxy_port_id = port_id; 12433 return 0; 12434 } 12435 } 12436 return rte_flow_error_set(error, ENODEV, 12437 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12438 NULL, "unable to find a proxy port"); 12439 } 12440