1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 struct tunnel_default_miss_ctx { 43 uint16_t *queue; 44 __extension__ 45 union { 46 struct rte_flow_action_rss action_rss; 47 struct rte_flow_action_queue miss_queue; 48 struct rte_flow_action_jump miss_jump; 49 uint8_t raw[0]; 50 }; 51 }; 52 53 void 54 mlx5_indirect_list_handles_release(struct rte_eth_dev *dev) 55 { 56 struct mlx5_priv *priv = dev->data->dev_private; 57 #ifdef HAVE_MLX5_HWS_SUPPORT 58 struct rte_flow_error error; 59 #endif 60 61 while (!LIST_EMPTY(&priv->indirect_list_head)) { 62 struct mlx5_indirect_list *e = 63 LIST_FIRST(&priv->indirect_list_head); 64 65 LIST_REMOVE(e, entry); 66 switch (e->type) { 67 #ifdef HAVE_MLX5_HWS_SUPPORT 68 case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR: 69 mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)e); 70 break; 71 case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY: 72 mlx5_destroy_legacy_indirect(dev, e); 73 break; 74 case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT: 75 mlx5_reformat_action_destroy(dev, 76 (struct rte_flow_action_list_handle *)e, &error); 77 break; 78 #endif 79 default: 80 DRV_LOG(ERR, "invalid indirect list type"); 81 MLX5_ASSERT(false); 82 break; 83 } 84 } 85 } 86 87 static int 88 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 89 struct rte_flow *flow, 90 const struct rte_flow_attr *attr, 91 const struct rte_flow_action *app_actions, 92 uint32_t flow_idx, 93 const struct mlx5_flow_tunnel *tunnel, 94 struct tunnel_default_miss_ctx *ctx, 95 struct rte_flow_error *error); 96 static struct mlx5_flow_tunnel * 97 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 98 static void 99 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 100 static uint32_t 101 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 102 const struct mlx5_flow_tunnel *tunnel, 103 uint32_t group, uint32_t *table, 104 struct rte_flow_error *error); 105 106 /** Device flow drivers. */ 107 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 108 109 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 110 111 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 112 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 113 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 114 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 115 #endif 116 #ifdef HAVE_MLX5_HWS_SUPPORT 117 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 118 #endif 119 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 120 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 121 }; 122 123 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 124 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 125 (const int []){ \ 126 __VA_ARGS__, 0, \ 127 } 128 129 /** Node object of input graph for mlx5_flow_expand_rss(). */ 130 struct mlx5_flow_expand_node { 131 const int *const next; 132 /**< 133 * List of next node indexes. Index 0 is interpreted as a terminator. 134 */ 135 const enum rte_flow_item_type type; 136 /**< Pattern item type of current node. */ 137 uint64_t rss_types; 138 /**< 139 * RSS types bit-field associated with this node 140 * (see RTE_ETH_RSS_* definitions). 141 */ 142 uint64_t node_flags; 143 /**< 144 * Bit-fields that define how the node is used in the expansion. 145 * (see MLX5_EXPANSION_NODE_* definitions). 146 */ 147 }; 148 149 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 150 struct mlx5_flow_expand_sqn { 151 uint32_t entries; /** Number of entries */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 /* Optional expand field. The expansion alg will not go deeper. */ 159 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 160 161 /* The node is not added implicitly as expansion to the flow pattern. 162 * If the node type does not match the flow pattern item type, the 163 * expansion alg will go deeper to its next items. 164 * In the current implementation, the list of next nodes indexes can 165 * have up to one node with this flag set and it has to be the last 166 * node index (before the list terminator). 167 */ 168 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 169 170 /** Object returned by mlx5_flow_expand_rss(). */ 171 struct mlx5_flow_expand_rss { 172 uint32_t entries; 173 /**< Number of entries @p patterns and @p priorities. */ 174 struct { 175 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 176 uint32_t priority; /**< Priority offset for each expansion. */ 177 } entry[]; 178 }; 179 180 static void 181 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 182 183 static const struct mlx5_flow_expand_node * 184 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 185 unsigned int item_idx, 186 const struct mlx5_flow_expand_node graph[], 187 const struct mlx5_flow_expand_node *node); 188 189 static __rte_always_inline int 190 mlx5_need_cache_flow(const struct mlx5_priv *priv, 191 const struct rte_flow_attr *attr) 192 { 193 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 194 (attr ? !attr->group : true) && 195 priv->mode_info.mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY && 196 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 197 } 198 199 static bool 200 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 201 { 202 switch (item->type) { 203 case RTE_FLOW_ITEM_TYPE_ETH: 204 case RTE_FLOW_ITEM_TYPE_VLAN: 205 case RTE_FLOW_ITEM_TYPE_IPV4: 206 case RTE_FLOW_ITEM_TYPE_IPV6: 207 case RTE_FLOW_ITEM_TYPE_UDP: 208 case RTE_FLOW_ITEM_TYPE_TCP: 209 case RTE_FLOW_ITEM_TYPE_ESP: 210 case RTE_FLOW_ITEM_TYPE_ICMP: 211 case RTE_FLOW_ITEM_TYPE_ICMP6: 212 case RTE_FLOW_ITEM_TYPE_VXLAN: 213 case RTE_FLOW_ITEM_TYPE_NVGRE: 214 case RTE_FLOW_ITEM_TYPE_GRE: 215 case RTE_FLOW_ITEM_TYPE_GENEVE: 216 case RTE_FLOW_ITEM_TYPE_MPLS: 217 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 218 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 219 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 220 case RTE_FLOW_ITEM_TYPE_GTP: 221 return true; 222 default: 223 break; 224 } 225 return false; 226 } 227 228 /** 229 * Network Service Header (NSH) and its next protocol values 230 * are described in RFC-8393. 231 */ 232 static enum rte_flow_item_type 233 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 234 { 235 enum rte_flow_item_type type; 236 237 switch (proto_mask & proto_spec) { 238 case 0: 239 type = RTE_FLOW_ITEM_TYPE_VOID; 240 break; 241 case RTE_VXLAN_GPE_TYPE_IPV4: 242 type = RTE_FLOW_ITEM_TYPE_IPV4; 243 break; 244 case RTE_VXLAN_GPE_TYPE_IPV6: 245 type = RTE_VXLAN_GPE_TYPE_IPV6; 246 break; 247 case RTE_VXLAN_GPE_TYPE_ETH: 248 type = RTE_FLOW_ITEM_TYPE_ETH; 249 break; 250 default: 251 type = RTE_FLOW_ITEM_TYPE_END; 252 } 253 return type; 254 } 255 256 static enum rte_flow_item_type 257 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 258 { 259 enum rte_flow_item_type type; 260 261 switch (proto_mask & proto_spec) { 262 case 0: 263 type = RTE_FLOW_ITEM_TYPE_VOID; 264 break; 265 case IPPROTO_UDP: 266 type = RTE_FLOW_ITEM_TYPE_UDP; 267 break; 268 case IPPROTO_TCP: 269 type = RTE_FLOW_ITEM_TYPE_TCP; 270 break; 271 case IPPROTO_IPIP: 272 type = RTE_FLOW_ITEM_TYPE_IPV4; 273 break; 274 case IPPROTO_IPV6: 275 type = RTE_FLOW_ITEM_TYPE_IPV6; 276 break; 277 case IPPROTO_ESP: 278 type = RTE_FLOW_ITEM_TYPE_ESP; 279 break; 280 default: 281 type = RTE_FLOW_ITEM_TYPE_END; 282 } 283 return type; 284 } 285 286 static enum rte_flow_item_type 287 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 288 rte_be16_t type_mask, bool is_tunnel) 289 { 290 enum rte_flow_item_type type; 291 292 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 293 case 0: 294 type = RTE_FLOW_ITEM_TYPE_VOID; 295 break; 296 case RTE_ETHER_TYPE_TEB: 297 type = is_tunnel ? 298 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 299 break; 300 case RTE_ETHER_TYPE_VLAN: 301 type = !is_tunnel ? 302 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 303 break; 304 case RTE_ETHER_TYPE_IPV4: 305 type = RTE_FLOW_ITEM_TYPE_IPV4; 306 break; 307 case RTE_ETHER_TYPE_IPV6: 308 type = RTE_FLOW_ITEM_TYPE_IPV6; 309 break; 310 default: 311 type = RTE_FLOW_ITEM_TYPE_END; 312 } 313 return type; 314 } 315 316 static enum rte_flow_item_type 317 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 318 { 319 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 320 do { \ 321 const void *m = item->mask; \ 322 const void *s = item->spec; \ 323 mask = m ? \ 324 ((const struct rte_flow_item_##type *)m)->fld : \ 325 rte_flow_item_##type##_mask.fld; \ 326 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 327 } while (0) 328 329 enum rte_flow_item_type ret; 330 uint16_t spec, mask; 331 332 if (item == NULL || item->spec == NULL) 333 return RTE_FLOW_ITEM_TYPE_VOID; 334 switch (item->type) { 335 case RTE_FLOW_ITEM_TYPE_ETH: 336 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 337 if (!mask) 338 return RTE_FLOW_ITEM_TYPE_VOID; 339 ret = mlx5_ethertype_to_item_type(spec, mask, false); 340 break; 341 case RTE_FLOW_ITEM_TYPE_VLAN: 342 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 343 if (!mask) 344 return RTE_FLOW_ITEM_TYPE_VOID; 345 ret = mlx5_ethertype_to_item_type(spec, mask, false); 346 break; 347 case RTE_FLOW_ITEM_TYPE_IPV4: 348 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 349 if (!mask) 350 return RTE_FLOW_ITEM_TYPE_VOID; 351 ret = mlx5_inet_proto_to_item_type(spec, mask); 352 break; 353 case RTE_FLOW_ITEM_TYPE_IPV6: 354 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 355 if (!mask) 356 return RTE_FLOW_ITEM_TYPE_VOID; 357 ret = mlx5_inet_proto_to_item_type(spec, mask); 358 break; 359 case RTE_FLOW_ITEM_TYPE_GENEVE: 360 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 361 ret = mlx5_ethertype_to_item_type(spec, mask, true); 362 break; 363 case RTE_FLOW_ITEM_TYPE_GRE: 364 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 365 ret = mlx5_ethertype_to_item_type(spec, mask, true); 366 break; 367 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 368 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 369 ret = mlx5_nsh_proto_to_item_type(spec, mask); 370 break; 371 default: 372 ret = RTE_FLOW_ITEM_TYPE_VOID; 373 break; 374 } 375 return ret; 376 #undef MLX5_XSET_ITEM_MASK_SPEC 377 } 378 379 static const int * 380 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 381 const int *next_node) 382 { 383 const struct mlx5_flow_expand_node *node = NULL; 384 const int *next = next_node; 385 386 while (next && *next) { 387 /* 388 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 389 * flag set, because they were not found in the flow pattern. 390 */ 391 node = &graph[*next]; 392 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 393 break; 394 next = node->next; 395 } 396 return next; 397 } 398 399 #define MLX5_RSS_EXP_ELT_N 32 400 401 /** 402 * Expand RSS flows into several possible flows according to the RSS hash 403 * fields requested and the driver capabilities. 404 * 405 * @param[out] buf 406 * Buffer to store the result expansion. 407 * @param[in] size 408 * Buffer size in bytes. If 0, @p buf can be NULL. 409 * @param[in] pattern 410 * User flow pattern. 411 * @param[in] types 412 * RSS types to expand (see RTE_ETH_RSS_* definitions). 413 * @param[in] graph 414 * Input graph to expand @p pattern according to @p types. 415 * @param[in] graph_root_index 416 * Index of root node in @p graph, typically 0. 417 * 418 * @return 419 * A positive value representing the size of @p buf in bytes regardless of 420 * @p size on success, a negative errno value otherwise and rte_errno is 421 * set, the following errors are defined: 422 * 423 * -E2BIG: graph-depth @p graph is too deep. 424 * -EINVAL: @p size has not enough space for expanded pattern. 425 */ 426 static int 427 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 428 const struct rte_flow_item *pattern, uint64_t types, 429 const struct mlx5_flow_expand_node graph[], 430 int graph_root_index) 431 { 432 const struct rte_flow_item *item; 433 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 434 const int *next_node; 435 const int *stack[MLX5_RSS_EXP_ELT_N]; 436 int stack_pos = 0; 437 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 438 unsigned int i, item_idx, last_expand_item_idx = 0; 439 size_t lsize; 440 size_t user_pattern_size = 0; 441 void *addr = NULL; 442 const struct mlx5_flow_expand_node *next = NULL; 443 struct rte_flow_item missed_item; 444 int missed = 0; 445 int elt = 0; 446 const struct rte_flow_item *last_expand_item = NULL; 447 448 memset(&missed_item, 0, sizeof(missed_item)); 449 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 450 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 451 if (lsize > size) 452 return -EINVAL; 453 buf->entry[0].priority = 0; 454 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 455 buf->entries = 0; 456 addr = buf->entry[0].pattern; 457 for (item = pattern, item_idx = 0; 458 item->type != RTE_FLOW_ITEM_TYPE_END; 459 item++, item_idx++) { 460 if (!mlx5_flow_is_rss_expandable_item(item)) { 461 user_pattern_size += sizeof(*item); 462 continue; 463 } 464 last_expand_item = item; 465 last_expand_item_idx = item_idx; 466 i = 0; 467 while (node->next && node->next[i]) { 468 next = &graph[node->next[i]]; 469 if (next->type == item->type) 470 break; 471 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 472 node = next; 473 i = 0; 474 } else { 475 ++i; 476 } 477 } 478 if (next) 479 node = next; 480 user_pattern_size += sizeof(*item); 481 } 482 user_pattern_size += sizeof(*item); /* Handle END item. */ 483 lsize += user_pattern_size; 484 if (lsize > size) 485 return -EINVAL; 486 /* Copy the user pattern in the first entry of the buffer. */ 487 rte_memcpy(addr, pattern, user_pattern_size); 488 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 489 buf->entries = 1; 490 /* Start expanding. */ 491 memset(flow_items, 0, sizeof(flow_items)); 492 user_pattern_size -= sizeof(*item); 493 /* 494 * Check if the last valid item has spec set, need complete pattern, 495 * and the pattern can be used for expansion. 496 */ 497 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 498 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 499 /* Item type END indicates expansion is not required. */ 500 return lsize; 501 } 502 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 503 next = NULL; 504 missed = 1; 505 i = 0; 506 while (node->next && node->next[i]) { 507 next = &graph[node->next[i]]; 508 if (next->type == missed_item.type) { 509 flow_items[0].type = missed_item.type; 510 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 511 break; 512 } 513 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 514 node = next; 515 i = 0; 516 } else { 517 ++i; 518 } 519 next = NULL; 520 } 521 } 522 if (next && missed) { 523 elt = 2; /* missed item + item end. */ 524 node = next; 525 lsize += elt * sizeof(*item) + user_pattern_size; 526 if (lsize > size) 527 return -EINVAL; 528 if (node->rss_types & types) { 529 buf->entry[buf->entries].priority = 1; 530 buf->entry[buf->entries].pattern = addr; 531 buf->entries++; 532 rte_memcpy(addr, buf->entry[0].pattern, 533 user_pattern_size); 534 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 535 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 536 addr = (void *)(((uintptr_t)addr) + 537 elt * sizeof(*item)); 538 } 539 } else if (last_expand_item != NULL) { 540 node = mlx5_flow_expand_rss_adjust_node(pattern, 541 last_expand_item_idx, graph, node); 542 } 543 memset(flow_items, 0, sizeof(flow_items)); 544 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 545 node->next); 546 stack[stack_pos] = next_node; 547 node = next_node ? &graph[*next_node] : NULL; 548 while (node) { 549 flow_items[stack_pos].type = node->type; 550 if (node->rss_types & types) { 551 size_t n; 552 /* 553 * compute the number of items to copy from the 554 * expansion and copy it. 555 * When the stack_pos is 0, there are 1 element in it, 556 * plus the addition END item. 557 */ 558 elt = stack_pos + 2; 559 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 560 lsize += elt * sizeof(*item) + user_pattern_size; 561 if (lsize > size) 562 return -EINVAL; 563 n = elt * sizeof(*item); 564 MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); 565 buf->entry[buf->entries].priority = 566 stack_pos + 1 + missed; 567 buf->entry[buf->entries].pattern = addr; 568 buf->entries++; 569 rte_memcpy(addr, buf->entry[0].pattern, 570 user_pattern_size); 571 addr = (void *)(((uintptr_t)addr) + 572 user_pattern_size); 573 rte_memcpy(addr, &missed_item, 574 missed * sizeof(*item)); 575 addr = (void *)(((uintptr_t)addr) + 576 missed * sizeof(*item)); 577 rte_memcpy(addr, flow_items, n); 578 addr = (void *)(((uintptr_t)addr) + n); 579 } 580 /* Go deeper. */ 581 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 582 node->next) { 583 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 584 node->next); 585 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 586 rte_errno = E2BIG; 587 return -rte_errno; 588 } 589 stack[stack_pos] = next_node; 590 } else if (*(next_node + 1)) { 591 /* Follow up with the next possibility. */ 592 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 593 ++next_node); 594 } else if (!stack_pos) { 595 /* 596 * Completing the traverse over the different paths. 597 * The next_node is advanced to the terminator. 598 */ 599 ++next_node; 600 } else { 601 /* Move to the next path. */ 602 while (stack_pos) { 603 next_node = stack[--stack_pos]; 604 next_node++; 605 if (*next_node) 606 break; 607 } 608 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 609 next_node); 610 stack[stack_pos] = next_node; 611 } 612 node = next_node && *next_node ? &graph[*next_node] : NULL; 613 }; 614 return lsize; 615 } 616 617 /** 618 * Expand SQN flows into several possible flows according to the Tx queue 619 * number 620 * 621 * @param[in] buf 622 * Buffer to store the result expansion. 623 * @param[in] size 624 * Buffer size in bytes. If 0, @p buf can be NULL. 625 * @param[in] pattern 626 * User flow pattern. 627 * @param[in] sq_specs 628 * Buffer to store sq spec. 629 * 630 * @return 631 * 0 for success and negative value for failure 632 * 633 */ 634 static int 635 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 636 const struct rte_flow_item *pattern, 637 struct mlx5_rte_flow_item_sq *sq_specs) 638 { 639 const struct rte_flow_item *item; 640 bool port_representor = false; 641 size_t user_pattern_size = 0; 642 struct rte_eth_dev *dev; 643 struct mlx5_priv *priv; 644 void *addr = NULL; 645 uint16_t port_id; 646 size_t lsize; 647 int elt = 2; 648 uint16_t i; 649 650 buf->entries = 0; 651 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 652 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 653 const struct rte_flow_item_ethdev *pid_v = item->spec; 654 655 if (!pid_v) 656 return 0; 657 port_id = pid_v->port_id; 658 port_representor = true; 659 } 660 user_pattern_size += sizeof(*item); 661 } 662 if (!port_representor) 663 return 0; 664 dev = &rte_eth_devices[port_id]; 665 priv = dev->data->dev_private; 666 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 667 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 668 sizeof(buf->entry[0]) * priv->txqs_n; 669 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 670 return -EINVAL; 671 addr = buf->entry[0].pattern; 672 for (i = 0; i != priv->txqs_n; ++i) { 673 struct rte_flow_item pattern_add[] = { 674 { 675 .type = (enum rte_flow_item_type) 676 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 677 .spec = &sq_specs[i], 678 }, 679 { 680 .type = RTE_FLOW_ITEM_TYPE_END, 681 }, 682 }; 683 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 684 685 if (txq == NULL) 686 return -EINVAL; 687 buf->entry[i].pattern = addr; 688 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 689 mlx5_txq_release(dev, i); 690 rte_memcpy(addr, pattern, user_pattern_size); 691 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 692 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 693 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 694 buf->entries++; 695 } 696 return 0; 697 } 698 699 enum mlx5_expansion { 700 MLX5_EXPANSION_ROOT, 701 MLX5_EXPANSION_ROOT_OUTER, 702 MLX5_EXPANSION_OUTER_ETH, 703 MLX5_EXPANSION_OUTER_VLAN, 704 MLX5_EXPANSION_OUTER_IPV4, 705 MLX5_EXPANSION_OUTER_IPV4_UDP, 706 MLX5_EXPANSION_OUTER_IPV4_TCP, 707 MLX5_EXPANSION_OUTER_IPV4_ESP, 708 MLX5_EXPANSION_OUTER_IPV4_ICMP, 709 MLX5_EXPANSION_OUTER_IPV6, 710 MLX5_EXPANSION_OUTER_IPV6_UDP, 711 MLX5_EXPANSION_OUTER_IPV6_TCP, 712 MLX5_EXPANSION_OUTER_IPV6_ESP, 713 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 714 MLX5_EXPANSION_VXLAN, 715 MLX5_EXPANSION_STD_VXLAN, 716 MLX5_EXPANSION_L3_VXLAN, 717 MLX5_EXPANSION_VXLAN_GPE, 718 MLX5_EXPANSION_GRE, 719 MLX5_EXPANSION_NVGRE, 720 MLX5_EXPANSION_GRE_KEY, 721 MLX5_EXPANSION_MPLS, 722 MLX5_EXPANSION_ETH, 723 MLX5_EXPANSION_VLAN, 724 MLX5_EXPANSION_IPV4, 725 MLX5_EXPANSION_IPV4_UDP, 726 MLX5_EXPANSION_IPV4_TCP, 727 MLX5_EXPANSION_IPV4_ESP, 728 MLX5_EXPANSION_IPV4_ICMP, 729 MLX5_EXPANSION_IPV6, 730 MLX5_EXPANSION_IPV6_UDP, 731 MLX5_EXPANSION_IPV6_TCP, 732 MLX5_EXPANSION_IPV6_ESP, 733 MLX5_EXPANSION_IPV6_ICMP6, 734 MLX5_EXPANSION_IPV6_FRAG_EXT, 735 MLX5_EXPANSION_GTP, 736 MLX5_EXPANSION_GENEVE, 737 }; 738 739 /** Supported expansion of items. */ 740 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 741 [MLX5_EXPANSION_ROOT] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 743 MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_END, 746 }, 747 [MLX5_EXPANSION_ROOT_OUTER] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 749 MLX5_EXPANSION_OUTER_IPV4, 750 MLX5_EXPANSION_OUTER_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_END, 752 }, 753 [MLX5_EXPANSION_OUTER_ETH] = { 754 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 755 .type = RTE_FLOW_ITEM_TYPE_ETH, 756 .rss_types = 0, 757 }, 758 [MLX5_EXPANSION_OUTER_VLAN] = { 759 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 760 MLX5_EXPANSION_OUTER_IPV6), 761 .type = RTE_FLOW_ITEM_TYPE_VLAN, 762 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 763 }, 764 [MLX5_EXPANSION_OUTER_IPV4] = { 765 .next = MLX5_FLOW_EXPAND_RSS_NEXT 766 (MLX5_EXPANSION_OUTER_IPV4_UDP, 767 MLX5_EXPANSION_OUTER_IPV4_TCP, 768 MLX5_EXPANSION_OUTER_IPV4_ESP, 769 MLX5_EXPANSION_OUTER_IPV4_ICMP, 770 MLX5_EXPANSION_GRE, 771 MLX5_EXPANSION_NVGRE, 772 MLX5_EXPANSION_IPV4, 773 MLX5_EXPANSION_IPV6), 774 .type = RTE_FLOW_ITEM_TYPE_IPV4, 775 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 776 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 777 }, 778 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 779 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 780 MLX5_EXPANSION_VXLAN_GPE, 781 MLX5_EXPANSION_MPLS, 782 MLX5_EXPANSION_GENEVE, 783 MLX5_EXPANSION_GTP), 784 .type = RTE_FLOW_ITEM_TYPE_UDP, 785 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 786 }, 787 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 788 .type = RTE_FLOW_ITEM_TYPE_TCP, 789 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 790 }, 791 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 792 .type = RTE_FLOW_ITEM_TYPE_ESP, 793 .rss_types = RTE_ETH_RSS_ESP, 794 }, 795 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 796 .type = RTE_FLOW_ITEM_TYPE_ICMP, 797 }, 798 [MLX5_EXPANSION_OUTER_IPV6] = { 799 .next = MLX5_FLOW_EXPAND_RSS_NEXT 800 (MLX5_EXPANSION_OUTER_IPV6_UDP, 801 MLX5_EXPANSION_OUTER_IPV6_TCP, 802 MLX5_EXPANSION_OUTER_IPV6_ESP, 803 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 804 MLX5_EXPANSION_IPV4, 805 MLX5_EXPANSION_IPV6, 806 MLX5_EXPANSION_GRE, 807 MLX5_EXPANSION_NVGRE), 808 .type = RTE_FLOW_ITEM_TYPE_IPV6, 809 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 810 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 811 }, 812 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 813 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 814 MLX5_EXPANSION_VXLAN_GPE, 815 MLX5_EXPANSION_MPLS, 816 MLX5_EXPANSION_GENEVE, 817 MLX5_EXPANSION_GTP), 818 .type = RTE_FLOW_ITEM_TYPE_UDP, 819 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 820 }, 821 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 822 .type = RTE_FLOW_ITEM_TYPE_TCP, 823 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 824 }, 825 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 826 .type = RTE_FLOW_ITEM_TYPE_ESP, 827 .rss_types = RTE_ETH_RSS_ESP, 828 }, 829 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 830 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 831 }, 832 [MLX5_EXPANSION_VXLAN] = { 833 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 834 MLX5_EXPANSION_IPV4, 835 MLX5_EXPANSION_IPV6), 836 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 837 }, 838 [MLX5_EXPANSION_STD_VXLAN] = { 839 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 840 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 841 }, 842 [MLX5_EXPANSION_L3_VXLAN] = { 843 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 844 MLX5_EXPANSION_IPV6), 845 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 846 }, 847 [MLX5_EXPANSION_VXLAN_GPE] = { 848 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 849 MLX5_EXPANSION_IPV4, 850 MLX5_EXPANSION_IPV6), 851 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 852 }, 853 [MLX5_EXPANSION_GRE] = { 854 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 855 MLX5_EXPANSION_IPV4, 856 MLX5_EXPANSION_IPV6, 857 MLX5_EXPANSION_GRE_KEY, 858 MLX5_EXPANSION_MPLS), 859 .type = RTE_FLOW_ITEM_TYPE_GRE, 860 }, 861 [MLX5_EXPANSION_GRE_KEY] = { 862 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 863 MLX5_EXPANSION_IPV6, 864 MLX5_EXPANSION_MPLS), 865 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 866 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 867 }, 868 [MLX5_EXPANSION_NVGRE] = { 869 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 870 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 871 }, 872 [MLX5_EXPANSION_MPLS] = { 873 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 874 MLX5_EXPANSION_IPV6, 875 MLX5_EXPANSION_ETH), 876 .type = RTE_FLOW_ITEM_TYPE_MPLS, 877 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 878 }, 879 [MLX5_EXPANSION_ETH] = { 880 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 881 .type = RTE_FLOW_ITEM_TYPE_ETH, 882 }, 883 [MLX5_EXPANSION_VLAN] = { 884 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 885 MLX5_EXPANSION_IPV6), 886 .type = RTE_FLOW_ITEM_TYPE_VLAN, 887 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 888 }, 889 [MLX5_EXPANSION_IPV4] = { 890 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 891 MLX5_EXPANSION_IPV4_TCP, 892 MLX5_EXPANSION_IPV4_ESP, 893 MLX5_EXPANSION_IPV4_ICMP), 894 .type = RTE_FLOW_ITEM_TYPE_IPV4, 895 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 896 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 897 }, 898 [MLX5_EXPANSION_IPV4_UDP] = { 899 .type = RTE_FLOW_ITEM_TYPE_UDP, 900 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 901 }, 902 [MLX5_EXPANSION_IPV4_TCP] = { 903 .type = RTE_FLOW_ITEM_TYPE_TCP, 904 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 905 }, 906 [MLX5_EXPANSION_IPV4_ESP] = { 907 .type = RTE_FLOW_ITEM_TYPE_ESP, 908 .rss_types = RTE_ETH_RSS_ESP, 909 }, 910 [MLX5_EXPANSION_IPV4_ICMP] = { 911 .type = RTE_FLOW_ITEM_TYPE_ICMP, 912 }, 913 [MLX5_EXPANSION_IPV6] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 915 MLX5_EXPANSION_IPV6_TCP, 916 MLX5_EXPANSION_IPV6_ESP, 917 MLX5_EXPANSION_IPV6_ICMP6, 918 MLX5_EXPANSION_IPV6_FRAG_EXT), 919 .type = RTE_FLOW_ITEM_TYPE_IPV6, 920 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 921 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 922 }, 923 [MLX5_EXPANSION_IPV6_UDP] = { 924 .type = RTE_FLOW_ITEM_TYPE_UDP, 925 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 926 }, 927 [MLX5_EXPANSION_IPV6_TCP] = { 928 .type = RTE_FLOW_ITEM_TYPE_TCP, 929 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 930 }, 931 [MLX5_EXPANSION_IPV6_ESP] = { 932 .type = RTE_FLOW_ITEM_TYPE_ESP, 933 .rss_types = RTE_ETH_RSS_ESP, 934 }, 935 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 936 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 937 }, 938 [MLX5_EXPANSION_IPV6_ICMP6] = { 939 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 940 }, 941 [MLX5_EXPANSION_GTP] = { 942 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 943 MLX5_EXPANSION_IPV6), 944 .type = RTE_FLOW_ITEM_TYPE_GTP, 945 }, 946 [MLX5_EXPANSION_GENEVE] = { 947 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 948 MLX5_EXPANSION_IPV4, 949 MLX5_EXPANSION_IPV6), 950 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 951 }, 952 }; 953 954 static struct rte_flow_action_handle * 955 mlx5_action_handle_create(struct rte_eth_dev *dev, 956 const struct rte_flow_indir_action_conf *conf, 957 const struct rte_flow_action *action, 958 struct rte_flow_error *error); 959 static int mlx5_action_handle_destroy 960 (struct rte_eth_dev *dev, 961 struct rte_flow_action_handle *handle, 962 struct rte_flow_error *error); 963 static int mlx5_action_handle_update 964 (struct rte_eth_dev *dev, 965 struct rte_flow_action_handle *handle, 966 const void *update, 967 struct rte_flow_error *error); 968 static int mlx5_action_handle_query 969 (struct rte_eth_dev *dev, 970 const struct rte_flow_action_handle *handle, 971 void *data, 972 struct rte_flow_error *error); 973 static int 974 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 975 struct rte_flow_tunnel *app_tunnel, 976 struct rte_flow_action **actions, 977 uint32_t *num_of_actions, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 981 struct rte_flow_tunnel *app_tunnel, 982 struct rte_flow_item **items, 983 uint32_t *num_of_items, 984 struct rte_flow_error *error); 985 static int 986 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 987 struct rte_flow_item *pmd_items, 988 uint32_t num_items, struct rte_flow_error *err); 989 static int 990 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 991 struct rte_flow_action *pmd_actions, 992 uint32_t num_actions, 993 struct rte_flow_error *err); 994 static int 995 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 996 struct rte_mbuf *m, 997 struct rte_flow_restore_info *info, 998 struct rte_flow_error *err); 999 static struct rte_flow_item_flex_handle * 1000 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 1001 const struct rte_flow_item_flex_conf *conf, 1002 struct rte_flow_error *error); 1003 static int 1004 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 1005 const struct rte_flow_item_flex_handle *handle, 1006 struct rte_flow_error *error); 1007 static int 1008 mlx5_flow_info_get(struct rte_eth_dev *dev, 1009 struct rte_flow_port_info *port_info, 1010 struct rte_flow_queue_info *queue_info, 1011 struct rte_flow_error *error); 1012 static int 1013 mlx5_flow_port_configure(struct rte_eth_dev *dev, 1014 const struct rte_flow_port_attr *port_attr, 1015 uint16_t nb_queue, 1016 const struct rte_flow_queue_attr *queue_attr[], 1017 struct rte_flow_error *err); 1018 1019 static struct rte_flow_pattern_template * 1020 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 1021 const struct rte_flow_pattern_template_attr *attr, 1022 const struct rte_flow_item items[], 1023 struct rte_flow_error *error); 1024 1025 static int 1026 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1027 struct rte_flow_pattern_template *template, 1028 struct rte_flow_error *error); 1029 static struct rte_flow_actions_template * 1030 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1031 const struct rte_flow_actions_template_attr *attr, 1032 const struct rte_flow_action actions[], 1033 const struct rte_flow_action masks[], 1034 struct rte_flow_error *error); 1035 static int 1036 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1037 struct rte_flow_actions_template *template, 1038 struct rte_flow_error *error); 1039 1040 static struct rte_flow_template_table * 1041 mlx5_flow_table_create(struct rte_eth_dev *dev, 1042 const struct rte_flow_template_table_attr *attr, 1043 struct rte_flow_pattern_template *item_templates[], 1044 uint8_t nb_item_templates, 1045 struct rte_flow_actions_template *action_templates[], 1046 uint8_t nb_action_templates, 1047 struct rte_flow_error *error); 1048 static int 1049 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1050 struct rte_flow_template_table *table, 1051 struct rte_flow_error *error); 1052 static int 1053 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 1054 uint32_t group_id, 1055 const struct rte_flow_group_attr *attr, 1056 const struct rte_flow_action actions[], 1057 struct rte_flow_error *error); 1058 1059 static int 1060 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1061 struct rte_flow_action_handle *handle, 1062 const void *update, void *query, 1063 enum rte_flow_query_update_mode qu_mode, 1064 struct rte_flow_error *error); 1065 1066 static struct rte_flow_action_list_handle * 1067 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 1068 const struct rte_flow_indir_action_conf *conf, 1069 const struct rte_flow_action *actions, 1070 struct rte_flow_error *error); 1071 1072 static int 1073 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 1074 struct rte_flow_action_list_handle *handle, 1075 struct rte_flow_error *error); 1076 1077 static int 1078 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 1079 const 1080 struct rte_flow_action_list_handle *handle, 1081 const void **update, void **query, 1082 enum rte_flow_query_update_mode mode, 1083 struct rte_flow_error *error); 1084 1085 static int 1086 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 1087 const struct rte_flow_template_table *table, 1088 const struct rte_flow_item pattern[], 1089 uint8_t pattern_template_index, 1090 uint32_t *hash, struct rte_flow_error *error); 1091 static int 1092 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 1093 const struct rte_flow_item pattern[], 1094 enum rte_flow_encap_hash_field dest_field, 1095 uint8_t *hash, 1096 struct rte_flow_error *error); 1097 1098 static int 1099 mlx5_template_table_resize(struct rte_eth_dev *dev, 1100 struct rte_flow_template_table *table, 1101 uint32_t nb_rules, struct rte_flow_error *error); 1102 static int 1103 mlx5_flow_async_update_resized(struct rte_eth_dev *dev, uint32_t queue, 1104 const struct rte_flow_op_attr *attr, 1105 struct rte_flow *rule, void *user_data, 1106 struct rte_flow_error *error); 1107 static int 1108 mlx5_table_resize_complete(struct rte_eth_dev *dev, 1109 struct rte_flow_template_table *table, 1110 struct rte_flow_error *error); 1111 1112 static const struct rte_flow_ops mlx5_flow_ops = { 1113 .validate = mlx5_flow_validate, 1114 .create = mlx5_flow_create, 1115 .destroy = mlx5_flow_destroy, 1116 .flush = mlx5_flow_flush, 1117 .isolate = mlx5_flow_isolate, 1118 .query = mlx5_flow_query, 1119 .dev_dump = mlx5_flow_dev_dump, 1120 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1121 .get_aged_flows = mlx5_flow_get_aged_flows, 1122 .action_handle_create = mlx5_action_handle_create, 1123 .action_handle_destroy = mlx5_action_handle_destroy, 1124 .action_handle_update = mlx5_action_handle_update, 1125 .action_handle_query = mlx5_action_handle_query, 1126 .action_handle_query_update = mlx5_action_handle_query_update, 1127 .action_list_handle_create = mlx5_action_list_handle_create, 1128 .action_list_handle_destroy = mlx5_action_list_handle_destroy, 1129 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1130 .tunnel_match = mlx5_flow_tunnel_match, 1131 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1132 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1133 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1134 .flex_item_create = mlx5_flow_flex_item_create, 1135 .flex_item_release = mlx5_flow_flex_item_release, 1136 .info_get = mlx5_flow_info_get, 1137 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1138 .configure = mlx5_flow_port_configure, 1139 .pattern_template_create = mlx5_flow_pattern_template_create, 1140 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1141 .actions_template_create = mlx5_flow_actions_template_create, 1142 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1143 .template_table_create = mlx5_flow_table_create, 1144 .template_table_destroy = mlx5_flow_table_destroy, 1145 .group_set_miss_actions = mlx5_flow_group_set_miss_actions, 1146 .action_list_handle_query_update = 1147 mlx5_flow_action_list_handle_query_update, 1148 .flow_calc_table_hash = mlx5_flow_calc_table_hash, 1149 .flow_calc_encap_hash = mlx5_flow_calc_encap_hash, 1150 .flow_template_table_resize = mlx5_template_table_resize, 1151 .flow_update_resized = mlx5_flow_async_update_resized, 1152 .flow_template_table_resize_complete = mlx5_table_resize_complete, 1153 }; 1154 1155 /* Tunnel information. */ 1156 struct mlx5_flow_tunnel_info { 1157 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1158 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1159 }; 1160 1161 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1162 { 1163 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1164 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1165 }, 1166 { 1167 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1168 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1169 }, 1170 { 1171 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1172 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1173 }, 1174 { 1175 .tunnel = MLX5_FLOW_LAYER_GRE, 1176 .ptype = RTE_PTYPE_TUNNEL_GRE, 1177 }, 1178 { 1179 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1180 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1181 }, 1182 { 1183 .tunnel = MLX5_FLOW_LAYER_MPLS, 1184 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1185 }, 1186 { 1187 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1188 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1189 }, 1190 { 1191 .tunnel = MLX5_FLOW_LAYER_IPIP, 1192 .ptype = RTE_PTYPE_TUNNEL_IP, 1193 }, 1194 { 1195 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1196 .ptype = RTE_PTYPE_TUNNEL_IP, 1197 }, 1198 { 1199 .tunnel = MLX5_FLOW_LAYER_GTP, 1200 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1201 }, 1202 }; 1203 1204 1205 1206 /** 1207 * Translate tag ID to register. 1208 * 1209 * @param[in] dev 1210 * Pointer to the Ethernet device structure. 1211 * @param[in] feature 1212 * The feature that request the register. 1213 * @param[in] id 1214 * The request register ID. 1215 * @param[out] error 1216 * Error description in case of any. 1217 * 1218 * @return 1219 * The request register on success, a negative errno 1220 * value otherwise and rte_errno is set. 1221 */ 1222 int 1223 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1224 enum mlx5_feature_name feature, 1225 uint32_t id, 1226 struct rte_flow_error *error) 1227 { 1228 struct mlx5_priv *priv = dev->data->dev_private; 1229 struct mlx5_sh_config *config = &priv->sh->config; 1230 struct mlx5_dev_registers *reg = &priv->sh->registers; 1231 enum modify_reg start_reg; 1232 bool skip_mtr_reg = false; 1233 1234 switch (feature) { 1235 case MLX5_HAIRPIN_RX: 1236 return REG_B; 1237 case MLX5_HAIRPIN_TX: 1238 return REG_A; 1239 case MLX5_METADATA_RX: 1240 switch (config->dv_xmeta_en) { 1241 case MLX5_XMETA_MODE_LEGACY: 1242 return REG_B; 1243 case MLX5_XMETA_MODE_META16: 1244 return REG_C_0; 1245 case MLX5_XMETA_MODE_META32: 1246 return REG_C_1; 1247 case MLX5_XMETA_MODE_META32_HWS: 1248 return REG_C_1; 1249 } 1250 break; 1251 case MLX5_METADATA_TX: 1252 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1253 return REG_C_1; 1254 } else { 1255 return REG_A; 1256 } 1257 case MLX5_METADATA_FDB: 1258 switch (config->dv_xmeta_en) { 1259 case MLX5_XMETA_MODE_LEGACY: 1260 return REG_NON; 1261 case MLX5_XMETA_MODE_META16: 1262 return REG_C_0; 1263 case MLX5_XMETA_MODE_META32: 1264 return REG_C_1; 1265 case MLX5_XMETA_MODE_META32_HWS: 1266 return REG_C_1; 1267 } 1268 break; 1269 case MLX5_FLOW_MARK: 1270 switch (config->dv_xmeta_en) { 1271 case MLX5_XMETA_MODE_LEGACY: 1272 case MLX5_XMETA_MODE_META32_HWS: 1273 return REG_NON; 1274 case MLX5_XMETA_MODE_META16: 1275 return REG_C_1; 1276 case MLX5_XMETA_MODE_META32: 1277 return REG_C_0; 1278 } 1279 break; 1280 case MLX5_MTR_ID: 1281 /* 1282 * If meter color and meter id share one register, flow match 1283 * should use the meter color register for match. 1284 */ 1285 if (priv->mtr_reg_share) 1286 return reg->aso_reg; 1287 else 1288 return reg->aso_reg != REG_C_2 ? REG_C_2 : 1289 REG_C_3; 1290 case MLX5_MTR_COLOR: 1291 case MLX5_ASO_FLOW_HIT: 1292 case MLX5_ASO_CONNTRACK: 1293 case MLX5_SAMPLE_ID: 1294 /* All features use the same REG_C. */ 1295 MLX5_ASSERT(reg->aso_reg != REG_NON); 1296 return reg->aso_reg; 1297 case MLX5_COPY_MARK: 1298 /* 1299 * Metadata COPY_MARK register using is in meter suffix sub 1300 * flow while with meter. It's safe to share the same register. 1301 */ 1302 return reg->aso_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1303 case MLX5_APP_TAG: 1304 /* 1305 * If meter is enable, it will engage the register for color 1306 * match and flow match. If meter color match is not using the 1307 * REG_C_2, need to skip the REG_C_x be used by meter color 1308 * match. 1309 * If meter is disable, free to use all available registers. 1310 */ 1311 start_reg = reg->aso_reg != REG_C_2 ? REG_C_2 : 1312 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1313 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1314 if (id > (uint32_t)(REG_C_7 - start_reg)) 1315 return rte_flow_error_set(error, EINVAL, 1316 RTE_FLOW_ERROR_TYPE_ITEM, 1317 NULL, "invalid tag id"); 1318 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1319 return rte_flow_error_set(error, ENOTSUP, 1320 RTE_FLOW_ERROR_TYPE_ITEM, 1321 NULL, "unsupported tag id"); 1322 /* 1323 * This case means meter is using the REG_C_x great than 2. 1324 * Take care not to conflict with meter color REG_C_x. 1325 * If the available index REG_C_y >= REG_C_x, skip the 1326 * color register. 1327 */ 1328 if (skip_mtr_reg && priv->sh->flow_mreg_c 1329 [id + start_reg - REG_C_0] >= reg->aso_reg) { 1330 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1331 return rte_flow_error_set(error, EINVAL, 1332 RTE_FLOW_ERROR_TYPE_ITEM, 1333 NULL, "invalid tag id"); 1334 if (priv->sh->flow_mreg_c 1335 [id + 1 + start_reg - REG_C_0] != REG_NON) 1336 return priv->sh->flow_mreg_c 1337 [id + 1 + start_reg - REG_C_0]; 1338 return rte_flow_error_set(error, ENOTSUP, 1339 RTE_FLOW_ERROR_TYPE_ITEM, 1340 NULL, "unsupported tag id"); 1341 } 1342 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1343 } 1344 MLX5_ASSERT(false); 1345 return rte_flow_error_set(error, EINVAL, 1346 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1347 NULL, "invalid feature name"); 1348 } 1349 1350 /** 1351 * Check extensive flow metadata register support. 1352 * 1353 * @param dev 1354 * Pointer to rte_eth_dev structure. 1355 * 1356 * @return 1357 * True if device supports extensive flow metadata register, otherwise false. 1358 */ 1359 bool 1360 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1361 { 1362 struct mlx5_priv *priv = dev->data->dev_private; 1363 1364 /* 1365 * Having available reg_c can be regarded inclusively as supporting 1366 * extensive flow metadata register, which could mean, 1367 * - metadata register copy action by modify header. 1368 * - 16 modify header actions is supported. 1369 * - reg_c's are preserved across different domain (FDB and NIC) on 1370 * packet loopback by flow lookup miss. 1371 */ 1372 return priv->sh->flow_mreg_c[2] != REG_NON; 1373 } 1374 1375 /** 1376 * Get the lowest priority. 1377 * 1378 * @param[in] dev 1379 * Pointer to the Ethernet device structure. 1380 * @param[in] attributes 1381 * Pointer to device flow rule attributes. 1382 * 1383 * @return 1384 * The value of lowest priority of flow. 1385 */ 1386 uint32_t 1387 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1388 const struct rte_flow_attr *attr) 1389 { 1390 struct mlx5_priv *priv = dev->data->dev_private; 1391 1392 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1393 return priv->sh->flow_max_priority - 2; 1394 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1395 } 1396 1397 /** 1398 * Calculate matcher priority of the flow. 1399 * 1400 * @param[in] dev 1401 * Pointer to the Ethernet device structure. 1402 * @param[in] attr 1403 * Pointer to device flow rule attributes. 1404 * @param[in] subpriority 1405 * The priority based on the items. 1406 * @param[in] external 1407 * Flow is user flow. 1408 * @return 1409 * The matcher priority of the flow. 1410 */ 1411 uint16_t 1412 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1413 const struct rte_flow_attr *attr, 1414 uint32_t subpriority, bool external) 1415 { 1416 uint16_t priority = (uint16_t)attr->priority; 1417 struct mlx5_priv *priv = dev->data->dev_private; 1418 1419 /* NIC root rules */ 1420 if (!attr->group && !attr->transfer) { 1421 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1422 priority = priv->sh->flow_max_priority - 1; 1423 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1424 /* FDB root rules */ 1425 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1426 attr->group == 0 && 1427 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1428 return (priv->sh->flow_max_priority - 1) * 3; 1429 } 1430 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1431 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1432 return priority * 3 + subpriority; 1433 } 1434 1435 /** 1436 * Verify the @p item specifications (spec, last, mask) are compatible with the 1437 * NIC capabilities. 1438 * 1439 * @param[in] item 1440 * Item specification. 1441 * @param[in] mask 1442 * @p item->mask or flow default bit-masks. 1443 * @param[in] nic_mask 1444 * Bit-masks covering supported fields by the NIC to compare with user mask. 1445 * @param[in] size 1446 * Bit-masks size in bytes. 1447 * @param[in] range_accepted 1448 * True if range of values is accepted for specific fields, false otherwise. 1449 * @param[out] error 1450 * Pointer to error structure. 1451 * 1452 * @return 1453 * 0 on success, a negative errno value otherwise and rte_errno is set. 1454 */ 1455 int 1456 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1457 const uint8_t *mask, 1458 const uint8_t *nic_mask, 1459 unsigned int size, 1460 bool range_accepted, 1461 struct rte_flow_error *error) 1462 { 1463 unsigned int i; 1464 1465 MLX5_ASSERT(nic_mask); 1466 for (i = 0; i < size; ++i) 1467 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1468 return rte_flow_error_set(error, ENOTSUP, 1469 RTE_FLOW_ERROR_TYPE_ITEM, 1470 item, 1471 "mask enables non supported" 1472 " bits"); 1473 if (!item->spec && (item->mask || item->last)) 1474 return rte_flow_error_set(error, EINVAL, 1475 RTE_FLOW_ERROR_TYPE_ITEM, item, 1476 "mask/last without a spec is not" 1477 " supported"); 1478 if (item->spec && item->last && !range_accepted) { 1479 uint8_t spec[size]; 1480 uint8_t last[size]; 1481 unsigned int i; 1482 int ret; 1483 1484 for (i = 0; i < size; ++i) { 1485 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1486 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1487 } 1488 ret = memcmp(spec, last, size); 1489 if (ret != 0) 1490 return rte_flow_error_set(error, EINVAL, 1491 RTE_FLOW_ERROR_TYPE_ITEM, 1492 item, 1493 "range is not valid"); 1494 } 1495 return 0; 1496 } 1497 1498 /** 1499 * Adjust the hash fields according to the @p flow information. 1500 * 1501 * @param[in] dev_flow. 1502 * Pointer to the mlx5_flow. 1503 * @param[in] tunnel 1504 * 1 when the hash field is for a tunnel item. 1505 * @param[in] layer_types 1506 * RTE_ETH_RSS_* types. 1507 * @param[in] hash_fields 1508 * Item hash fields. 1509 * 1510 * @return 1511 * The hash fields that should be used. 1512 */ 1513 uint64_t 1514 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1515 int tunnel __rte_unused, uint64_t layer_types, 1516 uint64_t hash_fields) 1517 { 1518 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1519 int rss_request_inner = rss_desc->level >= 2; 1520 1521 /* Check RSS hash level for tunnel. */ 1522 if (tunnel && rss_request_inner) 1523 hash_fields |= IBV_RX_HASH_INNER; 1524 else if (tunnel || rss_request_inner) 1525 return 0; 1526 #endif 1527 /* Check if requested layer matches RSS hash fields. */ 1528 if (!(rss_desc->types & layer_types)) 1529 return 0; 1530 return hash_fields; 1531 } 1532 1533 /** 1534 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1535 * if several tunnel rules are used on this queue, the tunnel ptype will be 1536 * cleared. 1537 * 1538 * @param rxq_ctrl 1539 * Rx queue to update. 1540 */ 1541 static void 1542 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1543 { 1544 unsigned int i; 1545 uint32_t tunnel_ptype = 0; 1546 1547 /* Look up for the ptype to use. */ 1548 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1549 if (!rxq_ctrl->flow_tunnels_n[i]) 1550 continue; 1551 if (!tunnel_ptype) { 1552 tunnel_ptype = tunnels_info[i].ptype; 1553 } else { 1554 tunnel_ptype = 0; 1555 break; 1556 } 1557 } 1558 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1559 } 1560 1561 /** 1562 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1563 * flow. 1564 * 1565 * @param[in] dev 1566 * Pointer to the Ethernet device structure. 1567 * @param[in] dev_handle 1568 * Pointer to device flow handle structure. 1569 */ 1570 void 1571 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1572 struct mlx5_flow_handle *dev_handle) 1573 { 1574 struct mlx5_priv *priv = dev->data->dev_private; 1575 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1576 struct mlx5_ind_table_obj *ind_tbl = NULL; 1577 unsigned int i; 1578 1579 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1580 struct mlx5_hrxq *hrxq; 1581 1582 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1583 dev_handle->rix_hrxq); 1584 if (hrxq) 1585 ind_tbl = hrxq->ind_table; 1586 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1587 struct mlx5_shared_action_rss *shared_rss; 1588 1589 shared_rss = mlx5_ipool_get 1590 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1591 dev_handle->rix_srss); 1592 if (shared_rss) 1593 ind_tbl = shared_rss->ind_tbl; 1594 } 1595 if (!ind_tbl) 1596 return; 1597 for (i = 0; i != ind_tbl->queues_n; ++i) { 1598 int idx = ind_tbl->queues[i]; 1599 struct mlx5_rxq_ctrl *rxq_ctrl; 1600 1601 if (mlx5_is_external_rxq(dev, idx)) 1602 continue; 1603 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1604 MLX5_ASSERT(rxq_ctrl != NULL); 1605 if (rxq_ctrl == NULL) 1606 continue; 1607 /* 1608 * To support metadata register copy on Tx loopback, 1609 * this must be always enabled (metadata may arive 1610 * from other port - not from local flows only. 1611 */ 1612 if (tunnel) { 1613 unsigned int j; 1614 1615 /* Increase the counter matching the flow. */ 1616 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1617 if ((tunnels_info[j].tunnel & 1618 dev_handle->layers) == 1619 tunnels_info[j].tunnel) { 1620 rxq_ctrl->flow_tunnels_n[j]++; 1621 break; 1622 } 1623 } 1624 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1625 } 1626 } 1627 } 1628 1629 static void 1630 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1631 { 1632 struct mlx5_priv *priv = dev->data->dev_private; 1633 struct mlx5_rxq_ctrl *rxq_ctrl; 1634 uint16_t port_id; 1635 1636 if (priv->sh->shared_mark_enabled) 1637 return; 1638 if (priv->master || priv->representor) { 1639 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1640 struct mlx5_priv *opriv = 1641 rte_eth_devices[port_id].data->dev_private; 1642 1643 if (!opriv || 1644 opriv->sh != priv->sh || 1645 opriv->domain_id != priv->domain_id || 1646 opriv->mark_enabled) 1647 continue; 1648 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1649 rxq_ctrl->rxq.mark = 1; 1650 } 1651 opriv->mark_enabled = 1; 1652 } 1653 } else { 1654 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1655 rxq_ctrl->rxq.mark = 1; 1656 } 1657 priv->mark_enabled = 1; 1658 } 1659 priv->sh->shared_mark_enabled = 1; 1660 } 1661 1662 /** 1663 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1664 * 1665 * @param[in] dev 1666 * Pointer to the Ethernet device structure. 1667 * @param[in] flow 1668 * Pointer to flow structure. 1669 */ 1670 static void 1671 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1672 { 1673 struct mlx5_priv *priv = dev->data->dev_private; 1674 uint32_t handle_idx; 1675 struct mlx5_flow_handle *dev_handle; 1676 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1677 1678 MLX5_ASSERT(wks); 1679 if (wks->mark) 1680 flow_rxq_mark_flag_set(dev); 1681 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1682 handle_idx, dev_handle, next) 1683 flow_drv_rxq_flags_set(dev, dev_handle); 1684 } 1685 1686 /** 1687 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1688 * device flow if no other flow uses it with the same kind of request. 1689 * 1690 * @param dev 1691 * Pointer to Ethernet device. 1692 * @param[in] dev_handle 1693 * Pointer to the device flow handle structure. 1694 */ 1695 static void 1696 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1697 struct mlx5_flow_handle *dev_handle) 1698 { 1699 struct mlx5_priv *priv = dev->data->dev_private; 1700 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1701 struct mlx5_ind_table_obj *ind_tbl = NULL; 1702 unsigned int i; 1703 1704 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1705 struct mlx5_hrxq *hrxq; 1706 1707 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1708 dev_handle->rix_hrxq); 1709 if (hrxq) 1710 ind_tbl = hrxq->ind_table; 1711 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1712 struct mlx5_shared_action_rss *shared_rss; 1713 1714 shared_rss = mlx5_ipool_get 1715 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1716 dev_handle->rix_srss); 1717 if (shared_rss) 1718 ind_tbl = shared_rss->ind_tbl; 1719 } 1720 if (!ind_tbl) 1721 return; 1722 MLX5_ASSERT(dev->data->dev_started); 1723 for (i = 0; i != ind_tbl->queues_n; ++i) { 1724 int idx = ind_tbl->queues[i]; 1725 struct mlx5_rxq_ctrl *rxq_ctrl; 1726 1727 if (mlx5_is_external_rxq(dev, idx)) 1728 continue; 1729 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1730 MLX5_ASSERT(rxq_ctrl != NULL); 1731 if (rxq_ctrl == NULL) 1732 continue; 1733 if (tunnel) { 1734 unsigned int j; 1735 1736 /* Decrease the counter matching the flow. */ 1737 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1738 if ((tunnels_info[j].tunnel & 1739 dev_handle->layers) == 1740 tunnels_info[j].tunnel) { 1741 rxq_ctrl->flow_tunnels_n[j]--; 1742 break; 1743 } 1744 } 1745 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1746 } 1747 } 1748 } 1749 1750 /** 1751 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1752 * @p flow if no other flow uses it with the same kind of request. 1753 * 1754 * @param dev 1755 * Pointer to Ethernet device. 1756 * @param[in] flow 1757 * Pointer to the flow. 1758 */ 1759 static void 1760 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1761 { 1762 struct mlx5_priv *priv = dev->data->dev_private; 1763 uint32_t handle_idx; 1764 struct mlx5_flow_handle *dev_handle; 1765 1766 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1767 handle_idx, dev_handle, next) 1768 flow_drv_rxq_flags_trim(dev, dev_handle); 1769 } 1770 1771 /** 1772 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1773 * 1774 * @param dev 1775 * Pointer to Ethernet device. 1776 */ 1777 static void 1778 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1779 { 1780 struct mlx5_priv *priv = dev->data->dev_private; 1781 unsigned int i; 1782 1783 for (i = 0; i != priv->rxqs_n; ++i) { 1784 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1785 unsigned int j; 1786 1787 if (rxq == NULL || rxq->ctrl == NULL) 1788 continue; 1789 rxq->ctrl->rxq.mark = 0; 1790 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1791 rxq->ctrl->flow_tunnels_n[j] = 0; 1792 rxq->ctrl->rxq.tunnel = 0; 1793 } 1794 priv->mark_enabled = 0; 1795 priv->sh->shared_mark_enabled = 0; 1796 } 1797 1798 static uint64_t mlx5_restore_info_dynflag; 1799 1800 int 1801 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 1802 { 1803 struct mlx5_priv *priv = dev->data->dev_private; 1804 uint64_t supported = 0; 1805 1806 if (!is_tunnel_offload_active(dev)) { 1807 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 1808 supported |= RTE_ETH_RX_METADATA_USER_MARK; 1809 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) { 1810 DRV_LOG(DEBUG, 1811 "tunnel offload was not activated, consider setting dv_xmeta_en=%d", 1812 MLX5_XMETA_MODE_MISS_INFO); 1813 } 1814 } else { 1815 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 1816 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 && 1817 mlx5_restore_info_dynflag == 0) 1818 mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag(); 1819 } 1820 1821 if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) 1822 priv->tunnel_enabled = 1; 1823 else 1824 priv->tunnel_enabled = 0; 1825 1826 *features &= supported; 1827 return 0; 1828 } 1829 1830 /** 1831 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1832 * 1833 * @param[in] dev 1834 * Pointer to the Ethernet device structure. 1835 */ 1836 void 1837 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) 1838 { 1839 struct mlx5_priv *priv = dev->data->dev_private; 1840 uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID; 1841 unsigned int i; 1842 1843 if (priv->tunnel_enabled) 1844 mark_flag |= mlx5_restore_info_dynflag; 1845 1846 for (i = 0; i != priv->rxqs_n; ++i) { 1847 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1848 struct mlx5_rxq_data *data; 1849 1850 if (rxq == NULL || rxq->ctrl == NULL) 1851 continue; 1852 data = &rxq->ctrl->rxq; 1853 if (!rte_flow_dynf_metadata_avail()) { 1854 data->dynf_meta = 0; 1855 data->flow_meta_mask = 0; 1856 data->flow_meta_offset = -1; 1857 data->flow_meta_port_mask = 0; 1858 } else { 1859 data->dynf_meta = 1; 1860 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1861 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1862 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1863 } 1864 data->mark_flag = mark_flag; 1865 } 1866 } 1867 1868 /* 1869 * return a pointer to the desired action in the list of actions. 1870 * 1871 * @param[in] actions 1872 * The list of actions to search the action in. 1873 * @param[in] action 1874 * The action to find. 1875 * 1876 * @return 1877 * Pointer to the action in the list, if found. NULL otherwise. 1878 */ 1879 const struct rte_flow_action * 1880 mlx5_flow_find_action(const struct rte_flow_action *actions, 1881 enum rte_flow_action_type action) 1882 { 1883 if (actions == NULL) 1884 return NULL; 1885 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1886 if (actions->type == action) 1887 return actions; 1888 return NULL; 1889 } 1890 1891 /* 1892 * Validate the flag action. 1893 * 1894 * @param[in] action_flags 1895 * Bit-fields that holds the actions detected until now. 1896 * @param[in] attr 1897 * Attributes of flow that includes this action. 1898 * @param[out] error 1899 * Pointer to error structure. 1900 * 1901 * @return 1902 * 0 on success, a negative errno value otherwise and rte_errno is set. 1903 */ 1904 int 1905 mlx5_flow_validate_action_flag(uint64_t action_flags, 1906 const struct rte_flow_attr *attr, 1907 struct rte_flow_error *error) 1908 { 1909 if (action_flags & MLX5_FLOW_ACTION_MARK) 1910 return rte_flow_error_set(error, EINVAL, 1911 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1912 "can't mark and flag in same flow"); 1913 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1914 return rte_flow_error_set(error, EINVAL, 1915 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1916 "can't have 2 flag" 1917 " actions in same flow"); 1918 if (attr->egress) 1919 return rte_flow_error_set(error, ENOTSUP, 1920 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1921 "flag action not supported for " 1922 "egress"); 1923 return 0; 1924 } 1925 1926 /* 1927 * Validate the mark action. 1928 * 1929 * @param[in] action 1930 * Pointer to the queue action. 1931 * @param[in] action_flags 1932 * Bit-fields that holds the actions detected until now. 1933 * @param[in] attr 1934 * Attributes of flow that includes this action. 1935 * @param[out] error 1936 * Pointer to error structure. 1937 * 1938 * @return 1939 * 0 on success, a negative errno value otherwise and rte_errno is set. 1940 */ 1941 int 1942 mlx5_flow_validate_action_mark(struct rte_eth_dev *dev, 1943 const struct rte_flow_action *action, 1944 uint64_t action_flags, 1945 const struct rte_flow_attr *attr, 1946 struct rte_flow_error *error) 1947 { 1948 const struct rte_flow_action_mark *mark = action->conf; 1949 1950 if (!mark) 1951 return rte_flow_error_set(error, EINVAL, 1952 RTE_FLOW_ERROR_TYPE_ACTION, 1953 action, 1954 "configuration cannot be null"); 1955 if (mark->id >= MLX5_FLOW_MARK_MAX) 1956 return rte_flow_error_set(error, EINVAL, 1957 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1958 &mark->id, 1959 "mark id must in 0 <= id < " 1960 RTE_STR(MLX5_FLOW_MARK_MAX)); 1961 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1962 return rte_flow_error_set(error, EINVAL, 1963 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1964 "can't flag and mark in same flow"); 1965 if (action_flags & MLX5_FLOW_ACTION_MARK) 1966 return rte_flow_error_set(error, EINVAL, 1967 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1968 "can't have 2 mark actions in same" 1969 " flow"); 1970 if (attr->egress) 1971 return rte_flow_error_set(error, ENOTSUP, 1972 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1973 "mark action not supported for " 1974 "egress"); 1975 if (attr->transfer && mlx5_hws_active(dev)) 1976 return rte_flow_error_set(error, ENOTSUP, 1977 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1978 "non-template mark action not supported for transfer"); 1979 return 0; 1980 } 1981 1982 /* 1983 * Validate the drop action. 1984 * 1985 * @param[in] dev 1986 * Pointer to the Ethernet device structure. 1987 * @param[in] is_root 1988 * True if flow is validated for root table. False otherwise. 1989 * @param[in] attr 1990 * Attributes of flow that includes this action. 1991 * @param[out] error 1992 * Pointer to error structure. 1993 * 1994 * @return 1995 * 0 on success, a negative errno value otherwise and rte_errno is set. 1996 */ 1997 int 1998 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 1999 bool is_root, 2000 const struct rte_flow_attr *attr, 2001 struct rte_flow_error *error) 2002 { 2003 struct mlx5_priv *priv = dev->data->dev_private; 2004 2005 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 2006 return rte_flow_error_set(error, ENOTSUP, 2007 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2008 "drop action not supported for " 2009 "egress"); 2010 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 2011 !priv->sh->dr_root_drop_action_en) { 2012 return rte_flow_error_set(error, ENOTSUP, 2013 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 2014 "drop action not supported for " 2015 "egress and transfer on group 0"); 2016 } 2017 return 0; 2018 } 2019 2020 /* 2021 * Validate the queue action. 2022 * 2023 * @param[in] action 2024 * Pointer to the queue action. 2025 * @param[in] action_flags 2026 * Bit-fields that holds the actions detected until now. 2027 * @param[in] dev 2028 * Pointer to the Ethernet device structure. 2029 * @param[in] attr 2030 * Attributes of flow that includes this action. 2031 * @param[out] error 2032 * Pointer to error structure. 2033 * 2034 * @return 2035 * 0 on success, a negative errno value otherwise and rte_errno is set. 2036 */ 2037 int 2038 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 2039 uint64_t action_flags, 2040 struct rte_eth_dev *dev, 2041 const struct rte_flow_attr *attr, 2042 struct rte_flow_error *error) 2043 { 2044 struct mlx5_priv *priv = dev->data->dev_private; 2045 const struct rte_flow_action_queue *queue = action->conf; 2046 2047 if (!queue) 2048 return rte_flow_error_set(error, EINVAL, 2049 RTE_FLOW_ERROR_TYPE_ACTION, action, 2050 "no QUEUE action configuration"); 2051 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2052 return rte_flow_error_set(error, EINVAL, 2053 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2054 "can't have 2 fate actions in" 2055 " same flow"); 2056 if (attr->egress) 2057 return rte_flow_error_set(error, ENOTSUP, 2058 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2059 "queue action not supported for egress."); 2060 if (mlx5_is_external_rxq(dev, queue->index)) 2061 return 0; 2062 if (!priv->rxqs_n) 2063 return rte_flow_error_set(error, EINVAL, 2064 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2065 NULL, "No Rx queues configured"); 2066 if (queue->index >= priv->rxqs_n) 2067 return rte_flow_error_set(error, EINVAL, 2068 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2069 &queue->index, 2070 "queue index out of range"); 2071 if (mlx5_rxq_get(dev, queue->index) == NULL) 2072 return rte_flow_error_set(error, EINVAL, 2073 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2074 &queue->index, 2075 "queue is not configured"); 2076 return 0; 2077 } 2078 2079 /** 2080 * Validate queue numbers for device RSS. 2081 * 2082 * @param[in] dev 2083 * Configured device. 2084 * @param[in] queues 2085 * Array of queue numbers. 2086 * @param[in] queues_n 2087 * Size of the @p queues array. 2088 * @param[out] error 2089 * On error, filled with a textual error description. 2090 * @param[out] queue_idx 2091 * On error, filled with an offending queue index in @p queues array. 2092 * 2093 * @return 2094 * 0 on success, a negative errno code on error. 2095 */ 2096 static int 2097 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2098 const uint16_t *queues, uint32_t queues_n, 2099 const char **error, uint32_t *queue_idx) 2100 { 2101 const struct mlx5_priv *priv = dev->data->dev_private; 2102 bool is_hairpin = false; 2103 bool is_ext_rss = false; 2104 uint32_t i; 2105 2106 for (i = 0; i != queues_n; ++i) { 2107 struct mlx5_rxq_ctrl *rxq_ctrl; 2108 2109 if (mlx5_is_external_rxq(dev, queues[0])) { 2110 is_ext_rss = true; 2111 continue; 2112 } 2113 if (is_ext_rss) { 2114 *error = "Combining external and regular RSS queues is not supported"; 2115 *queue_idx = i; 2116 return -ENOTSUP; 2117 } 2118 if (queues[i] >= priv->rxqs_n) { 2119 *error = "queue index out of range"; 2120 *queue_idx = i; 2121 return -EINVAL; 2122 } 2123 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2124 if (rxq_ctrl == NULL) { 2125 *error = "queue is not configured"; 2126 *queue_idx = i; 2127 return -EINVAL; 2128 } 2129 if (i == 0 && rxq_ctrl->is_hairpin) 2130 is_hairpin = true; 2131 if (is_hairpin != rxq_ctrl->is_hairpin) { 2132 *error = "combining hairpin and regular RSS queues is not supported"; 2133 *queue_idx = i; 2134 return -ENOTSUP; 2135 } 2136 } 2137 return 0; 2138 } 2139 2140 /* 2141 * Validate the rss action. 2142 * 2143 * @param[in] dev 2144 * Pointer to the Ethernet device structure. 2145 * @param[in] action 2146 * Pointer to the queue action. 2147 * @param[out] error 2148 * Pointer to error structure. 2149 * 2150 * @return 2151 * 0 on success, a negative errno value otherwise and rte_errno is set. 2152 */ 2153 int 2154 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2155 const struct rte_flow_action *action, 2156 struct rte_flow_error *error) 2157 { 2158 struct mlx5_priv *priv = dev->data->dev_private; 2159 const struct rte_flow_action_rss *rss = action->conf; 2160 int ret; 2161 const char *message; 2162 uint32_t queue_idx; 2163 2164 if (!rss) 2165 return rte_flow_error_set 2166 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 2167 action, "no RSS action configuration"); 2168 if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 2169 DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT", 2170 dev->data->port_id); 2171 } else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2172 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2173 return rte_flow_error_set(error, ENOTSUP, 2174 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2175 &rss->func, 2176 "RSS hash function not supported"); 2177 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2178 if (rss->level > 2) 2179 #else 2180 if (rss->level > 1) 2181 #endif 2182 return rte_flow_error_set(error, ENOTSUP, 2183 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2184 &rss->level, 2185 "tunnel RSS is not supported"); 2186 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2187 if (rss->key_len == 0 && rss->key != NULL) 2188 return rte_flow_error_set(error, ENOTSUP, 2189 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2190 &rss->key_len, 2191 "RSS hash key length 0"); 2192 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2193 return rte_flow_error_set(error, ENOTSUP, 2194 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2195 &rss->key_len, 2196 "RSS hash key too small"); 2197 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2198 return rte_flow_error_set(error, ENOTSUP, 2199 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2200 &rss->key_len, 2201 "RSS hash key too large"); 2202 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2203 return rte_flow_error_set(error, ENOTSUP, 2204 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2205 &rss->queue_num, 2206 "number of queues too large"); 2207 if (rss->types & MLX5_RSS_HF_MASK) 2208 return rte_flow_error_set(error, ENOTSUP, 2209 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2210 &rss->types, 2211 "some RSS protocols are not" 2212 " supported"); 2213 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2214 !(rss->types & RTE_ETH_RSS_IP)) 2215 return rte_flow_error_set(error, EINVAL, 2216 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2217 "L3 partial RSS requested but L3 RSS" 2218 " type not specified"); 2219 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2220 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2221 return rte_flow_error_set(error, EINVAL, 2222 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2223 "L4 partial RSS requested but L4 RSS" 2224 " type not specified"); 2225 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2226 return rte_flow_error_set(error, EINVAL, 2227 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2228 NULL, "No Rx queues configured"); 2229 if (!rss->queue_num) 2230 return rte_flow_error_set(error, EINVAL, 2231 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2232 NULL, "No queues configured"); 2233 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2234 &message, &queue_idx); 2235 if (ret != 0) { 2236 return rte_flow_error_set(error, -ret, 2237 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2238 &rss->queue[queue_idx], message); 2239 } 2240 return 0; 2241 } 2242 2243 /* 2244 * Validate the rss action. 2245 * 2246 * @param[in] action 2247 * Pointer to the queue action. 2248 * @param[in] action_flags 2249 * Bit-fields that holds the actions detected until now. 2250 * @param[in] dev 2251 * Pointer to the Ethernet device structure. 2252 * @param[in] attr 2253 * Attributes of flow that includes this action. 2254 * @param[in] item_flags 2255 * Items that were detected. 2256 * @param[out] error 2257 * Pointer to error structure. 2258 * 2259 * @return 2260 * 0 on success, a negative errno value otherwise and rte_errno is set. 2261 */ 2262 int 2263 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2264 uint64_t action_flags, 2265 struct rte_eth_dev *dev, 2266 const struct rte_flow_attr *attr, 2267 uint64_t item_flags, 2268 struct rte_flow_error *error) 2269 { 2270 const struct rte_flow_action_rss *rss = action->conf; 2271 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2272 int ret; 2273 2274 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2275 return rte_flow_error_set(error, EINVAL, 2276 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2277 "can't have 2 fate actions" 2278 " in same flow"); 2279 ret = mlx5_validate_action_rss(dev, action, error); 2280 if (ret) 2281 return ret; 2282 if (attr->egress) 2283 return rte_flow_error_set(error, ENOTSUP, 2284 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2285 "rss action not supported for " 2286 "egress"); 2287 if (rss->level > 1 && !tunnel) 2288 return rte_flow_error_set(error, EINVAL, 2289 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2290 "inner RSS is not supported for " 2291 "non-tunnel flows"); 2292 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2293 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2294 return rte_flow_error_set(error, EINVAL, 2295 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2296 "RSS on eCPRI is not supported now"); 2297 } 2298 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2299 !(item_flags & 2300 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2301 rss->level > 1) 2302 return rte_flow_error_set(error, EINVAL, 2303 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2304 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2305 return 0; 2306 } 2307 2308 /* 2309 * Validate the default miss action. 2310 * 2311 * @param[in] action_flags 2312 * Bit-fields that holds the actions detected until now. 2313 * @param[out] error 2314 * Pointer to error structure. 2315 * 2316 * @return 2317 * 0 on success, a negative errno value otherwise and rte_errno is set. 2318 */ 2319 int 2320 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2321 const struct rte_flow_attr *attr, 2322 struct rte_flow_error *error) 2323 { 2324 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2325 return rte_flow_error_set(error, EINVAL, 2326 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2327 "can't have 2 fate actions in" 2328 " same flow"); 2329 if (attr->egress) 2330 return rte_flow_error_set(error, ENOTSUP, 2331 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2332 "default miss action not supported " 2333 "for egress"); 2334 if (attr->group) 2335 return rte_flow_error_set(error, ENOTSUP, 2336 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2337 "only group 0 is supported"); 2338 if (attr->transfer) 2339 return rte_flow_error_set(error, ENOTSUP, 2340 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2341 NULL, "transfer is not supported"); 2342 return 0; 2343 } 2344 2345 /* 2346 * Validate the count action. 2347 * 2348 * @param[in] dev 2349 * Pointer to the Ethernet device structure. 2350 * @param[in] attr 2351 * Attributes of flow that includes this action. 2352 * @param[out] error 2353 * Pointer to error structure. 2354 * 2355 * @return 2356 * 0 on success, a negative errno value otherwise and rte_errno is set. 2357 */ 2358 int 2359 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2360 const struct rte_flow_attr *attr, 2361 struct rte_flow_error *error) 2362 { 2363 if (attr->egress) 2364 return rte_flow_error_set(error, ENOTSUP, 2365 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2366 "count action not supported for " 2367 "egress"); 2368 return 0; 2369 } 2370 2371 /* 2372 * Validate the ASO CT action. 2373 * 2374 * @param[in] dev 2375 * Pointer to the Ethernet device structure. 2376 * @param[in] conntrack 2377 * Pointer to the CT action profile. 2378 * @param[out] error 2379 * Pointer to error structure. 2380 * 2381 * @return 2382 * 0 on success, a negative errno value otherwise and rte_errno is set. 2383 */ 2384 int 2385 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2386 const struct rte_flow_action_conntrack *conntrack, 2387 struct rte_flow_error *error) 2388 { 2389 RTE_SET_USED(dev); 2390 2391 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2392 return rte_flow_error_set(error, EINVAL, 2393 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2394 "Invalid CT state"); 2395 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2396 return rte_flow_error_set(error, EINVAL, 2397 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2398 "Invalid last TCP packet flag"); 2399 return 0; 2400 } 2401 2402 /** 2403 * Validate the level value for modify field action. 2404 * 2405 * @param[in] data 2406 * Pointer to the rte_flow_field_data structure either src or dst. 2407 * @param[out] error 2408 * Pointer to error structure. 2409 * 2410 * @return 2411 * 0 on success, a negative errno value otherwise and rte_errno is set. 2412 */ 2413 int 2414 flow_validate_modify_field_level(const struct rte_flow_field_data *data, 2415 struct rte_flow_error *error) 2416 { 2417 if (data->level == 0 || data->field == RTE_FLOW_FIELD_FLEX_ITEM) 2418 return 0; 2419 if (data->field != RTE_FLOW_FIELD_TAG && 2420 data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) { 2421 if (data->level > 1) 2422 return rte_flow_error_set(error, ENOTSUP, 2423 RTE_FLOW_ERROR_TYPE_ACTION, 2424 NULL, 2425 "inner header fields modification is not supported"); 2426 return 0; 2427 } 2428 if (data->tag_index != 0) 2429 return rte_flow_error_set(error, EINVAL, 2430 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2431 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2432 /* 2433 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2434 * 'tag_index' field. In old API, it was provided using 'level' field 2435 * and it is still supported for backwards compatibility. 2436 */ 2437 DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field."); 2438 return 0; 2439 } 2440 2441 /** 2442 * Validate ICMP6 item. 2443 * 2444 * @param[in] item 2445 * Item specification. 2446 * @param[in] item_flags 2447 * Bit-fields that holds the items detected until now. 2448 * @param[in] ext_vlan_sup 2449 * Whether extended VLAN features are supported or not. 2450 * @param[out] error 2451 * Pointer to error structure. 2452 * 2453 * @return 2454 * 0 on success, a negative errno value otherwise and rte_errno is set. 2455 */ 2456 int 2457 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2458 uint64_t item_flags, 2459 uint8_t target_protocol, 2460 struct rte_flow_error *error) 2461 { 2462 const struct rte_flow_item_icmp6 *mask = item->mask; 2463 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2464 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2465 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2466 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2467 MLX5_FLOW_LAYER_OUTER_L4; 2468 int ret; 2469 2470 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2471 return rte_flow_error_set(error, EINVAL, 2472 RTE_FLOW_ERROR_TYPE_ITEM, item, 2473 "protocol filtering not compatible" 2474 " with ICMP6 layer"); 2475 if (!(item_flags & l3m)) 2476 return rte_flow_error_set(error, EINVAL, 2477 RTE_FLOW_ERROR_TYPE_ITEM, item, 2478 "IPv6 is mandatory to filter on" 2479 " ICMP6"); 2480 if (item_flags & l4m) 2481 return rte_flow_error_set(error, EINVAL, 2482 RTE_FLOW_ERROR_TYPE_ITEM, item, 2483 "multiple L4 layers not supported"); 2484 if (!mask) 2485 mask = &rte_flow_item_icmp6_mask; 2486 ret = mlx5_flow_item_acceptable 2487 (item, (const uint8_t *)mask, 2488 (const uint8_t *)&rte_flow_item_icmp6_mask, 2489 sizeof(struct rte_flow_item_icmp6), 2490 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2491 if (ret < 0) 2492 return ret; 2493 return 0; 2494 } 2495 2496 /** 2497 * Validate ICMP6 echo request/reply item. 2498 * 2499 * @param[in] item 2500 * Item specification. 2501 * @param[in] item_flags 2502 * Bit-fields that holds the items detected until now. 2503 * @param[in] ext_vlan_sup 2504 * Whether extended VLAN features are supported or not. 2505 * @param[out] error 2506 * Pointer to error structure. 2507 * 2508 * @return 2509 * 0 on success, a negative errno value otherwise and rte_errno is set. 2510 */ 2511 int 2512 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2513 uint64_t item_flags, 2514 uint8_t target_protocol, 2515 struct rte_flow_error *error) 2516 { 2517 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2518 const struct rte_flow_item_icmp6_echo nic_mask = { 2519 .hdr.base.type = 0xff, 2520 .hdr.base.code = 0xff, 2521 .hdr.identifier = RTE_BE16(0xffff), 2522 .hdr.sequence = RTE_BE16(0xffff), 2523 }; 2524 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2525 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2526 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2527 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2528 MLX5_FLOW_LAYER_OUTER_L4; 2529 int ret; 2530 2531 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2532 return rte_flow_error_set(error, EINVAL, 2533 RTE_FLOW_ERROR_TYPE_ITEM, item, 2534 "protocol filtering not compatible" 2535 " with ICMP6 layer"); 2536 if (!(item_flags & l3m)) 2537 return rte_flow_error_set(error, EINVAL, 2538 RTE_FLOW_ERROR_TYPE_ITEM, item, 2539 "IPv6 is mandatory to filter on" 2540 " ICMP6"); 2541 if (item_flags & l4m) 2542 return rte_flow_error_set(error, EINVAL, 2543 RTE_FLOW_ERROR_TYPE_ITEM, item, 2544 "multiple L4 layers not supported"); 2545 if (!mask) 2546 mask = &nic_mask; 2547 ret = mlx5_flow_item_acceptable 2548 (item, (const uint8_t *)mask, 2549 (const uint8_t *)&nic_mask, 2550 sizeof(struct rte_flow_item_icmp6_echo), 2551 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2552 if (ret < 0) 2553 return ret; 2554 return 0; 2555 } 2556 2557 /** 2558 * Validate ICMP item. 2559 * 2560 * @param[in] item 2561 * Item specification. 2562 * @param[in] item_flags 2563 * Bit-fields that holds the items detected until now. 2564 * @param[out] error 2565 * Pointer to error structure. 2566 * 2567 * @return 2568 * 0 on success, a negative errno value otherwise and rte_errno is set. 2569 */ 2570 int 2571 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2572 uint64_t item_flags, 2573 uint8_t target_protocol, 2574 struct rte_flow_error *error) 2575 { 2576 const struct rte_flow_item_icmp *mask = item->mask; 2577 const struct rte_flow_item_icmp nic_mask = { 2578 .hdr.icmp_type = 0xff, 2579 .hdr.icmp_code = 0xff, 2580 .hdr.icmp_ident = RTE_BE16(0xffff), 2581 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2582 }; 2583 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2584 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2585 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2586 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2587 MLX5_FLOW_LAYER_OUTER_L4; 2588 int ret; 2589 2590 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2591 return rte_flow_error_set(error, EINVAL, 2592 RTE_FLOW_ERROR_TYPE_ITEM, item, 2593 "protocol filtering not compatible" 2594 " with ICMP layer"); 2595 if (!(item_flags & l3m)) 2596 return rte_flow_error_set(error, EINVAL, 2597 RTE_FLOW_ERROR_TYPE_ITEM, item, 2598 "IPv4 is mandatory to filter" 2599 " on ICMP"); 2600 if (item_flags & l4m) 2601 return rte_flow_error_set(error, EINVAL, 2602 RTE_FLOW_ERROR_TYPE_ITEM, item, 2603 "multiple L4 layers not supported"); 2604 if (!mask) 2605 mask = &nic_mask; 2606 ret = mlx5_flow_item_acceptable 2607 (item, (const uint8_t *)mask, 2608 (const uint8_t *)&nic_mask, 2609 sizeof(struct rte_flow_item_icmp), 2610 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2611 if (ret < 0) 2612 return ret; 2613 return 0; 2614 } 2615 2616 /** 2617 * Validate Ethernet item. 2618 * 2619 * @param[in] item 2620 * Item specification. 2621 * @param[in] item_flags 2622 * Bit-fields that holds the items detected until now. 2623 * @param[out] error 2624 * Pointer to error structure. 2625 * 2626 * @return 2627 * 0 on success, a negative errno value otherwise and rte_errno is set. 2628 */ 2629 int 2630 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2631 uint64_t item_flags, bool ext_vlan_sup, 2632 struct rte_flow_error *error) 2633 { 2634 const struct rte_flow_item_eth *mask = item->mask; 2635 const struct rte_flow_item_eth nic_mask = { 2636 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2637 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2638 .hdr.ether_type = RTE_BE16(0xffff), 2639 .has_vlan = ext_vlan_sup ? 1 : 0, 2640 }; 2641 int ret; 2642 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2643 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2644 MLX5_FLOW_LAYER_OUTER_L2; 2645 2646 if (item_flags & ethm) 2647 return rte_flow_error_set(error, ENOTSUP, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "multiple L2 layers not supported"); 2650 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2651 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2652 return rte_flow_error_set(error, EINVAL, 2653 RTE_FLOW_ERROR_TYPE_ITEM, item, 2654 "L2 layer should not follow " 2655 "L3 layers"); 2656 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2657 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2658 return rte_flow_error_set(error, EINVAL, 2659 RTE_FLOW_ERROR_TYPE_ITEM, item, 2660 "L2 layer should not follow VLAN"); 2661 if (item_flags & MLX5_FLOW_LAYER_GTP) 2662 return rte_flow_error_set(error, EINVAL, 2663 RTE_FLOW_ERROR_TYPE_ITEM, item, 2664 "L2 layer should not follow GTP"); 2665 if (!mask) 2666 mask = &rte_flow_item_eth_mask; 2667 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2668 (const uint8_t *)&nic_mask, 2669 sizeof(struct rte_flow_item_eth), 2670 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2671 return ret; 2672 } 2673 2674 /** 2675 * Validate VLAN item. 2676 * 2677 * @param[in] item 2678 * Item specification. 2679 * @param[in] item_flags 2680 * Bit-fields that holds the items detected until now. 2681 * @param[in] dev 2682 * Ethernet device flow is being created on. 2683 * @param[out] error 2684 * Pointer to error structure. 2685 * 2686 * @return 2687 * 0 on success, a negative errno value otherwise and rte_errno is set. 2688 */ 2689 int 2690 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2691 uint64_t item_flags, 2692 struct rte_eth_dev *dev, 2693 struct rte_flow_error *error) 2694 { 2695 const struct rte_flow_item_vlan *spec = item->spec; 2696 const struct rte_flow_item_vlan *mask = item->mask; 2697 const struct rte_flow_item_vlan nic_mask = { 2698 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2699 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2700 }; 2701 uint16_t vlan_tag = 0; 2702 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2703 int ret; 2704 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2705 MLX5_FLOW_LAYER_INNER_L4) : 2706 (MLX5_FLOW_LAYER_OUTER_L3 | 2707 MLX5_FLOW_LAYER_OUTER_L4); 2708 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2709 MLX5_FLOW_LAYER_OUTER_VLAN; 2710 2711 if (item_flags & vlanm) 2712 return rte_flow_error_set(error, EINVAL, 2713 RTE_FLOW_ERROR_TYPE_ITEM, item, 2714 "multiple VLAN layers not supported"); 2715 else if ((item_flags & l34m) != 0) 2716 return rte_flow_error_set(error, EINVAL, 2717 RTE_FLOW_ERROR_TYPE_ITEM, item, 2718 "VLAN cannot follow L3/L4 layer"); 2719 if (!mask) 2720 mask = &rte_flow_item_vlan_mask; 2721 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2722 (const uint8_t *)&nic_mask, 2723 sizeof(struct rte_flow_item_vlan), 2724 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2725 if (ret) 2726 return ret; 2727 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2728 struct mlx5_priv *priv = dev->data->dev_private; 2729 2730 if (priv->vmwa_context) { 2731 /* 2732 * Non-NULL context means we have a virtual machine 2733 * and SR-IOV enabled, we have to create VLAN interface 2734 * to make hypervisor to setup E-Switch vport 2735 * context correctly. We avoid creating the multiple 2736 * VLAN interfaces, so we cannot support VLAN tag mask. 2737 */ 2738 return rte_flow_error_set(error, EINVAL, 2739 RTE_FLOW_ERROR_TYPE_ITEM, 2740 item, 2741 "VLAN tag mask is not" 2742 " supported in virtual" 2743 " environment"); 2744 } 2745 } 2746 if (spec) { 2747 vlan_tag = spec->hdr.vlan_tci; 2748 vlan_tag &= mask->hdr.vlan_tci; 2749 } 2750 /* 2751 * From verbs perspective an empty VLAN is equivalent 2752 * to a packet without VLAN layer. 2753 */ 2754 if (!vlan_tag) 2755 return rte_flow_error_set(error, EINVAL, 2756 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2757 item->spec, 2758 "VLAN cannot be empty"); 2759 return 0; 2760 } 2761 2762 /** 2763 * Validate IPV4 item. 2764 * 2765 * @param[in] item 2766 * Item specification. 2767 * @param[in] item_flags 2768 * Bit-fields that holds the items detected until now. 2769 * @param[in] last_item 2770 * Previous validated item in the pattern items. 2771 * @param[in] ether_type 2772 * Type in the ethernet layer header (including dot1q). 2773 * @param[in] acc_mask 2774 * Acceptable mask, if NULL default internal default mask 2775 * will be used to check whether item fields are supported. 2776 * @param[in] range_accepted 2777 * True if range of values is accepted for specific fields, false otherwise. 2778 * @param[out] error 2779 * Pointer to error structure. 2780 * 2781 * @return 2782 * 0 on success, a negative errno value otherwise and rte_errno is set. 2783 */ 2784 int 2785 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2786 uint64_t item_flags, 2787 uint64_t last_item, 2788 uint16_t ether_type, 2789 const struct rte_flow_item_ipv4 *acc_mask, 2790 bool range_accepted, 2791 struct rte_flow_error *error) 2792 { 2793 const struct rte_flow_item_ipv4 *mask = item->mask; 2794 const struct rte_flow_item_ipv4 *spec = item->spec; 2795 const struct rte_flow_item_ipv4 nic_mask = { 2796 .hdr = { 2797 .src_addr = RTE_BE32(0xffffffff), 2798 .dst_addr = RTE_BE32(0xffffffff), 2799 .type_of_service = 0xff, 2800 .next_proto_id = 0xff, 2801 }, 2802 }; 2803 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2804 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2805 MLX5_FLOW_LAYER_OUTER_L3; 2806 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2807 MLX5_FLOW_LAYER_OUTER_L4; 2808 int ret; 2809 uint8_t next_proto = 0xFF; 2810 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2811 MLX5_FLOW_LAYER_OUTER_VLAN | 2812 MLX5_FLOW_LAYER_INNER_VLAN); 2813 2814 if ((last_item & l2_vlan) && ether_type && 2815 ether_type != RTE_ETHER_TYPE_IPV4) 2816 return rte_flow_error_set(error, EINVAL, 2817 RTE_FLOW_ERROR_TYPE_ITEM, item, 2818 "IPv4 cannot follow L2/VLAN layer " 2819 "which ether type is not IPv4"); 2820 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2821 if (mask && spec) 2822 next_proto = mask->hdr.next_proto_id & 2823 spec->hdr.next_proto_id; 2824 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2825 return rte_flow_error_set(error, EINVAL, 2826 RTE_FLOW_ERROR_TYPE_ITEM, 2827 item, 2828 "multiple tunnel " 2829 "not supported"); 2830 } 2831 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2832 return rte_flow_error_set(error, EINVAL, 2833 RTE_FLOW_ERROR_TYPE_ITEM, item, 2834 "wrong tunnel type - IPv6 specified " 2835 "but IPv4 item provided"); 2836 if (item_flags & l3m) 2837 return rte_flow_error_set(error, ENOTSUP, 2838 RTE_FLOW_ERROR_TYPE_ITEM, item, 2839 "multiple L3 layers not supported"); 2840 else if (item_flags & l4m) 2841 return rte_flow_error_set(error, EINVAL, 2842 RTE_FLOW_ERROR_TYPE_ITEM, item, 2843 "L3 cannot follow an L4 layer."); 2844 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2845 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2846 return rte_flow_error_set(error, EINVAL, 2847 RTE_FLOW_ERROR_TYPE_ITEM, item, 2848 "L3 cannot follow an NVGRE layer."); 2849 if (!mask) 2850 mask = &rte_flow_item_ipv4_mask; 2851 else if (mask->hdr.next_proto_id != 0 && 2852 mask->hdr.next_proto_id != 0xff) 2853 return rte_flow_error_set(error, EINVAL, 2854 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2855 "partial mask is not supported" 2856 " for protocol"); 2857 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2858 acc_mask ? (const uint8_t *)acc_mask 2859 : (const uint8_t *)&nic_mask, 2860 sizeof(struct rte_flow_item_ipv4), 2861 range_accepted, error); 2862 if (ret < 0) 2863 return ret; 2864 return 0; 2865 } 2866 2867 /** 2868 * Validate IPV6 item. 2869 * 2870 * @param[in] item 2871 * Item specification. 2872 * @param[in] item_flags 2873 * Bit-fields that holds the items detected until now. 2874 * @param[in] last_item 2875 * Previous validated item in the pattern items. 2876 * @param[in] ether_type 2877 * Type in the ethernet layer header (including dot1q). 2878 * @param[in] acc_mask 2879 * Acceptable mask, if NULL default internal default mask 2880 * will be used to check whether item fields are supported. 2881 * @param[out] error 2882 * Pointer to error structure. 2883 * 2884 * @return 2885 * 0 on success, a negative errno value otherwise and rte_errno is set. 2886 */ 2887 int 2888 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2889 uint64_t item_flags, 2890 uint64_t last_item, 2891 uint16_t ether_type, 2892 const struct rte_flow_item_ipv6 *acc_mask, 2893 struct rte_flow_error *error) 2894 { 2895 const struct rte_flow_item_ipv6 *mask = item->mask; 2896 const struct rte_flow_item_ipv6 *spec = item->spec; 2897 const struct rte_flow_item_ipv6 nic_mask = { 2898 .hdr = { 2899 .src_addr = 2900 "\xff\xff\xff\xff\xff\xff\xff\xff" 2901 "\xff\xff\xff\xff\xff\xff\xff\xff", 2902 .dst_addr = 2903 "\xff\xff\xff\xff\xff\xff\xff\xff" 2904 "\xff\xff\xff\xff\xff\xff\xff\xff", 2905 .vtc_flow = RTE_BE32(0xffffffff), 2906 .proto = 0xff, 2907 }, 2908 }; 2909 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2910 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2911 MLX5_FLOW_LAYER_OUTER_L3; 2912 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2913 MLX5_FLOW_LAYER_OUTER_L4; 2914 int ret; 2915 uint8_t next_proto = 0xFF; 2916 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2917 MLX5_FLOW_LAYER_OUTER_VLAN | 2918 MLX5_FLOW_LAYER_INNER_VLAN); 2919 2920 if ((last_item & l2_vlan) && ether_type && 2921 ether_type != RTE_ETHER_TYPE_IPV6) 2922 return rte_flow_error_set(error, EINVAL, 2923 RTE_FLOW_ERROR_TYPE_ITEM, item, 2924 "IPv6 cannot follow L2/VLAN layer " 2925 "which ether type is not IPv6"); 2926 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2927 next_proto = spec->hdr.proto; 2928 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2929 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2930 return rte_flow_error_set(error, EINVAL, 2931 RTE_FLOW_ERROR_TYPE_ITEM, 2932 item, 2933 "multiple tunnel " 2934 "not supported"); 2935 } 2936 if (next_proto == IPPROTO_HOPOPTS || 2937 next_proto == IPPROTO_ROUTING || 2938 next_proto == IPPROTO_FRAGMENT || 2939 next_proto == IPPROTO_ESP || 2940 next_proto == IPPROTO_AH || 2941 next_proto == IPPROTO_DSTOPTS) 2942 return rte_flow_error_set(error, EINVAL, 2943 RTE_FLOW_ERROR_TYPE_ITEM, item, 2944 "IPv6 proto (next header) should " 2945 "not be set as extension header"); 2946 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2947 return rte_flow_error_set(error, EINVAL, 2948 RTE_FLOW_ERROR_TYPE_ITEM, item, 2949 "wrong tunnel type - IPv4 specified " 2950 "but IPv6 item provided"); 2951 if (item_flags & l3m) 2952 return rte_flow_error_set(error, ENOTSUP, 2953 RTE_FLOW_ERROR_TYPE_ITEM, item, 2954 "multiple L3 layers not supported"); 2955 else if (item_flags & l4m) 2956 return rte_flow_error_set(error, EINVAL, 2957 RTE_FLOW_ERROR_TYPE_ITEM, item, 2958 "L3 cannot follow an L4 layer."); 2959 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2960 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2961 return rte_flow_error_set(error, EINVAL, 2962 RTE_FLOW_ERROR_TYPE_ITEM, item, 2963 "L3 cannot follow an NVGRE layer."); 2964 if (!mask) 2965 mask = &rte_flow_item_ipv6_mask; 2966 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2967 acc_mask ? (const uint8_t *)acc_mask 2968 : (const uint8_t *)&nic_mask, 2969 sizeof(struct rte_flow_item_ipv6), 2970 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2971 if (ret < 0) 2972 return ret; 2973 return 0; 2974 } 2975 2976 /** 2977 * Validate UDP item. 2978 * 2979 * @param[in] item 2980 * Item specification. 2981 * @param[in] item_flags 2982 * Bit-fields that holds the items detected until now. 2983 * @param[in] target_protocol 2984 * The next protocol in the previous item. 2985 * @param[in] flow_mask 2986 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2987 * @param[out] error 2988 * Pointer to error structure. 2989 * 2990 * @return 2991 * 0 on success, a negative errno value otherwise and rte_errno is set. 2992 */ 2993 int 2994 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2995 uint64_t item_flags, 2996 uint8_t target_protocol, 2997 struct rte_flow_error *error) 2998 { 2999 const struct rte_flow_item_udp *mask = item->mask; 3000 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3001 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3002 MLX5_FLOW_LAYER_OUTER_L3; 3003 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3004 MLX5_FLOW_LAYER_OUTER_L4; 3005 int ret; 3006 3007 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 3008 return rte_flow_error_set(error, EINVAL, 3009 RTE_FLOW_ERROR_TYPE_ITEM, item, 3010 "protocol filtering not compatible" 3011 " with UDP layer"); 3012 if (!(item_flags & l3m)) 3013 return rte_flow_error_set(error, EINVAL, 3014 RTE_FLOW_ERROR_TYPE_ITEM, item, 3015 "L3 is mandatory to filter on L4"); 3016 if (item_flags & l4m) 3017 return rte_flow_error_set(error, EINVAL, 3018 RTE_FLOW_ERROR_TYPE_ITEM, item, 3019 "multiple L4 layers not supported"); 3020 if (!mask) 3021 mask = &rte_flow_item_udp_mask; 3022 ret = mlx5_flow_item_acceptable 3023 (item, (const uint8_t *)mask, 3024 (const uint8_t *)&rte_flow_item_udp_mask, 3025 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3026 error); 3027 if (ret < 0) 3028 return ret; 3029 return 0; 3030 } 3031 3032 /** 3033 * Validate TCP item. 3034 * 3035 * @param[in] item 3036 * Item specification. 3037 * @param[in] item_flags 3038 * Bit-fields that holds the items detected until now. 3039 * @param[in] target_protocol 3040 * The next protocol in the previous item. 3041 * @param[out] error 3042 * Pointer to error structure. 3043 * 3044 * @return 3045 * 0 on success, a negative errno value otherwise and rte_errno is set. 3046 */ 3047 int 3048 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 3049 uint64_t item_flags, 3050 uint8_t target_protocol, 3051 const struct rte_flow_item_tcp *flow_mask, 3052 struct rte_flow_error *error) 3053 { 3054 const struct rte_flow_item_tcp *mask = item->mask; 3055 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3056 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3057 MLX5_FLOW_LAYER_OUTER_L3; 3058 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3059 MLX5_FLOW_LAYER_OUTER_L4; 3060 int ret; 3061 3062 MLX5_ASSERT(flow_mask); 3063 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3064 return rte_flow_error_set(error, EINVAL, 3065 RTE_FLOW_ERROR_TYPE_ITEM, item, 3066 "protocol filtering not compatible" 3067 " with TCP layer"); 3068 if (!(item_flags & l3m)) 3069 return rte_flow_error_set(error, EINVAL, 3070 RTE_FLOW_ERROR_TYPE_ITEM, item, 3071 "L3 is mandatory to filter on L4"); 3072 if (item_flags & l4m) 3073 return rte_flow_error_set(error, EINVAL, 3074 RTE_FLOW_ERROR_TYPE_ITEM, item, 3075 "multiple L4 layers not supported"); 3076 if (!mask) 3077 mask = &rte_flow_item_tcp_mask; 3078 ret = mlx5_flow_item_acceptable 3079 (item, (const uint8_t *)mask, 3080 (const uint8_t *)flow_mask, 3081 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3082 error); 3083 if (ret < 0) 3084 return ret; 3085 return 0; 3086 } 3087 3088 /** 3089 * Validate VXLAN item. 3090 * 3091 * @param[in] dev 3092 * Pointer to the Ethernet device structure. 3093 * @param[in] udp_dport 3094 * UDP destination port 3095 * @param[in] item 3096 * Item specification. 3097 * @param[in] item_flags 3098 * Bit-fields that holds the items detected until now. 3099 * @param root 3100 * Whether action is on root table. 3101 * @param[out] error 3102 * Pointer to error structure. 3103 * 3104 * @return 3105 * 0 on success, a negative errno value otherwise and rte_errno is set. 3106 */ 3107 int 3108 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3109 uint16_t udp_dport, 3110 const struct rte_flow_item *item, 3111 uint64_t item_flags, 3112 bool root, 3113 struct rte_flow_error *error) 3114 { 3115 const struct rte_flow_item_vxlan *spec = item->spec; 3116 const struct rte_flow_item_vxlan *mask = item->mask; 3117 int ret; 3118 struct mlx5_priv *priv = dev->data->dev_private; 3119 union vni { 3120 uint32_t vlan_id; 3121 uint8_t vni[4]; 3122 } id = { .vlan_id = 0, }; 3123 const struct rte_flow_item_vxlan nic_mask = { 3124 .hdr.vni = "\xff\xff\xff", 3125 .hdr.rsvd1 = 0xff, 3126 }; 3127 const struct rte_flow_item_vxlan *valid_mask; 3128 3129 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3130 return rte_flow_error_set(error, ENOTSUP, 3131 RTE_FLOW_ERROR_TYPE_ITEM, item, 3132 "multiple tunnel layers not" 3133 " supported"); 3134 valid_mask = &rte_flow_item_vxlan_mask; 3135 /* 3136 * Verify only UDPv4 is present as defined in 3137 * https://tools.ietf.org/html/rfc7348 3138 */ 3139 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3140 return rte_flow_error_set(error, EINVAL, 3141 RTE_FLOW_ERROR_TYPE_ITEM, item, 3142 "no outer UDP layer found"); 3143 if (!mask) 3144 mask = &rte_flow_item_vxlan_mask; 3145 3146 if (priv->sh->steering_format_version != 3147 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3148 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3149 /* non-root table */ 3150 if (!root && priv->sh->misc5_cap) 3151 valid_mask = &nic_mask; 3152 /* Group zero in NIC domain */ 3153 if (!root && priv->sh->tunnel_header_0_1) 3154 valid_mask = &nic_mask; 3155 } 3156 ret = mlx5_flow_item_acceptable 3157 (item, (const uint8_t *)mask, 3158 (const uint8_t *)valid_mask, 3159 sizeof(struct rte_flow_item_vxlan), 3160 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3161 if (ret < 0) 3162 return ret; 3163 if (spec) { 3164 memcpy(&id.vni[1], spec->hdr.vni, 3); 3165 memcpy(&id.vni[1], mask->hdr.vni, 3); 3166 } 3167 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3168 return rte_flow_error_set(error, ENOTSUP, 3169 RTE_FLOW_ERROR_TYPE_ITEM, item, 3170 "VXLAN tunnel must be fully defined"); 3171 return 0; 3172 } 3173 3174 /** 3175 * Validate VXLAN_GPE item. 3176 * 3177 * @param[in] item 3178 * Item specification. 3179 * @param[in] item_flags 3180 * Bit-fields that holds the items detected until now. 3181 * @param[in] priv 3182 * Pointer to the private data structure. 3183 * @param[in] target_protocol 3184 * The next protocol in the previous item. 3185 * @param[out] error 3186 * Pointer to error structure. 3187 * 3188 * @return 3189 * 0 on success, a negative errno value otherwise and rte_errno is set. 3190 */ 3191 int 3192 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3193 uint64_t item_flags, 3194 struct rte_eth_dev *dev, 3195 struct rte_flow_error *error) 3196 { 3197 struct mlx5_priv *priv = dev->data->dev_private; 3198 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3199 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3200 int ret; 3201 union vni { 3202 uint32_t vlan_id; 3203 uint8_t vni[4]; 3204 } id = { .vlan_id = 0, }; 3205 3206 struct rte_flow_item_vxlan_gpe nic_mask = { 3207 .vni = "\xff\xff\xff", 3208 .protocol = 0xff, 3209 .flags = 0xff, 3210 }; 3211 3212 if (!priv->sh->config.l3_vxlan_en) 3213 return rte_flow_error_set(error, ENOTSUP, 3214 RTE_FLOW_ERROR_TYPE_ITEM, item, 3215 "L3 VXLAN is not enabled by device" 3216 " parameter and/or not configured in" 3217 " firmware"); 3218 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3219 return rte_flow_error_set(error, ENOTSUP, 3220 RTE_FLOW_ERROR_TYPE_ITEM, item, 3221 "multiple tunnel layers not" 3222 " supported"); 3223 /* 3224 * Verify only UDPv4 is present as defined in 3225 * https://tools.ietf.org/html/rfc7348 3226 */ 3227 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3228 return rte_flow_error_set(error, EINVAL, 3229 RTE_FLOW_ERROR_TYPE_ITEM, item, 3230 "no outer UDP layer found"); 3231 if (!mask) 3232 mask = &rte_flow_item_vxlan_gpe_mask; 3233 if (priv->sh->misc5_cap && priv->sh->tunnel_header_0_1) { 3234 nic_mask.rsvd0[0] = 0xff; 3235 nic_mask.rsvd0[1] = 0xff; 3236 nic_mask.rsvd1 = 0xff; 3237 } 3238 ret = mlx5_flow_item_acceptable 3239 (item, (const uint8_t *)mask, 3240 (const uint8_t *)&nic_mask, 3241 sizeof(struct rte_flow_item_vxlan_gpe), 3242 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3243 if (ret < 0) 3244 return ret; 3245 if (spec) { 3246 memcpy(&id.vni[1], spec->hdr.vni, 3); 3247 memcpy(&id.vni[1], mask->hdr.vni, 3); 3248 } 3249 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3250 return rte_flow_error_set(error, ENOTSUP, 3251 RTE_FLOW_ERROR_TYPE_ITEM, item, 3252 "VXLAN-GPE tunnel must be fully" 3253 " defined"); 3254 return 0; 3255 } 3256 /** 3257 * Validate GRE Key item. 3258 * 3259 * @param[in] item 3260 * Item specification. 3261 * @param[in] item_flags 3262 * Bit flags to mark detected items. 3263 * @param[in] gre_item 3264 * Pointer to gre_item 3265 * @param[out] error 3266 * Pointer to error structure. 3267 * 3268 * @return 3269 * 0 on success, a negative errno value otherwise and rte_errno is set. 3270 */ 3271 int 3272 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3273 uint64_t item_flags, 3274 const struct rte_flow_item *gre_item, 3275 struct rte_flow_error *error) 3276 { 3277 const rte_be32_t *mask = item->mask; 3278 int ret = 0; 3279 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3280 const struct rte_flow_item_gre *gre_spec; 3281 const struct rte_flow_item_gre *gre_mask; 3282 3283 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3284 return rte_flow_error_set(error, ENOTSUP, 3285 RTE_FLOW_ERROR_TYPE_ITEM, item, 3286 "Multiple GRE key not support"); 3287 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3288 return rte_flow_error_set(error, ENOTSUP, 3289 RTE_FLOW_ERROR_TYPE_ITEM, item, 3290 "No preceding GRE header"); 3291 if (item_flags & MLX5_FLOW_LAYER_INNER) 3292 return rte_flow_error_set(error, ENOTSUP, 3293 RTE_FLOW_ERROR_TYPE_ITEM, item, 3294 "GRE key following a wrong item"); 3295 gre_mask = gre_item->mask; 3296 if (!gre_mask) 3297 gre_mask = &rte_flow_item_gre_mask; 3298 gre_spec = gre_item->spec; 3299 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3300 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3301 return rte_flow_error_set(error, EINVAL, 3302 RTE_FLOW_ERROR_TYPE_ITEM, item, 3303 "Key bit must be on"); 3304 3305 if (!mask) 3306 mask = &gre_key_default_mask; 3307 ret = mlx5_flow_item_acceptable 3308 (item, (const uint8_t *)mask, 3309 (const uint8_t *)&gre_key_default_mask, 3310 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3311 return ret; 3312 } 3313 3314 /** 3315 * Validate GRE optional item. 3316 * 3317 * @param[in] dev 3318 * Pointer to the Ethernet device structure. 3319 * @param[in] item 3320 * Item specification. 3321 * @param[in] item_flags 3322 * Bit flags to mark detected items. 3323 * @param[in] attr 3324 * Flow rule attributes. 3325 * @param[in] gre_item 3326 * Pointer to gre_item 3327 * @param[out] error 3328 * Pointer to error structure. 3329 * 3330 * @return 3331 * 0 on success, a negative errno value otherwise and rte_errno is set. 3332 */ 3333 int 3334 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3335 const struct rte_flow_item *item, 3336 uint64_t item_flags, 3337 const struct rte_flow_attr *attr, 3338 const struct rte_flow_item *gre_item, 3339 struct rte_flow_error *error) 3340 { 3341 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3342 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3343 const struct rte_flow_item_gre_opt *spec = item->spec; 3344 const struct rte_flow_item_gre_opt *mask = item->mask; 3345 struct mlx5_priv *priv = dev->data->dev_private; 3346 int ret = 0; 3347 struct rte_flow_item_gre_opt nic_mask = { 3348 .checksum_rsvd = { 3349 .checksum = RTE_BE16(UINT16_MAX), 3350 .reserved1 = 0x0, 3351 }, 3352 .key = { 3353 .key = RTE_BE32(UINT32_MAX), 3354 }, 3355 .sequence = { 3356 .sequence = RTE_BE32(UINT32_MAX), 3357 }, 3358 }; 3359 3360 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3361 return rte_flow_error_set(error, ENOTSUP, 3362 RTE_FLOW_ERROR_TYPE_ITEM, item, 3363 "No preceding GRE header"); 3364 if (item_flags & MLX5_FLOW_LAYER_INNER) 3365 return rte_flow_error_set(error, ENOTSUP, 3366 RTE_FLOW_ERROR_TYPE_ITEM, item, 3367 "GRE option following a wrong item"); 3368 if (!spec || !mask) 3369 return rte_flow_error_set(error, EINVAL, 3370 RTE_FLOW_ERROR_TYPE_ITEM, item, 3371 "At least one field gre_option(checksum/key/sequence) must be specified"); 3372 if (!gre_mask) 3373 gre_mask = &rte_flow_item_gre_mask; 3374 if (mask->checksum_rsvd.checksum) 3375 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3376 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3377 return rte_flow_error_set(error, EINVAL, 3378 RTE_FLOW_ERROR_TYPE_ITEM, 3379 item, 3380 "Checksum bit must be on"); 3381 if (mask->key.key) 3382 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3383 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3384 return rte_flow_error_set(error, EINVAL, 3385 RTE_FLOW_ERROR_TYPE_ITEM, 3386 item, "Key bit must be on"); 3387 if (mask->sequence.sequence) 3388 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3389 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3390 return rte_flow_error_set(error, EINVAL, 3391 RTE_FLOW_ERROR_TYPE_ITEM, 3392 item, 3393 "Sequence bit must be on"); 3394 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3395 if (priv->sh->steering_format_version == 3396 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3397 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3398 !priv->sh->misc5_cap) || 3399 (!(priv->sh->tunnel_header_0_1 && 3400 priv->sh->tunnel_header_2_3) && 3401 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3402 return rte_flow_error_set(error, EINVAL, 3403 RTE_FLOW_ERROR_TYPE_ITEM, 3404 item, 3405 "Checksum/Sequence not supported"); 3406 } 3407 ret = mlx5_flow_item_acceptable 3408 (item, (const uint8_t *)mask, 3409 (const uint8_t *)&nic_mask, 3410 sizeof(struct rte_flow_item_gre_opt), 3411 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3412 return ret; 3413 } 3414 3415 /** 3416 * Validate GRE item. 3417 * 3418 * @param[in] item 3419 * Item specification. 3420 * @param[in] item_flags 3421 * Bit flags to mark detected items. 3422 * @param[in] target_protocol 3423 * The next protocol in the previous item. 3424 * @param[out] error 3425 * Pointer to error structure. 3426 * 3427 * @return 3428 * 0 on success, a negative errno value otherwise and rte_errno is set. 3429 */ 3430 int 3431 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3432 uint64_t item_flags, 3433 uint8_t target_protocol, 3434 struct rte_flow_error *error) 3435 { 3436 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3437 const struct rte_flow_item_gre *mask = item->mask; 3438 int ret; 3439 const struct rte_flow_item_gre nic_mask = { 3440 .c_rsvd0_ver = RTE_BE16(0xB000), 3441 .protocol = RTE_BE16(UINT16_MAX), 3442 }; 3443 3444 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3445 return rte_flow_error_set(error, EINVAL, 3446 RTE_FLOW_ERROR_TYPE_ITEM, item, 3447 "protocol filtering not compatible" 3448 " with this GRE layer"); 3449 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3450 return rte_flow_error_set(error, ENOTSUP, 3451 RTE_FLOW_ERROR_TYPE_ITEM, item, 3452 "multiple tunnel layers not" 3453 " supported"); 3454 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3455 return rte_flow_error_set(error, ENOTSUP, 3456 RTE_FLOW_ERROR_TYPE_ITEM, item, 3457 "L3 Layer is missing"); 3458 if (!mask) 3459 mask = &rte_flow_item_gre_mask; 3460 ret = mlx5_flow_item_acceptable 3461 (item, (const uint8_t *)mask, 3462 (const uint8_t *)&nic_mask, 3463 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3464 error); 3465 if (ret < 0) 3466 return ret; 3467 #ifndef HAVE_MLX5DV_DR 3468 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3469 if (spec && (spec->protocol & mask->protocol)) 3470 return rte_flow_error_set(error, ENOTSUP, 3471 RTE_FLOW_ERROR_TYPE_ITEM, item, 3472 "without MPLS support the" 3473 " specification cannot be used for" 3474 " filtering"); 3475 #endif 3476 #endif 3477 return 0; 3478 } 3479 3480 /** 3481 * Validate Geneve item. 3482 * 3483 * @param[in] item 3484 * Item specification. 3485 * @param[in] itemFlags 3486 * Bit-fields that holds the items detected until now. 3487 * @param[in] enPriv 3488 * Pointer to the private data structure. 3489 * @param[out] error 3490 * Pointer to error structure. 3491 * 3492 * @return 3493 * 0 on success, a negative errno value otherwise and rte_errno is set. 3494 */ 3495 3496 int 3497 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3498 uint64_t item_flags, 3499 struct rte_eth_dev *dev, 3500 struct rte_flow_error *error) 3501 { 3502 struct mlx5_priv *priv = dev->data->dev_private; 3503 const struct rte_flow_item_geneve *spec = item->spec; 3504 const struct rte_flow_item_geneve *mask = item->mask; 3505 int ret; 3506 uint16_t gbhdr; 3507 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3508 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3509 const struct rte_flow_item_geneve nic_mask = { 3510 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3511 .vni = "\xff\xff\xff", 3512 .protocol = RTE_BE16(UINT16_MAX), 3513 }; 3514 3515 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3516 return rte_flow_error_set(error, ENOTSUP, 3517 RTE_FLOW_ERROR_TYPE_ITEM, item, 3518 "L3 Geneve is not enabled by device" 3519 " parameter and/or not configured in" 3520 " firmware"); 3521 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3522 return rte_flow_error_set(error, ENOTSUP, 3523 RTE_FLOW_ERROR_TYPE_ITEM, item, 3524 "multiple tunnel layers not" 3525 " supported"); 3526 /* 3527 * Verify only UDPv4 is present as defined in 3528 * https://tools.ietf.org/html/rfc7348 3529 */ 3530 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3531 return rte_flow_error_set(error, EINVAL, 3532 RTE_FLOW_ERROR_TYPE_ITEM, item, 3533 "no outer UDP layer found"); 3534 if (!mask) 3535 mask = &rte_flow_item_geneve_mask; 3536 ret = mlx5_flow_item_acceptable 3537 (item, (const uint8_t *)mask, 3538 (const uint8_t *)&nic_mask, 3539 sizeof(struct rte_flow_item_geneve), 3540 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3541 if (ret) 3542 return ret; 3543 if (spec) { 3544 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3545 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3546 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3547 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3548 return rte_flow_error_set(error, ENOTSUP, 3549 RTE_FLOW_ERROR_TYPE_ITEM, 3550 item, 3551 "Geneve protocol unsupported" 3552 " fields are being used"); 3553 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3554 return rte_flow_error_set 3555 (error, ENOTSUP, 3556 RTE_FLOW_ERROR_TYPE_ITEM, 3557 item, 3558 "Unsupported Geneve options length"); 3559 } 3560 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3561 return rte_flow_error_set 3562 (error, ENOTSUP, 3563 RTE_FLOW_ERROR_TYPE_ITEM, item, 3564 "Geneve tunnel must be fully defined"); 3565 return 0; 3566 } 3567 3568 /** 3569 * Validate Geneve TLV option item. 3570 * 3571 * @param[in] item 3572 * Item specification. 3573 * @param[in] last_item 3574 * Previous validated item in the pattern items. 3575 * @param[in] geneve_item 3576 * Previous GENEVE item specification. 3577 * @param[in] dev 3578 * Pointer to the rte_eth_dev structure. 3579 * @param[out] error 3580 * Pointer to error structure. 3581 * 3582 * @return 3583 * 0 on success, a negative errno value otherwise and rte_errno is set. 3584 */ 3585 int 3586 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3587 uint64_t last_item, 3588 const struct rte_flow_item *geneve_item, 3589 struct rte_eth_dev *dev, 3590 struct rte_flow_error *error) 3591 { 3592 struct mlx5_priv *priv = dev->data->dev_private; 3593 struct mlx5_dev_ctx_shared *sh = priv->sh; 3594 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3595 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3596 uint8_t data_max_supported = 3597 hca_attr->max_geneve_tlv_option_data_len * 4; 3598 const struct rte_flow_item_geneve *geneve_spec; 3599 const struct rte_flow_item_geneve *geneve_mask; 3600 const struct rte_flow_item_geneve_opt *spec = item->spec; 3601 const struct rte_flow_item_geneve_opt *mask = item->mask; 3602 unsigned int i; 3603 unsigned int data_len; 3604 uint8_t tlv_option_len; 3605 uint16_t optlen_m, optlen_v; 3606 const struct rte_flow_item_geneve_opt full_mask = { 3607 .option_class = RTE_BE16(0xffff), 3608 .option_type = 0xff, 3609 .option_len = 0x1f, 3610 }; 3611 3612 if (!mask) 3613 mask = &rte_flow_item_geneve_opt_mask; 3614 if (!spec) 3615 return rte_flow_error_set 3616 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3617 "Geneve TLV opt class/type/length must be specified"); 3618 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3619 return rte_flow_error_set 3620 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3621 "Geneve TLV opt length exceeds the limit (31)"); 3622 /* Check if class type and length masks are full. */ 3623 if (full_mask.option_class != mask->option_class || 3624 full_mask.option_type != mask->option_type || 3625 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3626 return rte_flow_error_set 3627 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3628 "Geneve TLV opt class/type/length masks must be full"); 3629 /* Check if length is supported */ 3630 if ((uint32_t)spec->option_len > 3631 hca_attr->max_geneve_tlv_option_data_len) 3632 return rte_flow_error_set 3633 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3634 "Geneve TLV opt length not supported"); 3635 if (hca_attr->max_geneve_tlv_options > 1) 3636 DRV_LOG(DEBUG, 3637 "max_geneve_tlv_options supports more than 1 option"); 3638 /* Check GENEVE item preceding. */ 3639 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3640 return rte_flow_error_set 3641 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3642 "Geneve opt item must be preceded with Geneve item"); 3643 geneve_spec = geneve_item->spec; 3644 geneve_mask = geneve_item->mask ? geneve_item->mask : 3645 &rte_flow_item_geneve_mask; 3646 /* Check if GENEVE TLV option size doesn't exceed option length */ 3647 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3648 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3649 tlv_option_len = spec->option_len & mask->option_len; 3650 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3651 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3652 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3653 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3654 if ((optlen_v & optlen_m) <= tlv_option_len) 3655 return rte_flow_error_set 3656 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3657 "GENEVE TLV option length exceeds optlen"); 3658 } 3659 /* Check if length is 0 or data is 0. */ 3660 if (spec->data == NULL || spec->option_len == 0) 3661 return rte_flow_error_set 3662 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3663 "Geneve TLV opt with zero data/length not supported"); 3664 /* Check not all data & mask are 0. */ 3665 data_len = spec->option_len * 4; 3666 if (mask->data == NULL) { 3667 for (i = 0; i < data_len; i++) 3668 if (spec->data[i]) 3669 break; 3670 if (i == data_len) 3671 return rte_flow_error_set(error, ENOTSUP, 3672 RTE_FLOW_ERROR_TYPE_ITEM, item, 3673 "Can't match on Geneve option data 0"); 3674 } else { 3675 for (i = 0; i < data_len; i++) 3676 if (spec->data[i] & mask->data[i]) 3677 break; 3678 if (i == data_len) 3679 return rte_flow_error_set(error, ENOTSUP, 3680 RTE_FLOW_ERROR_TYPE_ITEM, item, 3681 "Can't match on Geneve option data and mask 0"); 3682 /* Check data mask supported. */ 3683 for (i = data_max_supported; i < data_len ; i++) 3684 if (mask->data[i]) 3685 return rte_flow_error_set(error, ENOTSUP, 3686 RTE_FLOW_ERROR_TYPE_ITEM, item, 3687 "Data mask is of unsupported size"); 3688 } 3689 /* Check GENEVE option is supported in NIC. */ 3690 if (!hca_attr->geneve_tlv_opt) 3691 return rte_flow_error_set 3692 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3693 "Geneve TLV opt not supported"); 3694 /* Check if we already have geneve option with different type/class. */ 3695 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3696 geneve_opt_resource = sh->geneve_tlv_option_resource; 3697 if (geneve_opt_resource != NULL) 3698 if (geneve_opt_resource->option_class != spec->option_class || 3699 geneve_opt_resource->option_type != spec->option_type || 3700 geneve_opt_resource->length != spec->option_len) { 3701 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3702 return rte_flow_error_set(error, ENOTSUP, 3703 RTE_FLOW_ERROR_TYPE_ITEM, item, 3704 "Only one Geneve TLV option supported"); 3705 } 3706 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3707 return 0; 3708 } 3709 3710 /** 3711 * Validate MPLS item. 3712 * 3713 * @param[in] dev 3714 * Pointer to the rte_eth_dev structure. 3715 * @param[in] item 3716 * Item specification. 3717 * @param[in] item_flags 3718 * Bit-fields that holds the items detected until now. 3719 * @param[in] prev_layer 3720 * The protocol layer indicated in previous item. 3721 * @param[out] error 3722 * Pointer to error structure. 3723 * 3724 * @return 3725 * 0 on success, a negative errno value otherwise and rte_errno is set. 3726 */ 3727 int 3728 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3729 const struct rte_flow_item *item __rte_unused, 3730 uint64_t item_flags __rte_unused, 3731 uint64_t prev_layer __rte_unused, 3732 struct rte_flow_error *error) 3733 { 3734 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3735 const struct rte_flow_item_mpls *mask = item->mask; 3736 struct mlx5_priv *priv = dev->data->dev_private; 3737 int ret; 3738 3739 if (!priv->sh->dev_cap.mpls_en) 3740 return rte_flow_error_set(error, ENOTSUP, 3741 RTE_FLOW_ERROR_TYPE_ITEM, item, 3742 "MPLS not supported or" 3743 " disabled in firmware" 3744 " configuration."); 3745 /* MPLS over UDP, GRE is allowed */ 3746 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3747 MLX5_FLOW_LAYER_GRE | 3748 MLX5_FLOW_LAYER_GRE_KEY))) 3749 return rte_flow_error_set(error, EINVAL, 3750 RTE_FLOW_ERROR_TYPE_ITEM, item, 3751 "protocol filtering not compatible" 3752 " with MPLS layer"); 3753 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3754 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3755 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3756 return rte_flow_error_set(error, ENOTSUP, 3757 RTE_FLOW_ERROR_TYPE_ITEM, item, 3758 "multiple tunnel layers not" 3759 " supported"); 3760 if (!mask) 3761 mask = &rte_flow_item_mpls_mask; 3762 ret = mlx5_flow_item_acceptable 3763 (item, (const uint8_t *)mask, 3764 (const uint8_t *)&rte_flow_item_mpls_mask, 3765 sizeof(struct rte_flow_item_mpls), 3766 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3767 if (ret < 0) 3768 return ret; 3769 return 0; 3770 #else 3771 return rte_flow_error_set(error, ENOTSUP, 3772 RTE_FLOW_ERROR_TYPE_ITEM, item, 3773 "MPLS is not supported by Verbs, please" 3774 " update."); 3775 #endif 3776 } 3777 3778 /** 3779 * Validate NVGRE item. 3780 * 3781 * @param[in] item 3782 * Item specification. 3783 * @param[in] item_flags 3784 * Bit flags to mark detected items. 3785 * @param[in] target_protocol 3786 * The next protocol in the previous item. 3787 * @param[out] error 3788 * Pointer to error structure. 3789 * 3790 * @return 3791 * 0 on success, a negative errno value otherwise and rte_errno is set. 3792 */ 3793 int 3794 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3795 uint64_t item_flags, 3796 uint8_t target_protocol, 3797 struct rte_flow_error *error) 3798 { 3799 const struct rte_flow_item_nvgre *mask = item->mask; 3800 int ret; 3801 3802 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3803 return rte_flow_error_set(error, EINVAL, 3804 RTE_FLOW_ERROR_TYPE_ITEM, item, 3805 "protocol filtering not compatible" 3806 " with this GRE layer"); 3807 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3808 return rte_flow_error_set(error, ENOTSUP, 3809 RTE_FLOW_ERROR_TYPE_ITEM, item, 3810 "multiple tunnel layers not" 3811 " supported"); 3812 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3813 return rte_flow_error_set(error, ENOTSUP, 3814 RTE_FLOW_ERROR_TYPE_ITEM, item, 3815 "L3 Layer is missing"); 3816 if (!mask) 3817 mask = &rte_flow_item_nvgre_mask; 3818 ret = mlx5_flow_item_acceptable 3819 (item, (const uint8_t *)mask, 3820 (const uint8_t *)&rte_flow_item_nvgre_mask, 3821 sizeof(struct rte_flow_item_nvgre), 3822 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3823 if (ret < 0) 3824 return ret; 3825 return 0; 3826 } 3827 3828 /** 3829 * Validate eCPRI item. 3830 * 3831 * @param[in] item 3832 * Item specification. 3833 * @param[in] item_flags 3834 * Bit-fields that holds the items detected until now. 3835 * @param[in] last_item 3836 * Previous validated item in the pattern items. 3837 * @param[in] ether_type 3838 * Type in the ethernet layer header (including dot1q). 3839 * @param[in] acc_mask 3840 * Acceptable mask, if NULL default internal default mask 3841 * will be used to check whether item fields are supported. 3842 * @param[out] error 3843 * Pointer to error structure. 3844 * 3845 * @return 3846 * 0 on success, a negative errno value otherwise and rte_errno is set. 3847 */ 3848 int 3849 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3850 uint64_t item_flags, 3851 uint64_t last_item, 3852 uint16_t ether_type, 3853 const struct rte_flow_item_ecpri *acc_mask, 3854 struct rte_flow_error *error) 3855 { 3856 const struct rte_flow_item_ecpri *mask = item->mask; 3857 const struct rte_flow_item_ecpri nic_mask = { 3858 .hdr = { 3859 .common = { 3860 .u32 = 3861 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3862 .type = 0xFF, 3863 }).u32), 3864 }, 3865 .dummy[0] = 0xFFFFFFFF, 3866 }, 3867 }; 3868 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3869 MLX5_FLOW_LAYER_OUTER_VLAN); 3870 struct rte_flow_item_ecpri mask_lo; 3871 3872 if (!(last_item & outer_l2_vlan) && 3873 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3874 return rte_flow_error_set(error, EINVAL, 3875 RTE_FLOW_ERROR_TYPE_ITEM, item, 3876 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3877 if ((last_item & outer_l2_vlan) && ether_type && 3878 ether_type != RTE_ETHER_TYPE_ECPRI) 3879 return rte_flow_error_set(error, EINVAL, 3880 RTE_FLOW_ERROR_TYPE_ITEM, item, 3881 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3882 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3883 return rte_flow_error_set(error, EINVAL, 3884 RTE_FLOW_ERROR_TYPE_ITEM, item, 3885 "eCPRI with tunnel is not supported right now"); 3886 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3887 return rte_flow_error_set(error, ENOTSUP, 3888 RTE_FLOW_ERROR_TYPE_ITEM, item, 3889 "multiple L3 layers not supported"); 3890 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3891 return rte_flow_error_set(error, EINVAL, 3892 RTE_FLOW_ERROR_TYPE_ITEM, item, 3893 "eCPRI cannot coexist with a TCP layer"); 3894 /* In specification, eCPRI could be over UDP layer. */ 3895 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3896 return rte_flow_error_set(error, EINVAL, 3897 RTE_FLOW_ERROR_TYPE_ITEM, item, 3898 "eCPRI over UDP layer is not yet supported right now"); 3899 /* Mask for type field in common header could be zero. */ 3900 if (!mask) 3901 mask = &rte_flow_item_ecpri_mask; 3902 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3903 /* Input mask is in big-endian format. */ 3904 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3905 return rte_flow_error_set(error, EINVAL, 3906 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3907 "partial mask is not supported for protocol"); 3908 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3909 return rte_flow_error_set(error, EINVAL, 3910 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3911 "message header mask must be after a type mask"); 3912 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3913 acc_mask ? (const uint8_t *)acc_mask 3914 : (const uint8_t *)&nic_mask, 3915 sizeof(struct rte_flow_item_ecpri), 3916 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3917 } 3918 3919 /** 3920 * Validate the NSH item. 3921 * 3922 * @param[in] dev 3923 * Pointer to Ethernet device on which flow rule is being created on. 3924 * @param[out] error 3925 * Pointer to error structure. 3926 * 3927 * @return 3928 * 0 on success, a negative errno value otherwise and rte_errno is set. 3929 */ 3930 int 3931 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev, 3932 const struct rte_flow_item *item, 3933 struct rte_flow_error *error) 3934 { 3935 struct mlx5_priv *priv = dev->data->dev_private; 3936 3937 if (item->mask) { 3938 return rte_flow_error_set(error, ENOTSUP, 3939 RTE_FLOW_ERROR_TYPE_ITEM, item, 3940 "NSH fields matching is not supported"); 3941 } 3942 3943 if (!priv->sh->config.dv_flow_en) { 3944 return rte_flow_error_set(error, ENOTSUP, 3945 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3946 NULL, "NSH support requires DV flow interface"); 3947 } 3948 3949 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) { 3950 return rte_flow_error_set(error, ENOTSUP, 3951 RTE_FLOW_ERROR_TYPE_ITEM, item, 3952 "Current FW does not support matching on NSH"); 3953 } 3954 3955 return 0; 3956 } 3957 3958 static int 3959 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3960 const struct rte_flow_attr *attr __rte_unused, 3961 const struct rte_flow_item items[] __rte_unused, 3962 const struct rte_flow_action actions[] __rte_unused, 3963 bool external __rte_unused, 3964 int hairpin __rte_unused, 3965 struct rte_flow_error *error) 3966 { 3967 return rte_flow_error_set(error, ENOTSUP, 3968 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3969 } 3970 3971 static struct mlx5_flow * 3972 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3973 const struct rte_flow_attr *attr __rte_unused, 3974 const struct rte_flow_item items[] __rte_unused, 3975 const struct rte_flow_action actions[] __rte_unused, 3976 struct rte_flow_error *error) 3977 { 3978 rte_flow_error_set(error, ENOTSUP, 3979 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3980 return NULL; 3981 } 3982 3983 static int 3984 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3985 struct mlx5_flow *dev_flow __rte_unused, 3986 const struct rte_flow_attr *attr __rte_unused, 3987 const struct rte_flow_item items[] __rte_unused, 3988 const struct rte_flow_action actions[] __rte_unused, 3989 struct rte_flow_error *error) 3990 { 3991 return rte_flow_error_set(error, ENOTSUP, 3992 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3993 } 3994 3995 static int 3996 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3997 struct rte_flow *flow __rte_unused, 3998 struct rte_flow_error *error) 3999 { 4000 return rte_flow_error_set(error, ENOTSUP, 4001 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4002 } 4003 4004 static void 4005 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 4006 struct rte_flow *flow __rte_unused) 4007 { 4008 } 4009 4010 static void 4011 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 4012 struct rte_flow *flow __rte_unused) 4013 { 4014 } 4015 4016 static int 4017 flow_null_query(struct rte_eth_dev *dev __rte_unused, 4018 struct rte_flow *flow __rte_unused, 4019 const struct rte_flow_action *actions __rte_unused, 4020 void *data __rte_unused, 4021 struct rte_flow_error *error) 4022 { 4023 return rte_flow_error_set(error, ENOTSUP, 4024 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4025 } 4026 4027 static int 4028 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 4029 uint32_t domains __rte_unused, 4030 uint32_t flags __rte_unused) 4031 { 4032 return 0; 4033 } 4034 4035 int 4036 flow_null_get_aged_flows(struct rte_eth_dev *dev, 4037 void **context __rte_unused, 4038 uint32_t nb_contexts __rte_unused, 4039 struct rte_flow_error *error __rte_unused) 4040 { 4041 DRV_LOG(ERR, "port %u get aged flows is not supported.", 4042 dev->data->port_id); 4043 return -ENOTSUP; 4044 } 4045 4046 uint32_t 4047 flow_null_counter_allocate(struct rte_eth_dev *dev) 4048 { 4049 DRV_LOG(ERR, "port %u counter allocate is not supported.", 4050 dev->data->port_id); 4051 return 0; 4052 } 4053 4054 void 4055 flow_null_counter_free(struct rte_eth_dev *dev, 4056 uint32_t counter __rte_unused) 4057 { 4058 DRV_LOG(ERR, "port %u counter free is not supported.", 4059 dev->data->port_id); 4060 } 4061 4062 int 4063 flow_null_counter_query(struct rte_eth_dev *dev, 4064 uint32_t counter __rte_unused, 4065 bool clear __rte_unused, 4066 uint64_t *pkts __rte_unused, 4067 uint64_t *bytes __rte_unused, 4068 void **action __rte_unused) 4069 { 4070 DRV_LOG(ERR, "port %u counter query is not supported.", 4071 dev->data->port_id); 4072 return -ENOTSUP; 4073 } 4074 4075 /* Void driver to protect from null pointer reference. */ 4076 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 4077 .validate = flow_null_validate, 4078 .prepare = flow_null_prepare, 4079 .translate = flow_null_translate, 4080 .apply = flow_null_apply, 4081 .remove = flow_null_remove, 4082 .destroy = flow_null_destroy, 4083 .query = flow_null_query, 4084 .sync_domain = flow_null_sync_domain, 4085 .get_aged_flows = flow_null_get_aged_flows, 4086 .counter_alloc = flow_null_counter_allocate, 4087 .counter_free = flow_null_counter_free, 4088 .counter_query = flow_null_counter_query 4089 }; 4090 4091 /** 4092 * Select flow driver type according to flow attributes and device 4093 * configuration. 4094 * 4095 * @param[in] dev 4096 * Pointer to the dev structure. 4097 * @param[in] attr 4098 * Pointer to the flow attributes. 4099 * 4100 * @return 4101 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4102 */ 4103 static enum mlx5_flow_drv_type 4104 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4105 { 4106 struct mlx5_priv *priv = dev->data->dev_private; 4107 /* The OS can determine first a specific flow type (DV, VERBS) */ 4108 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4109 4110 if (type != MLX5_FLOW_TYPE_MAX) 4111 return type; 4112 /* 4113 * Currently when dv_flow_en == 2, only HW steering engine is 4114 * supported. New engines can also be chosen here if ready. 4115 */ 4116 if (priv->sh->config.dv_flow_en == 2) 4117 return MLX5_FLOW_TYPE_HW; 4118 if (!attr) 4119 return MLX5_FLOW_TYPE_MIN; 4120 /* If no OS specific type - continue with DV/VERBS selection */ 4121 if (attr->transfer && priv->sh->config.dv_esw_en) 4122 type = MLX5_FLOW_TYPE_DV; 4123 if (!attr->transfer) 4124 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4125 MLX5_FLOW_TYPE_VERBS; 4126 return type; 4127 } 4128 4129 #define flow_get_drv_ops(type) flow_drv_ops[type] 4130 4131 /** 4132 * Flow driver validation API. This abstracts calling driver specific functions. 4133 * The type of flow driver is determined according to flow attributes. 4134 * 4135 * @param[in] dev 4136 * Pointer to the dev structure. 4137 * @param[in] attr 4138 * Pointer to the flow attributes. 4139 * @param[in] items 4140 * Pointer to the list of items. 4141 * @param[in] actions 4142 * Pointer to the list of actions. 4143 * @param[in] external 4144 * This flow rule is created by request external to PMD. 4145 * @param[in] hairpin 4146 * Number of hairpin TX actions, 0 means classic flow. 4147 * @param[out] error 4148 * Pointer to the error structure. 4149 * 4150 * @return 4151 * 0 on success, a negative errno value otherwise and rte_errno is set. 4152 */ 4153 static inline int 4154 flow_drv_validate(struct rte_eth_dev *dev, 4155 const struct rte_flow_attr *attr, 4156 const struct rte_flow_item items[], 4157 const struct rte_flow_action actions[], 4158 bool external, int hairpin, struct rte_flow_error *error) 4159 { 4160 const struct mlx5_flow_driver_ops *fops; 4161 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4162 4163 fops = flow_get_drv_ops(type); 4164 return fops->validate(dev, attr, items, actions, external, 4165 hairpin, error); 4166 } 4167 4168 /** 4169 * Flow driver preparation API. This abstracts calling driver specific 4170 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4171 * calculates the size of memory required for device flow, allocates the memory, 4172 * initializes the device flow and returns the pointer. 4173 * 4174 * @note 4175 * This function initializes device flow structure such as dv or verbs in 4176 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4177 * rest. For example, adding returning device flow to flow->dev_flow list and 4178 * setting backward reference to the flow should be done out of this function. 4179 * layers field is not filled either. 4180 * 4181 * @param[in] dev 4182 * Pointer to the dev structure. 4183 * @param[in] attr 4184 * Pointer to the flow attributes. 4185 * @param[in] items 4186 * Pointer to the list of items. 4187 * @param[in] actions 4188 * Pointer to the list of actions. 4189 * @param[in] flow_idx 4190 * This memory pool index to the flow. 4191 * @param[out] error 4192 * Pointer to the error structure. 4193 * 4194 * @return 4195 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4196 */ 4197 static inline struct mlx5_flow * 4198 flow_drv_prepare(struct rte_eth_dev *dev, 4199 const struct rte_flow *flow, 4200 const struct rte_flow_attr *attr, 4201 const struct rte_flow_item items[], 4202 const struct rte_flow_action actions[], 4203 uint32_t flow_idx, 4204 struct rte_flow_error *error) 4205 { 4206 const struct mlx5_flow_driver_ops *fops; 4207 enum mlx5_flow_drv_type type = flow->drv_type; 4208 struct mlx5_flow *mlx5_flow = NULL; 4209 4210 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4211 fops = flow_get_drv_ops(type); 4212 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4213 if (mlx5_flow) 4214 mlx5_flow->flow_idx = flow_idx; 4215 return mlx5_flow; 4216 } 4217 4218 /** 4219 * Flow driver translation API. This abstracts calling driver specific 4220 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4221 * translates a generic flow into a driver flow. flow_drv_prepare() must 4222 * precede. 4223 * 4224 * @note 4225 * dev_flow->layers could be filled as a result of parsing during translation 4226 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4227 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4228 * flow->actions could be overwritten even though all the expanded dev_flows 4229 * have the same actions. 4230 * 4231 * @param[in] dev 4232 * Pointer to the rte dev structure. 4233 * @param[in, out] dev_flow 4234 * Pointer to the mlx5 flow. 4235 * @param[in] attr 4236 * Pointer to the flow attributes. 4237 * @param[in] items 4238 * Pointer to the list of items. 4239 * @param[in] actions 4240 * Pointer to the list of actions. 4241 * @param[out] error 4242 * Pointer to the error structure. 4243 * 4244 * @return 4245 * 0 on success, a negative errno value otherwise and rte_errno is set. 4246 */ 4247 static inline int 4248 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4249 const struct rte_flow_attr *attr, 4250 const struct rte_flow_item items[], 4251 const struct rte_flow_action actions[], 4252 struct rte_flow_error *error) 4253 { 4254 const struct mlx5_flow_driver_ops *fops; 4255 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4256 4257 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4258 fops = flow_get_drv_ops(type); 4259 return fops->translate(dev, dev_flow, attr, items, actions, error); 4260 } 4261 4262 /** 4263 * Flow driver apply API. This abstracts calling driver specific functions. 4264 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4265 * translated driver flows on to device. flow_drv_translate() must precede. 4266 * 4267 * @param[in] dev 4268 * Pointer to Ethernet device structure. 4269 * @param[in, out] flow 4270 * Pointer to flow structure. 4271 * @param[out] error 4272 * Pointer to error structure. 4273 * 4274 * @return 4275 * 0 on success, a negative errno value otherwise and rte_errno is set. 4276 */ 4277 static inline int 4278 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4279 struct rte_flow_error *error) 4280 { 4281 const struct mlx5_flow_driver_ops *fops; 4282 enum mlx5_flow_drv_type type = flow->drv_type; 4283 4284 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4285 fops = flow_get_drv_ops(type); 4286 return fops->apply(dev, flow, error); 4287 } 4288 4289 /** 4290 * Flow driver destroy API. This abstracts calling driver specific functions. 4291 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4292 * on device and releases resources of the flow. 4293 * 4294 * @param[in] dev 4295 * Pointer to Ethernet device. 4296 * @param[in, out] flow 4297 * Pointer to flow structure. 4298 */ 4299 static inline void 4300 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4301 { 4302 const struct mlx5_flow_driver_ops *fops; 4303 enum mlx5_flow_drv_type type = flow->drv_type; 4304 4305 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4306 fops = flow_get_drv_ops(type); 4307 fops->destroy(dev, flow); 4308 } 4309 4310 /** 4311 * Flow driver find RSS policy tbl API. This abstracts calling driver 4312 * specific functions. Parent flow (rte_flow) should have driver 4313 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4314 * 4315 * @param[in] dev 4316 * Pointer to Ethernet device. 4317 * @param[in, out] flow 4318 * Pointer to flow structure. 4319 * @param[in] policy 4320 * Pointer to meter policy table. 4321 * @param[in] rss_desc 4322 * Pointer to rss_desc 4323 */ 4324 static struct mlx5_flow_meter_sub_policy * 4325 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4326 struct rte_flow *flow, 4327 struct mlx5_flow_meter_policy *policy, 4328 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4329 { 4330 const struct mlx5_flow_driver_ops *fops; 4331 enum mlx5_flow_drv_type type = flow->drv_type; 4332 4333 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4334 fops = flow_get_drv_ops(type); 4335 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4336 } 4337 4338 /** 4339 * Flow driver color tag rule API. This abstracts calling driver 4340 * specific functions. Parent flow (rte_flow) should have driver 4341 * type (drv_type). It will create the color tag rules in hierarchy meter. 4342 * 4343 * @param[in] dev 4344 * Pointer to Ethernet device. 4345 * @param[in, out] flow 4346 * Pointer to flow structure. 4347 * @param[in] fm 4348 * Pointer to flow meter structure. 4349 * @param[in] src_port 4350 * The src port this extra rule should use. 4351 * @param[in] item 4352 * The src port id match item. 4353 * @param[out] error 4354 * Pointer to error structure. 4355 */ 4356 static int 4357 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4358 struct rte_flow *flow, 4359 struct mlx5_flow_meter_info *fm, 4360 int32_t src_port, 4361 const struct rte_flow_item *item, 4362 struct rte_flow_error *error) 4363 { 4364 const struct mlx5_flow_driver_ops *fops; 4365 enum mlx5_flow_drv_type type = flow->drv_type; 4366 4367 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4368 fops = flow_get_drv_ops(type); 4369 return fops->meter_hierarchy_rule_create(dev, fm, 4370 src_port, item, error); 4371 } 4372 4373 /** 4374 * Get RSS action from the action list. 4375 * 4376 * @param[in] dev 4377 * Pointer to Ethernet device. 4378 * @param[in] actions 4379 * Pointer to the list of actions. 4380 * @param[in] flow 4381 * Parent flow structure pointer. 4382 * 4383 * @return 4384 * Pointer to the RSS action if exist, else return NULL. 4385 */ 4386 static const struct rte_flow_action_rss* 4387 flow_get_rss_action(struct rte_eth_dev *dev, 4388 const struct rte_flow_action actions[]) 4389 { 4390 struct mlx5_priv *priv = dev->data->dev_private; 4391 const struct rte_flow_action_rss *rss = NULL; 4392 struct mlx5_meter_policy_action_container *acg; 4393 struct mlx5_meter_policy_action_container *acy; 4394 4395 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4396 switch (actions->type) { 4397 case RTE_FLOW_ACTION_TYPE_RSS: 4398 rss = actions->conf; 4399 break; 4400 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4401 { 4402 const struct rte_flow_action_sample *sample = 4403 actions->conf; 4404 const struct rte_flow_action *act = sample->actions; 4405 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4406 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4407 rss = act->conf; 4408 break; 4409 } 4410 case RTE_FLOW_ACTION_TYPE_METER: 4411 { 4412 uint32_t mtr_idx; 4413 struct mlx5_flow_meter_info *fm; 4414 struct mlx5_flow_meter_policy *policy; 4415 const struct rte_flow_action_meter *mtr = actions->conf; 4416 4417 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4418 if (fm && !fm->def_policy) { 4419 policy = mlx5_flow_meter_policy_find(dev, 4420 fm->policy_id, NULL); 4421 MLX5_ASSERT(policy); 4422 if (policy->is_hierarchy) { 4423 policy = 4424 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4425 policy); 4426 if (!policy) 4427 return NULL; 4428 } 4429 if (policy->is_rss) { 4430 acg = 4431 &policy->act_cnt[RTE_COLOR_GREEN]; 4432 acy = 4433 &policy->act_cnt[RTE_COLOR_YELLOW]; 4434 if (acg->fate_action == 4435 MLX5_FLOW_FATE_SHARED_RSS) 4436 rss = acg->rss->conf; 4437 else if (acy->fate_action == 4438 MLX5_FLOW_FATE_SHARED_RSS) 4439 rss = acy->rss->conf; 4440 } 4441 } 4442 break; 4443 } 4444 default: 4445 break; 4446 } 4447 } 4448 return rss; 4449 } 4450 4451 /** 4452 * Get ASO age action by index. 4453 * 4454 * @param[in] dev 4455 * Pointer to the Ethernet device structure. 4456 * @param[in] age_idx 4457 * Index to the ASO age action. 4458 * 4459 * @return 4460 * The specified ASO age action. 4461 */ 4462 struct mlx5_aso_age_action* 4463 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4464 { 4465 uint16_t pool_idx = age_idx & UINT16_MAX; 4466 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4467 struct mlx5_priv *priv = dev->data->dev_private; 4468 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4469 struct mlx5_aso_age_pool *pool; 4470 4471 rte_rwlock_read_lock(&mng->resize_rwl); 4472 pool = mng->pools[pool_idx]; 4473 rte_rwlock_read_unlock(&mng->resize_rwl); 4474 return &pool->actions[offset - 1]; 4475 } 4476 4477 /* maps indirect action to translated direct in some actions array */ 4478 struct mlx5_translated_action_handle { 4479 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4480 int index; /**< Index in related array of rte_flow_action. */ 4481 }; 4482 4483 /** 4484 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4485 * direct action if translation possible. 4486 * This functionality used to run same execution path for both direct and 4487 * indirect actions on flow create. All necessary preparations for indirect 4488 * action handling should be performed on *handle* actions list returned 4489 * from this call. 4490 * 4491 * @param[in] dev 4492 * Pointer to Ethernet device. 4493 * @param[in] actions 4494 * List of actions to translate. 4495 * @param[out] handle 4496 * List to store translated indirect action object handles. 4497 * @param[in, out] indir_n 4498 * Size of *handle* array. On return should be updated with number of 4499 * indirect actions retrieved from the *actions* list. 4500 * @param[out] translated_actions 4501 * List of actions where all indirect actions were translated to direct 4502 * if possible. NULL if no translation took place. 4503 * @param[out] error 4504 * Pointer to the error structure. 4505 * 4506 * @return 4507 * 0 on success, a negative errno value otherwise and rte_errno is set. 4508 */ 4509 static int 4510 flow_action_handles_translate(struct rte_eth_dev *dev, 4511 const struct rte_flow_action actions[], 4512 struct mlx5_translated_action_handle *handle, 4513 int *indir_n, 4514 struct rte_flow_action **translated_actions, 4515 struct rte_flow_error *error) 4516 { 4517 struct mlx5_priv *priv = dev->data->dev_private; 4518 struct rte_flow_action *translated = NULL; 4519 size_t actions_size; 4520 int n; 4521 int copied_n = 0; 4522 struct mlx5_translated_action_handle *handle_end = NULL; 4523 4524 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4525 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4526 continue; 4527 if (copied_n == *indir_n) { 4528 return rte_flow_error_set 4529 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4530 NULL, "too many shared actions"); 4531 } 4532 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4533 sizeof(actions[n].conf)); 4534 handle[copied_n].index = n; 4535 copied_n++; 4536 } 4537 n++; 4538 *indir_n = copied_n; 4539 if (!copied_n) 4540 return 0; 4541 actions_size = sizeof(struct rte_flow_action) * n; 4542 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4543 if (!translated) { 4544 rte_errno = ENOMEM; 4545 return -ENOMEM; 4546 } 4547 memcpy(translated, actions, actions_size); 4548 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4549 struct mlx5_shared_action_rss *shared_rss; 4550 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4551 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4552 uint32_t idx = act_idx & 4553 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4554 4555 switch (type) { 4556 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4557 shared_rss = mlx5_ipool_get 4558 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4559 translated[handle->index].type = 4560 RTE_FLOW_ACTION_TYPE_RSS; 4561 translated[handle->index].conf = 4562 &shared_rss->origin; 4563 break; 4564 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4565 translated[handle->index].type = 4566 (enum rte_flow_action_type) 4567 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4568 translated[handle->index].conf = (void *)(uintptr_t)idx; 4569 break; 4570 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4571 translated[handle->index].type = 4572 (enum rte_flow_action_type) 4573 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4574 translated[handle->index].conf = (void *)(uintptr_t)idx; 4575 break; 4576 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4577 if (priv->sh->flow_hit_aso_en) { 4578 translated[handle->index].type = 4579 (enum rte_flow_action_type) 4580 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4581 translated[handle->index].conf = 4582 (void *)(uintptr_t)idx; 4583 break; 4584 } 4585 /* Fall-through */ 4586 case MLX5_INDIRECT_ACTION_TYPE_CT: 4587 if (priv->sh->ct_aso_en) { 4588 translated[handle->index].type = 4589 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4590 translated[handle->index].conf = 4591 (void *)(uintptr_t)idx; 4592 break; 4593 } 4594 /* Fall-through */ 4595 default: 4596 mlx5_free(translated); 4597 return rte_flow_error_set 4598 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4599 NULL, "invalid indirect action type"); 4600 } 4601 } 4602 *translated_actions = translated; 4603 return 0; 4604 } 4605 4606 /** 4607 * Get Shared RSS action from the action list. 4608 * 4609 * @param[in] dev 4610 * Pointer to Ethernet device. 4611 * @param[in] shared 4612 * Pointer to the list of actions. 4613 * @param[in] shared_n 4614 * Actions list length. 4615 * 4616 * @return 4617 * The MLX5 RSS action ID if exists, otherwise return 0. 4618 */ 4619 static uint32_t 4620 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4621 struct mlx5_translated_action_handle *handle, 4622 int shared_n) 4623 { 4624 struct mlx5_translated_action_handle *handle_end; 4625 struct mlx5_priv *priv = dev->data->dev_private; 4626 struct mlx5_shared_action_rss *shared_rss; 4627 4628 4629 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4630 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4631 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4632 uint32_t idx = act_idx & 4633 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4634 switch (type) { 4635 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4636 shared_rss = mlx5_ipool_get 4637 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4638 idx); 4639 rte_atomic_fetch_add_explicit(&shared_rss->refcnt, 1, 4640 rte_memory_order_relaxed); 4641 return idx; 4642 default: 4643 break; 4644 } 4645 } 4646 return 0; 4647 } 4648 4649 static unsigned int 4650 find_graph_root(uint32_t rss_level) 4651 { 4652 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4653 MLX5_EXPANSION_ROOT_OUTER; 4654 } 4655 4656 /** 4657 * Get layer flags from the prefix flow. 4658 * 4659 * Some flows may be split to several subflows, the prefix subflow gets the 4660 * match items and the suffix sub flow gets the actions. 4661 * Some actions need the user defined match item flags to get the detail for 4662 * the action. 4663 * This function helps the suffix flow to get the item layer flags from prefix 4664 * subflow. 4665 * 4666 * @param[in] dev_flow 4667 * Pointer the created prefix subflow. 4668 * 4669 * @return 4670 * The layers get from prefix subflow. 4671 */ 4672 static inline uint64_t 4673 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4674 { 4675 uint64_t layers = 0; 4676 4677 /* 4678 * Layers bits could be localization, but usually the compiler will 4679 * help to do the optimization work for source code. 4680 * If no decap actions, use the layers directly. 4681 */ 4682 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4683 return dev_flow->handle->layers; 4684 /* Convert L3 layers with decap action. */ 4685 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4686 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4687 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4688 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4689 /* Convert L4 layers with decap action. */ 4690 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4691 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4692 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4693 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4694 return layers; 4695 } 4696 4697 /** 4698 * Get metadata split action information. 4699 * 4700 * @param[in] actions 4701 * Pointer to the list of actions. 4702 * @param[out] qrss 4703 * Pointer to the return pointer. 4704 * @param[out] qrss_type 4705 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4706 * if no QUEUE/RSS is found. 4707 * @param[out] encap_idx 4708 * Pointer to the index of the encap action if exists, otherwise the last 4709 * action index. 4710 * 4711 * @return 4712 * Total number of actions. 4713 */ 4714 static int 4715 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4716 const struct rte_flow_action **qrss, 4717 int *encap_idx) 4718 { 4719 const struct rte_flow_action_raw_encap *raw_encap; 4720 int actions_n = 0; 4721 int raw_decap_idx = -1; 4722 4723 *encap_idx = -1; 4724 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4725 switch (actions->type) { 4726 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4727 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4728 *encap_idx = actions_n; 4729 break; 4730 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4731 raw_decap_idx = actions_n; 4732 break; 4733 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4734 raw_encap = actions->conf; 4735 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4736 *encap_idx = raw_decap_idx != -1 ? 4737 raw_decap_idx : actions_n; 4738 break; 4739 case RTE_FLOW_ACTION_TYPE_QUEUE: 4740 case RTE_FLOW_ACTION_TYPE_RSS: 4741 *qrss = actions; 4742 break; 4743 default: 4744 break; 4745 } 4746 actions_n++; 4747 } 4748 if (*encap_idx == -1) 4749 *encap_idx = actions_n; 4750 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4751 return actions_n + 1; 4752 } 4753 4754 /** 4755 * Check if the action will change packet. 4756 * 4757 * @param dev 4758 * Pointer to Ethernet device. 4759 * @param[in] type 4760 * action type. 4761 * 4762 * @return 4763 * true if action will change packet, false otherwise. 4764 */ 4765 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4766 enum rte_flow_action_type type) 4767 { 4768 struct mlx5_priv *priv = dev->data->dev_private; 4769 4770 switch (type) { 4771 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4772 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4773 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4774 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4775 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4776 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4777 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4778 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4779 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4780 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4781 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4782 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4783 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4784 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4785 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4786 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4787 case RTE_FLOW_ACTION_TYPE_SET_META: 4788 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4789 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4790 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4791 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4792 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4793 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4794 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4795 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4796 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4797 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4798 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4799 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4800 return true; 4801 case RTE_FLOW_ACTION_TYPE_FLAG: 4802 case RTE_FLOW_ACTION_TYPE_MARK: 4803 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4804 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4805 return true; 4806 else 4807 return false; 4808 default: 4809 return false; 4810 } 4811 } 4812 4813 /** 4814 * Check meter action from the action list. 4815 * 4816 * @param dev 4817 * Pointer to Ethernet device. 4818 * @param[in] actions 4819 * Pointer to the list of actions. 4820 * @param[out] has_mtr 4821 * Pointer to the meter exist flag. 4822 * @param[out] has_modify 4823 * Pointer to the flag showing there's packet change action. 4824 * @param[out] meter_id 4825 * Pointer to the meter id. 4826 * 4827 * @return 4828 * Total number of actions. 4829 */ 4830 static int 4831 flow_check_meter_action(struct rte_eth_dev *dev, 4832 const struct rte_flow_action actions[], 4833 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4834 { 4835 const struct rte_flow_action_meter *mtr = NULL; 4836 int actions_n = 0; 4837 4838 MLX5_ASSERT(has_mtr); 4839 *has_mtr = false; 4840 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4841 switch (actions->type) { 4842 case RTE_FLOW_ACTION_TYPE_METER: 4843 mtr = actions->conf; 4844 *meter_id = mtr->mtr_id; 4845 *has_mtr = true; 4846 break; 4847 default: 4848 break; 4849 } 4850 if (!*has_mtr) 4851 *has_modify |= flow_check_modify_action_type(dev, 4852 actions->type); 4853 actions_n++; 4854 } 4855 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4856 return actions_n + 1; 4857 } 4858 4859 /** 4860 * Check if the flow should be split due to hairpin. 4861 * The reason for the split is that in current HW we can't 4862 * support encap and push-vlan on Rx, so if a flow contains 4863 * these actions we move it to Tx. 4864 * 4865 * @param dev 4866 * Pointer to Ethernet device. 4867 * @param[in] attr 4868 * Flow rule attributes. 4869 * @param[in] actions 4870 * Associated actions (list terminated by the END action). 4871 * 4872 * @return 4873 * > 0 the number of actions and the flow should be split, 4874 * 0 when no split required. 4875 */ 4876 static int 4877 flow_check_hairpin_split(struct rte_eth_dev *dev, 4878 const struct rte_flow_attr *attr, 4879 const struct rte_flow_action actions[]) 4880 { 4881 int queue_action = 0; 4882 int action_n = 0; 4883 int split = 0; 4884 int push_vlan = 0; 4885 const struct rte_flow_action_queue *queue; 4886 const struct rte_flow_action_rss *rss; 4887 const struct rte_flow_action_raw_encap *raw_encap; 4888 const struct rte_eth_hairpin_conf *conf; 4889 4890 if (!attr->ingress) 4891 return 0; 4892 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4893 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4894 push_vlan = 1; 4895 switch (actions->type) { 4896 case RTE_FLOW_ACTION_TYPE_QUEUE: 4897 queue = actions->conf; 4898 if (queue == NULL) 4899 return 0; 4900 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4901 if (conf == NULL || conf->tx_explicit != 0) 4902 return 0; 4903 queue_action = 1; 4904 action_n++; 4905 break; 4906 case RTE_FLOW_ACTION_TYPE_RSS: 4907 rss = actions->conf; 4908 if (rss == NULL || rss->queue_num == 0) 4909 return 0; 4910 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4911 if (conf == NULL || conf->tx_explicit != 0) 4912 return 0; 4913 queue_action = 1; 4914 action_n++; 4915 break; 4916 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4917 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4918 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4919 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4920 split++; 4921 action_n++; 4922 break; 4923 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4924 if (push_vlan) 4925 split++; 4926 action_n++; 4927 break; 4928 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4929 raw_encap = actions->conf; 4930 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4931 split++; 4932 action_n++; 4933 break; 4934 default: 4935 action_n++; 4936 break; 4937 } 4938 } 4939 if (split && queue_action) 4940 return action_n; 4941 return 0; 4942 } 4943 4944 /* Declare flow create/destroy prototype in advance. */ 4945 static uint32_t 4946 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4947 const struct rte_flow_attr *attr, 4948 const struct rte_flow_item items[], 4949 const struct rte_flow_action actions[], 4950 bool external, struct rte_flow_error *error); 4951 4952 static void 4953 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4954 uint32_t flow_idx); 4955 4956 int 4957 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4958 struct mlx5_list_entry *entry, void *cb_ctx) 4959 { 4960 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4961 struct mlx5_flow_mreg_copy_resource *mcp_res = 4962 container_of(entry, typeof(*mcp_res), hlist_ent); 4963 4964 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4965 } 4966 4967 struct mlx5_list_entry * 4968 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4969 { 4970 struct rte_eth_dev *dev = tool_ctx; 4971 struct mlx5_priv *priv = dev->data->dev_private; 4972 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4973 struct mlx5_flow_mreg_copy_resource *mcp_res; 4974 struct rte_flow_error *error = ctx->error; 4975 uint32_t idx = 0; 4976 int ret; 4977 uint32_t mark_id = *(uint32_t *)(ctx->data); 4978 struct rte_flow_attr attr = { 4979 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4980 .ingress = 1, 4981 }; 4982 struct mlx5_rte_flow_item_tag tag_spec = { 4983 .data = mark_id, 4984 }; 4985 struct rte_flow_item items[] = { 4986 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4987 }; 4988 struct rte_flow_action_mark ftag = { 4989 .id = mark_id, 4990 }; 4991 struct mlx5_flow_action_copy_mreg cp_mreg = { 4992 .dst = REG_B, 4993 .src = REG_NON, 4994 }; 4995 struct rte_flow_action_jump jump = { 4996 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4997 }; 4998 struct rte_flow_action actions[] = { 4999 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 5000 }; 5001 5002 /* Fill the register fields in the flow. */ 5003 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 5004 if (ret < 0) 5005 return NULL; 5006 tag_spec.id = ret; 5007 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5008 if (ret < 0) 5009 return NULL; 5010 cp_mreg.src = ret; 5011 /* Provide the full width of FLAG specific value. */ 5012 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 5013 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 5014 /* Build a new flow. */ 5015 if (mark_id != MLX5_DEFAULT_COPY_ID) { 5016 items[0] = (struct rte_flow_item){ 5017 .type = (enum rte_flow_item_type) 5018 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5019 .spec = &tag_spec, 5020 }; 5021 items[1] = (struct rte_flow_item){ 5022 .type = RTE_FLOW_ITEM_TYPE_END, 5023 }; 5024 actions[0] = (struct rte_flow_action){ 5025 .type = (enum rte_flow_action_type) 5026 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 5027 .conf = &ftag, 5028 }; 5029 actions[1] = (struct rte_flow_action){ 5030 .type = (enum rte_flow_action_type) 5031 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5032 .conf = &cp_mreg, 5033 }; 5034 actions[2] = (struct rte_flow_action){ 5035 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5036 .conf = &jump, 5037 }; 5038 actions[3] = (struct rte_flow_action){ 5039 .type = RTE_FLOW_ACTION_TYPE_END, 5040 }; 5041 } else { 5042 /* Default rule, wildcard match. */ 5043 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 5044 items[0] = (struct rte_flow_item){ 5045 .type = RTE_FLOW_ITEM_TYPE_END, 5046 }; 5047 actions[0] = (struct rte_flow_action){ 5048 .type = (enum rte_flow_action_type) 5049 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5050 .conf = &cp_mreg, 5051 }; 5052 actions[1] = (struct rte_flow_action){ 5053 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5054 .conf = &jump, 5055 }; 5056 actions[2] = (struct rte_flow_action){ 5057 .type = RTE_FLOW_ACTION_TYPE_END, 5058 }; 5059 } 5060 /* Build a new entry. */ 5061 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5062 if (!mcp_res) { 5063 rte_errno = ENOMEM; 5064 return NULL; 5065 } 5066 mcp_res->idx = idx; 5067 mcp_res->mark_id = mark_id; 5068 /* 5069 * The copy Flows are not included in any list. There 5070 * ones are referenced from other Flows and can not 5071 * be applied, removed, deleted in arbitrary order 5072 * by list traversing. 5073 */ 5074 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 5075 &attr, items, actions, false, error); 5076 if (!mcp_res->rix_flow) { 5077 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 5078 return NULL; 5079 } 5080 return &mcp_res->hlist_ent; 5081 } 5082 5083 struct mlx5_list_entry * 5084 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5085 void *cb_ctx __rte_unused) 5086 { 5087 struct rte_eth_dev *dev = tool_ctx; 5088 struct mlx5_priv *priv = dev->data->dev_private; 5089 struct mlx5_flow_mreg_copy_resource *mcp_res; 5090 uint32_t idx = 0; 5091 5092 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5093 if (!mcp_res) { 5094 rte_errno = ENOMEM; 5095 return NULL; 5096 } 5097 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 5098 mcp_res->idx = idx; 5099 return &mcp_res->hlist_ent; 5100 } 5101 5102 void 5103 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5104 { 5105 struct mlx5_flow_mreg_copy_resource *mcp_res = 5106 container_of(entry, typeof(*mcp_res), hlist_ent); 5107 struct rte_eth_dev *dev = tool_ctx; 5108 struct mlx5_priv *priv = dev->data->dev_private; 5109 5110 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5111 } 5112 5113 /** 5114 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5115 * 5116 * As mark_id is unique, if there's already a registered flow for the mark_id, 5117 * return by increasing the reference counter of the resource. Otherwise, create 5118 * the resource (mcp_res) and flow. 5119 * 5120 * Flow looks like, 5121 * - If ingress port is ANY and reg_c[1] is mark_id, 5122 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5123 * 5124 * For default flow (zero mark_id), flow is like, 5125 * - If ingress port is ANY, 5126 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5127 * 5128 * @param dev 5129 * Pointer to Ethernet device. 5130 * @param mark_id 5131 * ID of MARK action, zero means default flow for META. 5132 * @param[out] error 5133 * Perform verbose error reporting if not NULL. 5134 * 5135 * @return 5136 * Associated resource on success, NULL otherwise and rte_errno is set. 5137 */ 5138 static struct mlx5_flow_mreg_copy_resource * 5139 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5140 struct rte_flow_error *error) 5141 { 5142 struct mlx5_priv *priv = dev->data->dev_private; 5143 struct mlx5_list_entry *entry; 5144 struct mlx5_flow_cb_ctx ctx = { 5145 .dev = dev, 5146 .error = error, 5147 .data = &mark_id, 5148 }; 5149 5150 /* Check if already registered. */ 5151 MLX5_ASSERT(priv->mreg_cp_tbl); 5152 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5153 if (!entry) 5154 return NULL; 5155 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5156 hlist_ent); 5157 } 5158 5159 void 5160 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5161 { 5162 struct mlx5_flow_mreg_copy_resource *mcp_res = 5163 container_of(entry, typeof(*mcp_res), hlist_ent); 5164 struct rte_eth_dev *dev = tool_ctx; 5165 struct mlx5_priv *priv = dev->data->dev_private; 5166 5167 MLX5_ASSERT(mcp_res->rix_flow); 5168 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5169 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5170 } 5171 5172 /** 5173 * Release flow in RX_CP_TBL. 5174 * 5175 * @param dev 5176 * Pointer to Ethernet device. 5177 * @flow 5178 * Parent flow for wich copying is provided. 5179 */ 5180 static void 5181 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5182 struct rte_flow *flow) 5183 { 5184 struct mlx5_flow_mreg_copy_resource *mcp_res; 5185 struct mlx5_priv *priv = dev->data->dev_private; 5186 5187 if (!flow->rix_mreg_copy) 5188 return; 5189 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5190 flow->rix_mreg_copy); 5191 if (!mcp_res || !priv->mreg_cp_tbl) 5192 return; 5193 MLX5_ASSERT(mcp_res->rix_flow); 5194 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5195 flow->rix_mreg_copy = 0; 5196 } 5197 5198 /** 5199 * Remove the default copy action from RX_CP_TBL. 5200 * 5201 * This functions is called in the mlx5_dev_start(). No thread safe 5202 * is guaranteed. 5203 * 5204 * @param dev 5205 * Pointer to Ethernet device. 5206 */ 5207 static void 5208 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5209 { 5210 struct mlx5_list_entry *entry; 5211 struct mlx5_priv *priv = dev->data->dev_private; 5212 struct mlx5_flow_cb_ctx ctx; 5213 uint32_t mark_id; 5214 5215 /* Check if default flow is registered. */ 5216 if (!priv->mreg_cp_tbl) 5217 return; 5218 mark_id = MLX5_DEFAULT_COPY_ID; 5219 ctx.data = &mark_id; 5220 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5221 if (!entry) 5222 return; 5223 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5224 } 5225 5226 /** 5227 * Add the default copy action in RX_CP_TBL. 5228 * 5229 * This functions is called in the mlx5_dev_start(). No thread safe 5230 * is guaranteed. 5231 * 5232 * @param dev 5233 * Pointer to Ethernet device. 5234 * @param[out] error 5235 * Perform verbose error reporting if not NULL. 5236 * 5237 * @return 5238 * 0 for success, negative value otherwise and rte_errno is set. 5239 */ 5240 static int 5241 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5242 struct rte_flow_error *error) 5243 { 5244 struct mlx5_priv *priv = dev->data->dev_private; 5245 struct mlx5_flow_mreg_copy_resource *mcp_res; 5246 struct mlx5_flow_cb_ctx ctx; 5247 uint32_t mark_id; 5248 5249 /* Check whether extensive metadata feature is engaged. */ 5250 if (!priv->sh->config.dv_flow_en || 5251 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5252 !mlx5_flow_ext_mreg_supported(dev) || 5253 !priv->sh->dv_regc0_mask) 5254 return 0; 5255 /* 5256 * Add default mreg copy flow may be called multiple time, but 5257 * only be called once in stop. Avoid register it twice. 5258 */ 5259 mark_id = MLX5_DEFAULT_COPY_ID; 5260 ctx.data = &mark_id; 5261 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5262 return 0; 5263 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5264 if (!mcp_res) 5265 return -rte_errno; 5266 return 0; 5267 } 5268 5269 /** 5270 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5271 * 5272 * All the flow having Q/RSS action should be split by 5273 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5274 * performs the following, 5275 * - CQE->flow_tag := reg_c[1] (MARK) 5276 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5277 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5278 * but there should be a flow per each MARK ID set by MARK action. 5279 * 5280 * For the aforementioned reason, if there's a MARK action in flow's action 5281 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5282 * the MARK ID to CQE's flow_tag like, 5283 * - If reg_c[1] is mark_id, 5284 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5285 * 5286 * For SET_META action which stores value in reg_c[0], as the destination is 5287 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5288 * MARK ID means the default flow. The default flow looks like, 5289 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5290 * 5291 * @param dev 5292 * Pointer to Ethernet device. 5293 * @param flow 5294 * Pointer to flow structure. 5295 * @param[in] actions 5296 * Pointer to the list of actions. 5297 * @param[out] error 5298 * Perform verbose error reporting if not NULL. 5299 * 5300 * @return 5301 * 0 on success, negative value otherwise and rte_errno is set. 5302 */ 5303 static int 5304 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5305 struct rte_flow *flow, 5306 const struct rte_flow_action *actions, 5307 struct rte_flow_error *error) 5308 { 5309 struct mlx5_priv *priv = dev->data->dev_private; 5310 struct mlx5_sh_config *config = &priv->sh->config; 5311 struct mlx5_flow_mreg_copy_resource *mcp_res; 5312 const struct rte_flow_action_mark *mark; 5313 5314 /* Check whether extensive metadata feature is engaged. */ 5315 if (!config->dv_flow_en || 5316 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5317 !mlx5_flow_ext_mreg_supported(dev) || 5318 !priv->sh->dv_regc0_mask) 5319 return 0; 5320 /* Find MARK action. */ 5321 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5322 switch (actions->type) { 5323 case RTE_FLOW_ACTION_TYPE_FLAG: 5324 mcp_res = flow_mreg_add_copy_action 5325 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5326 if (!mcp_res) 5327 return -rte_errno; 5328 flow->rix_mreg_copy = mcp_res->idx; 5329 return 0; 5330 case RTE_FLOW_ACTION_TYPE_MARK: 5331 mark = (const struct rte_flow_action_mark *) 5332 actions->conf; 5333 mcp_res = 5334 flow_mreg_add_copy_action(dev, mark->id, error); 5335 if (!mcp_res) 5336 return -rte_errno; 5337 flow->rix_mreg_copy = mcp_res->idx; 5338 return 0; 5339 default: 5340 break; 5341 } 5342 } 5343 return 0; 5344 } 5345 5346 #define MLX5_MAX_SPLIT_ACTIONS 24 5347 #define MLX5_MAX_SPLIT_ITEMS 24 5348 5349 /** 5350 * Split the hairpin flow. 5351 * Since HW can't support encap and push-vlan on Rx, we move these 5352 * actions to Tx. 5353 * If the count action is after the encap then we also 5354 * move the count action. in this case the count will also measure 5355 * the outer bytes. 5356 * 5357 * @param dev 5358 * Pointer to Ethernet device. 5359 * @param[in] actions 5360 * Associated actions (list terminated by the END action). 5361 * @param[out] actions_rx 5362 * Rx flow actions. 5363 * @param[out] actions_tx 5364 * Tx flow actions.. 5365 * @param[out] pattern_tx 5366 * The pattern items for the Tx flow. 5367 * @param[out] flow_id 5368 * The flow ID connected to this flow. 5369 * 5370 * @return 5371 * 0 on success. 5372 */ 5373 static int 5374 flow_hairpin_split(struct rte_eth_dev *dev, 5375 const struct rte_flow_action actions[], 5376 struct rte_flow_action actions_rx[], 5377 struct rte_flow_action actions_tx[], 5378 struct rte_flow_item pattern_tx[], 5379 uint32_t flow_id) 5380 { 5381 const struct rte_flow_action_raw_encap *raw_encap; 5382 const struct rte_flow_action_raw_decap *raw_decap; 5383 struct mlx5_rte_flow_action_set_tag *set_tag; 5384 struct rte_flow_action *tag_action; 5385 struct mlx5_rte_flow_item_tag *tag_item; 5386 struct rte_flow_item *item; 5387 char *addr; 5388 int push_vlan = 0; 5389 int encap = 0; 5390 5391 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5392 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5393 push_vlan = 1; 5394 switch (actions->type) { 5395 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5396 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5397 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5398 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5399 rte_memcpy(actions_tx, actions, 5400 sizeof(struct rte_flow_action)); 5401 actions_tx++; 5402 break; 5403 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5404 if (push_vlan) { 5405 rte_memcpy(actions_tx, actions, 5406 sizeof(struct rte_flow_action)); 5407 actions_tx++; 5408 } else { 5409 rte_memcpy(actions_rx, actions, 5410 sizeof(struct rte_flow_action)); 5411 actions_rx++; 5412 } 5413 break; 5414 case RTE_FLOW_ACTION_TYPE_COUNT: 5415 case RTE_FLOW_ACTION_TYPE_AGE: 5416 if (encap) { 5417 rte_memcpy(actions_tx, actions, 5418 sizeof(struct rte_flow_action)); 5419 actions_tx++; 5420 } else { 5421 rte_memcpy(actions_rx, actions, 5422 sizeof(struct rte_flow_action)); 5423 actions_rx++; 5424 } 5425 break; 5426 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5427 raw_encap = actions->conf; 5428 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5429 memcpy(actions_tx, actions, 5430 sizeof(struct rte_flow_action)); 5431 actions_tx++; 5432 encap = 1; 5433 } else { 5434 rte_memcpy(actions_rx, actions, 5435 sizeof(struct rte_flow_action)); 5436 actions_rx++; 5437 } 5438 break; 5439 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5440 raw_decap = actions->conf; 5441 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5442 memcpy(actions_tx, actions, 5443 sizeof(struct rte_flow_action)); 5444 actions_tx++; 5445 } else { 5446 rte_memcpy(actions_rx, actions, 5447 sizeof(struct rte_flow_action)); 5448 actions_rx++; 5449 } 5450 break; 5451 default: 5452 rte_memcpy(actions_rx, actions, 5453 sizeof(struct rte_flow_action)); 5454 actions_rx++; 5455 break; 5456 } 5457 } 5458 /* Add set meta action and end action for the Rx flow. */ 5459 tag_action = actions_rx; 5460 tag_action->type = (enum rte_flow_action_type) 5461 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5462 actions_rx++; 5463 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5464 actions_rx++; 5465 set_tag = (void *)actions_rx; 5466 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5467 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5468 .data = flow_id, 5469 }; 5470 MLX5_ASSERT(set_tag->id > REG_NON); 5471 tag_action->conf = set_tag; 5472 /* Create Tx item list. */ 5473 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5474 addr = (void *)&pattern_tx[2]; 5475 item = pattern_tx; 5476 item->type = (enum rte_flow_item_type) 5477 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5478 tag_item = (void *)addr; 5479 tag_item->data = flow_id; 5480 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5481 MLX5_ASSERT(set_tag->id > REG_NON); 5482 item->spec = tag_item; 5483 addr += sizeof(struct mlx5_rte_flow_item_tag); 5484 tag_item = (void *)addr; 5485 tag_item->data = UINT32_MAX; 5486 tag_item->id = UINT16_MAX; 5487 item->mask = tag_item; 5488 item->last = NULL; 5489 item++; 5490 item->type = RTE_FLOW_ITEM_TYPE_END; 5491 return 0; 5492 } 5493 5494 /** 5495 * The last stage of splitting chain, just creates the subflow 5496 * without any modification. 5497 * 5498 * @param[in] dev 5499 * Pointer to Ethernet device. 5500 * @param[in] flow 5501 * Parent flow structure pointer. 5502 * @param[in, out] sub_flow 5503 * Pointer to return the created subflow, may be NULL. 5504 * @param[in] attr 5505 * Flow rule attributes. 5506 * @param[in] items 5507 * Pattern specification (list terminated by the END pattern item). 5508 * @param[in] actions 5509 * Associated actions (list terminated by the END action). 5510 * @param[in] flow_split_info 5511 * Pointer to flow split info structure. 5512 * @param[out] error 5513 * Perform verbose error reporting if not NULL. 5514 * @return 5515 * 0 on success, negative value otherwise 5516 */ 5517 static int 5518 flow_create_split_inner(struct rte_eth_dev *dev, 5519 struct rte_flow *flow, 5520 struct mlx5_flow **sub_flow, 5521 const struct rte_flow_attr *attr, 5522 const struct rte_flow_item items[], 5523 const struct rte_flow_action actions[], 5524 struct mlx5_flow_split_info *flow_split_info, 5525 struct rte_flow_error *error) 5526 { 5527 struct mlx5_flow *dev_flow; 5528 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5529 5530 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5531 flow_split_info->flow_idx, error); 5532 if (!dev_flow) 5533 return -rte_errno; 5534 dev_flow->flow = flow; 5535 dev_flow->external = flow_split_info->external; 5536 dev_flow->skip_scale = flow_split_info->skip_scale; 5537 /* Subflow object was created, we must include one in the list. */ 5538 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5539 dev_flow->handle, next); 5540 /* 5541 * If dev_flow is as one of the suffix flow, some actions in suffix 5542 * flow may need some user defined item layer flags, and pass the 5543 * Metadata rxq mark flag to suffix flow as well. 5544 */ 5545 if (flow_split_info->prefix_layers) 5546 dev_flow->handle->layers = flow_split_info->prefix_layers; 5547 if (flow_split_info->prefix_mark) { 5548 MLX5_ASSERT(wks); 5549 wks->mark = 1; 5550 } 5551 if (sub_flow) 5552 *sub_flow = dev_flow; 5553 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5554 dev_flow->dv.table_id = flow_split_info->table_id; 5555 #endif 5556 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5557 } 5558 5559 /** 5560 * Get the sub policy of a meter. 5561 * 5562 * @param[in] dev 5563 * Pointer to Ethernet device. 5564 * @param[in] flow 5565 * Parent flow structure pointer. 5566 * @param wks 5567 * Pointer to thread flow work space. 5568 * @param[in] attr 5569 * Flow rule attributes. 5570 * @param[in] items 5571 * Pattern specification (list terminated by the END pattern item). 5572 * @param[out] error 5573 * Perform verbose error reporting if not NULL. 5574 * 5575 * @return 5576 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5577 */ 5578 static struct mlx5_flow_meter_sub_policy * 5579 get_meter_sub_policy(struct rte_eth_dev *dev, 5580 struct rte_flow *flow, 5581 struct mlx5_flow_workspace *wks, 5582 const struct rte_flow_attr *attr, 5583 const struct rte_flow_item items[], 5584 struct rte_flow_error *error) 5585 { 5586 struct mlx5_flow_meter_policy *policy; 5587 struct mlx5_flow_meter_policy *final_policy; 5588 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5589 5590 policy = wks->policy; 5591 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5592 if (final_policy->is_rss || final_policy->is_queue) { 5593 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5594 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5595 uint32_t i; 5596 5597 /* 5598 * This is a tmp dev_flow, 5599 * no need to register any matcher for it in translate. 5600 */ 5601 wks->skip_matcher_reg = 1; 5602 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5603 struct mlx5_flow dev_flow = {0}; 5604 struct mlx5_flow_handle dev_handle = { {0} }; 5605 uint8_t fate = final_policy->act_cnt[i].fate_action; 5606 5607 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5608 const struct rte_flow_action_rss *rss_act = 5609 final_policy->act_cnt[i].rss->conf; 5610 struct rte_flow_action rss_actions[2] = { 5611 [0] = { 5612 .type = RTE_FLOW_ACTION_TYPE_RSS, 5613 .conf = rss_act, 5614 }, 5615 [1] = { 5616 .type = RTE_FLOW_ACTION_TYPE_END, 5617 .conf = NULL, 5618 } 5619 }; 5620 5621 dev_flow.handle = &dev_handle; 5622 dev_flow.ingress = attr->ingress; 5623 dev_flow.flow = flow; 5624 dev_flow.external = 0; 5625 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5626 dev_flow.dv.transfer = attr->transfer; 5627 #endif 5628 /** 5629 * Translate RSS action to get rss hash fields. 5630 */ 5631 if (flow_drv_translate(dev, &dev_flow, attr, 5632 items, rss_actions, error)) 5633 goto exit; 5634 rss_desc_v[i] = wks->rss_desc; 5635 rss_desc_v[i].symmetric_hash_function = 5636 dev_flow.symmetric_hash_function; 5637 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5638 rss_desc_v[i].hash_fields = 5639 dev_flow.hash_fields; 5640 rss_desc_v[i].queue_num = 5641 rss_desc_v[i].hash_fields ? 5642 rss_desc_v[i].queue_num : 1; 5643 rss_desc_v[i].tunnel = 5644 !!(dev_flow.handle->layers & 5645 MLX5_FLOW_LAYER_TUNNEL); 5646 /* Use the RSS queues in the containers. */ 5647 rss_desc_v[i].queue = 5648 (uint16_t *)(uintptr_t)rss_act->queue; 5649 rss_desc[i] = &rss_desc_v[i]; 5650 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5651 /* This is queue action. */ 5652 rss_desc_v[i] = wks->rss_desc; 5653 rss_desc_v[i].key_len = 0; 5654 rss_desc_v[i].hash_fields = 0; 5655 rss_desc_v[i].queue = 5656 &final_policy->act_cnt[i].queue; 5657 rss_desc_v[i].queue_num = 1; 5658 rss_desc[i] = &rss_desc_v[i]; 5659 } else { 5660 rss_desc[i] = NULL; 5661 } 5662 } 5663 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5664 flow, policy, rss_desc); 5665 } else { 5666 enum mlx5_meter_domain mtr_domain = 5667 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5668 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5669 MLX5_MTR_DOMAIN_INGRESS); 5670 sub_policy = policy->sub_policys[mtr_domain][0]; 5671 } 5672 if (!sub_policy) 5673 rte_flow_error_set(error, EINVAL, 5674 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5675 "Failed to get meter sub-policy."); 5676 exit: 5677 return sub_policy; 5678 } 5679 5680 /** 5681 * Split the meter flow. 5682 * 5683 * As meter flow will split to three sub flow, other than meter 5684 * action, the other actions make sense to only meter accepts 5685 * the packet. If it need to be dropped, no other additional 5686 * actions should be take. 5687 * 5688 * One kind of special action which decapsulates the L3 tunnel 5689 * header will be in the prefix sub flow, as not to take the 5690 * L3 tunnel header into account. 5691 * 5692 * @param[in] dev 5693 * Pointer to Ethernet device. 5694 * @param[in] flow 5695 * Parent flow structure pointer. 5696 * @param wks 5697 * Pointer to thread flow work space. 5698 * @param[in] attr 5699 * Flow rule attributes. 5700 * @param[in] items 5701 * Pattern specification (list terminated by the END pattern item). 5702 * @param[out] sfx_items 5703 * Suffix flow match items (list terminated by the END pattern item). 5704 * @param[in] actions 5705 * Associated actions (list terminated by the END action). 5706 * @param[out] actions_sfx 5707 * Suffix flow actions. 5708 * @param[out] actions_pre 5709 * Prefix flow actions. 5710 * @param[out] mtr_flow_id 5711 * Pointer to meter flow id. 5712 * @param[out] error 5713 * Perform verbose error reporting if not NULL. 5714 * 5715 * @return 5716 * 0 on success, a negative errno value otherwise and rte_errno is set. 5717 */ 5718 static int 5719 flow_meter_split_prep(struct rte_eth_dev *dev, 5720 struct rte_flow *flow, 5721 struct mlx5_flow_workspace *wks, 5722 const struct rte_flow_attr *attr, 5723 const struct rte_flow_item items[], 5724 struct rte_flow_item sfx_items[], 5725 const struct rte_flow_action actions[], 5726 struct rte_flow_action actions_sfx[], 5727 struct rte_flow_action actions_pre[], 5728 uint32_t *mtr_flow_id, 5729 struct rte_flow_error *error) 5730 { 5731 struct mlx5_priv *priv = dev->data->dev_private; 5732 struct mlx5_flow_meter_info *fm = wks->fm; 5733 struct rte_flow_action *tag_action = NULL; 5734 struct rte_flow_item *tag_item; 5735 struct mlx5_rte_flow_action_set_tag *set_tag; 5736 const struct rte_flow_action_raw_encap *raw_encap; 5737 const struct rte_flow_action_raw_decap *raw_decap; 5738 struct mlx5_rte_flow_item_tag *tag_item_spec; 5739 struct mlx5_rte_flow_item_tag *tag_item_mask; 5740 uint32_t tag_id = 0; 5741 bool vlan_actions; 5742 struct rte_flow_item *orig_sfx_items = sfx_items; 5743 const struct rte_flow_item *orig_items = items; 5744 struct rte_flow_action *hw_mtr_action; 5745 struct rte_flow_action *action_pre_head = NULL; 5746 uint16_t flow_src_port = priv->representor_id; 5747 bool mtr_first; 5748 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5749 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5750 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5751 uint32_t flow_id = 0; 5752 uint32_t flow_id_reversed = 0; 5753 uint8_t flow_id_bits = 0; 5754 bool after_meter = false; 5755 int shift; 5756 5757 /* Prepare the suffix subflow items. */ 5758 tag_item = sfx_items++; 5759 tag_item->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5760 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5761 int item_type = items->type; 5762 5763 switch (item_type) { 5764 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5765 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5766 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5767 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5768 return -rte_errno; 5769 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5770 flow_src_port != priv->representor_id) { 5771 if (flow_drv_mtr_hierarchy_rule_create(dev, 5772 flow, fm, 5773 flow_src_port, 5774 items, 5775 error)) 5776 return -rte_errno; 5777 } 5778 memcpy(sfx_items, items, sizeof(*sfx_items)); 5779 sfx_items++; 5780 break; 5781 case RTE_FLOW_ITEM_TYPE_VLAN: 5782 /* 5783 * Copy VLAN items in case VLAN actions are performed. 5784 * If there are no VLAN actions, these items will be VOID. 5785 */ 5786 memcpy(sfx_items, items, sizeof(*sfx_items)); 5787 sfx_items->type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5788 sfx_items++; 5789 break; 5790 default: 5791 break; 5792 } 5793 } 5794 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5795 sfx_items++; 5796 mtr_first = priv->sh->meter_aso_en && 5797 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5798 /* For ASO meter, meter must be before tag in TX direction. */ 5799 if (mtr_first) { 5800 action_pre_head = actions_pre++; 5801 /* Leave space for tag action. */ 5802 tag_action = actions_pre++; 5803 } 5804 /* Prepare the actions for prefix and suffix flow. */ 5805 vlan_actions = false; 5806 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5807 struct rte_flow_action *action_cur = NULL; 5808 5809 switch (actions->type) { 5810 case RTE_FLOW_ACTION_TYPE_METER: 5811 if (mtr_first) { 5812 action_cur = action_pre_head; 5813 } else { 5814 /* Leave space for tag action. */ 5815 tag_action = actions_pre++; 5816 action_cur = actions_pre++; 5817 } 5818 after_meter = true; 5819 break; 5820 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5821 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5822 action_cur = actions_pre++; 5823 break; 5824 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5825 raw_encap = actions->conf; 5826 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5827 action_cur = actions_pre++; 5828 break; 5829 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5830 raw_decap = actions->conf; 5831 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5832 action_cur = actions_pre++; 5833 break; 5834 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5835 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5836 vlan_actions = true; 5837 break; 5838 case RTE_FLOW_ACTION_TYPE_COUNT: 5839 if (fm->def_policy) 5840 action_cur = after_meter ? 5841 actions_sfx++ : actions_pre++; 5842 break; 5843 default: 5844 break; 5845 } 5846 if (!action_cur) 5847 action_cur = (fm->def_policy) ? 5848 actions_sfx++ : actions_pre++; 5849 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5850 } 5851 /* If there are no VLAN actions, convert VLAN items to VOID in suffix flow items. */ 5852 if (!vlan_actions) { 5853 struct rte_flow_item *it = orig_sfx_items; 5854 5855 for (; it->type != RTE_FLOW_ITEM_TYPE_END; it++) 5856 if (it->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 5857 it->type = RTE_FLOW_ITEM_TYPE_VOID; 5858 } 5859 /* Add end action to the actions. */ 5860 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5861 if (priv->sh->meter_aso_en) { 5862 /** 5863 * For ASO meter, need to add an extra jump action explicitly, 5864 * to jump from meter to policer table. 5865 */ 5866 struct mlx5_flow_meter_sub_policy *sub_policy; 5867 struct mlx5_flow_tbl_data_entry *tbl_data; 5868 5869 if (!fm->def_policy) { 5870 sub_policy = get_meter_sub_policy(dev, flow, wks, 5871 attr, orig_items, 5872 error); 5873 if (!sub_policy) 5874 return -rte_errno; 5875 } else { 5876 enum mlx5_meter_domain mtr_domain = 5877 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5878 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5879 MLX5_MTR_DOMAIN_INGRESS); 5880 5881 sub_policy = 5882 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5883 } 5884 tbl_data = container_of(sub_policy->tbl_rsc, 5885 struct mlx5_flow_tbl_data_entry, tbl); 5886 hw_mtr_action = actions_pre++; 5887 hw_mtr_action->type = (enum rte_flow_action_type) 5888 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5889 hw_mtr_action->conf = tbl_data->jump.action; 5890 } 5891 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5892 actions_pre++; 5893 if (!tag_action) 5894 return rte_flow_error_set(error, ENOMEM, 5895 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5896 NULL, "No tag action space."); 5897 if (!mtr_flow_id) { 5898 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5899 goto exit; 5900 } 5901 /* Only default-policy Meter creates mtr flow id. */ 5902 if (fm->def_policy) { 5903 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5904 if (!tag_id) 5905 return rte_flow_error_set(error, ENOMEM, 5906 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5907 "Failed to allocate meter flow id."); 5908 flow_id = tag_id - 1; 5909 flow_id_bits = (!flow_id) ? 1 : 5910 (MLX5_REG_BITS - rte_clz32(flow_id)); 5911 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5912 mtr_reg_bits) { 5913 mlx5_ipool_free(fm->flow_ipool, tag_id); 5914 return rte_flow_error_set(error, EINVAL, 5915 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5916 "Meter flow id exceeds max limit."); 5917 } 5918 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5919 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5920 } 5921 /* Build tag actions and items for meter_id/meter flow_id. */ 5922 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5923 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5924 tag_item_mask = tag_item_spec + 1; 5925 /* Both flow_id and meter_id share the same register. */ 5926 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5927 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5928 0, error), 5929 .offset = mtr_id_offset, 5930 .length = mtr_reg_bits, 5931 .data = flow->meter, 5932 }; 5933 /* 5934 * The color Reg bits used by flow_id are growing from 5935 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5936 */ 5937 for (shift = 0; shift < flow_id_bits; shift++) 5938 flow_id_reversed = (flow_id_reversed << 1) | 5939 ((flow_id >> shift) & 0x1); 5940 set_tag->data |= 5941 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5942 tag_item_spec->id = set_tag->id; 5943 tag_item_spec->data = set_tag->data << mtr_id_offset; 5944 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5945 tag_action->type = (enum rte_flow_action_type) 5946 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5947 tag_action->conf = set_tag; 5948 tag_item->spec = tag_item_spec; 5949 tag_item->last = NULL; 5950 tag_item->mask = tag_item_mask; 5951 exit: 5952 if (mtr_flow_id) 5953 *mtr_flow_id = tag_id; 5954 return 0; 5955 } 5956 5957 /** 5958 * Split action list having QUEUE/RSS for metadata register copy. 5959 * 5960 * Once Q/RSS action is detected in user's action list, the flow action 5961 * should be split in order to copy metadata registers, which will happen in 5962 * RX_CP_TBL like, 5963 * - CQE->flow_tag := reg_c[1] (MARK) 5964 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5965 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5966 * This is because the last action of each flow must be a terminal action 5967 * (QUEUE, RSS or DROP). 5968 * 5969 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5970 * stored and kept in the mlx5_flow structure per each sub_flow. 5971 * 5972 * The Q/RSS action is replaced with, 5973 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5974 * And the following JUMP action is added at the end, 5975 * - JUMP, to RX_CP_TBL. 5976 * 5977 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5978 * flow_create_split_metadata() routine. The flow will look like, 5979 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5980 * 5981 * @param dev 5982 * Pointer to Ethernet device. 5983 * @param[out] split_actions 5984 * Pointer to store split actions to jump to CP_TBL. 5985 * @param[in] actions 5986 * Pointer to the list of original flow actions. 5987 * @param[in] qrss 5988 * Pointer to the Q/RSS action. 5989 * @param[in] actions_n 5990 * Number of original actions. 5991 * @param[in] mtr_sfx 5992 * Check if it is in meter suffix table. 5993 * @param[out] error 5994 * Perform verbose error reporting if not NULL. 5995 * 5996 * @return 5997 * non-zero unique flow_id on success, otherwise 0 and 5998 * error/rte_error are set. 5999 */ 6000 static uint32_t 6001 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 6002 struct rte_flow_action *split_actions, 6003 const struct rte_flow_action *actions, 6004 const struct rte_flow_action *qrss, 6005 int actions_n, int mtr_sfx, 6006 struct rte_flow_error *error) 6007 { 6008 struct mlx5_priv *priv = dev->data->dev_private; 6009 struct mlx5_rte_flow_action_set_tag *set_tag; 6010 struct rte_flow_action_jump *jump; 6011 const int qrss_idx = qrss - actions; 6012 uint32_t flow_id = 0; 6013 int ret = 0; 6014 6015 /* 6016 * Given actions will be split 6017 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 6018 * - Add jump to mreg CP_TBL. 6019 * As a result, there will be one more action. 6020 */ 6021 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 6022 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 6023 ++actions_n; 6024 set_tag = (void *)(split_actions + actions_n); 6025 /* 6026 * If we are not the meter suffix flow, add the tag action. 6027 * Since meter suffix flow already has the tag added. 6028 */ 6029 if (!mtr_sfx) { 6030 /* 6031 * Allocate the new subflow ID. This one is unique within 6032 * device and not shared with representors. Otherwise, 6033 * we would have to resolve multi-thread access synch 6034 * issue. Each flow on the shared device is appended 6035 * with source vport identifier, so the resulting 6036 * flows will be unique in the shared (by master and 6037 * representors) domain even if they have coinciding 6038 * IDs. 6039 */ 6040 mlx5_ipool_malloc(priv->sh->ipool 6041 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 6042 if (!flow_id) 6043 return rte_flow_error_set(error, ENOMEM, 6044 RTE_FLOW_ERROR_TYPE_ACTION, 6045 NULL, "can't allocate id " 6046 "for split Q/RSS subflow"); 6047 /* Internal SET_TAG action to set flow ID. */ 6048 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 6049 .data = flow_id, 6050 }; 6051 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 6052 if (ret < 0) 6053 return ret; 6054 set_tag->id = ret; 6055 /* Construct new actions array. */ 6056 /* Replace QUEUE/RSS action. */ 6057 split_actions[qrss_idx] = (struct rte_flow_action){ 6058 .type = (enum rte_flow_action_type) 6059 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6060 .conf = set_tag, 6061 }; 6062 } else { 6063 /* 6064 * If we are the suffix flow of meter, tag already exist. 6065 * Set the QUEUE/RSS action to void. 6066 */ 6067 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 6068 } 6069 /* JUMP action to jump to mreg copy table (CP_TBL). */ 6070 jump = (void *)(set_tag + 1); 6071 *jump = (struct rte_flow_action_jump){ 6072 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6073 }; 6074 split_actions[actions_n - 2] = (struct rte_flow_action){ 6075 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6076 .conf = jump, 6077 }; 6078 split_actions[actions_n - 1] = (struct rte_flow_action){ 6079 .type = RTE_FLOW_ACTION_TYPE_END, 6080 }; 6081 return flow_id; 6082 } 6083 6084 /** 6085 * Extend the given action list for Tx metadata copy. 6086 * 6087 * Copy the given action list to the ext_actions and add flow metadata register 6088 * copy action in order to copy reg_a set by WQE to reg_c[0]. 6089 * 6090 * @param[out] ext_actions 6091 * Pointer to the extended action list. 6092 * @param[in] actions 6093 * Pointer to the list of actions. 6094 * @param[in] actions_n 6095 * Number of actions in the list. 6096 * @param[out] error 6097 * Perform verbose error reporting if not NULL. 6098 * @param[in] encap_idx 6099 * The encap action index. 6100 * 6101 * @return 6102 * 0 on success, negative value otherwise 6103 */ 6104 static int 6105 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6106 struct rte_flow_action *ext_actions, 6107 const struct rte_flow_action *actions, 6108 int actions_n, struct rte_flow_error *error, 6109 int encap_idx) 6110 { 6111 struct mlx5_flow_action_copy_mreg *cp_mreg = 6112 (struct mlx5_flow_action_copy_mreg *) 6113 (ext_actions + actions_n + 1); 6114 int ret; 6115 6116 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6117 if (ret < 0) 6118 return ret; 6119 cp_mreg->dst = ret; 6120 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6121 if (ret < 0) 6122 return ret; 6123 cp_mreg->src = ret; 6124 if (encap_idx != 0) 6125 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6126 if (encap_idx == actions_n - 1) { 6127 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6128 .type = (enum rte_flow_action_type) 6129 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6130 .conf = cp_mreg, 6131 }; 6132 ext_actions[actions_n] = (struct rte_flow_action){ 6133 .type = RTE_FLOW_ACTION_TYPE_END, 6134 }; 6135 } else { 6136 ext_actions[encap_idx] = (struct rte_flow_action){ 6137 .type = (enum rte_flow_action_type) 6138 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6139 .conf = cp_mreg, 6140 }; 6141 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6142 sizeof(*ext_actions) * (actions_n - encap_idx)); 6143 } 6144 return 0; 6145 } 6146 6147 /** 6148 * Check the match action from the action list. 6149 * 6150 * @param[in] actions 6151 * Pointer to the list of actions. 6152 * @param[in] attr 6153 * Flow rule attributes. 6154 * @param[in] action 6155 * The action to be check if exist. 6156 * @param[out] match_action_pos 6157 * Pointer to the position of the matched action if exists, otherwise is -1. 6158 * @param[out] qrss_action_pos 6159 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6160 * @param[out] modify_after_mirror 6161 * Pointer to the flag of modify action after FDB mirroring. 6162 * 6163 * @return 6164 * > 0 the total number of actions. 6165 * 0 if not found match action in action list. 6166 */ 6167 static int 6168 flow_check_match_action(const struct rte_flow_action actions[], 6169 const struct rte_flow_attr *attr, 6170 enum rte_flow_action_type action, 6171 int *match_action_pos, int *qrss_action_pos, 6172 int *modify_after_mirror) 6173 { 6174 const struct rte_flow_action_sample *sample; 6175 const struct rte_flow_action_raw_decap *decap; 6176 const struct rte_flow_action *action_cur = NULL; 6177 int actions_n = 0; 6178 uint32_t ratio = 0; 6179 int sub_type = 0; 6180 int flag = 0; 6181 int fdb_mirror = 0; 6182 6183 *match_action_pos = -1; 6184 *qrss_action_pos = -1; 6185 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6186 if (actions->type == action) { 6187 flag = 1; 6188 *match_action_pos = actions_n; 6189 } 6190 switch (actions->type) { 6191 case RTE_FLOW_ACTION_TYPE_QUEUE: 6192 case RTE_FLOW_ACTION_TYPE_RSS: 6193 *qrss_action_pos = actions_n; 6194 break; 6195 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6196 sample = actions->conf; 6197 ratio = sample->ratio; 6198 sub_type = ((const struct rte_flow_action *) 6199 (sample->actions))->type; 6200 if (ratio == 1 && attr->transfer && 6201 sub_type != RTE_FLOW_ACTION_TYPE_END) 6202 fdb_mirror = 1; 6203 break; 6204 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6205 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6206 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6207 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6208 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6209 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6210 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6211 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6212 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6213 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6214 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6215 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6216 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6217 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6218 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6219 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6220 case RTE_FLOW_ACTION_TYPE_FLAG: 6221 case RTE_FLOW_ACTION_TYPE_MARK: 6222 case RTE_FLOW_ACTION_TYPE_SET_META: 6223 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6224 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6225 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6226 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6227 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6228 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6229 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6230 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6231 case RTE_FLOW_ACTION_TYPE_METER: 6232 if (fdb_mirror) 6233 *modify_after_mirror = 1; 6234 break; 6235 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6236 decap = actions->conf; 6237 action_cur = actions; 6238 while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID) 6239 ; 6240 if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6241 const struct rte_flow_action_raw_encap *encap = 6242 action_cur->conf; 6243 if (decap->size <= 6244 MLX5_ENCAPSULATION_DECISION_SIZE && 6245 encap->size > 6246 MLX5_ENCAPSULATION_DECISION_SIZE) 6247 /* L3 encap. */ 6248 break; 6249 } 6250 if (fdb_mirror) 6251 *modify_after_mirror = 1; 6252 break; 6253 default: 6254 break; 6255 } 6256 actions_n++; 6257 } 6258 if (flag && fdb_mirror && !*modify_after_mirror) { 6259 /* FDB mirroring uses the destination array to implement 6260 * instead of FLOW_SAMPLER object. 6261 */ 6262 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6263 flag = 0; 6264 } 6265 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6266 return flag ? actions_n + 1 : 0; 6267 } 6268 6269 #define SAMPLE_SUFFIX_ITEM 3 6270 6271 /** 6272 * Split the sample flow. 6273 * 6274 * As sample flow will split to two sub flow, sample flow with 6275 * sample action, the other actions will move to new suffix flow. 6276 * 6277 * Also add unique tag id with tag action in the sample flow, 6278 * the same tag id will be as match in the suffix flow. 6279 * 6280 * @param dev 6281 * Pointer to Ethernet device. 6282 * @param[in] add_tag 6283 * Add extra tag action flag. 6284 * @param[out] sfx_items 6285 * Suffix flow match items (list terminated by the END pattern item). 6286 * @param[in] actions 6287 * Associated actions (list terminated by the END action). 6288 * @param[out] actions_sfx 6289 * Suffix flow actions. 6290 * @param[out] actions_pre 6291 * Prefix flow actions. 6292 * @param[in] actions_n 6293 * The total number of actions. 6294 * @param[in] sample_action_pos 6295 * The sample action position. 6296 * @param[in] qrss_action_pos 6297 * The Queue/RSS action position. 6298 * @param[in] jump_table 6299 * Add extra jump action flag. 6300 * @param[out] error 6301 * Perform verbose error reporting if not NULL. 6302 * 6303 * @return 6304 * 0 on success, or unique flow_id, a negative errno value 6305 * otherwise and rte_errno is set. 6306 */ 6307 static int 6308 flow_sample_split_prep(struct rte_eth_dev *dev, 6309 int add_tag, 6310 const struct rte_flow_item items[], 6311 struct rte_flow_item sfx_items[], 6312 const struct rte_flow_action actions[], 6313 struct rte_flow_action actions_sfx[], 6314 struct rte_flow_action actions_pre[], 6315 int actions_n, 6316 int sample_action_pos, 6317 int qrss_action_pos, 6318 int jump_table, 6319 struct rte_flow_error *error) 6320 { 6321 struct mlx5_priv *priv = dev->data->dev_private; 6322 struct mlx5_rte_flow_action_set_tag *set_tag; 6323 struct mlx5_rte_flow_item_tag *tag_spec; 6324 struct mlx5_rte_flow_item_tag *tag_mask; 6325 struct rte_flow_action_jump *jump_action; 6326 uint32_t tag_id = 0; 6327 int append_index = 0; 6328 int set_tag_idx = -1; 6329 int index; 6330 int ret; 6331 6332 if (sample_action_pos < 0) 6333 return rte_flow_error_set(error, EINVAL, 6334 RTE_FLOW_ERROR_TYPE_ACTION, 6335 NULL, "invalid position of sample " 6336 "action in list"); 6337 /* Prepare the actions for prefix and suffix flow. */ 6338 if (add_tag) { 6339 /* Update the new added tag action index preceding 6340 * the PUSH_VLAN or ENCAP action. 6341 */ 6342 const struct rte_flow_action_raw_encap *raw_encap; 6343 const struct rte_flow_action *action = actions; 6344 int encap_idx; 6345 int action_idx = 0; 6346 int raw_decap_idx = -1; 6347 int push_vlan_idx = -1; 6348 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6349 switch (action->type) { 6350 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6351 raw_decap_idx = action_idx; 6352 break; 6353 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6354 raw_encap = action->conf; 6355 if (raw_encap->size > 6356 MLX5_ENCAPSULATION_DECISION_SIZE) { 6357 encap_idx = raw_decap_idx != -1 ? 6358 raw_decap_idx : action_idx; 6359 if (encap_idx < sample_action_pos && 6360 push_vlan_idx == -1) 6361 set_tag_idx = encap_idx; 6362 } 6363 break; 6364 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6365 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6366 encap_idx = action_idx; 6367 if (encap_idx < sample_action_pos && 6368 push_vlan_idx == -1) 6369 set_tag_idx = encap_idx; 6370 break; 6371 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6372 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6373 if (action_idx < sample_action_pos && 6374 push_vlan_idx == -1) { 6375 set_tag_idx = action_idx; 6376 push_vlan_idx = action_idx; 6377 } 6378 break; 6379 default: 6380 break; 6381 } 6382 action_idx++; 6383 } 6384 } 6385 /* Prepare the actions for prefix and suffix flow. */ 6386 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6387 index = qrss_action_pos; 6388 /* Put the preceding the Queue/RSS action into prefix flow. */ 6389 if (index != 0) 6390 memcpy(actions_pre, actions, 6391 sizeof(struct rte_flow_action) * index); 6392 /* Put others preceding the sample action into prefix flow. */ 6393 if (sample_action_pos > index + 1) 6394 memcpy(actions_pre + index, actions + index + 1, 6395 sizeof(struct rte_flow_action) * 6396 (sample_action_pos - index - 1)); 6397 index = sample_action_pos - 1; 6398 /* Put Queue/RSS action into Suffix flow. */ 6399 memcpy(actions_sfx, actions + qrss_action_pos, 6400 sizeof(struct rte_flow_action)); 6401 actions_sfx++; 6402 } else if (add_tag && set_tag_idx >= 0) { 6403 if (set_tag_idx > 0) 6404 memcpy(actions_pre, actions, 6405 sizeof(struct rte_flow_action) * set_tag_idx); 6406 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6407 sizeof(struct rte_flow_action) * 6408 (sample_action_pos - set_tag_idx)); 6409 index = sample_action_pos; 6410 } else { 6411 index = sample_action_pos; 6412 if (index != 0) 6413 memcpy(actions_pre, actions, 6414 sizeof(struct rte_flow_action) * index); 6415 } 6416 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6417 * For CX6DX and above, metadata registers Cx preserve their value, 6418 * add an extra tag action for NIC-RX and E-Switch Domain. 6419 */ 6420 if (add_tag) { 6421 /* Prepare the prefix tag action. */ 6422 append_index++; 6423 set_tag = (void *)(actions_pre + actions_n + append_index); 6424 /* Trust VF/SF on CX5 not supported meter so that the reserved 6425 * metadata regC is REG_NON, back to use application tag 6426 * index 0. 6427 */ 6428 if (unlikely(priv->sh->registers.aso_reg == REG_NON)) 6429 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6430 else 6431 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6432 if (ret < 0) 6433 return ret; 6434 mlx5_ipool_malloc(priv->sh->ipool 6435 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6436 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6437 .id = ret, 6438 .data = tag_id, 6439 }; 6440 /* Prepare the suffix subflow items. */ 6441 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6442 tag_spec->data = tag_id; 6443 tag_spec->id = set_tag->id; 6444 tag_mask = tag_spec + 1; 6445 tag_mask->data = UINT32_MAX; 6446 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6447 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6448 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6449 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6450 memcpy(sfx_items, items, sizeof(*sfx_items)); 6451 sfx_items++; 6452 break; 6453 } 6454 } 6455 sfx_items[0] = (struct rte_flow_item){ 6456 .type = (enum rte_flow_item_type) 6457 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6458 .spec = tag_spec, 6459 .last = NULL, 6460 .mask = tag_mask, 6461 }; 6462 sfx_items[1] = (struct rte_flow_item){ 6463 .type = (enum rte_flow_item_type) 6464 RTE_FLOW_ITEM_TYPE_END, 6465 }; 6466 /* Prepare the tag action in prefix subflow. */ 6467 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6468 actions_pre[set_tag_idx] = 6469 (struct rte_flow_action){ 6470 .type = (enum rte_flow_action_type) 6471 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6472 .conf = set_tag, 6473 }; 6474 /* Update next sample position due to add one tag action */ 6475 index += 1; 6476 } 6477 /* Copy the sample action into prefix flow. */ 6478 memcpy(actions_pre + index, actions + sample_action_pos, 6479 sizeof(struct rte_flow_action)); 6480 index += 1; 6481 /* For the modify action after the sample action in E-Switch mirroring, 6482 * Add the extra jump action in prefix subflow and jump into the next 6483 * table, then do the modify action in the new table. 6484 */ 6485 if (jump_table) { 6486 /* Prepare the prefix jump action. */ 6487 append_index++; 6488 jump_action = (void *)(actions_pre + actions_n + append_index); 6489 jump_action->group = jump_table; 6490 actions_pre[index++] = 6491 (struct rte_flow_action){ 6492 .type = (enum rte_flow_action_type) 6493 RTE_FLOW_ACTION_TYPE_JUMP, 6494 .conf = jump_action, 6495 }; 6496 } 6497 actions_pre[index] = (struct rte_flow_action){ 6498 .type = (enum rte_flow_action_type) 6499 RTE_FLOW_ACTION_TYPE_END, 6500 }; 6501 /* Put the actions after sample into Suffix flow. */ 6502 memcpy(actions_sfx, actions + sample_action_pos + 1, 6503 sizeof(struct rte_flow_action) * 6504 (actions_n - sample_action_pos - 1)); 6505 return tag_id; 6506 } 6507 6508 /** 6509 * The splitting for metadata feature. 6510 * 6511 * - Q/RSS action on NIC Rx should be split in order to pass by 6512 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6513 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6514 * 6515 * - All the actions on NIC Tx should have a mreg copy action to 6516 * copy reg_a from WQE to reg_c[0]. 6517 * 6518 * @param dev 6519 * Pointer to Ethernet device. 6520 * @param[in] flow 6521 * Parent flow structure pointer. 6522 * @param[in] attr 6523 * Flow rule attributes. 6524 * @param[in] items 6525 * Pattern specification (list terminated by the END pattern item). 6526 * @param[in] actions 6527 * Associated actions (list terminated by the END action). 6528 * @param[in] flow_split_info 6529 * Pointer to flow split info structure. 6530 * @param[out] error 6531 * Perform verbose error reporting if not NULL. 6532 * @return 6533 * 0 on success, negative value otherwise 6534 */ 6535 static int 6536 flow_create_split_metadata(struct rte_eth_dev *dev, 6537 struct rte_flow *flow, 6538 const struct rte_flow_attr *attr, 6539 const struct rte_flow_item items[], 6540 const struct rte_flow_action actions[], 6541 struct mlx5_flow_split_info *flow_split_info, 6542 struct rte_flow_error *error) 6543 { 6544 struct mlx5_priv *priv = dev->data->dev_private; 6545 struct mlx5_sh_config *config = &priv->sh->config; 6546 const struct rte_flow_action *qrss = NULL; 6547 struct rte_flow_action *ext_actions = NULL; 6548 struct mlx5_flow *dev_flow = NULL; 6549 uint32_t qrss_id = 0; 6550 int mtr_sfx = 0; 6551 size_t act_size; 6552 int actions_n; 6553 int encap_idx; 6554 int ret; 6555 6556 /* Check whether extensive metadata feature is engaged. */ 6557 if (!config->dv_flow_en || 6558 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6559 !mlx5_flow_ext_mreg_supported(dev)) 6560 return flow_create_split_inner(dev, flow, NULL, attr, items, 6561 actions, flow_split_info, error); 6562 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6563 &encap_idx); 6564 if (qrss) { 6565 /* Exclude hairpin flows from splitting. */ 6566 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6567 const struct rte_flow_action_queue *queue; 6568 6569 queue = qrss->conf; 6570 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6571 qrss = NULL; 6572 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6573 const struct rte_flow_action_rss *rss; 6574 6575 rss = qrss->conf; 6576 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6577 qrss = NULL; 6578 } 6579 } 6580 if (qrss) { 6581 /* Check if it is in meter suffix table. */ 6582 mtr_sfx = attr->group == 6583 ((attr->transfer && priv->fdb_def_rule) ? 6584 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6585 MLX5_FLOW_TABLE_LEVEL_METER); 6586 /* 6587 * Q/RSS action on NIC Rx should be split in order to pass by 6588 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6589 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6590 */ 6591 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6592 sizeof(struct rte_flow_action_set_tag) + 6593 sizeof(struct rte_flow_action_jump); 6594 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6595 SOCKET_ID_ANY); 6596 if (!ext_actions) 6597 return rte_flow_error_set(error, ENOMEM, 6598 RTE_FLOW_ERROR_TYPE_ACTION, 6599 NULL, "no memory to split " 6600 "metadata flow"); 6601 /* 6602 * Create the new actions list with removed Q/RSS action 6603 * and appended set tag and jump to register copy table 6604 * (RX_CP_TBL). We should preallocate unique tag ID here 6605 * in advance, because it is needed for set tag action. 6606 */ 6607 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6608 qrss, actions_n, 6609 mtr_sfx, error); 6610 if (!mtr_sfx && !qrss_id) { 6611 ret = -rte_errno; 6612 goto exit; 6613 } 6614 } else if (attr->egress) { 6615 /* 6616 * All the actions on NIC Tx should have a metadata register 6617 * copy action to copy reg_a from WQE to reg_c[meta] 6618 */ 6619 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6620 sizeof(struct mlx5_flow_action_copy_mreg); 6621 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6622 SOCKET_ID_ANY); 6623 if (!ext_actions) 6624 return rte_flow_error_set(error, ENOMEM, 6625 RTE_FLOW_ERROR_TYPE_ACTION, 6626 NULL, "no memory to split " 6627 "metadata flow"); 6628 /* Create the action list appended with copy register. */ 6629 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6630 actions_n, error, encap_idx); 6631 if (ret < 0) 6632 goto exit; 6633 } 6634 /* Add the unmodified original or prefix subflow. */ 6635 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6636 items, ext_actions ? ext_actions : 6637 actions, flow_split_info, error); 6638 if (ret < 0) 6639 goto exit; 6640 MLX5_ASSERT(dev_flow); 6641 if (qrss) { 6642 const struct rte_flow_attr q_attr = { 6643 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6644 .ingress = 1, 6645 }; 6646 /* Internal PMD action to set register. */ 6647 struct mlx5_rte_flow_item_tag q_tag_spec = { 6648 .data = qrss_id, 6649 .id = REG_NON, 6650 }; 6651 struct rte_flow_item q_items[] = { 6652 { 6653 .type = (enum rte_flow_item_type) 6654 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6655 .spec = &q_tag_spec, 6656 .last = NULL, 6657 .mask = NULL, 6658 }, 6659 { 6660 .type = RTE_FLOW_ITEM_TYPE_END, 6661 }, 6662 }; 6663 struct rte_flow_action q_actions[] = { 6664 { 6665 .type = qrss->type, 6666 .conf = qrss->conf, 6667 }, 6668 { 6669 .type = RTE_FLOW_ACTION_TYPE_END, 6670 }, 6671 }; 6672 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6673 6674 /* 6675 * Configure the tag item only if there is no meter subflow. 6676 * Since tag is already marked in the meter suffix subflow 6677 * we can just use the meter suffix items as is. 6678 */ 6679 if (qrss_id) { 6680 /* Not meter subflow. */ 6681 MLX5_ASSERT(!mtr_sfx); 6682 /* 6683 * Put unique id in prefix flow due to it is destroyed 6684 * after suffix flow and id will be freed after there 6685 * is no actual flows with this id and identifier 6686 * reallocation becomes possible (for example, for 6687 * other flows in other threads). 6688 */ 6689 dev_flow->handle->split_flow_id = qrss_id; 6690 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6691 error); 6692 if (ret < 0) 6693 goto exit; 6694 q_tag_spec.id = ret; 6695 } 6696 dev_flow = NULL; 6697 /* Add suffix subflow to execute Q/RSS. */ 6698 flow_split_info->prefix_layers = layers; 6699 flow_split_info->prefix_mark = 0; 6700 flow_split_info->table_id = 0; 6701 ret = flow_create_split_inner(dev, flow, &dev_flow, 6702 &q_attr, mtr_sfx ? items : 6703 q_items, q_actions, 6704 flow_split_info, error); 6705 if (ret < 0) 6706 goto exit; 6707 /* qrss ID should be freed if failed. */ 6708 qrss_id = 0; 6709 MLX5_ASSERT(dev_flow); 6710 } 6711 6712 exit: 6713 /* 6714 * We do not destroy the partially created sub_flows in case of error. 6715 * These ones are included into parent flow list and will be destroyed 6716 * by flow_drv_destroy. 6717 */ 6718 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6719 qrss_id); 6720 mlx5_free(ext_actions); 6721 return ret; 6722 } 6723 6724 /** 6725 * Create meter internal drop flow with the original pattern. 6726 * 6727 * @param dev 6728 * Pointer to Ethernet device. 6729 * @param[in] flow 6730 * Parent flow structure pointer. 6731 * @param[in] attr 6732 * Flow rule attributes. 6733 * @param[in] items 6734 * Pattern specification (list terminated by the END pattern item). 6735 * @param[in] flow_split_info 6736 * Pointer to flow split info structure. 6737 * @param[in] fm 6738 * Pointer to flow meter structure. 6739 * @param[out] error 6740 * Perform verbose error reporting if not NULL. 6741 * @return 6742 * 0 on success, negative value otherwise 6743 */ 6744 static uint32_t 6745 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6746 struct rte_flow *flow, 6747 const struct rte_flow_attr *attr, 6748 const struct rte_flow_item items[], 6749 struct mlx5_flow_split_info *flow_split_info, 6750 struct mlx5_flow_meter_info *fm, 6751 struct rte_flow_error *error) 6752 { 6753 struct mlx5_flow *dev_flow = NULL; 6754 struct rte_flow_attr drop_attr = *attr; 6755 struct rte_flow_action drop_actions[3]; 6756 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6757 6758 MLX5_ASSERT(fm->drop_cnt); 6759 drop_actions[0].type = 6760 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6761 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6762 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6763 drop_actions[1].conf = NULL; 6764 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6765 drop_actions[2].conf = NULL; 6766 drop_split_info.external = false; 6767 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6768 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6769 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6770 return flow_create_split_inner(dev, flow, &dev_flow, 6771 &drop_attr, items, drop_actions, 6772 &drop_split_info, error); 6773 } 6774 6775 static int 6776 flow_count_vlan_items(const struct rte_flow_item items[]) 6777 { 6778 int items_n = 0; 6779 6780 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6781 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN || 6782 items->type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_VLAN) 6783 items_n++; 6784 } 6785 return items_n; 6786 } 6787 6788 /** 6789 * The splitting for meter feature. 6790 * 6791 * - The meter flow will be split to two flows as prefix and 6792 * suffix flow. The packets make sense only it pass the prefix 6793 * meter action. 6794 * 6795 * - Reg_C_5 is used for the packet to match betweend prefix and 6796 * suffix flow. 6797 * 6798 * @param dev 6799 * Pointer to Ethernet device. 6800 * @param[in] flow 6801 * Parent flow structure pointer. 6802 * @param[in] attr 6803 * Flow rule attributes. 6804 * @param[in] items 6805 * Pattern specification (list terminated by the END pattern item). 6806 * @param[in] actions 6807 * Associated actions (list terminated by the END action). 6808 * @param[in] flow_split_info 6809 * Pointer to flow split info structure. 6810 * @param[out] error 6811 * Perform verbose error reporting if not NULL. 6812 * @return 6813 * 0 on success, negative value otherwise 6814 */ 6815 static int 6816 flow_create_split_meter(struct rte_eth_dev *dev, 6817 struct rte_flow *flow, 6818 const struct rte_flow_attr *attr, 6819 const struct rte_flow_item items[], 6820 const struct rte_flow_action actions[], 6821 struct mlx5_flow_split_info *flow_split_info, 6822 struct rte_flow_error *error) 6823 { 6824 struct mlx5_priv *priv = dev->data->dev_private; 6825 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6826 struct rte_flow_action *sfx_actions = NULL; 6827 struct rte_flow_action *pre_actions = NULL; 6828 struct rte_flow_item *sfx_items = NULL; 6829 struct mlx5_flow *dev_flow = NULL; 6830 struct rte_flow_attr sfx_attr = *attr; 6831 struct mlx5_flow_meter_info *fm = NULL; 6832 uint8_t skip_scale_restore; 6833 bool has_mtr = false; 6834 bool has_modify = false; 6835 bool set_mtr_reg = true; 6836 bool is_mtr_hierarchy = false; 6837 uint32_t meter_id = 0; 6838 uint32_t mtr_idx = 0; 6839 uint32_t mtr_flow_id = 0; 6840 size_t act_size; 6841 size_t item_size; 6842 int actions_n = 0; 6843 int vlan_items_n = 0; 6844 int ret = 0; 6845 6846 if (priv->mtr_en) 6847 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6848 &has_modify, &meter_id); 6849 if (has_mtr) { 6850 if (flow->meter) { 6851 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6852 if (!fm) 6853 return rte_flow_error_set(error, EINVAL, 6854 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6855 NULL, "Meter not found."); 6856 } else { 6857 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6858 if (!fm) 6859 return rte_flow_error_set(error, EINVAL, 6860 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6861 NULL, "Meter not found."); 6862 ret = mlx5_flow_meter_attach(priv, fm, 6863 &sfx_attr, error); 6864 if (ret) 6865 return -rte_errno; 6866 flow->meter = mtr_idx; 6867 } 6868 MLX5_ASSERT(wks); 6869 wks->fm = fm; 6870 if (!fm->def_policy) { 6871 wks->policy = mlx5_flow_meter_policy_find(dev, 6872 fm->policy_id, 6873 NULL); 6874 MLX5_ASSERT(wks->policy); 6875 if (wks->policy->mark) 6876 wks->mark = 1; 6877 if (wks->policy->is_hierarchy) { 6878 wks->final_policy = 6879 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6880 wks->policy); 6881 if (!wks->final_policy) 6882 return rte_flow_error_set(error, 6883 EINVAL, 6884 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6885 "Failed to find terminal policy of hierarchy."); 6886 is_mtr_hierarchy = true; 6887 } 6888 } 6889 /* 6890 * If it isn't default-policy Meter, and 6891 * 1. Not meter hierarchy and there's no action in flow to change 6892 * packet (modify/encap/decap etc.), OR 6893 * 2. No drop count needed for this meter. 6894 * Then no need to use regC to save meter id anymore. 6895 */ 6896 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6897 set_mtr_reg = false; 6898 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6899 #define METER_PREFIX_ACTION 7 6900 act_size = (sizeof(struct rte_flow_action) * 6901 (actions_n + METER_PREFIX_ACTION)) + 6902 sizeof(struct mlx5_rte_flow_action_set_tag); 6903 /* Flow can have multiple VLAN items. Account for them in suffix items. */ 6904 vlan_items_n = flow_count_vlan_items(items); 6905 /* Suffix items: tag, [vlans], port id, end. */ 6906 #define METER_SUFFIX_ITEM 3 6907 item_size = sizeof(struct rte_flow_item) * (METER_SUFFIX_ITEM + vlan_items_n) + 6908 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6909 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6910 0, SOCKET_ID_ANY); 6911 if (!sfx_actions) 6912 return rte_flow_error_set(error, ENOMEM, 6913 RTE_FLOW_ERROR_TYPE_ACTION, 6914 NULL, "no memory to split " 6915 "meter flow"); 6916 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6917 act_size); 6918 /* There's no suffix flow for meter of non-default policy. */ 6919 if (!fm->def_policy) 6920 pre_actions = sfx_actions + 1; 6921 else 6922 pre_actions = sfx_actions + actions_n; 6923 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6924 items, sfx_items, actions, 6925 sfx_actions, pre_actions, 6926 (set_mtr_reg ? &mtr_flow_id : NULL), 6927 error); 6928 if (ret) { 6929 ret = -rte_errno; 6930 goto exit; 6931 } 6932 /* Add the prefix subflow. */ 6933 skip_scale_restore = flow_split_info->skip_scale; 6934 flow_split_info->skip_scale |= 6935 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6936 ret = flow_create_split_inner(dev, flow, &dev_flow, 6937 attr, items, pre_actions, 6938 flow_split_info, error); 6939 flow_split_info->skip_scale = skip_scale_restore; 6940 if (ret) { 6941 if (mtr_flow_id) 6942 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6943 ret = -rte_errno; 6944 goto exit; 6945 } 6946 if (mtr_flow_id) { 6947 dev_flow->handle->split_flow_id = mtr_flow_id; 6948 dev_flow->handle->is_meter_flow_id = 1; 6949 } 6950 if (!fm->def_policy) { 6951 if (!set_mtr_reg && fm->drop_cnt) 6952 ret = 6953 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6954 &sfx_attr, items, 6955 flow_split_info, 6956 fm, error); 6957 goto exit; 6958 } 6959 /* Setting the sfx group atrr. */ 6960 sfx_attr.group = sfx_attr.transfer ? 6961 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6962 MLX5_FLOW_TABLE_LEVEL_METER; 6963 flow_split_info->prefix_layers = 6964 flow_get_prefix_layer_flags(dev_flow); 6965 flow_split_info->prefix_mark |= wks->mark; 6966 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6967 } 6968 /* Add the prefix subflow. */ 6969 ret = flow_create_split_metadata(dev, flow, 6970 &sfx_attr, sfx_items ? 6971 sfx_items : items, 6972 sfx_actions ? sfx_actions : actions, 6973 flow_split_info, error); 6974 exit: 6975 if (sfx_actions) 6976 mlx5_free(sfx_actions); 6977 return ret; 6978 } 6979 6980 /** 6981 * The splitting for sample feature. 6982 * 6983 * Once Sample action is detected in the action list, the flow actions should 6984 * be split into prefix sub flow and suffix sub flow. 6985 * 6986 * The original items remain in the prefix sub flow, all actions preceding the 6987 * sample action and the sample action itself will be copied to the prefix 6988 * sub flow, the actions following the sample action will be copied to the 6989 * suffix sub flow, Queue action always be located in the suffix sub flow. 6990 * 6991 * In order to make the packet from prefix sub flow matches with suffix sub 6992 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6993 * flow uses tag item with the unique flow id. 6994 * 6995 * @param dev 6996 * Pointer to Ethernet device. 6997 * @param[in] flow 6998 * Parent flow structure pointer. 6999 * @param[in] attr 7000 * Flow rule attributes. 7001 * @param[in] items 7002 * Pattern specification (list terminated by the END pattern item). 7003 * @param[in] actions 7004 * Associated actions (list terminated by the END action). 7005 * @param[in] flow_split_info 7006 * Pointer to flow split info structure. 7007 * @param[out] error 7008 * Perform verbose error reporting if not NULL. 7009 * @return 7010 * 0 on success, negative value otherwise 7011 */ 7012 static int 7013 flow_create_split_sample(struct rte_eth_dev *dev, 7014 struct rte_flow *flow, 7015 const struct rte_flow_attr *attr, 7016 const struct rte_flow_item items[], 7017 const struct rte_flow_action actions[], 7018 struct mlx5_flow_split_info *flow_split_info, 7019 struct rte_flow_error *error) 7020 { 7021 struct mlx5_priv *priv = dev->data->dev_private; 7022 struct rte_flow_action *sfx_actions = NULL; 7023 struct rte_flow_action *pre_actions = NULL; 7024 struct rte_flow_item *sfx_items = NULL; 7025 struct mlx5_flow *dev_flow = NULL; 7026 struct rte_flow_attr sfx_attr = *attr; 7027 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7028 struct mlx5_flow_dv_sample_resource *sample_res; 7029 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 7030 struct mlx5_flow_tbl_resource *sfx_tbl; 7031 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 7032 #endif 7033 size_t act_size; 7034 size_t item_size; 7035 uint32_t fdb_tx = 0; 7036 int32_t tag_id = 0; 7037 int actions_n = 0; 7038 int sample_action_pos; 7039 int qrss_action_pos; 7040 int add_tag = 0; 7041 int modify_after_mirror = 0; 7042 uint16_t jump_table = 0; 7043 const uint32_t next_ft_step = 1; 7044 int ret = 0; 7045 struct mlx5_priv *item_port_priv = NULL; 7046 const struct rte_flow_item *item; 7047 7048 if (priv->sampler_en) 7049 actions_n = flow_check_match_action(actions, attr, 7050 RTE_FLOW_ACTION_TYPE_SAMPLE, 7051 &sample_action_pos, &qrss_action_pos, 7052 &modify_after_mirror); 7053 if (actions_n) { 7054 /* The prefix actions must includes sample, tag, end. */ 7055 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 7056 + sizeof(struct mlx5_rte_flow_action_set_tag); 7057 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 7058 sizeof(struct mlx5_rte_flow_item_tag) * 2; 7059 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 7060 item_size), 0, SOCKET_ID_ANY); 7061 if (!sfx_actions) 7062 return rte_flow_error_set(error, ENOMEM, 7063 RTE_FLOW_ERROR_TYPE_ACTION, 7064 NULL, "no memory to split " 7065 "sample flow"); 7066 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 7067 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 7068 const struct rte_flow_item_port_id *spec; 7069 7070 spec = (const struct rte_flow_item_port_id *)item->spec; 7071 if (spec) 7072 item_port_priv = 7073 mlx5_port_to_eswitch_info(spec->id, true); 7074 break; 7075 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 7076 const struct rte_flow_item_ethdev *spec; 7077 7078 spec = (const struct rte_flow_item_ethdev *)item->spec; 7079 if (spec) 7080 item_port_priv = 7081 mlx5_port_to_eswitch_info(spec->port_id, true); 7082 break; 7083 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 7084 const struct rte_flow_item_ethdev *spec; 7085 7086 spec = (const struct rte_flow_item_ethdev *)item->spec; 7087 if (spec) 7088 item_port_priv = 7089 mlx5_port_to_eswitch_info(spec->port_id, true); 7090 break; 7091 } 7092 } 7093 /* The representor_id is UINT16_MAX for uplink. */ 7094 fdb_tx = (attr->transfer && 7095 flow_source_vport_representor(priv, item_port_priv)); 7096 /* 7097 * When reg_c_preserve is set, metadata registers Cx preserve 7098 * their value even through packet duplication. 7099 */ 7100 add_tag = (!fdb_tx || 7101 priv->sh->cdev->config.hca_attr.reg_c_preserve); 7102 if (add_tag) 7103 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 7104 + act_size); 7105 if (modify_after_mirror) 7106 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 7107 next_ft_step; 7108 pre_actions = sfx_actions + actions_n; 7109 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 7110 actions, sfx_actions, 7111 pre_actions, actions_n, 7112 sample_action_pos, 7113 qrss_action_pos, jump_table, 7114 error); 7115 if (tag_id < 0 || (add_tag && !tag_id)) { 7116 ret = -rte_errno; 7117 goto exit; 7118 } 7119 if (modify_after_mirror) 7120 flow_split_info->skip_scale = 7121 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 7122 /* Add the prefix subflow. */ 7123 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7124 items, pre_actions, 7125 flow_split_info, error); 7126 if (ret) { 7127 ret = -rte_errno; 7128 goto exit; 7129 } 7130 dev_flow->handle->split_flow_id = tag_id; 7131 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7132 if (!modify_after_mirror) { 7133 /* Set the sfx group attr. */ 7134 sample_res = (struct mlx5_flow_dv_sample_resource *) 7135 dev_flow->dv.sample_res; 7136 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7137 sample_res->normal_path_tbl; 7138 sfx_tbl_data = container_of(sfx_tbl, 7139 struct mlx5_flow_tbl_data_entry, 7140 tbl); 7141 sfx_attr.group = sfx_attr.transfer ? 7142 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7143 } else { 7144 MLX5_ASSERT(attr->transfer); 7145 sfx_attr.group = jump_table; 7146 } 7147 flow_split_info->prefix_layers = 7148 flow_get_prefix_layer_flags(dev_flow); 7149 MLX5_ASSERT(wks); 7150 flow_split_info->prefix_mark |= wks->mark; 7151 /* Suffix group level already be scaled with factor, set 7152 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7153 * again in translation. 7154 */ 7155 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7156 #endif 7157 } 7158 /* Add the suffix subflow. */ 7159 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7160 sfx_items ? sfx_items : items, 7161 sfx_actions ? sfx_actions : actions, 7162 flow_split_info, error); 7163 exit: 7164 if (sfx_actions) 7165 mlx5_free(sfx_actions); 7166 return ret; 7167 } 7168 7169 /** 7170 * Split the flow to subflow set. The splitters might be linked 7171 * in the chain, like this: 7172 * flow_create_split_outer() calls: 7173 * flow_create_split_meter() calls: 7174 * flow_create_split_metadata(meter_subflow_0) calls: 7175 * flow_create_split_inner(metadata_subflow_0) 7176 * flow_create_split_inner(metadata_subflow_1) 7177 * flow_create_split_inner(metadata_subflow_2) 7178 * flow_create_split_metadata(meter_subflow_1) calls: 7179 * flow_create_split_inner(metadata_subflow_0) 7180 * flow_create_split_inner(metadata_subflow_1) 7181 * flow_create_split_inner(metadata_subflow_2) 7182 * 7183 * This provide flexible way to add new levels of flow splitting. 7184 * The all of successfully created subflows are included to the 7185 * parent flow dev_flow list. 7186 * 7187 * @param dev 7188 * Pointer to Ethernet device. 7189 * @param[in] flow 7190 * Parent flow structure pointer. 7191 * @param[in] attr 7192 * Flow rule attributes. 7193 * @param[in] items 7194 * Pattern specification (list terminated by the END pattern item). 7195 * @param[in] actions 7196 * Associated actions (list terminated by the END action). 7197 * @param[in] flow_split_info 7198 * Pointer to flow split info structure. 7199 * @param[out] error 7200 * Perform verbose error reporting if not NULL. 7201 * @return 7202 * 0 on success, negative value otherwise 7203 */ 7204 static int 7205 flow_create_split_outer(struct rte_eth_dev *dev, 7206 struct rte_flow *flow, 7207 const struct rte_flow_attr *attr, 7208 const struct rte_flow_item items[], 7209 const struct rte_flow_action actions[], 7210 struct mlx5_flow_split_info *flow_split_info, 7211 struct rte_flow_error *error) 7212 { 7213 int ret; 7214 7215 ret = flow_create_split_sample(dev, flow, attr, items, 7216 actions, flow_split_info, error); 7217 MLX5_ASSERT(ret <= 0); 7218 return ret; 7219 } 7220 7221 static inline struct mlx5_flow_tunnel * 7222 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7223 { 7224 struct mlx5_flow_tunnel *tunnel; 7225 7226 #pragma GCC diagnostic push 7227 #pragma GCC diagnostic ignored "-Wcast-qual" 7228 tunnel = (typeof(tunnel))flow->tunnel; 7229 #pragma GCC diagnostic pop 7230 7231 return tunnel; 7232 } 7233 7234 /** 7235 * Create a flow and add it to @p list. 7236 * 7237 * @param dev 7238 * Pointer to Ethernet device. 7239 * @param list 7240 * Pointer to a TAILQ flow list. If this parameter NULL, 7241 * no list insertion occurred, flow is just created, 7242 * this is caller's responsibility to track the 7243 * created flow. 7244 * @param[in] attr 7245 * Flow rule attributes. 7246 * @param[in] items 7247 * Pattern specification (list terminated by the END pattern item). 7248 * @param[in] actions 7249 * Associated actions (list terminated by the END action). 7250 * @param[in] external 7251 * This flow rule is created by request external to PMD. 7252 * @param[out] error 7253 * Perform verbose error reporting if not NULL. 7254 * 7255 * @return 7256 * A flow index on success, 0 otherwise and rte_errno is set. 7257 */ 7258 static uint32_t 7259 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7260 const struct rte_flow_attr *attr, 7261 const struct rte_flow_item items[], 7262 const struct rte_flow_action original_actions[], 7263 bool external, struct rte_flow_error *error) 7264 { 7265 struct mlx5_priv *priv = dev->data->dev_private; 7266 struct rte_flow *flow = NULL; 7267 struct mlx5_flow *dev_flow; 7268 const struct rte_flow_action_rss *rss = NULL; 7269 struct mlx5_translated_action_handle 7270 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7271 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7272 union { 7273 struct mlx5_flow_expand_rss buf; 7274 uint8_t buffer[8192]; 7275 } expand_buffer; 7276 union { 7277 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7278 uint8_t buffer[2048]; 7279 } actions_rx; 7280 union { 7281 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7282 uint8_t buffer[2048]; 7283 } actions_hairpin_tx; 7284 union { 7285 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7286 uint8_t buffer[2048]; 7287 } items_tx; 7288 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7289 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7290 struct mlx5_flow_rss_desc *rss_desc; 7291 const struct rte_flow_action *p_actions_rx; 7292 uint32_t i; 7293 uint32_t idx = 0; 7294 int hairpin_flow; 7295 struct rte_flow_attr attr_tx = { .priority = 0 }; 7296 const struct rte_flow_action *actions; 7297 struct rte_flow_action *translated_actions = NULL; 7298 struct mlx5_flow_tunnel *tunnel; 7299 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7300 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7301 struct mlx5_flow_split_info flow_split_info = { 7302 .external = !!external, 7303 .skip_scale = 0, 7304 .flow_idx = 0, 7305 .prefix_mark = 0, 7306 .prefix_layers = 0, 7307 .table_id = 0 7308 }; 7309 int ret; 7310 7311 MLX5_ASSERT(wks); 7312 rss_desc = &wks->rss_desc; 7313 ret = flow_action_handles_translate(dev, original_actions, 7314 indir_actions, 7315 &indir_actions_n, 7316 &translated_actions, error); 7317 if (ret < 0) { 7318 MLX5_ASSERT(translated_actions == NULL); 7319 return 0; 7320 } 7321 actions = translated_actions ? translated_actions : original_actions; 7322 p_actions_rx = actions; 7323 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7324 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7325 external, hairpin_flow, error); 7326 if (ret < 0) 7327 goto error_before_hairpin_split; 7328 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7329 if (!flow) { 7330 rte_errno = ENOMEM; 7331 goto error_before_hairpin_split; 7332 } 7333 if (hairpin_flow > 0) { 7334 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7335 rte_errno = EINVAL; 7336 goto error_before_hairpin_split; 7337 } 7338 flow_hairpin_split(dev, actions, actions_rx.actions, 7339 actions_hairpin_tx.actions, items_tx.items, 7340 idx); 7341 p_actions_rx = actions_rx.actions; 7342 } 7343 flow_split_info.flow_idx = idx; 7344 flow->drv_type = flow_get_drv_type(dev, attr); 7345 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7346 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7347 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7348 /* RSS Action only works on NIC RX domain */ 7349 if (attr->ingress) 7350 rss = flow_get_rss_action(dev, p_actions_rx); 7351 if (rss) { 7352 MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); 7353 rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func); 7354 /* 7355 * The following information is required by 7356 * mlx5_flow_hashfields_adjust() in advance. 7357 */ 7358 rss_desc->level = rss->level; 7359 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7360 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7361 } 7362 flow->dev_handles = 0; 7363 if (rss && rss->types) { 7364 unsigned int graph_root; 7365 7366 graph_root = find_graph_root(rss->level); 7367 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7368 items, rss->types, 7369 mlx5_support_expansion, graph_root); 7370 MLX5_ASSERT(ret > 0 && 7371 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7372 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7373 for (i = 0; i < buf->entries; ++i) 7374 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7375 } 7376 } else { 7377 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7378 sizeof(expand_buffer.buffer), 7379 items, sq_specs); 7380 if (ret) { 7381 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7382 NULL, "not enough memory for rte_flow"); 7383 goto error; 7384 } 7385 if (buf->entries == 0) { 7386 buf->entries = 1; 7387 buf->entry[0].pattern = (void *)(uintptr_t)items; 7388 } 7389 } 7390 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7391 indir_actions_n); 7392 for (i = 0; i < buf->entries; ++i) { 7393 /* Initialize flow split data. */ 7394 flow_split_info.prefix_layers = 0; 7395 flow_split_info.prefix_mark = 0; 7396 flow_split_info.skip_scale = 0; 7397 /* 7398 * The splitter may create multiple dev_flows, 7399 * depending on configuration. In the simplest 7400 * case it just creates unmodified original flow. 7401 */ 7402 ret = flow_create_split_outer(dev, flow, attr, 7403 buf->entry[i].pattern, 7404 p_actions_rx, &flow_split_info, 7405 error); 7406 if (ret < 0) 7407 goto error; 7408 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7409 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7410 p_actions_rx, 7411 idx, 7412 wks->flows[0].tunnel, 7413 &default_miss_ctx, 7414 error); 7415 if (ret < 0) { 7416 mlx5_free(default_miss_ctx.queue); 7417 goto error; 7418 } 7419 } 7420 } 7421 /* Create the tx flow. */ 7422 if (hairpin_flow) { 7423 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7424 attr_tx.ingress = 0; 7425 attr_tx.egress = 1; 7426 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7427 actions_hairpin_tx.actions, 7428 idx, error); 7429 if (!dev_flow) 7430 goto error; 7431 dev_flow->flow = flow; 7432 dev_flow->external = 0; 7433 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7434 dev_flow->handle, next); 7435 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7436 items_tx.items, 7437 actions_hairpin_tx.actions, error); 7438 if (ret < 0) 7439 goto error; 7440 } 7441 /* 7442 * Update the metadata register copy table. If extensive 7443 * metadata feature is enabled and registers are supported 7444 * we might create the extra rte_flow for each unique 7445 * MARK/FLAG action ID. 7446 * 7447 * The table is updated for ingress and transfer flows only, because 7448 * the egress Flows belong to the different device and 7449 * copy table should be updated in peer NIC Rx domain. 7450 */ 7451 if ((attr->ingress || attr->transfer) && 7452 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7453 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7454 if (ret) 7455 goto error; 7456 } 7457 /* 7458 * If the flow is external (from application) OR device is started, 7459 * OR mreg discover, then apply immediately. 7460 */ 7461 if (external || dev->data->dev_started || 7462 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7463 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7464 ret = flow_drv_apply(dev, flow, error); 7465 if (ret < 0) 7466 goto error; 7467 } 7468 flow->type = type; 7469 flow_rxq_flags_set(dev, flow); 7470 rte_free(translated_actions); 7471 tunnel = flow_tunnel_from_rule(wks->flows); 7472 if (tunnel) { 7473 flow->tunnel = 1; 7474 flow->tunnel_id = tunnel->tunnel_id; 7475 rte_atomic_fetch_add_explicit(&tunnel->refctn, 1, rte_memory_order_relaxed); 7476 mlx5_free(default_miss_ctx.queue); 7477 } 7478 mlx5_flow_pop_thread_workspace(); 7479 return idx; 7480 error: 7481 MLX5_ASSERT(flow); 7482 ret = rte_errno; /* Save rte_errno before cleanup. */ 7483 flow_mreg_del_copy_action(dev, flow); 7484 flow_drv_destroy(dev, flow); 7485 if (rss_desc->shared_rss) 7486 rte_atomic_fetch_sub_explicit(&((struct mlx5_shared_action_rss *) 7487 mlx5_ipool_get 7488 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7489 rss_desc->shared_rss))->refcnt, 1, rte_memory_order_relaxed); 7490 mlx5_ipool_free(priv->flows[type], idx); 7491 rte_errno = ret; /* Restore rte_errno. */ 7492 ret = rte_errno; 7493 rte_errno = ret; 7494 error_before_hairpin_split: 7495 mlx5_flow_pop_thread_workspace(); 7496 rte_free(translated_actions); 7497 return 0; 7498 } 7499 7500 /** 7501 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7502 * incoming packets to table 1. 7503 * 7504 * Other flow rules, requested for group n, will be created in 7505 * e-switch table n+1. 7506 * Jump action to e-switch group n will be created to group n+1. 7507 * 7508 * Used when working in switchdev mode, to utilise advantages of table 1 7509 * and above. 7510 * 7511 * @param dev 7512 * Pointer to Ethernet device. 7513 * 7514 * @return 7515 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7516 */ 7517 struct rte_flow * 7518 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7519 { 7520 const struct rte_flow_attr attr = { 7521 .group = 0, 7522 .priority = 0, 7523 .ingress = 0, 7524 .egress = 0, 7525 .transfer = 1, 7526 }; 7527 const struct rte_flow_item pattern = { 7528 .type = RTE_FLOW_ITEM_TYPE_END, 7529 }; 7530 struct rte_flow_action_jump jump = { 7531 .group = 1, 7532 }; 7533 const struct rte_flow_action actions[] = { 7534 { 7535 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7536 .conf = &jump, 7537 }, 7538 { 7539 .type = RTE_FLOW_ACTION_TYPE_END, 7540 }, 7541 }; 7542 struct rte_flow_error error; 7543 7544 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7545 &attr, &pattern, 7546 actions, false, &error); 7547 } 7548 7549 /** 7550 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7551 * and sq number, directs all packets to peer vport. 7552 * 7553 * @param dev 7554 * Pointer to Ethernet device. 7555 * @param sq_num 7556 * SQ number. 7557 * 7558 * @return 7559 * Flow ID on success, 0 otherwise and rte_errno is set. 7560 */ 7561 uint32_t 7562 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7563 { 7564 struct rte_flow_attr attr = { 7565 .group = 0, 7566 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7567 .ingress = 0, 7568 .egress = 0, 7569 .transfer = 1, 7570 }; 7571 struct rte_flow_item_port_id port_spec = { 7572 .id = MLX5_PORT_ESW_MGR, 7573 }; 7574 struct mlx5_rte_flow_item_sq sq_spec = { 7575 .queue = sq_num, 7576 }; 7577 struct rte_flow_item pattern[] = { 7578 { 7579 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7580 .spec = &port_spec, 7581 }, 7582 { 7583 .type = (enum rte_flow_item_type) 7584 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7585 .spec = &sq_spec, 7586 }, 7587 { 7588 .type = RTE_FLOW_ITEM_TYPE_END, 7589 }, 7590 }; 7591 struct rte_flow_action_jump jump = { 7592 .group = 1, 7593 }; 7594 struct rte_flow_action_port_id port = { 7595 .id = dev->data->port_id, 7596 }; 7597 struct rte_flow_action actions[] = { 7598 { 7599 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7600 .conf = &jump, 7601 }, 7602 { 7603 .type = RTE_FLOW_ACTION_TYPE_END, 7604 }, 7605 }; 7606 struct rte_flow_error error; 7607 7608 /* 7609 * Creates group 0, highest priority jump flow. 7610 * Matches txq to bypass kernel packets. 7611 */ 7612 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7613 false, &error) == 0) 7614 return 0; 7615 /* Create group 1, lowest priority redirect flow for txq. */ 7616 attr.group = 1; 7617 actions[0].conf = &port; 7618 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7619 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7620 actions, false, &error); 7621 } 7622 7623 /** 7624 * Validate a flow supported by the NIC. 7625 * 7626 * @see rte_flow_validate() 7627 * @see rte_flow_ops 7628 */ 7629 int 7630 mlx5_flow_validate(struct rte_eth_dev *dev, 7631 const struct rte_flow_attr *attr, 7632 const struct rte_flow_item items[], 7633 const struct rte_flow_action original_actions[], 7634 struct rte_flow_error *error) 7635 { 7636 int hairpin_flow; 7637 struct mlx5_translated_action_handle 7638 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7639 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7640 const struct rte_flow_action *actions; 7641 struct rte_flow_action *translated_actions = NULL; 7642 int ret = flow_action_handles_translate(dev, original_actions, 7643 indir_actions, 7644 &indir_actions_n, 7645 &translated_actions, error); 7646 7647 if (ret) 7648 return ret; 7649 actions = translated_actions ? translated_actions : original_actions; 7650 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7651 ret = flow_drv_validate(dev, attr, items, actions, 7652 true, hairpin_flow, error); 7653 rte_free(translated_actions); 7654 return ret; 7655 } 7656 7657 static int 7658 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7659 const struct rte_flow_attr *attr, 7660 const uint32_t orig_prio, 7661 const struct rte_flow_item *items, 7662 const struct rte_flow_action *actions, 7663 uint32_t flow_idx) 7664 { 7665 struct mlx5_priv *priv = dev->data->dev_private; 7666 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7667 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7668 struct rte_flow_error error; 7669 int len, ret; 7670 7671 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7672 if (!flow_info) { 7673 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7674 return -1; 7675 } 7676 flow_info->orig_prio = orig_prio; 7677 flow_info->attr = *attr; 7678 /* Standby mode rule awlays saves it in low priority entry. */ 7679 flow_info->flow_idx_low_prio = flow_idx; 7680 7681 /* Store matching items. */ 7682 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7683 if (ret <= 0) { 7684 DRV_LOG(ERR, "Can't get items length."); 7685 goto end; 7686 } 7687 len = RTE_ALIGN(ret, 16); 7688 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7689 if (!flow_info->items) { 7690 DRV_LOG(ERR, "No enough memory for items caching."); 7691 goto end; 7692 } 7693 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7694 if (ret <= 0) { 7695 DRV_LOG(ERR, "Can't duplicate items."); 7696 goto end; 7697 } 7698 7699 /* Store flow actions. */ 7700 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7701 if (ret <= 0) { 7702 DRV_LOG(ERR, "Can't get actions length."); 7703 goto end; 7704 } 7705 len = RTE_ALIGN(ret, 16); 7706 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7707 if (!flow_info->actions) { 7708 DRV_LOG(ERR, "No enough memory for actions caching."); 7709 goto end; 7710 } 7711 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7712 if (ret <= 0) { 7713 DRV_LOG(ERR, "Can't duplicate actions."); 7714 goto end; 7715 } 7716 7717 /* Insert to the list end. */ 7718 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7719 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7720 } else { 7721 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7722 while (LIST_NEXT(tmp_info, next)) 7723 tmp_info = LIST_NEXT(tmp_info, next); 7724 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7725 } 7726 return 0; 7727 end: 7728 if (flow_info->items) 7729 mlx5_free(flow_info->items); 7730 if (flow_info->actions) 7731 mlx5_free(flow_info->actions); 7732 mlx5_free(flow_info); 7733 return -1; 7734 } 7735 7736 static int 7737 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7738 { 7739 struct mlx5_priv *priv = dev->data->dev_private; 7740 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7741 struct mlx5_dv_flow_info *flow_info; 7742 struct rte_flow_attr attr; 7743 struct rte_flow_error error; 7744 struct rte_flow *high, *low; 7745 7746 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7747 while (flow_info) { 7748 /* DUP flow may have the same priority. */ 7749 if (flow_info->orig_prio != flow_info->attr.priority) { 7750 attr = flow_info->attr; 7751 if (orig_prio) 7752 attr.priority = flow_info->orig_prio; 7753 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7754 &attr, flow_info->items, flow_info->actions, 7755 true, &error); 7756 if (!flow_info->flow_idx_high_prio) { 7757 DRV_LOG(ERR, "Priority toggle failed internally."); 7758 goto err; 7759 } 7760 } 7761 flow_info = LIST_NEXT(flow_info, next); 7762 } 7763 /* Delete the low priority rules and swap the flow handle. */ 7764 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7765 while (flow_info) { 7766 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7767 if (flow_info->orig_prio != flow_info->attr.priority) { 7768 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7769 flow_info->flow_idx_high_prio); 7770 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7771 flow_info->flow_idx_low_prio); 7772 if (high && low) { 7773 RTE_SWAP(*low, *high); 7774 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7775 flow_info->flow_idx_low_prio); 7776 flow_info->flow_idx_high_prio = 0; 7777 } 7778 } 7779 flow_info = LIST_NEXT(flow_info, next); 7780 } 7781 return 0; 7782 err: 7783 /* Destroy preceding successful high priority rules. */ 7784 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7785 while (flow_info) { 7786 if (flow_info->orig_prio != flow_info->attr.priority) { 7787 if (flow_info->flow_idx_high_prio) 7788 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7789 flow_info->flow_idx_high_prio); 7790 else 7791 break; 7792 flow_info->flow_idx_high_prio = 0; 7793 } 7794 flow_info = LIST_NEXT(flow_info, next); 7795 } 7796 return -1; 7797 } 7798 7799 /** 7800 * Set the mode of the flow engine of a process to active or standby during live migration. 7801 * 7802 * @param[in] mode 7803 * MLX5 flow engine mode, @see `enum rte_pmd_mlx5_flow_engine_mode`. 7804 * @param[in] flags 7805 * Flow engine mode specific flags. 7806 * 7807 * @return 7808 * Negative value on error, positive on success. 7809 */ 7810 int 7811 rte_pmd_mlx5_flow_engine_set_mode(enum rte_pmd_mlx5_flow_engine_mode mode, uint32_t flags) 7812 { 7813 struct mlx5_priv *priv; 7814 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info; 7815 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7816 uint16_t port, port_id; 7817 uint16_t toggle_num = 0; 7818 struct rte_eth_dev *dev; 7819 enum rte_pmd_mlx5_flow_engine_mode orig_mode; 7820 uint32_t orig_flags; 7821 bool need_toggle = false; 7822 7823 /* Check if flags combinations are supported. */ 7824 if (flags && flags != RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7825 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7826 return -1; 7827 } 7828 MLX5_ETH_FOREACH_DEV(port, NULL) { 7829 dev = &rte_eth_devices[port]; 7830 priv = dev->data->dev_private; 7831 mode_info = &priv->mode_info; 7832 /* No mode change. Assume all devices hold the same mode. */ 7833 if (mode_info->mode == mode) { 7834 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7835 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7836 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7837 port, mode_info->mode_flag); 7838 orig_mode = mode_info->mode; 7839 orig_flags = mode_info->mode_flag; 7840 goto err; 7841 } 7842 mode_info->mode_flag = flags; 7843 toggle_num++; 7844 continue; 7845 } 7846 /* Active -> standby. */ 7847 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_STANDBY) { 7848 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7849 DRV_LOG(ERR, "Cached rule existed"); 7850 orig_mode = mode_info->mode; 7851 orig_flags = mode_info->mode_flag; 7852 goto err; 7853 } 7854 mode_info->mode_flag = flags; 7855 mode_info->mode = mode; 7856 toggle_num++; 7857 /* Standby -> active. */ 7858 } else if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7859 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7860 DRV_LOG(INFO, "No cached rule existed"); 7861 } else { 7862 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7863 orig_mode = mode_info->mode; 7864 orig_flags = mode_info->mode_flag; 7865 need_toggle = true; 7866 goto err; 7867 } 7868 } 7869 toggle_num++; 7870 } 7871 } 7872 if (mode == RTE_PMD_MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7873 /* Clear cache flow rules. */ 7874 MLX5_ETH_FOREACH_DEV(port, NULL) { 7875 priv = rte_eth_devices[port].data->dev_private; 7876 mode_info = &priv->mode_info; 7877 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7878 while (flow_info) { 7879 tmp_info = LIST_NEXT(flow_info, next); 7880 LIST_REMOVE(flow_info, next); 7881 mlx5_free(flow_info->actions); 7882 mlx5_free(flow_info->items); 7883 mlx5_free(flow_info); 7884 flow_info = tmp_info; 7885 } 7886 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7887 } 7888 } 7889 return toggle_num; 7890 err: 7891 /* Rollback all preceding successful ports. */ 7892 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7893 if (port_id == port) 7894 break; 7895 priv = rte_eth_devices[port_id].data->dev_private; 7896 mode_info = &priv->mode_info; 7897 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7898 mlx5_flow_cache_flow_toggle(dev, false)) 7899 return -EPERM; 7900 mode_info->mode = orig_mode; 7901 mode_info->mode_flag = orig_flags; 7902 } 7903 return -EINVAL; 7904 } 7905 /** 7906 * Create a flow. 7907 * 7908 * @see rte_flow_create() 7909 * @see rte_flow_ops 7910 */ 7911 struct rte_flow * 7912 mlx5_flow_create(struct rte_eth_dev *dev, 7913 const struct rte_flow_attr *attr, 7914 const struct rte_flow_item items[], 7915 const struct rte_flow_action actions[], 7916 struct rte_flow_error *error) 7917 { 7918 struct mlx5_priv *priv = dev->data->dev_private; 7919 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7920 uint32_t prio = attr->priority; 7921 uint32_t flow_idx; 7922 7923 if (priv->sh->config.dv_flow_en == 2) { 7924 rte_flow_error_set(error, ENOTSUP, 7925 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7926 NULL, 7927 "Flow non-Q creation not supported"); 7928 return NULL; 7929 } 7930 /* 7931 * If the device is not started yet, it is not allowed to created a 7932 * flow from application. PMD default flows and traffic control flows 7933 * are not affected. 7934 */ 7935 if (unlikely(!dev->data->dev_started)) { 7936 DRV_LOG(DEBUG, "port %u is not started when " 7937 "inserting a flow", dev->data->port_id); 7938 rte_flow_error_set(error, ENODEV, 7939 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7940 NULL, 7941 "port not started"); 7942 return NULL; 7943 } 7944 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7945 if (attr->transfer || 7946 (attr->ingress && !(priv->mode_info.mode_flag & 7947 RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 7948 new_attr->priority += 1; 7949 } 7950 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 7951 if (!flow_idx) 7952 return NULL; 7953 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7954 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 7955 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7956 flow_idx = 0; 7957 } 7958 } 7959 return (void *)(uintptr_t)flow_idx; 7960 } 7961 7962 /** 7963 * Destroy a flow in a list. 7964 * 7965 * @param dev 7966 * Pointer to Ethernet device. 7967 * @param[in] flow_idx 7968 * Index of flow to destroy. 7969 */ 7970 static void 7971 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7972 uint32_t flow_idx) 7973 { 7974 struct mlx5_priv *priv = dev->data->dev_private; 7975 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7976 7977 if (!flow) 7978 return; 7979 MLX5_ASSERT(flow->type == type); 7980 /* 7981 * Update RX queue flags only if port is started, otherwise it is 7982 * already clean. 7983 */ 7984 if (dev->data->dev_started) 7985 flow_rxq_flags_trim(dev, flow); 7986 flow_drv_destroy(dev, flow); 7987 if (flow->tunnel) { 7988 struct mlx5_flow_tunnel *tunnel; 7989 7990 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7991 RTE_VERIFY(tunnel); 7992 if (!(rte_atomic_fetch_sub_explicit(&tunnel->refctn, 1, 7993 rte_memory_order_relaxed) - 1)) 7994 mlx5_flow_tunnel_free(dev, tunnel); 7995 } 7996 flow_mreg_del_copy_action(dev, flow); 7997 mlx5_ipool_free(priv->flows[type], flow_idx); 7998 } 7999 8000 /** 8001 * Destroy all flows. 8002 * 8003 * @param dev 8004 * Pointer to Ethernet device. 8005 * @param type 8006 * Flow type to be flushed. 8007 * @param active 8008 * If flushing is called actively. 8009 */ 8010 void 8011 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 8012 bool active) 8013 { 8014 struct mlx5_priv *priv = dev->data->dev_private; 8015 uint32_t num_flushed = 0, fidx = 1; 8016 struct rte_flow *flow; 8017 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8018 struct mlx5_dv_flow_info *flow_info; 8019 8020 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8021 if (priv->sh->config.dv_flow_en == 2 && 8022 type == MLX5_FLOW_TYPE_GEN) { 8023 flow_hw_q_flow_flush(dev, NULL); 8024 return; 8025 } 8026 #endif 8027 8028 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 8029 flow_list_destroy(dev, type, fidx); 8030 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 8031 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8032 while (flow_info) { 8033 /* Romove the cache flow info. */ 8034 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 8035 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8036 LIST_REMOVE(flow_info, next); 8037 mlx5_free(flow_info->items); 8038 mlx5_free(flow_info->actions); 8039 mlx5_free(flow_info); 8040 break; 8041 } 8042 flow_info = LIST_NEXT(flow_info, next); 8043 } 8044 } 8045 num_flushed++; 8046 } 8047 if (active) { 8048 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 8049 dev->data->port_id, num_flushed); 8050 } 8051 } 8052 8053 /** 8054 * Stop all default actions for flows. 8055 * 8056 * @param dev 8057 * Pointer to Ethernet device. 8058 */ 8059 void 8060 mlx5_flow_stop_default(struct rte_eth_dev *dev) 8061 { 8062 flow_mreg_del_default_copy_action(dev); 8063 flow_rxq_flags_clear(dev); 8064 } 8065 8066 /** 8067 * Set rxq flag. 8068 * 8069 * @param[in] dev 8070 * Pointer to the rte_eth_dev structure. 8071 * @param[in] enable 8072 * Flag to enable or not. 8073 */ 8074 void 8075 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 8076 { 8077 struct mlx5_priv *priv = dev->data->dev_private; 8078 unsigned int i; 8079 8080 if ((!priv->mark_enabled && !enable) || 8081 (priv->mark_enabled && enable)) 8082 return; 8083 for (i = 0; i < priv->rxqs_n; ++i) { 8084 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 8085 8086 /* With RXQ start/stop feature, RXQ might be stopped. */ 8087 if (!rxq_ctrl) 8088 continue; 8089 rxq_ctrl->rxq.mark = enable; 8090 } 8091 priv->mark_enabled = enable; 8092 } 8093 8094 /** 8095 * Start all default actions for flows. 8096 * 8097 * @param dev 8098 * Pointer to Ethernet device. 8099 * @return 8100 * 0 on success, a negative errno value otherwise and rte_errno is set. 8101 */ 8102 int 8103 mlx5_flow_start_default(struct rte_eth_dev *dev) 8104 { 8105 struct rte_flow_error error; 8106 8107 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8108 return flow_mreg_add_default_copy_action(dev, &error); 8109 } 8110 8111 /** 8112 * Release key of thread specific flow workspace data. 8113 */ 8114 void 8115 flow_release_workspace(void *data) 8116 { 8117 struct mlx5_flow_workspace *wks = data; 8118 struct mlx5_flow_workspace *next; 8119 8120 while (wks) { 8121 next = wks->next; 8122 free(wks); 8123 wks = next; 8124 } 8125 } 8126 8127 /** 8128 * Get thread specific current flow workspace. 8129 * 8130 * @return pointer to thread specific flow workspace data, NULL on error. 8131 */ 8132 struct mlx5_flow_workspace* 8133 mlx5_flow_get_thread_workspace(void) 8134 { 8135 struct mlx5_flow_workspace *data; 8136 8137 data = mlx5_flow_os_get_specific_workspace(); 8138 MLX5_ASSERT(data && data->inuse); 8139 if (!data || !data->inuse) 8140 DRV_LOG(ERR, "flow workspace not initialized."); 8141 return data; 8142 } 8143 8144 /** 8145 * Allocate and init new flow workspace. 8146 * 8147 * @return pointer to flow workspace data, NULL on error. 8148 */ 8149 static struct mlx5_flow_workspace* 8150 flow_alloc_thread_workspace(void) 8151 { 8152 size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); 8153 size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; 8154 struct mlx5_flow_workspace *data = calloc(1, data_size + 8155 rss_queue_array_size); 8156 8157 if (!data) { 8158 DRV_LOG(ERR, "Failed to allocate flow workspace memory."); 8159 return NULL; 8160 } 8161 data->rss_desc.queue = RTE_PTR_ADD(data, data_size); 8162 return data; 8163 } 8164 8165 /** 8166 * Get new thread specific flow workspace. 8167 * 8168 * If current workspace inuse, create new one and set as current. 8169 * 8170 * @return pointer to thread specific flow workspace data, NULL on error. 8171 */ 8172 struct mlx5_flow_workspace* 8173 mlx5_flow_push_thread_workspace(void) 8174 { 8175 struct mlx5_flow_workspace *curr; 8176 struct mlx5_flow_workspace *data; 8177 8178 curr = mlx5_flow_os_get_specific_workspace(); 8179 if (!curr) { 8180 data = flow_alloc_thread_workspace(); 8181 if (!data) 8182 return NULL; 8183 mlx5_flow_os_workspace_gc_add(data); 8184 } else if (!curr->inuse) { 8185 data = curr; 8186 } else if (curr->next) { 8187 data = curr->next; 8188 } else { 8189 data = flow_alloc_thread_workspace(); 8190 if (!data) 8191 return NULL; 8192 curr->next = data; 8193 data->prev = curr; 8194 } 8195 data->inuse = 1; 8196 data->flow_idx = 0; 8197 /* Set as current workspace */ 8198 if (mlx5_flow_os_set_specific_workspace(data)) 8199 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8200 return data; 8201 } 8202 8203 /** 8204 * Close current thread specific flow workspace. 8205 * 8206 * If previous workspace available, set it as current. 8207 * 8208 * @return pointer to thread specific flow workspace data, NULL on error. 8209 */ 8210 void 8211 mlx5_flow_pop_thread_workspace(void) 8212 { 8213 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8214 8215 if (!data) 8216 return; 8217 if (!data->inuse) { 8218 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8219 return; 8220 } 8221 data->inuse = 0; 8222 if (!data->prev) 8223 return; 8224 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8225 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8226 } 8227 8228 /** 8229 * Verify the flow list is empty 8230 * 8231 * @param dev 8232 * Pointer to Ethernet device. 8233 * 8234 * @return the number of flows not released. 8235 */ 8236 int 8237 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8238 { 8239 struct mlx5_priv *priv = dev->data->dev_private; 8240 struct rte_flow *flow; 8241 uint32_t idx = 0; 8242 int ret = 0, i; 8243 8244 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8245 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8246 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8247 dev->data->port_id, (void *)flow); 8248 ret++; 8249 } 8250 } 8251 return ret; 8252 } 8253 8254 /** 8255 * Enable default hairpin egress flow. 8256 * 8257 * @param dev 8258 * Pointer to Ethernet device. 8259 * @param sq_num 8260 * The SQ hw number. 8261 * 8262 * @return 8263 * 0 on success, a negative errno value otherwise and rte_errno is set. 8264 */ 8265 int 8266 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8267 uint32_t sq_num) 8268 { 8269 const struct rte_flow_attr attr = { 8270 .egress = 1, 8271 .priority = 0, 8272 }; 8273 struct mlx5_rte_flow_item_sq queue_spec = { 8274 .queue = sq_num, 8275 }; 8276 struct mlx5_rte_flow_item_sq queue_mask = { 8277 .queue = UINT32_MAX, 8278 }; 8279 struct rte_flow_item items[] = { 8280 { 8281 .type = (enum rte_flow_item_type) 8282 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8283 .spec = &queue_spec, 8284 .last = NULL, 8285 .mask = &queue_mask, 8286 }, 8287 { 8288 .type = RTE_FLOW_ITEM_TYPE_END, 8289 }, 8290 }; 8291 struct rte_flow_action_jump jump = { 8292 .group = MLX5_HAIRPIN_TX_TABLE, 8293 }; 8294 struct rte_flow_action actions[2]; 8295 uint32_t flow_idx; 8296 struct rte_flow_error error; 8297 8298 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8299 actions[0].conf = &jump; 8300 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8301 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8302 &attr, items, actions, false, &error); 8303 if (!flow_idx) { 8304 DRV_LOG(DEBUG, 8305 "Failed to create ctrl flow: rte_errno(%d)," 8306 " type(%d), message(%s)", 8307 rte_errno, error.type, 8308 error.message ? error.message : " (no stated reason)"); 8309 return -rte_errno; 8310 } 8311 return 0; 8312 } 8313 8314 /** 8315 * Enable a control flow configured from the control plane. 8316 * 8317 * @param dev 8318 * Pointer to Ethernet device. 8319 * @param eth_spec 8320 * An Ethernet flow spec to apply. 8321 * @param eth_mask 8322 * An Ethernet flow mask to apply. 8323 * @param vlan_spec 8324 * A VLAN flow spec to apply. 8325 * @param vlan_mask 8326 * A VLAN flow mask to apply. 8327 * 8328 * @return 8329 * 0 on success, a negative errno value otherwise and rte_errno is set. 8330 */ 8331 int 8332 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8333 struct rte_flow_item_eth *eth_spec, 8334 struct rte_flow_item_eth *eth_mask, 8335 struct rte_flow_item_vlan *vlan_spec, 8336 struct rte_flow_item_vlan *vlan_mask) 8337 { 8338 struct mlx5_priv *priv = dev->data->dev_private; 8339 const struct rte_flow_attr attr = { 8340 .ingress = 1, 8341 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8342 }; 8343 struct rte_flow_item items[] = { 8344 { 8345 .type = RTE_FLOW_ITEM_TYPE_ETH, 8346 .spec = eth_spec, 8347 .last = NULL, 8348 .mask = eth_mask, 8349 }, 8350 { 8351 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8352 RTE_FLOW_ITEM_TYPE_END, 8353 .spec = vlan_spec, 8354 .last = NULL, 8355 .mask = vlan_mask, 8356 }, 8357 { 8358 .type = RTE_FLOW_ITEM_TYPE_END, 8359 }, 8360 }; 8361 uint16_t queue[priv->reta_idx_n]; 8362 struct rte_flow_action_rss action_rss = { 8363 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8364 .level = 0, 8365 .types = priv->rss_conf.rss_hf, 8366 .key_len = priv->rss_conf.rss_key_len, 8367 .queue_num = priv->reta_idx_n, 8368 .key = priv->rss_conf.rss_key, 8369 .queue = queue, 8370 }; 8371 struct rte_flow_action actions[] = { 8372 { 8373 .type = RTE_FLOW_ACTION_TYPE_RSS, 8374 .conf = &action_rss, 8375 }, 8376 { 8377 .type = RTE_FLOW_ACTION_TYPE_END, 8378 }, 8379 }; 8380 uint32_t flow_idx; 8381 struct rte_flow_error error; 8382 unsigned int i; 8383 8384 if (!priv->reta_idx_n || !priv->rxqs_n) { 8385 return 0; 8386 } 8387 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8388 action_rss.types = 0; 8389 for (i = 0; i != priv->reta_idx_n; ++i) 8390 queue[i] = (*priv->reta_idx)[i]; 8391 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8392 &attr, items, actions, false, &error); 8393 if (!flow_idx) 8394 return -rte_errno; 8395 return 0; 8396 } 8397 8398 /** 8399 * Enable a flow control configured from the control plane. 8400 * 8401 * @param dev 8402 * Pointer to Ethernet device. 8403 * @param eth_spec 8404 * An Ethernet flow spec to apply. 8405 * @param eth_mask 8406 * An Ethernet flow mask to apply. 8407 * 8408 * @return 8409 * 0 on success, a negative errno value otherwise and rte_errno is set. 8410 */ 8411 int 8412 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8413 struct rte_flow_item_eth *eth_spec, 8414 struct rte_flow_item_eth *eth_mask) 8415 { 8416 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8417 } 8418 8419 /** 8420 * Create default miss flow rule matching lacp traffic 8421 * 8422 * @param dev 8423 * Pointer to Ethernet device. 8424 * @param eth_spec 8425 * An Ethernet flow spec to apply. 8426 * 8427 * @return 8428 * 0 on success, a negative errno value otherwise and rte_errno is set. 8429 */ 8430 int 8431 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8432 { 8433 /* 8434 * The LACP matching is done by only using ether type since using 8435 * a multicast dst mac causes kernel to give low priority to this flow. 8436 */ 8437 static const struct rte_flow_item_eth lacp_spec = { 8438 .hdr.ether_type = RTE_BE16(0x8809), 8439 }; 8440 static const struct rte_flow_item_eth lacp_mask = { 8441 .hdr.ether_type = 0xffff, 8442 }; 8443 const struct rte_flow_attr attr = { 8444 .ingress = 1, 8445 }; 8446 struct rte_flow_item items[] = { 8447 { 8448 .type = RTE_FLOW_ITEM_TYPE_ETH, 8449 .spec = &lacp_spec, 8450 .mask = &lacp_mask, 8451 }, 8452 { 8453 .type = RTE_FLOW_ITEM_TYPE_END, 8454 }, 8455 }; 8456 struct rte_flow_action actions[] = { 8457 { 8458 .type = (enum rte_flow_action_type) 8459 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8460 }, 8461 { 8462 .type = RTE_FLOW_ACTION_TYPE_END, 8463 }, 8464 }; 8465 struct rte_flow_error error; 8466 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8467 &attr, items, actions, 8468 false, &error); 8469 8470 if (!flow_idx) 8471 return -rte_errno; 8472 return 0; 8473 } 8474 8475 /** 8476 * Destroy a flow. 8477 * 8478 * @see rte_flow_destroy() 8479 * @see rte_flow_ops 8480 */ 8481 int 8482 mlx5_flow_destroy(struct rte_eth_dev *dev, 8483 struct rte_flow *flow, 8484 struct rte_flow_error *error __rte_unused) 8485 { 8486 struct mlx5_priv *priv = dev->data->dev_private; 8487 struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8488 struct mlx5_dv_flow_info *flow_info; 8489 8490 if (priv->sh->config.dv_flow_en == 2) 8491 return rte_flow_error_set(error, ENOTSUP, 8492 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8493 NULL, 8494 "Flow non-Q destruction not supported"); 8495 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8496 (uintptr_t)(void *)flow); 8497 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8498 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8499 while (flow_info) { 8500 /* Romove the cache flow info. */ 8501 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8502 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8503 LIST_REMOVE(flow_info, next); 8504 mlx5_free(flow_info->items); 8505 mlx5_free(flow_info->actions); 8506 mlx5_free(flow_info); 8507 break; 8508 } 8509 flow_info = LIST_NEXT(flow_info, next); 8510 } 8511 } 8512 return 0; 8513 } 8514 8515 /** 8516 * Destroy all flows. 8517 * 8518 * @see rte_flow_flush() 8519 * @see rte_flow_ops 8520 */ 8521 int 8522 mlx5_flow_flush(struct rte_eth_dev *dev, 8523 struct rte_flow_error *error __rte_unused) 8524 { 8525 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8526 return 0; 8527 } 8528 8529 /** 8530 * Isolated mode. 8531 * 8532 * @see rte_flow_isolate() 8533 * @see rte_flow_ops 8534 */ 8535 int 8536 mlx5_flow_isolate(struct rte_eth_dev *dev, 8537 int enable, 8538 struct rte_flow_error *error) 8539 { 8540 struct mlx5_priv *priv = dev->data->dev_private; 8541 8542 if (dev->data->dev_started) { 8543 rte_flow_error_set(error, EBUSY, 8544 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8545 NULL, 8546 "port must be stopped first"); 8547 return -rte_errno; 8548 } 8549 if (!enable && !priv->sh->config.repr_matching) 8550 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8551 "isolated mode cannot be disabled when " 8552 "representor matching is disabled"); 8553 priv->isolated = !!enable; 8554 if (enable) 8555 dev->dev_ops = &mlx5_dev_ops_isolate; 8556 else 8557 dev->dev_ops = &mlx5_dev_ops; 8558 8559 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8560 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8561 8562 return 0; 8563 } 8564 8565 /** 8566 * Query a flow. 8567 * 8568 * @see rte_flow_query() 8569 * @see rte_flow_ops 8570 */ 8571 static int 8572 flow_drv_query(struct rte_eth_dev *dev, 8573 struct rte_flow *eflow, 8574 const struct rte_flow_action *actions, 8575 void *data, 8576 struct rte_flow_error *error) 8577 { 8578 struct mlx5_priv *priv = dev->data->dev_private; 8579 const struct mlx5_flow_driver_ops *fops; 8580 struct rte_flow *flow = NULL; 8581 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8582 8583 if (priv->sh->config.dv_flow_en == 2) { 8584 #ifdef HAVE_MLX5_HWS_SUPPORT 8585 flow = eflow; 8586 ftype = MLX5_FLOW_TYPE_HW; 8587 #endif 8588 } else { 8589 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8590 (uintptr_t)(void *)eflow); 8591 } 8592 if (!flow) { 8593 return rte_flow_error_set(error, ENOENT, 8594 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8595 NULL, 8596 "invalid flow handle"); 8597 } 8598 if (ftype == MLX5_FLOW_TYPE_MIN) 8599 ftype = flow->drv_type; 8600 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8601 fops = flow_get_drv_ops(ftype); 8602 8603 return fops->query(dev, flow, actions, data, error); 8604 } 8605 8606 /** 8607 * Query a flow. 8608 * 8609 * @see rte_flow_query() 8610 * @see rte_flow_ops 8611 */ 8612 int 8613 mlx5_flow_query(struct rte_eth_dev *dev, 8614 struct rte_flow *flow, 8615 const struct rte_flow_action *actions, 8616 void *data, 8617 struct rte_flow_error *error) 8618 { 8619 int ret; 8620 8621 ret = flow_drv_query(dev, flow, actions, data, 8622 error); 8623 if (ret < 0) 8624 return ret; 8625 return 0; 8626 } 8627 8628 /** 8629 * Get rte_flow callbacks. 8630 * 8631 * @param dev 8632 * Pointer to Ethernet device structure. 8633 * @param ops 8634 * Pointer to operation-specific structure. 8635 * 8636 * @return 0 8637 */ 8638 int 8639 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8640 const struct rte_flow_ops **ops) 8641 { 8642 *ops = &mlx5_flow_ops; 8643 return 0; 8644 } 8645 8646 /** 8647 * Validate meter policy actions. 8648 * Dispatcher for action type specific validation. 8649 * 8650 * @param[in] dev 8651 * Pointer to the Ethernet device structure. 8652 * @param[in] action 8653 * The meter policy action object to validate. 8654 * @param[in] attr 8655 * Attributes of flow to determine steering domain. 8656 * @param[out] is_rss 8657 * Is RSS or not. 8658 * @param[out] domain_bitmap 8659 * Domain bitmap. 8660 * @param[out] is_def_policy 8661 * Is default policy or not. 8662 * @param[out] error 8663 * Perform verbose error reporting if not NULL. Initialized in case of 8664 * error only. 8665 * 8666 * @return 8667 * 0 on success, otherwise negative errno value. 8668 */ 8669 int 8670 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8671 const struct rte_flow_action *actions[RTE_COLORS], 8672 struct rte_flow_attr *attr, 8673 bool *is_rss, 8674 uint8_t *domain_bitmap, 8675 uint8_t *policy_mode, 8676 struct rte_mtr_error *error) 8677 { 8678 const struct mlx5_flow_driver_ops *fops; 8679 8680 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8681 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8682 domain_bitmap, policy_mode, error); 8683 } 8684 8685 /** 8686 * Destroy the meter table set. 8687 * 8688 * @param[in] dev 8689 * Pointer to Ethernet device. 8690 * @param[in] mtr_policy 8691 * Meter policy struct. 8692 */ 8693 void 8694 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8695 struct mlx5_flow_meter_policy *mtr_policy) 8696 { 8697 const struct mlx5_flow_driver_ops *fops; 8698 8699 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8700 fops->destroy_mtr_acts(dev, mtr_policy); 8701 } 8702 8703 /** 8704 * Create policy action, lock free, 8705 * (mutex should be acquired by caller). 8706 * Dispatcher for action type specific call. 8707 * 8708 * @param[in] dev 8709 * Pointer to the Ethernet device structure. 8710 * @param[in] mtr_policy 8711 * Meter policy struct. 8712 * @param[in] action 8713 * Action specification used to create meter actions. 8714 * @param[in] attr 8715 * Flow rule attributes. 8716 * @param[out] error 8717 * Perform verbose error reporting if not NULL. Initialized in case of 8718 * error only. 8719 * 8720 * @return 8721 * 0 on success, otherwise negative errno value. 8722 */ 8723 int 8724 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8725 struct mlx5_flow_meter_policy *mtr_policy, 8726 const struct rte_flow_action *actions[RTE_COLORS], 8727 struct rte_flow_attr *attr, 8728 struct rte_mtr_error *error) 8729 { 8730 const struct mlx5_flow_driver_ops *fops; 8731 8732 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8733 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8734 } 8735 8736 /** 8737 * Create policy rules, lock free, 8738 * (mutex should be acquired by caller). 8739 * Dispatcher for action type specific call. 8740 * 8741 * @param[in] dev 8742 * Pointer to the Ethernet device structure. 8743 * @param[in] mtr_policy 8744 * Meter policy struct. 8745 * 8746 * @return 8747 * 0 on success, -1 otherwise. 8748 */ 8749 int 8750 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8751 struct mlx5_flow_meter_policy *mtr_policy) 8752 { 8753 const struct mlx5_flow_driver_ops *fops; 8754 8755 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8756 return fops->create_policy_rules(dev, mtr_policy); 8757 } 8758 8759 /** 8760 * Destroy policy rules, lock free, 8761 * (mutex should be acquired by caller). 8762 * Dispatcher for action type specific call. 8763 * 8764 * @param[in] dev 8765 * Pointer to the Ethernet device structure. 8766 * @param[in] mtr_policy 8767 * Meter policy struct. 8768 */ 8769 void 8770 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8771 struct mlx5_flow_meter_policy *mtr_policy) 8772 { 8773 const struct mlx5_flow_driver_ops *fops; 8774 8775 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8776 fops->destroy_policy_rules(dev, mtr_policy); 8777 } 8778 8779 /** 8780 * Destroy the default policy table set. 8781 * 8782 * @param[in] dev 8783 * Pointer to Ethernet device. 8784 */ 8785 void 8786 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8787 { 8788 const struct mlx5_flow_driver_ops *fops; 8789 8790 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8791 fops->destroy_def_policy(dev); 8792 } 8793 8794 /** 8795 * Destroy the default policy table set. 8796 * 8797 * @param[in] dev 8798 * Pointer to Ethernet device. 8799 * 8800 * @return 8801 * 0 on success, -1 otherwise. 8802 */ 8803 int 8804 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8805 { 8806 const struct mlx5_flow_driver_ops *fops; 8807 8808 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8809 return fops->create_def_policy(dev); 8810 } 8811 8812 /** 8813 * Create the needed meter and suffix tables. 8814 * 8815 * @param[in] dev 8816 * Pointer to Ethernet device. 8817 * 8818 * @return 8819 * 0 on success, -1 otherwise. 8820 */ 8821 int 8822 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8823 struct mlx5_flow_meter_info *fm, 8824 uint32_t mtr_idx, 8825 uint8_t domain_bitmap) 8826 { 8827 const struct mlx5_flow_driver_ops *fops; 8828 8829 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8830 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8831 } 8832 8833 /** 8834 * Destroy the meter table set. 8835 * 8836 * @param[in] dev 8837 * Pointer to Ethernet device. 8838 * @param[in] tbl 8839 * Pointer to the meter table set. 8840 */ 8841 void 8842 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8843 struct mlx5_flow_meter_info *fm) 8844 { 8845 const struct mlx5_flow_driver_ops *fops; 8846 8847 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8848 fops->destroy_mtr_tbls(dev, fm); 8849 } 8850 8851 /** 8852 * Destroy the global meter drop table. 8853 * 8854 * @param[in] dev 8855 * Pointer to Ethernet device. 8856 */ 8857 void 8858 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8859 { 8860 const struct mlx5_flow_driver_ops *fops; 8861 8862 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8863 fops->destroy_mtr_drop_tbls(dev); 8864 } 8865 8866 /** 8867 * Destroy the sub policy table with RX queue. 8868 * 8869 * @param[in] dev 8870 * Pointer to Ethernet device. 8871 * @param[in] mtr_policy 8872 * Pointer to meter policy table. 8873 */ 8874 void 8875 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8876 struct mlx5_flow_meter_policy *mtr_policy) 8877 { 8878 const struct mlx5_flow_driver_ops *fops; 8879 8880 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8881 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8882 } 8883 8884 /** 8885 * Allocate the needed aso flow meter id. 8886 * 8887 * @param[in] dev 8888 * Pointer to Ethernet device. 8889 * 8890 * @return 8891 * Index to aso flow meter on success, NULL otherwise. 8892 */ 8893 uint32_t 8894 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8895 { 8896 const struct mlx5_flow_driver_ops *fops; 8897 8898 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8899 return fops->create_meter(dev); 8900 } 8901 8902 /** 8903 * Free the aso flow meter id. 8904 * 8905 * @param[in] dev 8906 * Pointer to Ethernet device. 8907 * @param[in] mtr_idx 8908 * Index to aso flow meter to be free. 8909 * 8910 * @return 8911 * 0 on success. 8912 */ 8913 void 8914 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8915 { 8916 const struct mlx5_flow_driver_ops *fops; 8917 8918 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8919 fops->free_meter(dev, mtr_idx); 8920 } 8921 8922 /** 8923 * Allocate a counter. 8924 * 8925 * @param[in] dev 8926 * Pointer to Ethernet device structure. 8927 * 8928 * @return 8929 * Index to allocated counter on success, 0 otherwise. 8930 */ 8931 uint32_t 8932 mlx5_counter_alloc(struct rte_eth_dev *dev) 8933 { 8934 struct rte_flow_attr attr = { .transfer = 0 }; 8935 8936 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8937 (dev); 8938 } 8939 8940 /** 8941 * Free a counter. 8942 * 8943 * @param[in] dev 8944 * Pointer to Ethernet device structure. 8945 * @param[in] cnt 8946 * Index to counter to be free. 8947 */ 8948 void 8949 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8950 { 8951 struct rte_flow_attr attr = { .transfer = 0 }; 8952 8953 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8954 } 8955 8956 /** 8957 * Query counter statistics. 8958 * 8959 * @param[in] dev 8960 * Pointer to Ethernet device structure. 8961 * @param[in] cnt 8962 * Index to counter to query. 8963 * @param[in] clear 8964 * Set to clear counter statistics. 8965 * @param[out] pkts 8966 * The counter hits packets number to save. 8967 * @param[out] bytes 8968 * The counter hits bytes number to save. 8969 * 8970 * @return 8971 * 0 on success, a negative errno value otherwise. 8972 */ 8973 int 8974 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8975 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8976 { 8977 struct rte_flow_attr attr = { .transfer = 0 }; 8978 8979 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8980 (dev, cnt, clear, pkts, bytes, action); 8981 } 8982 8983 /** 8984 * Get information about HWS pre-configurable resources. 8985 * 8986 * @param[in] dev 8987 * Pointer to the rte_eth_dev structure. 8988 * @param[out] port_info 8989 * Pointer to port information. 8990 * @param[out] queue_info 8991 * Pointer to queue information. 8992 * @param[out] error 8993 * Pointer to error structure. 8994 * 8995 * @return 8996 * 0 on success, a negative errno value otherwise and rte_errno is set. 8997 */ 8998 static int 8999 mlx5_flow_info_get(struct rte_eth_dev *dev, 9000 struct rte_flow_port_info *port_info, 9001 struct rte_flow_queue_info *queue_info, 9002 struct rte_flow_error *error) 9003 { 9004 const struct mlx5_flow_driver_ops *fops; 9005 struct rte_flow_attr attr = {0}; 9006 9007 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9008 return rte_flow_error_set(error, ENOTSUP, 9009 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9010 NULL, 9011 "info get with incorrect steering mode"); 9012 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9013 return fops->info_get(dev, port_info, queue_info, error); 9014 } 9015 9016 /** 9017 * Configure port HWS resources. 9018 * 9019 * @param[in] dev 9020 * Pointer to the rte_eth_dev structure. 9021 * @param[in] port_attr 9022 * Port configuration attributes. 9023 * @param[in] nb_queue 9024 * Number of queue. 9025 * @param[in] queue_attr 9026 * Array that holds attributes for each flow queue. 9027 * @param[out] error 9028 * Pointer to error structure. 9029 * 9030 * @return 9031 * 0 on success, a negative errno value otherwise and rte_errno is set. 9032 */ 9033 static int 9034 mlx5_flow_port_configure(struct rte_eth_dev *dev, 9035 const struct rte_flow_port_attr *port_attr, 9036 uint16_t nb_queue, 9037 const struct rte_flow_queue_attr *queue_attr[], 9038 struct rte_flow_error *error) 9039 { 9040 const struct mlx5_flow_driver_ops *fops; 9041 struct rte_flow_attr attr = {0}; 9042 9043 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9044 return rte_flow_error_set(error, ENOTSUP, 9045 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9046 NULL, 9047 "port configure with incorrect steering mode"); 9048 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9049 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 9050 } 9051 9052 /** 9053 * Validate item template. 9054 * 9055 * @param[in] dev 9056 * Pointer to the rte_eth_dev structure. 9057 * @param[in] attr 9058 * Pointer to the item template attributes. 9059 * @param[in] items 9060 * The template item pattern. 9061 * @param[out] error 9062 * Pointer to error structure. 9063 * 9064 * @return 9065 * 0 on success, a negative errno value otherwise and rte_errno is set. 9066 */ 9067 int 9068 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 9069 const struct rte_flow_pattern_template_attr *attr, 9070 const struct rte_flow_item items[], 9071 struct rte_flow_error *error) 9072 { 9073 const struct mlx5_flow_driver_ops *fops; 9074 struct rte_flow_attr fattr = {0}; 9075 9076 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9077 rte_flow_error_set(error, ENOTSUP, 9078 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9079 "pattern validate with incorrect steering mode"); 9080 return -ENOTSUP; 9081 } 9082 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9083 return fops->pattern_validate(dev, attr, items, error); 9084 } 9085 9086 /** 9087 * Create flow item template. 9088 * 9089 * @param[in] dev 9090 * Pointer to the rte_eth_dev structure. 9091 * @param[in] attr 9092 * Pointer to the item template attributes. 9093 * @param[in] items 9094 * The template item pattern. 9095 * @param[out] error 9096 * Pointer to error structure. 9097 * 9098 * @return 9099 * 0 on success, a negative errno value otherwise and rte_errno is set. 9100 */ 9101 static struct rte_flow_pattern_template * 9102 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9103 const struct rte_flow_pattern_template_attr *attr, 9104 const struct rte_flow_item items[], 9105 struct rte_flow_error *error) 9106 { 9107 const struct mlx5_flow_driver_ops *fops; 9108 struct rte_flow_attr fattr = {0}; 9109 9110 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9111 rte_flow_error_set(error, ENOTSUP, 9112 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9113 NULL, 9114 "pattern create with incorrect steering mode"); 9115 return NULL; 9116 } 9117 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9118 return fops->pattern_template_create(dev, attr, items, error); 9119 } 9120 9121 /** 9122 * Destroy flow item template. 9123 * 9124 * @param[in] dev 9125 * Pointer to the rte_eth_dev structure. 9126 * @param[in] template 9127 * Pointer to the item template to be destroyed. 9128 * @param[out] error 9129 * Pointer to error structure. 9130 * 9131 * @return 9132 * 0 on success, a negative errno value otherwise and rte_errno is set. 9133 */ 9134 static int 9135 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9136 struct rte_flow_pattern_template *template, 9137 struct rte_flow_error *error) 9138 { 9139 const struct mlx5_flow_driver_ops *fops; 9140 struct rte_flow_attr attr = {0}; 9141 9142 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9143 return rte_flow_error_set(error, ENOTSUP, 9144 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9145 NULL, 9146 "pattern destroy with incorrect steering mode"); 9147 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9148 return fops->pattern_template_destroy(dev, template, error); 9149 } 9150 9151 /** 9152 * Validate flow actions template. 9153 * 9154 * @param[in] dev 9155 * Pointer to the rte_eth_dev structure. 9156 * @param[in] attr 9157 * Pointer to the action template attributes. 9158 * @param[in] actions 9159 * Associated actions (list terminated by the END action). 9160 * @param[in] masks 9161 * List of actions that marks which of the action's member is constant. 9162 * @param[out] error 9163 * Pointer to error structure. 9164 * 9165 * @return 9166 * 0 on success, a negative errno value otherwise and rte_errno is set. 9167 */ 9168 int 9169 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9170 const struct rte_flow_actions_template_attr *attr, 9171 const struct rte_flow_action actions[], 9172 const struct rte_flow_action masks[], 9173 struct rte_flow_error *error) 9174 { 9175 const struct mlx5_flow_driver_ops *fops; 9176 struct rte_flow_attr fattr = {0}; 9177 9178 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9179 rte_flow_error_set(error, ENOTSUP, 9180 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9181 "actions validate with incorrect steering mode"); 9182 return -ENOTSUP; 9183 } 9184 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9185 return fops->actions_validate(dev, attr, actions, masks, error); 9186 } 9187 9188 /** 9189 * Create flow item template. 9190 * 9191 * @param[in] dev 9192 * Pointer to the rte_eth_dev structure. 9193 * @param[in] attr 9194 * Pointer to the action template attributes. 9195 * @param[in] actions 9196 * Associated actions (list terminated by the END action). 9197 * @param[in] masks 9198 * List of actions that marks which of the action's member is constant. 9199 * @param[out] error 9200 * Pointer to error structure. 9201 * 9202 * @return 9203 * 0 on success, a negative errno value otherwise and rte_errno is set. 9204 */ 9205 static struct rte_flow_actions_template * 9206 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9207 const struct rte_flow_actions_template_attr *attr, 9208 const struct rte_flow_action actions[], 9209 const struct rte_flow_action masks[], 9210 struct rte_flow_error *error) 9211 { 9212 const struct mlx5_flow_driver_ops *fops; 9213 struct rte_flow_attr fattr = {0}; 9214 9215 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9216 rte_flow_error_set(error, ENOTSUP, 9217 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9218 NULL, 9219 "action create with incorrect steering mode"); 9220 return NULL; 9221 } 9222 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9223 return fops->actions_template_create(dev, attr, actions, masks, error); 9224 } 9225 9226 /** 9227 * Destroy flow action template. 9228 * 9229 * @param[in] dev 9230 * Pointer to the rte_eth_dev structure. 9231 * @param[in] template 9232 * Pointer to the action template to be destroyed. 9233 * @param[out] error 9234 * Pointer to error structure. 9235 * 9236 * @return 9237 * 0 on success, a negative errno value otherwise and rte_errno is set. 9238 */ 9239 static int 9240 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9241 struct rte_flow_actions_template *template, 9242 struct rte_flow_error *error) 9243 { 9244 const struct mlx5_flow_driver_ops *fops; 9245 struct rte_flow_attr attr = {0}; 9246 9247 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9248 return rte_flow_error_set(error, ENOTSUP, 9249 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9250 NULL, 9251 "action destroy with incorrect steering mode"); 9252 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9253 return fops->actions_template_destroy(dev, template, error); 9254 } 9255 9256 /** 9257 * Create flow table. 9258 * 9259 * @param[in] dev 9260 * Pointer to the rte_eth_dev structure. 9261 * @param[in] attr 9262 * Pointer to the table attributes. 9263 * @param[in] item_templates 9264 * Item template array to be binded to the table. 9265 * @param[in] nb_item_templates 9266 * Number of item template. 9267 * @param[in] action_templates 9268 * Action template array to be binded to the table. 9269 * @param[in] nb_action_templates 9270 * Number of action template. 9271 * @param[out] error 9272 * Pointer to error structure. 9273 * 9274 * @return 9275 * Table on success, NULL otherwise and rte_errno is set. 9276 */ 9277 static struct rte_flow_template_table * 9278 mlx5_flow_table_create(struct rte_eth_dev *dev, 9279 const struct rte_flow_template_table_attr *attr, 9280 struct rte_flow_pattern_template *item_templates[], 9281 uint8_t nb_item_templates, 9282 struct rte_flow_actions_template *action_templates[], 9283 uint8_t nb_action_templates, 9284 struct rte_flow_error *error) 9285 { 9286 const struct mlx5_flow_driver_ops *fops; 9287 struct rte_flow_attr fattr = {0}; 9288 9289 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9290 rte_flow_error_set(error, ENOTSUP, 9291 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9292 NULL, 9293 "table create with incorrect steering mode"); 9294 return NULL; 9295 } 9296 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9297 return fops->template_table_create(dev, 9298 attr, 9299 item_templates, 9300 nb_item_templates, 9301 action_templates, 9302 nb_action_templates, 9303 error); 9304 } 9305 9306 /** 9307 * PMD destroy flow table. 9308 * 9309 * @param[in] dev 9310 * Pointer to the rte_eth_dev structure. 9311 * @param[in] table 9312 * Pointer to the table to be destroyed. 9313 * @param[out] error 9314 * Pointer to error structure. 9315 * 9316 * @return 9317 * 0 on success, a negative errno value otherwise and rte_errno is set. 9318 */ 9319 static int 9320 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9321 struct rte_flow_template_table *table, 9322 struct rte_flow_error *error) 9323 { 9324 const struct mlx5_flow_driver_ops *fops; 9325 struct rte_flow_attr attr = {0}; 9326 9327 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9328 return rte_flow_error_set(error, ENOTSUP, 9329 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9330 NULL, 9331 "table destroy with incorrect steering mode"); 9332 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9333 return fops->template_table_destroy(dev, table, error); 9334 } 9335 9336 /** 9337 * PMD group set miss actions. 9338 * 9339 * @param[in] dev 9340 * Pointer to the rte_eth_dev structure. 9341 * @param[in] attr 9342 * Pointer to group attributes 9343 * @param[in] actions 9344 * Array of actions 9345 * @param[out] error 9346 * Pointer to error structure. 9347 * 9348 * @return 9349 * 0 on success, a negative errno value otherwise and rte_errno is set. 9350 */ 9351 static int 9352 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 9353 uint32_t group_id, 9354 const struct rte_flow_group_attr *attr, 9355 const struct rte_flow_action actions[], 9356 struct rte_flow_error *error) 9357 { 9358 const struct mlx5_flow_driver_ops *fops; 9359 struct rte_flow_attr fattr = {0}; 9360 9361 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9362 return rte_flow_error_set(error, ENOTSUP, 9363 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9364 NULL, 9365 "group set miss actions with incorrect steering mode"); 9366 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9367 return fops->group_set_miss_actions(dev, group_id, attr, actions, error); 9368 } 9369 9370 /** 9371 * Allocate a new memory for the counter values wrapped by all the needed 9372 * management. 9373 * 9374 * @param[in] sh 9375 * Pointer to mlx5_dev_ctx_shared object. 9376 * 9377 * @return 9378 * 0 on success, a negative errno value otherwise. 9379 */ 9380 static int 9381 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9382 { 9383 struct mlx5_counter_stats_mem_mng *mem_mng; 9384 volatile struct flow_counter_stats *raw_data; 9385 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9386 int size = (sizeof(struct flow_counter_stats) * 9387 MLX5_COUNTERS_PER_POOL + 9388 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9389 sizeof(struct mlx5_counter_stats_mem_mng); 9390 size_t pgsize = rte_mem_page_size(); 9391 uint8_t *mem; 9392 int ret; 9393 int i; 9394 9395 if (pgsize == (size_t)-1) { 9396 DRV_LOG(ERR, "Failed to get mem page size"); 9397 rte_errno = ENOMEM; 9398 return -ENOMEM; 9399 } 9400 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9401 if (!mem) { 9402 rte_errno = ENOMEM; 9403 return -ENOMEM; 9404 } 9405 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9406 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9407 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9408 sh->cdev->pdn, mem, size, 9409 &mem_mng->wm); 9410 if (ret) { 9411 rte_errno = errno; 9412 mlx5_free(mem); 9413 return -rte_errno; 9414 } 9415 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9416 raw_data = (volatile struct flow_counter_stats *)mem; 9417 for (i = 0; i < raws_n; ++i) { 9418 mem_mng->raws[i].mem_mng = mem_mng; 9419 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9420 } 9421 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9422 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9423 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9424 next); 9425 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9426 sh->sws_cmng.mem_mng = mem_mng; 9427 return 0; 9428 } 9429 9430 /** 9431 * Set the statistic memory to the new counter pool. 9432 * 9433 * @param[in] sh 9434 * Pointer to mlx5_dev_ctx_shared object. 9435 * @param[in] pool 9436 * Pointer to the pool to set the statistic memory. 9437 * 9438 * @return 9439 * 0 on success, a negative errno value otherwise. 9440 */ 9441 static int 9442 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9443 struct mlx5_flow_counter_pool *pool) 9444 { 9445 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9446 /* Resize statistic memory once used out. */ 9447 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9448 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9449 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9450 return -1; 9451 } 9452 rte_spinlock_lock(&pool->sl); 9453 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9454 rte_spinlock_unlock(&pool->sl); 9455 pool->raw_hw = NULL; 9456 return 0; 9457 } 9458 9459 #define MLX5_POOL_QUERY_FREQ_US 1000000 9460 9461 /** 9462 * Set the periodic procedure for triggering asynchronous batch queries for all 9463 * the counter pools. 9464 * 9465 * @param[in] sh 9466 * Pointer to mlx5_dev_ctx_shared object. 9467 */ 9468 void 9469 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9470 { 9471 uint32_t pools_n, us; 9472 9473 pools_n = rte_atomic_load_explicit(&sh->sws_cmng.n_valid, rte_memory_order_relaxed); 9474 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9475 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9476 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9477 sh->sws_cmng.query_thread_on = 0; 9478 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9479 } else { 9480 sh->sws_cmng.query_thread_on = 1; 9481 } 9482 } 9483 9484 /** 9485 * The periodic procedure for triggering asynchronous batch queries for all the 9486 * counter pools. This function is probably called by the host thread. 9487 * 9488 * @param[in] arg 9489 * The parameter for the alarm process. 9490 */ 9491 void 9492 mlx5_flow_query_alarm(void *arg) 9493 { 9494 struct mlx5_dev_ctx_shared *sh = arg; 9495 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9496 uint16_t pool_index = cmng->pool_index; 9497 struct mlx5_flow_counter_pool *pool; 9498 uint16_t n_valid; 9499 int ret; 9500 9501 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9502 goto set_alarm; 9503 rte_spinlock_lock(&cmng->pool_update_sl); 9504 pool = cmng->pools[pool_index]; 9505 n_valid = cmng->n_valid; 9506 rte_spinlock_unlock(&cmng->pool_update_sl); 9507 /* Set the statistic memory to the new created pool. */ 9508 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9509 goto set_alarm; 9510 if (pool->raw_hw) 9511 /* There is a pool query in progress. */ 9512 goto set_alarm; 9513 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9514 if (!pool->raw_hw) 9515 /* No free counter statistics raw memory. */ 9516 goto set_alarm; 9517 /* 9518 * Identify the counters released between query trigger and query 9519 * handle more efficiently. The counter released in this gap period 9520 * should wait for a new round of query as the new arrived packets 9521 * will not be taken into account. 9522 */ 9523 pool->query_gen++; 9524 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9525 MLX5_COUNTERS_PER_POOL, 9526 NULL, NULL, 9527 pool->raw_hw->mem_mng->wm.lkey, 9528 (void *)(uintptr_t) 9529 pool->raw_hw->data, 9530 sh->devx_comp, 9531 (uint64_t)(uintptr_t)pool); 9532 if (ret) { 9533 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9534 " %d", pool->min_dcs->id); 9535 pool->raw_hw = NULL; 9536 goto set_alarm; 9537 } 9538 LIST_REMOVE(pool->raw_hw, next); 9539 cmng->pending_queries++; 9540 pool_index++; 9541 if (pool_index >= n_valid) 9542 pool_index = 0; 9543 set_alarm: 9544 cmng->pool_index = pool_index; 9545 mlx5_set_query_alarm(sh); 9546 } 9547 9548 /** 9549 * Check and callback event for new aged flow in the counter pool 9550 * 9551 * @param[in] sh 9552 * Pointer to mlx5_dev_ctx_shared object. 9553 * @param[in] pool 9554 * Pointer to Current counter pool. 9555 */ 9556 static void 9557 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9558 struct mlx5_flow_counter_pool *pool) 9559 { 9560 struct mlx5_priv *priv; 9561 struct mlx5_flow_counter *cnt; 9562 struct mlx5_age_info *age_info; 9563 struct mlx5_age_param *age_param; 9564 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9565 struct mlx5_counter_stats_raw *prev = pool->raw; 9566 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9567 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9568 uint16_t expected = AGE_CANDIDATE; 9569 uint32_t i; 9570 9571 pool->time_of_last_age_check = curr_time; 9572 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9573 cnt = MLX5_POOL_GET_CNT(pool, i); 9574 age_param = MLX5_CNT_TO_AGE(cnt); 9575 if (rte_atomic_load_explicit(&age_param->state, 9576 rte_memory_order_relaxed) != AGE_CANDIDATE) 9577 continue; 9578 if (cur->data[i].hits != prev->data[i].hits) { 9579 rte_atomic_store_explicit(&age_param->sec_since_last_hit, 0, 9580 rte_memory_order_relaxed); 9581 continue; 9582 } 9583 if (rte_atomic_fetch_add_explicit(&age_param->sec_since_last_hit, 9584 time_delta, 9585 rte_memory_order_relaxed) + time_delta <= age_param->timeout) 9586 continue; 9587 /** 9588 * Hold the lock first, or if between the 9589 * state AGE_TMOUT and tailq operation the 9590 * release happened, the release procedure 9591 * may delete a non-existent tailq node. 9592 */ 9593 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9594 age_info = GET_PORT_AGE_INFO(priv); 9595 rte_spinlock_lock(&age_info->aged_sl); 9596 if (rte_atomic_compare_exchange_strong_explicit(&age_param->state, &expected, 9597 AGE_TMOUT, 9598 rte_memory_order_relaxed, 9599 rte_memory_order_relaxed)) { 9600 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9601 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9602 } 9603 rte_spinlock_unlock(&age_info->aged_sl); 9604 } 9605 mlx5_age_event_prepare(sh); 9606 } 9607 9608 /** 9609 * Handler for the HW respond about ready values from an asynchronous batch 9610 * query. This function is probably called by the host thread. 9611 * 9612 * @param[in] sh 9613 * The pointer to the shared device context. 9614 * @param[in] async_id 9615 * The Devx async ID. 9616 * @param[in] status 9617 * The status of the completion. 9618 */ 9619 void 9620 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9621 uint64_t async_id, int status) 9622 { 9623 struct mlx5_flow_counter_pool *pool = 9624 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9625 struct mlx5_counter_stats_raw *raw_to_free; 9626 uint8_t query_gen = pool->query_gen ^ 1; 9627 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9628 enum mlx5_counter_type cnt_type = 9629 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9630 MLX5_COUNTER_TYPE_ORIGIN; 9631 9632 if (unlikely(status)) { 9633 raw_to_free = pool->raw_hw; 9634 } else { 9635 raw_to_free = pool->raw; 9636 if (pool->is_aged) 9637 mlx5_flow_aging_check(sh, pool); 9638 rte_spinlock_lock(&pool->sl); 9639 pool->raw = pool->raw_hw; 9640 rte_spinlock_unlock(&pool->sl); 9641 /* Be sure the new raw counters data is updated in memory. */ 9642 rte_io_wmb(); 9643 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9644 rte_spinlock_lock(&cmng->csl[cnt_type]); 9645 TAILQ_CONCAT(&cmng->counters[cnt_type], 9646 &pool->counters[query_gen], next); 9647 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9648 } 9649 } 9650 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9651 pool->raw_hw = NULL; 9652 sh->sws_cmng.pending_queries--; 9653 } 9654 9655 static int 9656 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9657 const struct flow_grp_info *grp_info, 9658 struct rte_flow_error *error) 9659 { 9660 if (grp_info->transfer && grp_info->external && 9661 grp_info->fdb_def_rule) { 9662 if (group == UINT32_MAX) 9663 return rte_flow_error_set 9664 (error, EINVAL, 9665 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9666 NULL, 9667 "group index not supported"); 9668 *table = group + 1; 9669 } else { 9670 *table = group; 9671 } 9672 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9673 return 0; 9674 } 9675 9676 /** 9677 * Translate the rte_flow group index to HW table value. 9678 * 9679 * If tunnel offload is disabled, all group ids converted to flow table 9680 * id using the standard method. 9681 * If tunnel offload is enabled, group id can be converted using the 9682 * standard or tunnel conversion method. Group conversion method 9683 * selection depends on flags in `grp_info` parameter: 9684 * - Internal (grp_info.external == 0) groups conversion uses the 9685 * standard method. 9686 * - Group ids in JUMP action converted with the tunnel conversion. 9687 * - Group id in rule attribute conversion depends on a rule type and 9688 * group id value: 9689 * ** non zero group attributes converted with the tunnel method 9690 * ** zero group attribute in non-tunnel rule is converted using the 9691 * standard method - there's only one root table 9692 * ** zero group attribute in steer tunnel rule is converted with the 9693 * standard method - single root table 9694 * ** zero group attribute in match tunnel rule is a special OvS 9695 * case: that value is used for portability reasons. That group 9696 * id is converted with the tunnel conversion method. 9697 * 9698 * @param[in] dev 9699 * Port device 9700 * @param[in] tunnel 9701 * PMD tunnel offload object 9702 * @param[in] group 9703 * rte_flow group index value. 9704 * @param[out] table 9705 * HW table value. 9706 * @param[in] grp_info 9707 * flags used for conversion 9708 * @param[out] error 9709 * Pointer to error structure. 9710 * 9711 * @return 9712 * 0 on success, a negative errno value otherwise and rte_errno is set. 9713 */ 9714 int 9715 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9716 const struct mlx5_flow_tunnel *tunnel, 9717 uint32_t group, uint32_t *table, 9718 const struct flow_grp_info *grp_info, 9719 struct rte_flow_error *error) 9720 { 9721 int ret; 9722 bool standard_translation; 9723 9724 if (!grp_info->skip_scale && grp_info->external && 9725 group < MLX5_MAX_TABLES_EXTERNAL) 9726 group *= MLX5_FLOW_TABLE_FACTOR; 9727 if (is_tunnel_offload_active(dev)) { 9728 standard_translation = !grp_info->external || 9729 grp_info->std_tbl_fix; 9730 } else { 9731 standard_translation = true; 9732 } 9733 DRV_LOG(DEBUG, 9734 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9735 dev->data->port_id, group, grp_info->transfer, 9736 grp_info->external, grp_info->fdb_def_rule, 9737 standard_translation ? "STANDARD" : "TUNNEL"); 9738 if (standard_translation) 9739 ret = flow_group_to_table(dev->data->port_id, group, table, 9740 grp_info, error); 9741 else 9742 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9743 table, error); 9744 9745 return ret; 9746 } 9747 9748 /** 9749 * Discover availability of metadata reg_c's. 9750 * 9751 * Iteratively use test flows to check availability. 9752 * 9753 * @param[in] dev 9754 * Pointer to the Ethernet device structure. 9755 * 9756 * @return 9757 * 0 on success, a negative errno value otherwise and rte_errno is set. 9758 */ 9759 int 9760 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9761 { 9762 struct mlx5_priv *priv = dev->data->dev_private; 9763 enum modify_reg idx; 9764 int n = 0; 9765 9766 /* reg_c[0] and reg_c[1] are reserved. */ 9767 priv->sh->flow_mreg_c[n++] = REG_C_0; 9768 priv->sh->flow_mreg_c[n++] = REG_C_1; 9769 /* Discover availability of other reg_c's. */ 9770 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9771 struct rte_flow_attr attr = { 9772 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9773 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9774 .ingress = 1, 9775 }; 9776 struct rte_flow_item items[] = { 9777 [0] = { 9778 .type = RTE_FLOW_ITEM_TYPE_END, 9779 }, 9780 }; 9781 struct rte_flow_action actions[] = { 9782 [0] = { 9783 .type = (enum rte_flow_action_type) 9784 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9785 .conf = &(struct mlx5_flow_action_copy_mreg){ 9786 .src = REG_C_1, 9787 .dst = idx, 9788 }, 9789 }, 9790 [1] = { 9791 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9792 .conf = &(struct rte_flow_action_jump){ 9793 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9794 }, 9795 }, 9796 [2] = { 9797 .type = RTE_FLOW_ACTION_TYPE_END, 9798 }, 9799 }; 9800 uint32_t flow_idx; 9801 struct rte_flow *flow; 9802 struct rte_flow_error error; 9803 9804 if (!priv->sh->config.dv_flow_en) 9805 break; 9806 /* Create internal flow, validation skips copy action. */ 9807 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9808 items, actions, false, &error); 9809 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9810 flow_idx); 9811 if (!flow) 9812 continue; 9813 priv->sh->flow_mreg_c[n++] = idx; 9814 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9815 } 9816 for (; n < MLX5_MREG_C_NUM; ++n) 9817 priv->sh->flow_mreg_c[n] = REG_NON; 9818 priv->sh->metadata_regc_check_flag = 1; 9819 return 0; 9820 } 9821 9822 int 9823 save_dump_file(const uint8_t *data, uint32_t size, 9824 uint32_t type, uint64_t id, void *arg, FILE *file) 9825 { 9826 char line[BUF_SIZE]; 9827 uint32_t out = 0; 9828 uint32_t k; 9829 uint32_t actions_num; 9830 struct rte_flow_query_count *count; 9831 9832 memset(line, 0, BUF_SIZE); 9833 switch (type) { 9834 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9835 actions_num = *(uint32_t *)(arg); 9836 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9837 type, id, actions_num); 9838 break; 9839 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 9840 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 9841 type, id); 9842 break; 9843 case DR_DUMP_REC_TYPE_PMD_COUNTER: 9844 count = (struct rte_flow_query_count *)arg; 9845 fprintf(file, 9846 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 9847 type, id, count->hits, count->bytes); 9848 return 0; 9849 default: 9850 return -1; 9851 } 9852 9853 for (k = 0; k < size; k++) { 9854 /* Make sure we do not overrun the line buffer length. */ 9855 if (out >= BUF_SIZE - 4) { 9856 line[out] = '\0'; 9857 break; 9858 } 9859 out += snprintf(line + out, BUF_SIZE - out, "%02x", 9860 (data[k]) & 0xff); 9861 } 9862 fprintf(file, "%s\n", line); 9863 return 0; 9864 } 9865 9866 int 9867 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 9868 struct rte_flow_query_count *count, struct rte_flow_error *error) 9869 { 9870 struct rte_flow_action action[2]; 9871 enum mlx5_flow_drv_type ftype; 9872 const struct mlx5_flow_driver_ops *fops; 9873 9874 if (!flow) { 9875 return rte_flow_error_set(error, ENOENT, 9876 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9877 NULL, 9878 "invalid flow handle"); 9879 } 9880 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 9881 action[1].type = RTE_FLOW_ACTION_TYPE_END; 9882 if (flow->counter) { 9883 memset(count, 0, sizeof(struct rte_flow_query_count)); 9884 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 9885 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 9886 ftype < MLX5_FLOW_TYPE_MAX); 9887 fops = flow_get_drv_ops(ftype); 9888 return fops->query(dev, flow, action, count, error); 9889 } 9890 return -1; 9891 } 9892 9893 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 9894 /** 9895 * Dump flow ipool data to file 9896 * 9897 * @param[in] dev 9898 * The pointer to Ethernet device. 9899 * @param[in] file 9900 * A pointer to a file for output. 9901 * @param[out] error 9902 * Perform verbose error reporting if not NULL. PMDs initialize this 9903 * structure in case of error only. 9904 * @return 9905 * 0 on success, a negative value otherwise. 9906 */ 9907 int 9908 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 9909 struct rte_flow *flow, FILE *file, 9910 struct rte_flow_error *error) 9911 { 9912 struct mlx5_priv *priv = dev->data->dev_private; 9913 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 9914 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 9915 uint32_t handle_idx; 9916 struct mlx5_flow_handle *dh; 9917 struct rte_flow_query_count count; 9918 uint32_t actions_num; 9919 const uint8_t *data; 9920 size_t size; 9921 uint64_t id; 9922 uint32_t type; 9923 void *action = NULL; 9924 9925 if (!flow) { 9926 return rte_flow_error_set(error, ENOENT, 9927 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9928 NULL, 9929 "invalid flow handle"); 9930 } 9931 handle_idx = flow->dev_handles; 9932 /* query counter */ 9933 if (flow->counter && 9934 (!mlx5_counter_query(dev, flow->counter, false, 9935 &count.hits, &count.bytes, &action)) && action) { 9936 id = (uint64_t)(uintptr_t)action; 9937 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 9938 save_dump_file(NULL, 0, type, 9939 id, (void *)&count, file); 9940 } 9941 9942 while (handle_idx) { 9943 dh = mlx5_ipool_get(priv->sh->ipool 9944 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 9945 if (!dh) 9946 continue; 9947 handle_idx = dh->next.next; 9948 9949 /* Get modify_hdr and encap_decap buf from ipools. */ 9950 encap_decap = NULL; 9951 modify_hdr = dh->dvh.modify_hdr; 9952 9953 if (dh->dvh.rix_encap_decap) { 9954 encap_decap = mlx5_ipool_get(priv->sh->ipool 9955 [MLX5_IPOOL_DECAP_ENCAP], 9956 dh->dvh.rix_encap_decap); 9957 } 9958 if (modify_hdr) { 9959 data = (const uint8_t *)modify_hdr->actions; 9960 size = (size_t)(modify_hdr->actions_num) * 8; 9961 id = (uint64_t)(uintptr_t)modify_hdr->action; 9962 actions_num = modify_hdr->actions_num; 9963 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 9964 save_dump_file(data, size, type, id, 9965 (void *)(&actions_num), file); 9966 } 9967 if (encap_decap) { 9968 data = encap_decap->buf; 9969 size = encap_decap->size; 9970 id = (uint64_t)(uintptr_t)encap_decap->action; 9971 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 9972 save_dump_file(data, size, type, 9973 id, NULL, file); 9974 } 9975 } 9976 return 0; 9977 } 9978 9979 /** 9980 * Dump all flow's encap_decap/modify_hdr/counter data to file 9981 * 9982 * @param[in] dev 9983 * The pointer to Ethernet device. 9984 * @param[in] file 9985 * A pointer to a file for output. 9986 * @param[out] error 9987 * Perform verbose error reporting if not NULL. PMDs initialize this 9988 * structure in case of error only. 9989 * @return 9990 * 0 on success, a negative value otherwise. 9991 */ 9992 static int 9993 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 9994 FILE *file, struct rte_flow_error *error __rte_unused) 9995 { 9996 struct mlx5_priv *priv = dev->data->dev_private; 9997 struct mlx5_dev_ctx_shared *sh = priv->sh; 9998 struct mlx5_hlist *h; 9999 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10000 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10001 struct rte_flow_query_count count; 10002 uint32_t actions_num; 10003 const uint8_t *data; 10004 size_t size; 10005 uint64_t id; 10006 uint32_t type; 10007 uint32_t i; 10008 uint32_t j; 10009 struct mlx5_list_inconst *l_inconst; 10010 struct mlx5_list_entry *e; 10011 int lcore_index; 10012 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 10013 uint32_t max; 10014 void *action; 10015 10016 /* encap_decap hlist is lcore_share, get global core cache. */ 10017 i = MLX5_LIST_GLOBAL; 10018 h = sh->encaps_decaps; 10019 if (h) { 10020 for (j = 0; j <= h->mask; j++) { 10021 l_inconst = &h->buckets[j].l; 10022 if (!l_inconst || !l_inconst->cache[i]) 10023 continue; 10024 10025 e = LIST_FIRST(&l_inconst->cache[i]->h); 10026 while (e) { 10027 encap_decap = 10028 (struct mlx5_flow_dv_encap_decap_resource *)e; 10029 data = encap_decap->buf; 10030 size = encap_decap->size; 10031 id = (uint64_t)(uintptr_t)encap_decap->action; 10032 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10033 save_dump_file(data, size, type, 10034 id, NULL, file); 10035 e = LIST_NEXT(e, next); 10036 } 10037 } 10038 } 10039 10040 /* get modify_hdr */ 10041 h = sh->modify_cmds; 10042 if (h) { 10043 lcore_index = rte_lcore_index(rte_lcore_id()); 10044 if (unlikely(lcore_index == -1)) { 10045 lcore_index = MLX5_LIST_NLCORE; 10046 rte_spinlock_lock(&h->l_const.lcore_lock); 10047 } 10048 i = lcore_index; 10049 10050 if (lcore_index == MLX5_LIST_NLCORE) { 10051 for (i = 0; i <= (uint32_t)lcore_index; i++) { 10052 for (j = 0; j <= h->mask; j++) { 10053 l_inconst = &h->buckets[j].l; 10054 if (!l_inconst || !l_inconst->cache[i]) 10055 continue; 10056 10057 e = LIST_FIRST(&l_inconst->cache[i]->h); 10058 while (e) { 10059 modify_hdr = 10060 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10061 data = (const uint8_t *)modify_hdr->actions; 10062 size = (size_t)(modify_hdr->actions_num) * 8; 10063 actions_num = modify_hdr->actions_num; 10064 id = (uint64_t)(uintptr_t)modify_hdr->action; 10065 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10066 save_dump_file(data, size, type, id, 10067 (void *)(&actions_num), file); 10068 e = LIST_NEXT(e, next); 10069 } 10070 } 10071 } 10072 } else { 10073 for (j = 0; j <= h->mask; j++) { 10074 l_inconst = &h->buckets[j].l; 10075 if (!l_inconst || !l_inconst->cache[i]) 10076 continue; 10077 10078 e = LIST_FIRST(&l_inconst->cache[i]->h); 10079 while (e) { 10080 modify_hdr = 10081 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10082 data = (const uint8_t *)modify_hdr->actions; 10083 size = (size_t)(modify_hdr->actions_num) * 8; 10084 actions_num = modify_hdr->actions_num; 10085 id = (uint64_t)(uintptr_t)modify_hdr->action; 10086 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10087 save_dump_file(data, size, type, id, 10088 (void *)(&actions_num), file); 10089 e = LIST_NEXT(e, next); 10090 } 10091 } 10092 } 10093 10094 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10095 rte_spinlock_unlock(&h->l_const.lcore_lock); 10096 } 10097 10098 /* get counter */ 10099 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10100 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10101 for (j = 1; j <= max; j++) { 10102 action = NULL; 10103 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10104 &count.bytes, &action)) && action) { 10105 id = (uint64_t)(uintptr_t)action; 10106 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10107 save_dump_file(NULL, 0, type, 10108 id, (void *)&count, file); 10109 } 10110 } 10111 return 0; 10112 } 10113 #endif 10114 10115 /** 10116 * Dump flow raw hw data to file 10117 * 10118 * @param[in] dev 10119 * The pointer to Ethernet device. 10120 * @param[in] file 10121 * A pointer to a file for output. 10122 * @param[out] error 10123 * Perform verbose error reporting if not NULL. PMDs initialize this 10124 * structure in case of error only. 10125 * @return 10126 * 0 on success, a negative value otherwise. 10127 */ 10128 int 10129 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10130 FILE *file, 10131 struct rte_flow_error *error __rte_unused) 10132 { 10133 struct mlx5_priv *priv = dev->data->dev_private; 10134 struct mlx5_dev_ctx_shared *sh = priv->sh; 10135 uint32_t handle_idx; 10136 int ret; 10137 struct mlx5_flow_handle *dh; 10138 struct rte_flow *flow; 10139 10140 if (!sh->config.dv_flow_en) { 10141 if (fputs("device dv flow disabled\n", file) <= 0) 10142 return -errno; 10143 return -ENOTSUP; 10144 } 10145 10146 /* dump all */ 10147 if (!flow_idx) { 10148 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10149 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10150 return -EINVAL; 10151 10152 if (sh->config.dv_flow_en == 2) 10153 return mlx5dr_debug_dump(priv->dr_ctx, file); 10154 #endif 10155 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10156 sh->rx_domain, 10157 sh->tx_domain, file); 10158 } 10159 /* dump one */ 10160 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10161 (uintptr_t)(void *)flow_idx); 10162 if (!flow) 10163 return -EINVAL; 10164 10165 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10166 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10167 #endif 10168 handle_idx = flow->dev_handles; 10169 while (handle_idx) { 10170 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10171 handle_idx); 10172 if (!dh) 10173 return -ENOENT; 10174 if (dh->drv_flow) { 10175 if (sh->config.dv_flow_en == 2) 10176 return -ENOTSUP; 10177 10178 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10179 file); 10180 if (ret) 10181 return -ENOENT; 10182 } 10183 handle_idx = dh->next.next; 10184 } 10185 return 0; 10186 } 10187 10188 /** 10189 * Get aged-out flows. 10190 * 10191 * @param[in] dev 10192 * Pointer to the Ethernet device structure. 10193 * @param[in] context 10194 * The address of an array of pointers to the aged-out flows contexts. 10195 * @param[in] nb_countexts 10196 * The length of context array pointers. 10197 * @param[out] error 10198 * Perform verbose error reporting if not NULL. Initialized in case of 10199 * error only. 10200 * 10201 * @return 10202 * how many contexts get in success, otherwise negative errno value. 10203 * if nb_contexts is 0, return the amount of all aged contexts. 10204 * if nb_contexts is not 0 , return the amount of aged flows reported 10205 * in the context array. 10206 */ 10207 int 10208 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10209 uint32_t nb_contexts, struct rte_flow_error *error) 10210 { 10211 struct rte_flow_attr attr = { .transfer = 0 }; 10212 10213 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10214 (dev, contexts, nb_contexts, error); 10215 } 10216 10217 /** 10218 * Get aged-out flows per HWS queue. 10219 * 10220 * @param[in] dev 10221 * Pointer to the Ethernet device structure. 10222 * @param[in] queue_id 10223 * Flow queue to query. 10224 * @param[in] context 10225 * The address of an array of pointers to the aged-out flows contexts. 10226 * @param[in] nb_countexts 10227 * The length of context array pointers. 10228 * @param[out] error 10229 * Perform verbose error reporting if not NULL. Initialized in case of 10230 * error only. 10231 * 10232 * @return 10233 * how many contexts get in success, otherwise negative errno value. 10234 * if nb_contexts is 0, return the amount of all aged contexts. 10235 * if nb_contexts is not 0 , return the amount of aged flows reported 10236 * in the context array. 10237 */ 10238 int 10239 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10240 void **contexts, uint32_t nb_contexts, 10241 struct rte_flow_error *error) 10242 { 10243 const struct mlx5_flow_driver_ops *fops; 10244 struct rte_flow_attr attr = { 0 }; 10245 10246 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10247 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10248 return fops->get_q_aged_flows(dev, queue_id, contexts, 10249 nb_contexts, error); 10250 } 10251 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10252 dev->data->port_id, queue_id); 10253 return rte_flow_error_set(error, ENOTSUP, 10254 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10255 "get Q aged flows with incorrect steering mode"); 10256 } 10257 10258 /* Wrapper for driver action_validate op callback */ 10259 static int 10260 flow_drv_action_validate(struct rte_eth_dev *dev, 10261 const struct rte_flow_indir_action_conf *conf, 10262 const struct rte_flow_action *action, 10263 const struct mlx5_flow_driver_ops *fops, 10264 struct rte_flow_error *error) 10265 { 10266 static const char err_msg[] = "indirect action validation unsupported"; 10267 10268 if (!fops->action_validate) { 10269 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10270 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10271 NULL, err_msg); 10272 return -rte_errno; 10273 } 10274 return fops->action_validate(dev, conf, action, error); 10275 } 10276 10277 /** 10278 * Destroys the shared action by handle. 10279 * 10280 * @param dev 10281 * Pointer to Ethernet device structure. 10282 * @param[in] handle 10283 * Handle for the indirect action object to be destroyed. 10284 * @param[out] error 10285 * Perform verbose error reporting if not NULL. PMDs initialize this 10286 * structure in case of error only. 10287 * 10288 * @return 10289 * 0 on success, a negative errno value otherwise and rte_errno is set. 10290 * 10291 * @note: wrapper for driver action_create op callback. 10292 */ 10293 static int 10294 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10295 struct rte_flow_action_handle *handle, 10296 struct rte_flow_error *error) 10297 { 10298 static const char err_msg[] = "indirect action destruction unsupported"; 10299 struct rte_flow_attr attr = { .transfer = 0 }; 10300 const struct mlx5_flow_driver_ops *fops = 10301 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10302 10303 if (!fops->action_destroy) { 10304 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10305 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10306 NULL, err_msg); 10307 return -rte_errno; 10308 } 10309 return fops->action_destroy(dev, handle, error); 10310 } 10311 10312 /* Wrapper for driver action_destroy op callback */ 10313 static int 10314 flow_drv_action_update(struct rte_eth_dev *dev, 10315 struct rte_flow_action_handle *handle, 10316 const void *update, 10317 const struct mlx5_flow_driver_ops *fops, 10318 struct rte_flow_error *error) 10319 { 10320 static const char err_msg[] = "indirect action update unsupported"; 10321 10322 if (!fops->action_update) { 10323 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10324 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10325 NULL, err_msg); 10326 return -rte_errno; 10327 } 10328 return fops->action_update(dev, handle, update, error); 10329 } 10330 10331 /* Wrapper for driver action_destroy op callback */ 10332 static int 10333 flow_drv_action_query(struct rte_eth_dev *dev, 10334 const struct rte_flow_action_handle *handle, 10335 void *data, 10336 const struct mlx5_flow_driver_ops *fops, 10337 struct rte_flow_error *error) 10338 { 10339 static const char err_msg[] = "indirect action query unsupported"; 10340 10341 if (!fops->action_query) { 10342 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10343 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10344 NULL, err_msg); 10345 return -rte_errno; 10346 } 10347 return fops->action_query(dev, handle, data, error); 10348 } 10349 10350 /** 10351 * Create indirect action for reuse in multiple flow rules. 10352 * 10353 * @param dev 10354 * Pointer to Ethernet device structure. 10355 * @param conf 10356 * Pointer to indirect action object configuration. 10357 * @param[in] action 10358 * Action configuration for indirect action object creation. 10359 * @param[out] error 10360 * Perform verbose error reporting if not NULL. PMDs initialize this 10361 * structure in case of error only. 10362 * @return 10363 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10364 */ 10365 static struct rte_flow_action_handle * 10366 mlx5_action_handle_create(struct rte_eth_dev *dev, 10367 const struct rte_flow_indir_action_conf *conf, 10368 const struct rte_flow_action *action, 10369 struct rte_flow_error *error) 10370 { 10371 static const char err_msg[] = "indirect action creation unsupported"; 10372 struct rte_flow_attr attr = { .transfer = 0 }; 10373 const struct mlx5_flow_driver_ops *fops = 10374 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10375 10376 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10377 return NULL; 10378 if (!fops->action_create) { 10379 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10380 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10381 NULL, err_msg); 10382 return NULL; 10383 } 10384 return fops->action_create(dev, conf, action, error); 10385 } 10386 10387 /** 10388 * Updates inplace the indirect action configuration pointed by *handle* 10389 * with the configuration provided as *update* argument. 10390 * The update of the indirect action configuration effects all flow rules 10391 * reusing the action via handle. 10392 * 10393 * @param dev 10394 * Pointer to Ethernet device structure. 10395 * @param[in] handle 10396 * Handle for the indirect action to be updated. 10397 * @param[in] update 10398 * Action specification used to modify the action pointed by handle. 10399 * *update* could be of same type with the action pointed by the *handle* 10400 * handle argument, or some other structures like a wrapper, depending on 10401 * the indirect action type. 10402 * @param[out] error 10403 * Perform verbose error reporting if not NULL. PMDs initialize this 10404 * structure in case of error only. 10405 * 10406 * @return 10407 * 0 on success, a negative errno value otherwise and rte_errno is set. 10408 */ 10409 static int 10410 mlx5_action_handle_update(struct rte_eth_dev *dev, 10411 struct rte_flow_action_handle *handle, 10412 const void *update, 10413 struct rte_flow_error *error) 10414 { 10415 struct rte_flow_attr attr = { .transfer = 0 }; 10416 const struct mlx5_flow_driver_ops *fops = 10417 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10418 int ret; 10419 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 10420 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 10421 10422 switch (type) { 10423 case MLX5_INDIRECT_ACTION_TYPE_CT: 10424 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 10425 ret = 0; 10426 break; 10427 default: 10428 ret = flow_drv_action_validate(dev, NULL, 10429 (const struct rte_flow_action *)update, 10430 fops, error); 10431 } 10432 if (ret) 10433 return ret; 10434 return flow_drv_action_update(dev, handle, update, fops, 10435 error); 10436 } 10437 10438 /** 10439 * Query the indirect action by handle. 10440 * 10441 * This function allows retrieving action-specific data such as counters. 10442 * Data is gathered by special action which may be present/referenced in 10443 * more than one flow rule definition. 10444 * 10445 * see @RTE_FLOW_ACTION_TYPE_COUNT 10446 * 10447 * @param dev 10448 * Pointer to Ethernet device structure. 10449 * @param[in] handle 10450 * Handle for the indirect action to query. 10451 * @param[in, out] data 10452 * Pointer to storage for the associated query data type. 10453 * @param[out] error 10454 * Perform verbose error reporting if not NULL. PMDs initialize this 10455 * structure in case of error only. 10456 * 10457 * @return 10458 * 0 on success, a negative errno value otherwise and rte_errno is set. 10459 */ 10460 static int 10461 mlx5_action_handle_query(struct rte_eth_dev *dev, 10462 const struct rte_flow_action_handle *handle, 10463 void *data, 10464 struct rte_flow_error *error) 10465 { 10466 struct rte_flow_attr attr = { .transfer = 0 }; 10467 const struct mlx5_flow_driver_ops *fops = 10468 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10469 10470 return flow_drv_action_query(dev, handle, data, fops, error); 10471 } 10472 10473 static int 10474 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10475 struct rte_flow_action_handle *handle, 10476 const void *update, void *query, 10477 enum rte_flow_query_update_mode qu_mode, 10478 struct rte_flow_error *error) 10479 { 10480 struct rte_flow_attr attr = { .transfer = 0 }; 10481 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10482 const struct mlx5_flow_driver_ops *fops; 10483 10484 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10485 return rte_flow_error_set(error, ENOTSUP, 10486 RTE_FLOW_ERROR_TYPE_ACTION, 10487 NULL, "invalid driver type"); 10488 fops = flow_get_drv_ops(drv_type); 10489 if (!fops || !fops->action_query_update) 10490 return rte_flow_error_set(error, ENOTSUP, 10491 RTE_FLOW_ERROR_TYPE_ACTION, 10492 NULL, "no query_update handler"); 10493 return fops->action_query_update(dev, handle, update, 10494 query, qu_mode, error); 10495 } 10496 10497 10498 #define MLX5_DRV_FOPS_OR_ERR(dev, fops, drv_cb, ret) \ 10499 { \ 10500 struct rte_flow_attr attr = { .transfer = 0 }; \ 10501 enum mlx5_flow_drv_type drv_type = flow_get_drv_type((dev), &attr); \ 10502 if (drv_type == MLX5_FLOW_TYPE_MIN || \ 10503 drv_type == MLX5_FLOW_TYPE_MAX) { \ 10504 rte_flow_error_set(error, ENOTSUP, \ 10505 RTE_FLOW_ERROR_TYPE_ACTION, \ 10506 NULL, "invalid driver type"); \ 10507 return ret; \ 10508 } \ 10509 (fops) = flow_get_drv_ops(drv_type); \ 10510 if (!(fops) || !(fops)->drv_cb) { \ 10511 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, \ 10512 NULL, "no action_list handler"); \ 10513 return ret; \ 10514 } \ 10515 } 10516 10517 static struct rte_flow_action_list_handle * 10518 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 10519 const struct rte_flow_indir_action_conf *conf, 10520 const struct rte_flow_action *actions, 10521 struct rte_flow_error *error) 10522 { 10523 const struct mlx5_flow_driver_ops *fops; 10524 10525 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_create, NULL); 10526 return fops->action_list_handle_create(dev, conf, actions, error); 10527 } 10528 10529 static int 10530 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 10531 struct rte_flow_action_list_handle *handle, 10532 struct rte_flow_error *error) 10533 { 10534 const struct mlx5_flow_driver_ops *fops; 10535 10536 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_destroy, ENOTSUP); 10537 return fops->action_list_handle_destroy(dev, handle, error); 10538 } 10539 10540 static int 10541 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 10542 const 10543 struct rte_flow_action_list_handle *handle, 10544 const void **update, void **query, 10545 enum rte_flow_query_update_mode mode, 10546 struct rte_flow_error *error) 10547 { 10548 const struct mlx5_flow_driver_ops *fops; 10549 10550 MLX5_DRV_FOPS_OR_ERR(dev, fops, 10551 action_list_handle_query_update, ENOTSUP); 10552 return fops->action_list_handle_query_update(dev, handle, update, query, 10553 mode, error); 10554 } 10555 static int 10556 mlx5_flow_calc_table_hash(struct rte_eth_dev *dev, 10557 const struct rte_flow_template_table *table, 10558 const struct rte_flow_item pattern[], 10559 uint8_t pattern_template_index, 10560 uint32_t *hash, struct rte_flow_error *error) 10561 { 10562 struct rte_flow_attr attr = { .transfer = 0 }; 10563 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10564 const struct mlx5_flow_driver_ops *fops; 10565 10566 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10567 return rte_flow_error_set(error, ENOTSUP, 10568 RTE_FLOW_ERROR_TYPE_ACTION, 10569 NULL, "invalid driver type"); 10570 fops = flow_get_drv_ops(drv_type); 10571 if (!fops || !fops->action_query_update) 10572 return rte_flow_error_set(error, ENOTSUP, 10573 RTE_FLOW_ERROR_TYPE_ACTION, 10574 NULL, "no query_update handler"); 10575 return fops->flow_calc_table_hash(dev, table, pattern, pattern_template_index, 10576 hash, error); 10577 } 10578 10579 static int 10580 mlx5_flow_calc_encap_hash(struct rte_eth_dev *dev, 10581 const struct rte_flow_item pattern[], 10582 enum rte_flow_encap_hash_field dest_field, 10583 uint8_t *hash, 10584 struct rte_flow_error *error) 10585 { 10586 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, NULL); 10587 const struct mlx5_flow_driver_ops *fops; 10588 10589 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10590 return rte_flow_error_set(error, ENOTSUP, 10591 RTE_FLOW_ERROR_TYPE_ACTION, 10592 NULL, "invalid driver type"); 10593 fops = flow_get_drv_ops(drv_type); 10594 if (!fops || !fops->flow_calc_encap_hash) 10595 return rte_flow_error_set(error, ENOTSUP, 10596 RTE_FLOW_ERROR_TYPE_ACTION, 10597 NULL, "no calc encap hash handler"); 10598 return fops->flow_calc_encap_hash(dev, pattern, dest_field, hash, error); 10599 } 10600 10601 static int 10602 mlx5_template_table_resize(struct rte_eth_dev *dev, 10603 struct rte_flow_template_table *table, 10604 uint32_t nb_rules, struct rte_flow_error *error) 10605 { 10606 const struct mlx5_flow_driver_ops *fops; 10607 10608 MLX5_DRV_FOPS_OR_ERR(dev, fops, table_resize, ENOTSUP); 10609 return fops->table_resize(dev, table, nb_rules, error); 10610 } 10611 10612 static int 10613 mlx5_table_resize_complete(struct rte_eth_dev *dev, 10614 struct rte_flow_template_table *table, 10615 struct rte_flow_error *error) 10616 { 10617 const struct mlx5_flow_driver_ops *fops; 10618 10619 MLX5_DRV_FOPS_OR_ERR(dev, fops, table_resize_complete, ENOTSUP); 10620 return fops->table_resize_complete(dev, table, error); 10621 } 10622 10623 static int 10624 mlx5_flow_async_update_resized(struct rte_eth_dev *dev, uint32_t queue, 10625 const struct rte_flow_op_attr *op_attr, 10626 struct rte_flow *rule, void *user_data, 10627 struct rte_flow_error *error) 10628 { 10629 const struct mlx5_flow_driver_ops *fops; 10630 10631 MLX5_DRV_FOPS_OR_ERR(dev, fops, flow_update_resized, ENOTSUP); 10632 return fops->flow_update_resized(dev, queue, op_attr, rule, user_data, error); 10633 } 10634 10635 /** 10636 * Destroy all indirect actions (shared RSS). 10637 * 10638 * @param dev 10639 * Pointer to Ethernet device. 10640 * 10641 * @return 10642 * 0 on success, a negative errno value otherwise and rte_errno is set. 10643 */ 10644 int 10645 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10646 { 10647 struct rte_flow_error error; 10648 struct mlx5_priv *priv = dev->data->dev_private; 10649 struct mlx5_shared_action_rss *shared_rss; 10650 int ret = 0; 10651 uint32_t idx; 10652 10653 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10654 priv->rss_shared_actions, idx, shared_rss, next) { 10655 ret |= mlx5_action_handle_destroy(dev, 10656 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10657 } 10658 return ret; 10659 } 10660 10661 /** 10662 * Validate existing indirect actions against current device configuration 10663 * and attach them to device resources. 10664 * 10665 * @param dev 10666 * Pointer to Ethernet device. 10667 * 10668 * @return 10669 * 0 on success, a negative errno value otherwise and rte_errno is set. 10670 */ 10671 int 10672 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10673 { 10674 struct mlx5_priv *priv = dev->data->dev_private; 10675 int ret = 0; 10676 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10677 10678 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10679 const char *message; 10680 uint32_t queue_idx; 10681 10682 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10683 ind_tbl->queues_n, 10684 &message, &queue_idx); 10685 if (ret != 0) { 10686 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10687 dev->data->port_id, ind_tbl->queues[queue_idx], 10688 message); 10689 break; 10690 } 10691 } 10692 if (ret != 0) 10693 return ret; 10694 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10695 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10696 if (ret != 0) { 10697 DRV_LOG(ERR, "Port %u could not attach " 10698 "indirection table obj %p", 10699 dev->data->port_id, (void *)ind_tbl); 10700 goto error; 10701 } 10702 } 10703 10704 return 0; 10705 error: 10706 ind_tbl_last = ind_tbl; 10707 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10708 if (ind_tbl == ind_tbl_last) 10709 break; 10710 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10711 DRV_LOG(CRIT, "Port %u could not detach " 10712 "indirection table obj %p on rollback", 10713 dev->data->port_id, (void *)ind_tbl); 10714 } 10715 return ret; 10716 } 10717 10718 /** 10719 * Detach indirect actions of the device from its resources. 10720 * 10721 * @param dev 10722 * Pointer to Ethernet device. 10723 * 10724 * @return 10725 * 0 on success, a negative errno value otherwise and rte_errno is set. 10726 */ 10727 int 10728 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10729 { 10730 struct mlx5_priv *priv = dev->data->dev_private; 10731 int ret = 0; 10732 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10733 10734 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10735 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10736 if (ret != 0) { 10737 DRV_LOG(ERR, "Port %u could not detach " 10738 "indirection table obj %p", 10739 dev->data->port_id, (void *)ind_tbl); 10740 goto error; 10741 } 10742 } 10743 return 0; 10744 error: 10745 ind_tbl_last = ind_tbl; 10746 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10747 if (ind_tbl == ind_tbl_last) 10748 break; 10749 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10750 DRV_LOG(CRIT, "Port %u could not attach " 10751 "indirection table obj %p on rollback", 10752 dev->data->port_id, (void *)ind_tbl); 10753 } 10754 return ret; 10755 } 10756 10757 #ifndef HAVE_MLX5DV_DR 10758 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10759 #else 10760 #define MLX5_DOMAIN_SYNC_FLOW \ 10761 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10762 #endif 10763 10764 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10765 { 10766 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10767 const struct mlx5_flow_driver_ops *fops; 10768 int ret; 10769 struct rte_flow_attr attr = { .transfer = 0 }; 10770 10771 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10772 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10773 if (ret > 0) 10774 ret = -ret; 10775 return ret; 10776 } 10777 10778 const struct mlx5_flow_tunnel * 10779 mlx5_get_tof(const struct rte_flow_item *item, 10780 const struct rte_flow_action *action, 10781 enum mlx5_tof_rule_type *rule_type) 10782 { 10783 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10784 if (item->type == (typeof(item->type)) 10785 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10786 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10787 return flow_items_to_tunnel(item); 10788 } 10789 } 10790 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10791 if (action->type == (typeof(action->type)) 10792 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10793 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10794 return flow_actions_to_tunnel(action); 10795 } 10796 } 10797 return NULL; 10798 } 10799 10800 /** 10801 * tunnel offload functionality is defined for DV environment only 10802 */ 10803 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10804 __extension__ 10805 union tunnel_offload_mark { 10806 uint32_t val; 10807 struct { 10808 uint32_t app_reserve:8; 10809 uint32_t table_id:15; 10810 uint32_t transfer:1; 10811 uint32_t _unused_:8; 10812 }; 10813 }; 10814 10815 static bool 10816 mlx5_access_tunnel_offload_db 10817 (struct rte_eth_dev *dev, 10818 bool (*match)(struct rte_eth_dev *, 10819 struct mlx5_flow_tunnel *, const void *), 10820 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10821 void (*miss)(struct rte_eth_dev *, void *), 10822 void *ctx, bool lock_op); 10823 10824 static int 10825 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10826 struct rte_flow *flow, 10827 const struct rte_flow_attr *attr, 10828 const struct rte_flow_action *app_actions, 10829 uint32_t flow_idx, 10830 const struct mlx5_flow_tunnel *tunnel, 10831 struct tunnel_default_miss_ctx *ctx, 10832 struct rte_flow_error *error) 10833 { 10834 struct mlx5_priv *priv = dev->data->dev_private; 10835 struct mlx5_flow *dev_flow; 10836 struct rte_flow_attr miss_attr = *attr; 10837 const struct rte_flow_item miss_items[2] = { 10838 { 10839 .type = RTE_FLOW_ITEM_TYPE_ETH, 10840 .spec = NULL, 10841 .last = NULL, 10842 .mask = NULL 10843 }, 10844 { 10845 .type = RTE_FLOW_ITEM_TYPE_END, 10846 .spec = NULL, 10847 .last = NULL, 10848 .mask = NULL 10849 } 10850 }; 10851 union tunnel_offload_mark mark_id; 10852 struct rte_flow_action_mark miss_mark; 10853 struct rte_flow_action miss_actions[3] = { 10854 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10855 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10856 }; 10857 const struct rte_flow_action_jump *jump_data; 10858 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10859 struct flow_grp_info grp_info = { 10860 .external = 1, 10861 .transfer = attr->transfer, 10862 .fdb_def_rule = !!priv->fdb_def_rule, 10863 .std_tbl_fix = 0, 10864 }; 10865 int ret; 10866 10867 if (!attr->transfer) { 10868 uint32_t q_size; 10869 10870 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10871 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10872 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10873 0, SOCKET_ID_ANY); 10874 if (!ctx->queue) 10875 return rte_flow_error_set 10876 (error, ENOMEM, 10877 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10878 NULL, "invalid default miss RSS"); 10879 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10880 ctx->action_rss.level = 0, 10881 ctx->action_rss.types = priv->rss_conf.rss_hf, 10882 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10883 ctx->action_rss.queue_num = priv->reta_idx_n, 10884 ctx->action_rss.key = priv->rss_conf.rss_key, 10885 ctx->action_rss.queue = ctx->queue; 10886 if (!priv->reta_idx_n || !priv->rxqs_n) 10887 return rte_flow_error_set 10888 (error, EINVAL, 10889 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10890 NULL, "invalid port configuration"); 10891 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10892 ctx->action_rss.types = 0; 10893 for (i = 0; i != priv->reta_idx_n; ++i) 10894 ctx->queue[i] = (*priv->reta_idx)[i]; 10895 } else { 10896 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10897 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10898 } 10899 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10900 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10901 jump_data = app_actions->conf; 10902 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10903 miss_attr.group = jump_data->group; 10904 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10905 &flow_table, &grp_info, error); 10906 if (ret) 10907 return rte_flow_error_set(error, EINVAL, 10908 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10909 NULL, "invalid tunnel id"); 10910 mark_id.app_reserve = 0; 10911 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10912 mark_id.transfer = !!attr->transfer; 10913 mark_id._unused_ = 0; 10914 miss_mark.id = mark_id.val; 10915 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10916 miss_items, miss_actions, flow_idx, error); 10917 if (!dev_flow) 10918 return -rte_errno; 10919 dev_flow->flow = flow; 10920 dev_flow->external = true; 10921 dev_flow->tunnel = tunnel; 10922 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10923 /* Subflow object was created, we must include one in the list. */ 10924 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10925 dev_flow->handle, next); 10926 DRV_LOG(DEBUG, 10927 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10928 dev->data->port_id, tunnel->app_tunnel.type, 10929 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10930 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10931 miss_actions, error); 10932 if (!ret) 10933 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10934 error); 10935 10936 return ret; 10937 } 10938 10939 static const struct mlx5_flow_tbl_data_entry * 10940 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10941 { 10942 struct mlx5_priv *priv = dev->data->dev_private; 10943 struct mlx5_dev_ctx_shared *sh = priv->sh; 10944 struct mlx5_list_entry *he; 10945 union tunnel_offload_mark mbits = { .val = mark }; 10946 union mlx5_flow_tbl_key table_key = { 10947 { 10948 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10949 .id = 0, 10950 .reserved = 0, 10951 .dummy = 0, 10952 .is_fdb = !!mbits.transfer, 10953 .is_egress = 0, 10954 } 10955 }; 10956 struct mlx5_flow_cb_ctx ctx = { 10957 .data = &table_key.v64, 10958 }; 10959 10960 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10961 return he ? 10962 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10963 } 10964 10965 static void 10966 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10967 struct mlx5_list_entry *entry) 10968 { 10969 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10970 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10971 10972 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10973 tunnel_flow_tbl_to_id(tte->flow_table)); 10974 mlx5_free(tte); 10975 } 10976 10977 static int 10978 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10979 struct mlx5_list_entry *entry, void *cb_ctx) 10980 { 10981 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10982 union tunnel_tbl_key tbl = { 10983 .val = *(uint64_t *)(ctx->data), 10984 }; 10985 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10986 10987 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10988 } 10989 10990 static struct mlx5_list_entry * 10991 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10992 { 10993 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10994 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10995 struct tunnel_tbl_entry *tte; 10996 union tunnel_tbl_key tbl = { 10997 .val = *(uint64_t *)(ctx->data), 10998 }; 10999 11000 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 11001 sizeof(*tte), 0, 11002 SOCKET_ID_ANY); 11003 if (!tte) 11004 goto err; 11005 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11006 &tte->flow_table); 11007 if (tte->flow_table >= MLX5_MAX_TABLES) { 11008 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 11009 tte->flow_table); 11010 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11011 tte->flow_table); 11012 goto err; 11013 } else if (!tte->flow_table) { 11014 goto err; 11015 } 11016 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 11017 tte->tunnel_id = tbl.tunnel_id; 11018 tte->group = tbl.group; 11019 return &tte->hash; 11020 err: 11021 if (tte) 11022 mlx5_free(tte); 11023 return NULL; 11024 } 11025 11026 static struct mlx5_list_entry * 11027 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 11028 struct mlx5_list_entry *oentry, 11029 void *cb_ctx __rte_unused) 11030 { 11031 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 11032 0, SOCKET_ID_ANY); 11033 11034 if (!tte) 11035 return NULL; 11036 memcpy(tte, oentry, sizeof(*tte)); 11037 return &tte->hash; 11038 } 11039 11040 static void 11041 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 11042 struct mlx5_list_entry *entry) 11043 { 11044 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11045 11046 mlx5_free(tte); 11047 } 11048 11049 static uint32_t 11050 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 11051 const struct mlx5_flow_tunnel *tunnel, 11052 uint32_t group, uint32_t *table, 11053 struct rte_flow_error *error) 11054 { 11055 struct mlx5_list_entry *he; 11056 struct tunnel_tbl_entry *tte; 11057 union tunnel_tbl_key key = { 11058 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 11059 .group = group 11060 }; 11061 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11062 struct mlx5_hlist *group_hash; 11063 struct mlx5_flow_cb_ctx ctx = { 11064 .data = &key.val, 11065 }; 11066 11067 group_hash = tunnel ? tunnel->groups : thub->groups; 11068 he = mlx5_hlist_register(group_hash, key.val, &ctx); 11069 if (!he) 11070 return rte_flow_error_set(error, EINVAL, 11071 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 11072 NULL, 11073 "tunnel group index not supported"); 11074 tte = container_of(he, typeof(*tte), hash); 11075 *table = tte->flow_table; 11076 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 11077 dev->data->port_id, key.tunnel_id, group, *table); 11078 return 0; 11079 } 11080 11081 static void 11082 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 11083 struct mlx5_flow_tunnel *tunnel) 11084 { 11085 struct mlx5_priv *priv = dev->data->dev_private; 11086 struct mlx5_indexed_pool *ipool; 11087 11088 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11089 dev->data->port_id, tunnel->tunnel_id); 11090 LIST_REMOVE(tunnel, chain); 11091 mlx5_hlist_destroy(tunnel->groups); 11092 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11093 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11094 } 11095 11096 static bool 11097 mlx5_access_tunnel_offload_db 11098 (struct rte_eth_dev *dev, 11099 bool (*match)(struct rte_eth_dev *, 11100 struct mlx5_flow_tunnel *, const void *), 11101 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11102 void (*miss)(struct rte_eth_dev *, void *), 11103 void *ctx, bool lock_op) 11104 { 11105 bool verdict = false; 11106 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11107 struct mlx5_flow_tunnel *tunnel; 11108 11109 rte_spinlock_lock(&thub->sl); 11110 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11111 verdict = match(dev, tunnel, (const void *)ctx); 11112 if (verdict) 11113 break; 11114 } 11115 if (!lock_op) 11116 rte_spinlock_unlock(&thub->sl); 11117 if (verdict && hit) 11118 hit(dev, tunnel, ctx); 11119 if (!verdict && miss) 11120 miss(dev, ctx); 11121 if (lock_op) 11122 rte_spinlock_unlock(&thub->sl); 11123 11124 return verdict; 11125 } 11126 11127 struct tunnel_db_find_tunnel_id_ctx { 11128 uint32_t tunnel_id; 11129 struct mlx5_flow_tunnel *tunnel; 11130 }; 11131 11132 static bool 11133 find_tunnel_id_match(struct rte_eth_dev *dev, 11134 struct mlx5_flow_tunnel *tunnel, const void *x) 11135 { 11136 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11137 11138 RTE_SET_USED(dev); 11139 return tunnel->tunnel_id == ctx->tunnel_id; 11140 } 11141 11142 static void 11143 find_tunnel_id_hit(struct rte_eth_dev *dev, 11144 struct mlx5_flow_tunnel *tunnel, void *x) 11145 { 11146 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11147 RTE_SET_USED(dev); 11148 ctx->tunnel = tunnel; 11149 } 11150 11151 static struct mlx5_flow_tunnel * 11152 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11153 { 11154 struct tunnel_db_find_tunnel_id_ctx ctx = { 11155 .tunnel_id = id, 11156 }; 11157 11158 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11159 find_tunnel_id_hit, NULL, &ctx, true); 11160 11161 return ctx.tunnel; 11162 } 11163 11164 static struct mlx5_flow_tunnel * 11165 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11166 const struct rte_flow_tunnel *app_tunnel) 11167 { 11168 struct mlx5_priv *priv = dev->data->dev_private; 11169 struct mlx5_indexed_pool *ipool; 11170 struct mlx5_flow_tunnel *tunnel; 11171 uint32_t id; 11172 11173 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11174 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11175 if (!tunnel) 11176 return NULL; 11177 if (id >= MLX5_MAX_TUNNELS) { 11178 mlx5_ipool_free(ipool, id); 11179 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11180 return NULL; 11181 } 11182 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11183 priv->sh, 11184 mlx5_flow_tunnel_grp2tbl_create_cb, 11185 mlx5_flow_tunnel_grp2tbl_match_cb, 11186 mlx5_flow_tunnel_grp2tbl_remove_cb, 11187 mlx5_flow_tunnel_grp2tbl_clone_cb, 11188 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11189 if (!tunnel->groups) { 11190 mlx5_ipool_free(ipool, id); 11191 return NULL; 11192 } 11193 /* initiate new PMD tunnel */ 11194 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11195 tunnel->tunnel_id = id; 11196 tunnel->action.type = (typeof(tunnel->action.type)) 11197 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11198 tunnel->action.conf = tunnel; 11199 tunnel->item.type = (typeof(tunnel->item.type)) 11200 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11201 tunnel->item.spec = tunnel; 11202 tunnel->item.last = NULL; 11203 tunnel->item.mask = NULL; 11204 11205 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11206 dev->data->port_id, tunnel->tunnel_id); 11207 11208 return tunnel; 11209 } 11210 11211 struct tunnel_db_get_tunnel_ctx { 11212 const struct rte_flow_tunnel *app_tunnel; 11213 struct mlx5_flow_tunnel *tunnel; 11214 }; 11215 11216 static bool get_tunnel_match(struct rte_eth_dev *dev, 11217 struct mlx5_flow_tunnel *tunnel, const void *x) 11218 { 11219 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11220 11221 RTE_SET_USED(dev); 11222 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11223 sizeof(*ctx->app_tunnel)); 11224 } 11225 11226 static void get_tunnel_hit(struct rte_eth_dev *dev, 11227 struct mlx5_flow_tunnel *tunnel, void *x) 11228 { 11229 /* called under tunnel spinlock protection */ 11230 struct tunnel_db_get_tunnel_ctx *ctx = x; 11231 11232 RTE_SET_USED(dev); 11233 tunnel->refctn++; 11234 ctx->tunnel = tunnel; 11235 } 11236 11237 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11238 { 11239 /* called under tunnel spinlock protection */ 11240 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11241 struct tunnel_db_get_tunnel_ctx *ctx = x; 11242 11243 rte_spinlock_unlock(&thub->sl); 11244 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11245 rte_spinlock_lock(&thub->sl); 11246 if (ctx->tunnel) { 11247 ctx->tunnel->refctn = 1; 11248 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11249 } 11250 } 11251 11252 11253 static int 11254 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11255 const struct rte_flow_tunnel *app_tunnel, 11256 struct mlx5_flow_tunnel **tunnel) 11257 { 11258 struct tunnel_db_get_tunnel_ctx ctx = { 11259 .app_tunnel = app_tunnel, 11260 }; 11261 11262 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11263 get_tunnel_miss, &ctx, true); 11264 *tunnel = ctx.tunnel; 11265 return ctx.tunnel ? 0 : -ENOMEM; 11266 } 11267 11268 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11269 { 11270 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11271 11272 if (!thub) 11273 return; 11274 if (!LIST_EMPTY(&thub->tunnels)) 11275 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11276 mlx5_hlist_destroy(thub->groups); 11277 mlx5_free(thub); 11278 } 11279 11280 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11281 { 11282 int err; 11283 struct mlx5_flow_tunnel_hub *thub; 11284 11285 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11286 0, SOCKET_ID_ANY); 11287 if (!thub) 11288 return -ENOMEM; 11289 LIST_INIT(&thub->tunnels); 11290 rte_spinlock_init(&thub->sl); 11291 thub->groups = mlx5_hlist_create("flow groups", 64, 11292 false, true, sh, 11293 mlx5_flow_tunnel_grp2tbl_create_cb, 11294 mlx5_flow_tunnel_grp2tbl_match_cb, 11295 mlx5_flow_tunnel_grp2tbl_remove_cb, 11296 mlx5_flow_tunnel_grp2tbl_clone_cb, 11297 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11298 if (!thub->groups) { 11299 err = -rte_errno; 11300 goto err; 11301 } 11302 sh->tunnel_hub = thub; 11303 11304 return 0; 11305 11306 err: 11307 if (thub->groups) 11308 mlx5_hlist_destroy(thub->groups); 11309 if (thub) 11310 mlx5_free(thub); 11311 return err; 11312 } 11313 11314 static inline int 11315 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11316 struct rte_flow_tunnel *tunnel, 11317 struct rte_flow_error *error) 11318 { 11319 struct mlx5_priv *priv = dev->data->dev_private; 11320 11321 if (!priv->sh->config.dv_flow_en) 11322 return rte_flow_error_set(error, ENOTSUP, 11323 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11324 "flow DV interface is off"); 11325 if (!is_tunnel_offload_active(dev)) 11326 return rte_flow_error_set(error, ENOTSUP, 11327 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11328 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11329 if (!tunnel) 11330 return rte_flow_error_set(error, EINVAL, 11331 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11332 "no application tunnel"); 11333 switch (tunnel->type) { 11334 default: 11335 return rte_flow_error_set(error, EINVAL, 11336 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11337 "unsupported tunnel type"); 11338 case RTE_FLOW_ITEM_TYPE_VXLAN: 11339 case RTE_FLOW_ITEM_TYPE_GRE: 11340 case RTE_FLOW_ITEM_TYPE_NVGRE: 11341 case RTE_FLOW_ITEM_TYPE_GENEVE: 11342 break; 11343 } 11344 return 0; 11345 } 11346 11347 static int 11348 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11349 struct rte_flow_tunnel *app_tunnel, 11350 struct rte_flow_action **actions, 11351 uint32_t *num_of_actions, 11352 struct rte_flow_error *error) 11353 { 11354 struct mlx5_flow_tunnel *tunnel; 11355 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11356 11357 if (ret) 11358 return ret; 11359 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11360 if (ret < 0) { 11361 return rte_flow_error_set(error, ret, 11362 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11363 "failed to initialize pmd tunnel"); 11364 } 11365 *actions = &tunnel->action; 11366 *num_of_actions = 1; 11367 return 0; 11368 } 11369 11370 static int 11371 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11372 struct rte_flow_tunnel *app_tunnel, 11373 struct rte_flow_item **items, 11374 uint32_t *num_of_items, 11375 struct rte_flow_error *error) 11376 { 11377 struct mlx5_flow_tunnel *tunnel; 11378 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11379 11380 if (ret) 11381 return ret; 11382 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11383 if (ret < 0) { 11384 return rte_flow_error_set(error, ret, 11385 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11386 "failed to initialize pmd tunnel"); 11387 } 11388 *items = &tunnel->item; 11389 *num_of_items = 1; 11390 return 0; 11391 } 11392 11393 struct tunnel_db_element_release_ctx { 11394 struct rte_flow_item *items; 11395 struct rte_flow_action *actions; 11396 uint32_t num_elements; 11397 struct rte_flow_error *error; 11398 int ret; 11399 }; 11400 11401 static bool 11402 tunnel_element_release_match(struct rte_eth_dev *dev, 11403 struct mlx5_flow_tunnel *tunnel, const void *x) 11404 { 11405 const struct tunnel_db_element_release_ctx *ctx = x; 11406 11407 RTE_SET_USED(dev); 11408 if (ctx->num_elements != 1) 11409 return false; 11410 else if (ctx->items) 11411 return ctx->items == &tunnel->item; 11412 else if (ctx->actions) 11413 return ctx->actions == &tunnel->action; 11414 11415 return false; 11416 } 11417 11418 static void 11419 tunnel_element_release_hit(struct rte_eth_dev *dev, 11420 struct mlx5_flow_tunnel *tunnel, void *x) 11421 { 11422 struct tunnel_db_element_release_ctx *ctx = x; 11423 ctx->ret = 0; 11424 if (!(rte_atomic_fetch_sub_explicit(&tunnel->refctn, 1, rte_memory_order_relaxed) - 1)) 11425 mlx5_flow_tunnel_free(dev, tunnel); 11426 } 11427 11428 static void 11429 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11430 { 11431 struct tunnel_db_element_release_ctx *ctx = x; 11432 RTE_SET_USED(dev); 11433 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11434 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11435 "invalid argument"); 11436 } 11437 11438 static int 11439 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11440 struct rte_flow_item *pmd_items, 11441 uint32_t num_items, struct rte_flow_error *err) 11442 { 11443 struct tunnel_db_element_release_ctx ctx = { 11444 .items = pmd_items, 11445 .actions = NULL, 11446 .num_elements = num_items, 11447 .error = err, 11448 }; 11449 11450 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11451 tunnel_element_release_hit, 11452 tunnel_element_release_miss, &ctx, false); 11453 11454 return ctx.ret; 11455 } 11456 11457 static int 11458 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11459 struct rte_flow_action *pmd_actions, 11460 uint32_t num_actions, struct rte_flow_error *err) 11461 { 11462 struct tunnel_db_element_release_ctx ctx = { 11463 .items = NULL, 11464 .actions = pmd_actions, 11465 .num_elements = num_actions, 11466 .error = err, 11467 }; 11468 11469 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11470 tunnel_element_release_hit, 11471 tunnel_element_release_miss, &ctx, false); 11472 11473 return ctx.ret; 11474 } 11475 11476 static int 11477 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11478 struct rte_mbuf *m, 11479 struct rte_flow_restore_info *info, 11480 struct rte_flow_error *err) 11481 { 11482 uint64_t ol_flags = m->ol_flags; 11483 const struct mlx5_flow_tbl_data_entry *tble; 11484 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11485 struct mlx5_priv *priv = dev->data->dev_private; 11486 11487 if (priv->tunnel_enabled == 0) 11488 goto err; 11489 if ((ol_flags & mask) != mask) 11490 goto err; 11491 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11492 if (!tble) { 11493 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11494 dev->data->port_id, m->hash.fdir.hi); 11495 goto err; 11496 } 11497 MLX5_ASSERT(tble->tunnel); 11498 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11499 info->group_id = tble->group_id; 11500 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11501 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11502 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11503 11504 return 0; 11505 11506 err: 11507 return rte_flow_error_set(err, EINVAL, 11508 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11509 "failed to get restore info"); 11510 } 11511 11512 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11513 static int 11514 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11515 __rte_unused struct rte_flow_tunnel *app_tunnel, 11516 __rte_unused struct rte_flow_action **actions, 11517 __rte_unused uint32_t *num_of_actions, 11518 __rte_unused struct rte_flow_error *error) 11519 { 11520 return -ENOTSUP; 11521 } 11522 11523 static int 11524 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11525 __rte_unused struct rte_flow_tunnel *app_tunnel, 11526 __rte_unused struct rte_flow_item **items, 11527 __rte_unused uint32_t *num_of_items, 11528 __rte_unused struct rte_flow_error *error) 11529 { 11530 return -ENOTSUP; 11531 } 11532 11533 static int 11534 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11535 __rte_unused struct rte_flow_item *pmd_items, 11536 __rte_unused uint32_t num_items, 11537 __rte_unused struct rte_flow_error *err) 11538 { 11539 return -ENOTSUP; 11540 } 11541 11542 static int 11543 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11544 __rte_unused struct rte_flow_action *pmd_action, 11545 __rte_unused uint32_t num_actions, 11546 __rte_unused struct rte_flow_error *err) 11547 { 11548 return -ENOTSUP; 11549 } 11550 11551 static int 11552 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11553 __rte_unused struct rte_mbuf *m, 11554 __rte_unused struct rte_flow_restore_info *i, 11555 __rte_unused struct rte_flow_error *err) 11556 { 11557 return -ENOTSUP; 11558 } 11559 11560 static int 11561 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11562 __rte_unused struct rte_flow *flow, 11563 __rte_unused const struct rte_flow_attr *attr, 11564 __rte_unused const struct rte_flow_action *actions, 11565 __rte_unused uint32_t flow_idx, 11566 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11567 __rte_unused struct tunnel_default_miss_ctx *ctx, 11568 __rte_unused struct rte_flow_error *error) 11569 { 11570 return -ENOTSUP; 11571 } 11572 11573 static struct mlx5_flow_tunnel * 11574 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11575 __rte_unused uint32_t id) 11576 { 11577 return NULL; 11578 } 11579 11580 static void 11581 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11582 __rte_unused struct mlx5_flow_tunnel *tunnel) 11583 { 11584 } 11585 11586 static uint32_t 11587 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11588 __rte_unused const struct mlx5_flow_tunnel *t, 11589 __rte_unused uint32_t group, 11590 __rte_unused uint32_t *table, 11591 struct rte_flow_error *error) 11592 { 11593 return rte_flow_error_set(error, ENOTSUP, 11594 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11595 "tunnel offload requires DV support"); 11596 } 11597 11598 void 11599 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11600 __rte_unused uint16_t port_id) 11601 { 11602 } 11603 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11604 11605 /* Flex flow item API */ 11606 static struct rte_flow_item_flex_handle * 11607 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11608 const struct rte_flow_item_flex_conf *conf, 11609 struct rte_flow_error *error) 11610 { 11611 static const char err_msg[] = "flex item creation unsupported"; 11612 struct mlx5_priv *priv = dev->data->dev_private; 11613 struct rte_flow_attr attr = { .transfer = 0 }; 11614 const struct mlx5_flow_driver_ops *fops = 11615 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11616 11617 if (!priv->pci_dev) { 11618 rte_flow_error_set(error, ENOTSUP, 11619 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11620 "create flex item on PF only"); 11621 return NULL; 11622 } 11623 switch (priv->pci_dev->id.device_id) { 11624 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11625 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11626 break; 11627 default: 11628 rte_flow_error_set(error, ENOTSUP, 11629 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11630 "flex item available on BlueField ports only"); 11631 return NULL; 11632 } 11633 if (!fops->item_create) { 11634 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11635 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11636 NULL, err_msg); 11637 return NULL; 11638 } 11639 return fops->item_create(dev, conf, error); 11640 } 11641 11642 static int 11643 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11644 const struct rte_flow_item_flex_handle *handle, 11645 struct rte_flow_error *error) 11646 { 11647 static const char err_msg[] = "flex item release unsupported"; 11648 struct rte_flow_attr attr = { .transfer = 0 }; 11649 const struct mlx5_flow_driver_ops *fops = 11650 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11651 11652 if (!fops->item_release) { 11653 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11654 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11655 NULL, err_msg); 11656 return -rte_errno; 11657 } 11658 return fops->item_release(dev, handle, error); 11659 } 11660 11661 static void 11662 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11663 { 11664 int ret; 11665 struct rte_flow_error error; 11666 11667 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11668 char *item_name; 11669 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11670 sizeof(item_name), 11671 (void *)(uintptr_t)item->type, &error); 11672 if (ret > 0) 11673 printf("%s ", item_name); 11674 else 11675 printf("%d\n", (int)item->type); 11676 } 11677 printf("END\n"); 11678 } 11679 11680 static int 11681 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11682 { 11683 const struct rte_flow_item_udp *spec = udp_item->spec; 11684 const struct rte_flow_item_udp *mask = udp_item->mask; 11685 uint16_t udp_dport = 0; 11686 11687 if (spec != NULL) { 11688 if (!mask) 11689 mask = &rte_flow_item_udp_mask; 11690 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11691 mask->hdr.dst_port); 11692 } 11693 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11694 } 11695 11696 static const struct mlx5_flow_expand_node * 11697 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11698 unsigned int item_idx, 11699 const struct mlx5_flow_expand_node graph[], 11700 const struct mlx5_flow_expand_node *node) 11701 { 11702 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11703 11704 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11705 node != NULL && 11706 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11707 /* 11708 * The expansion node is VXLAN and it is also the last 11709 * expandable item in the pattern, so need to continue 11710 * expansion of the inner tunnel. 11711 */ 11712 MLX5_ASSERT(item_idx > 0); 11713 prev_item = pattern + item_idx - 1; 11714 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11715 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11716 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11717 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11718 } 11719 return node; 11720 } 11721 11722 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11723 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11724 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11725 }; 11726 11727 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11728 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11729 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11730 { 9, 10, 11 }, { 12, 13, 14 }, 11731 }; 11732 11733 /** 11734 * Discover the number of available flow priorities. 11735 * 11736 * @param dev 11737 * Ethernet device. 11738 * 11739 * @return 11740 * On success, number of available flow priorities. 11741 * On failure, a negative errno-style code and rte_errno is set. 11742 */ 11743 int 11744 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11745 { 11746 static const uint16_t vprio[] = {8, 16}; 11747 const struct mlx5_priv *priv = dev->data->dev_private; 11748 const struct mlx5_flow_driver_ops *fops; 11749 enum mlx5_flow_drv_type type; 11750 int ret; 11751 11752 type = mlx5_flow_os_get_type(); 11753 if (type == MLX5_FLOW_TYPE_MAX) { 11754 type = MLX5_FLOW_TYPE_VERBS; 11755 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11756 type = MLX5_FLOW_TYPE_DV; 11757 } 11758 fops = flow_get_drv_ops(type); 11759 if (fops->discover_priorities == NULL) { 11760 DRV_LOG(ERR, "Priority discovery not supported"); 11761 rte_errno = ENOTSUP; 11762 return -rte_errno; 11763 } 11764 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11765 if (ret < 0) 11766 return ret; 11767 switch (ret) { 11768 case 8: 11769 ret = RTE_DIM(priority_map_3); 11770 break; 11771 case 16: 11772 ret = RTE_DIM(priority_map_5); 11773 break; 11774 default: 11775 rte_errno = ENOTSUP; 11776 DRV_LOG(ERR, 11777 "port %u maximum priority: %d expected 8/16", 11778 dev->data->port_id, ret); 11779 return -rte_errno; 11780 } 11781 DRV_LOG(INFO, "port %u supported flow priorities:" 11782 " 0-%d for ingress or egress root table," 11783 " 0-%d for non-root table or transfer root table.", 11784 dev->data->port_id, ret - 2, 11785 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11786 return ret; 11787 } 11788 11789 /** 11790 * Adjust flow priority based on the highest layer and the request priority. 11791 * 11792 * @param[in] dev 11793 * Pointer to the Ethernet device structure. 11794 * @param[in] priority 11795 * The rule base priority. 11796 * @param[in] subpriority 11797 * The priority based on the items. 11798 * 11799 * @return 11800 * The new priority. 11801 */ 11802 uint32_t 11803 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11804 uint32_t subpriority) 11805 { 11806 uint32_t res = 0; 11807 struct mlx5_priv *priv = dev->data->dev_private; 11808 11809 switch (priv->sh->flow_max_priority) { 11810 case RTE_DIM(priority_map_3): 11811 res = priority_map_3[priority][subpriority]; 11812 break; 11813 case RTE_DIM(priority_map_5): 11814 res = priority_map_5[priority][subpriority]; 11815 break; 11816 } 11817 return res; 11818 } 11819 11820 /** 11821 * Get the priority for sending traffic to kernel table. 11822 * 11823 * @param[in] dev 11824 * Pointer to the Ethernet device structure. 11825 * 11826 * @return 11827 * On success: the value of priority for sending traffic to kernel table 11828 * On failure: -1 11829 */ 11830 uint32_t 11831 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11832 { 11833 struct mlx5_priv *priv = dev->data->dev_private; 11834 uint32_t res; 11835 11836 switch (priv->sh->flow_max_priority) { 11837 case RTE_DIM(priority_map_5): 11838 res = 15; 11839 break; 11840 case RTE_DIM(priority_map_3): 11841 res = 7; 11842 break; 11843 default: 11844 DRV_LOG(ERR, 11845 "port %u maximum priority: %d expected 8/16", 11846 dev->data->port_id, priv->sh->flow_max_priority); 11847 res = (uint32_t)-1; 11848 } 11849 return res; 11850 } 11851 11852 /** 11853 * Get the E-Switch Manager vport id. 11854 * 11855 * @param[in] dev 11856 * Pointer to the Ethernet device structure. 11857 * 11858 * @return 11859 * The vport id. 11860 */ 11861 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11862 { 11863 struct mlx5_priv *priv = dev->data->dev_private; 11864 struct mlx5_common_device *cdev = priv->sh->cdev; 11865 11866 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11867 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11868 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11869 11870 if (priv->pci_dev == NULL) 11871 return 0; 11872 switch (priv->pci_dev->id.device_id) { 11873 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 11874 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11875 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11876 /* 11877 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11878 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11879 * ECPF vport ID is selected and not the host port (0) in any BF case. 11880 */ 11881 return (int16_t)MLX5_ECPF_VPORT_ID; 11882 default: 11883 return MLX5_PF_VPORT_ID; 11884 } 11885 } 11886 11887 /** 11888 * Parse item to get the vport id. 11889 * 11890 * @param[in] dev 11891 * Pointer to the Ethernet device structure. 11892 * @param[in] item 11893 * The src port id match item. 11894 * @param[out] vport_id 11895 * Pointer to put the vport id. 11896 * @param[out] all_ports 11897 * Indicate if the item matches all ports. 11898 * @param[out] error 11899 * Pointer to error structure. 11900 * 11901 * @return 11902 * 0 on success, a negative errno value otherwise and rte_errno is set. 11903 */ 11904 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11905 const struct rte_flow_item *item, 11906 uint16_t *vport_id, 11907 bool *all_ports, 11908 struct rte_flow_error *error) 11909 { 11910 struct mlx5_priv *port_priv; 11911 const struct rte_flow_item_port_id *pid_v = NULL; 11912 const struct rte_flow_item_ethdev *dev_v = NULL; 11913 uint32_t esw_mgr_port; 11914 uint32_t src_port; 11915 11916 if (all_ports) 11917 *all_ports = false; 11918 switch (item->type) { 11919 case RTE_FLOW_ITEM_TYPE_PORT_ID: 11920 pid_v = item->spec; 11921 if (!pid_v) 11922 return 0; 11923 src_port = pid_v->id; 11924 esw_mgr_port = MLX5_PORT_ESW_MGR; 11925 break; 11926 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 11927 dev_v = item->spec; 11928 if (!dev_v) { 11929 if (all_ports) 11930 *all_ports = true; 11931 return 0; 11932 } 11933 src_port = dev_v->port_id; 11934 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11935 break; 11936 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 11937 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11938 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11939 break; 11940 default: 11941 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11942 NULL, "Incorrect item type."); 11943 } 11944 if (src_port == esw_mgr_port) { 11945 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11946 } else { 11947 port_priv = mlx5_port_to_eswitch_info(src_port, false); 11948 if (!port_priv) 11949 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11950 NULL, "Failed to get port info."); 11951 *vport_id = port_priv->representor_id; 11952 } 11953 11954 return 0; 11955 } 11956 11957 int 11958 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11959 uint16_t *proxy_port_id, 11960 struct rte_flow_error *error) 11961 { 11962 const struct mlx5_priv *priv = dev->data->dev_private; 11963 uint16_t port_id; 11964 11965 if (!priv->sh->config.dv_esw_en) 11966 return rte_flow_error_set(error, EINVAL, 11967 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11968 NULL, 11969 "unable to provide a proxy port" 11970 " without E-Switch configured"); 11971 if (!priv->master && !priv->representor) 11972 return rte_flow_error_set(error, EINVAL, 11973 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11974 NULL, 11975 "unable to provide a proxy port" 11976 " for port which is not a master" 11977 " or a representor port"); 11978 if (priv->master) { 11979 *proxy_port_id = dev->data->port_id; 11980 return 0; 11981 } 11982 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11983 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11984 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11985 11986 if (port_priv->master && 11987 port_priv->domain_id == priv->domain_id) { 11988 *proxy_port_id = port_id; 11989 return 0; 11990 } 11991 } 11992 return rte_flow_error_set(error, ENODEV, 11993 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11994 NULL, "unable to find a proxy port"); 11995 } 11996 11997 /** 11998 * Discover IPv6 traffic class ID support in rdma-core and firmware. 11999 * 12000 * @param dev 12001 * Ethernet device. 12002 * 12003 * @return 12004 * 0, rdma-core is good to work with firmware. 12005 * -EOPNOTSUPP, rdma-core could not work with new IPv6 TC ID. 12006 */ 12007 int 12008 mlx5_flow_discover_ipv6_tc_support(struct rte_eth_dev *dev) 12009 { 12010 struct rte_flow_action_set_dscp set_dscp; 12011 struct rte_flow_attr attr; 12012 struct rte_flow_action actions[2]; 12013 struct rte_flow_item items[3]; 12014 struct rte_flow_error error; 12015 uint32_t flow_idx; 12016 12017 memset(&attr, 0, sizeof(attr)); 12018 memset(actions, 0, sizeof(actions)); 12019 memset(items, 0, sizeof(items)); 12020 attr.group = 1; 12021 attr.egress = 1; 12022 items[0].type = RTE_FLOW_ITEM_TYPE_ETH; 12023 items[1].type = RTE_FLOW_ITEM_TYPE_IPV6; 12024 items[2].type = RTE_FLOW_ITEM_TYPE_END; 12025 /* Random value */ 12026 set_dscp.dscp = 9; 12027 actions[0].type = RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP; 12028 actions[0].conf = &set_dscp; 12029 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 12030 12031 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, items, actions, true, &error); 12032 if (!flow_idx) 12033 return -EOPNOTSUPP; 12034 12035 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 12036 return 0; 12037 } 12038 12039 void * 12040 rte_pmd_mlx5_create_geneve_tlv_parser(uint16_t port_id, 12041 const struct rte_pmd_mlx5_geneve_tlv tlv_list[], 12042 uint8_t nb_options) 12043 { 12044 #ifdef HAVE_MLX5_HWS_SUPPORT 12045 return mlx5_geneve_tlv_parser_create(port_id, tlv_list, nb_options); 12046 #else 12047 (void)port_id; 12048 (void)tlv_list; 12049 (void)nb_options; 12050 DRV_LOG(ERR, "%s is not supported.", __func__); 12051 rte_errno = ENOTSUP; 12052 return NULL; 12053 #endif 12054 } 12055 12056 int 12057 rte_pmd_mlx5_destroy_geneve_tlv_parser(void *handle) 12058 { 12059 #ifdef HAVE_MLX5_HWS_SUPPORT 12060 return mlx5_geneve_tlv_parser_destroy(handle); 12061 #else 12062 (void)handle; 12063 DRV_LOG(ERR, "%s is not supported.", __func__); 12064 rte_errno = ENOTSUP; 12065 return -rte_errno; 12066 #endif 12067 } 12068