1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 struct tunnel_default_miss_ctx { 43 uint16_t *queue; 44 __extension__ 45 union { 46 struct rte_flow_action_rss action_rss; 47 struct rte_flow_action_queue miss_queue; 48 struct rte_flow_action_jump miss_jump; 49 uint8_t raw[0]; 50 }; 51 }; 52 53 void 54 mlx5_indirect_list_handles_release(struct rte_eth_dev *dev) 55 { 56 struct mlx5_priv *priv = dev->data->dev_private; 57 #ifdef HAVE_MLX5_HWS_SUPPORT 58 struct rte_flow_error error; 59 #endif 60 61 while (!LIST_EMPTY(&priv->indirect_list_head)) { 62 struct mlx5_indirect_list *e = 63 LIST_FIRST(&priv->indirect_list_head); 64 65 LIST_REMOVE(e, entry); 66 switch (e->type) { 67 #ifdef HAVE_MLX5_HWS_SUPPORT 68 case MLX5_INDIRECT_ACTION_LIST_TYPE_MIRROR: 69 mlx5_hw_mirror_destroy(dev, (struct mlx5_mirror *)e); 70 break; 71 case MLX5_INDIRECT_ACTION_LIST_TYPE_LEGACY: 72 mlx5_destroy_legacy_indirect(dev, e); 73 break; 74 case MLX5_INDIRECT_ACTION_LIST_TYPE_REFORMAT: 75 mlx5_reformat_action_destroy(dev, 76 (struct rte_flow_action_list_handle *)e, &error); 77 break; 78 #endif 79 default: 80 DRV_LOG(ERR, "invalid indirect list type"); 81 MLX5_ASSERT(false); 82 break; 83 } 84 } 85 } 86 87 static int 88 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 89 struct rte_flow *flow, 90 const struct rte_flow_attr *attr, 91 const struct rte_flow_action *app_actions, 92 uint32_t flow_idx, 93 const struct mlx5_flow_tunnel *tunnel, 94 struct tunnel_default_miss_ctx *ctx, 95 struct rte_flow_error *error); 96 static struct mlx5_flow_tunnel * 97 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 98 static void 99 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 100 static uint32_t 101 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 102 const struct mlx5_flow_tunnel *tunnel, 103 uint32_t group, uint32_t *table, 104 struct rte_flow_error *error); 105 106 /** Device flow drivers. */ 107 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 108 109 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 110 111 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 112 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 113 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 114 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 115 #endif 116 #ifdef HAVE_MLX5_HWS_SUPPORT 117 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 118 #endif 119 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 120 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 121 }; 122 123 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 124 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 125 (const int []){ \ 126 __VA_ARGS__, 0, \ 127 } 128 129 /** Node object of input graph for mlx5_flow_expand_rss(). */ 130 struct mlx5_flow_expand_node { 131 const int *const next; 132 /**< 133 * List of next node indexes. Index 0 is interpreted as a terminator. 134 */ 135 const enum rte_flow_item_type type; 136 /**< Pattern item type of current node. */ 137 uint64_t rss_types; 138 /**< 139 * RSS types bit-field associated with this node 140 * (see RTE_ETH_RSS_* definitions). 141 */ 142 uint64_t node_flags; 143 /**< 144 * Bit-fields that define how the node is used in the expansion. 145 * (see MLX5_EXPANSION_NODE_* definitions). 146 */ 147 }; 148 149 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 150 struct mlx5_flow_expand_sqn { 151 uint32_t entries; /** Number of entries */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 /* Optional expand field. The expansion alg will not go deeper. */ 159 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 160 161 /* The node is not added implicitly as expansion to the flow pattern. 162 * If the node type does not match the flow pattern item type, the 163 * expansion alg will go deeper to its next items. 164 * In the current implementation, the list of next nodes indexes can 165 * have up to one node with this flag set and it has to be the last 166 * node index (before the list terminator). 167 */ 168 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 169 170 /** Object returned by mlx5_flow_expand_rss(). */ 171 struct mlx5_flow_expand_rss { 172 uint32_t entries; 173 /**< Number of entries @p patterns and @p priorities. */ 174 struct { 175 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 176 uint32_t priority; /**< Priority offset for each expansion. */ 177 } entry[]; 178 }; 179 180 static void 181 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 182 183 static const struct mlx5_flow_expand_node * 184 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 185 unsigned int item_idx, 186 const struct mlx5_flow_expand_node graph[], 187 const struct mlx5_flow_expand_node *node); 188 189 static __rte_always_inline int 190 mlx5_need_cache_flow(const struct mlx5_priv *priv, 191 const struct rte_flow_attr *attr) 192 { 193 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 194 (attr ? !attr->group : true) && 195 priv->mode_info.mode == MLX5_FLOW_ENGINE_MODE_STANDBY && 196 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 197 } 198 199 static bool 200 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 201 { 202 switch (item->type) { 203 case RTE_FLOW_ITEM_TYPE_ETH: 204 case RTE_FLOW_ITEM_TYPE_VLAN: 205 case RTE_FLOW_ITEM_TYPE_IPV4: 206 case RTE_FLOW_ITEM_TYPE_IPV6: 207 case RTE_FLOW_ITEM_TYPE_UDP: 208 case RTE_FLOW_ITEM_TYPE_TCP: 209 case RTE_FLOW_ITEM_TYPE_ESP: 210 case RTE_FLOW_ITEM_TYPE_ICMP: 211 case RTE_FLOW_ITEM_TYPE_ICMP6: 212 case RTE_FLOW_ITEM_TYPE_VXLAN: 213 case RTE_FLOW_ITEM_TYPE_NVGRE: 214 case RTE_FLOW_ITEM_TYPE_GRE: 215 case RTE_FLOW_ITEM_TYPE_GENEVE: 216 case RTE_FLOW_ITEM_TYPE_MPLS: 217 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 218 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 219 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 220 case RTE_FLOW_ITEM_TYPE_GTP: 221 return true; 222 default: 223 break; 224 } 225 return false; 226 } 227 228 /** 229 * Network Service Header (NSH) and its next protocol values 230 * are described in RFC-8393. 231 */ 232 static enum rte_flow_item_type 233 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 234 { 235 enum rte_flow_item_type type; 236 237 switch (proto_mask & proto_spec) { 238 case 0: 239 type = RTE_FLOW_ITEM_TYPE_VOID; 240 break; 241 case RTE_VXLAN_GPE_TYPE_IPV4: 242 type = RTE_FLOW_ITEM_TYPE_IPV4; 243 break; 244 case RTE_VXLAN_GPE_TYPE_IPV6: 245 type = RTE_VXLAN_GPE_TYPE_IPV6; 246 break; 247 case RTE_VXLAN_GPE_TYPE_ETH: 248 type = RTE_FLOW_ITEM_TYPE_ETH; 249 break; 250 default: 251 type = RTE_FLOW_ITEM_TYPE_END; 252 } 253 return type; 254 } 255 256 static enum rte_flow_item_type 257 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 258 { 259 enum rte_flow_item_type type; 260 261 switch (proto_mask & proto_spec) { 262 case 0: 263 type = RTE_FLOW_ITEM_TYPE_VOID; 264 break; 265 case IPPROTO_UDP: 266 type = RTE_FLOW_ITEM_TYPE_UDP; 267 break; 268 case IPPROTO_TCP: 269 type = RTE_FLOW_ITEM_TYPE_TCP; 270 break; 271 case IPPROTO_IPIP: 272 type = RTE_FLOW_ITEM_TYPE_IPV4; 273 break; 274 case IPPROTO_IPV6: 275 type = RTE_FLOW_ITEM_TYPE_IPV6; 276 break; 277 case IPPROTO_ESP: 278 type = RTE_FLOW_ITEM_TYPE_ESP; 279 break; 280 default: 281 type = RTE_FLOW_ITEM_TYPE_END; 282 } 283 return type; 284 } 285 286 static enum rte_flow_item_type 287 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 288 rte_be16_t type_mask, bool is_tunnel) 289 { 290 enum rte_flow_item_type type; 291 292 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 293 case 0: 294 type = RTE_FLOW_ITEM_TYPE_VOID; 295 break; 296 case RTE_ETHER_TYPE_TEB: 297 type = is_tunnel ? 298 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 299 break; 300 case RTE_ETHER_TYPE_VLAN: 301 type = !is_tunnel ? 302 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 303 break; 304 case RTE_ETHER_TYPE_IPV4: 305 type = RTE_FLOW_ITEM_TYPE_IPV4; 306 break; 307 case RTE_ETHER_TYPE_IPV6: 308 type = RTE_FLOW_ITEM_TYPE_IPV6; 309 break; 310 default: 311 type = RTE_FLOW_ITEM_TYPE_END; 312 } 313 return type; 314 } 315 316 static enum rte_flow_item_type 317 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 318 { 319 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 320 do { \ 321 const void *m = item->mask; \ 322 const void *s = item->spec; \ 323 mask = m ? \ 324 ((const struct rte_flow_item_##type *)m)->fld : \ 325 rte_flow_item_##type##_mask.fld; \ 326 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 327 } while (0) 328 329 enum rte_flow_item_type ret; 330 uint16_t spec, mask; 331 332 if (item == NULL || item->spec == NULL) 333 return RTE_FLOW_ITEM_TYPE_VOID; 334 switch (item->type) { 335 case RTE_FLOW_ITEM_TYPE_ETH: 336 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 337 if (!mask) 338 return RTE_FLOW_ITEM_TYPE_VOID; 339 ret = mlx5_ethertype_to_item_type(spec, mask, false); 340 break; 341 case RTE_FLOW_ITEM_TYPE_VLAN: 342 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 343 if (!mask) 344 return RTE_FLOW_ITEM_TYPE_VOID; 345 ret = mlx5_ethertype_to_item_type(spec, mask, false); 346 break; 347 case RTE_FLOW_ITEM_TYPE_IPV4: 348 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 349 if (!mask) 350 return RTE_FLOW_ITEM_TYPE_VOID; 351 ret = mlx5_inet_proto_to_item_type(spec, mask); 352 break; 353 case RTE_FLOW_ITEM_TYPE_IPV6: 354 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 355 if (!mask) 356 return RTE_FLOW_ITEM_TYPE_VOID; 357 ret = mlx5_inet_proto_to_item_type(spec, mask); 358 break; 359 case RTE_FLOW_ITEM_TYPE_GENEVE: 360 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 361 ret = mlx5_ethertype_to_item_type(spec, mask, true); 362 break; 363 case RTE_FLOW_ITEM_TYPE_GRE: 364 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 365 ret = mlx5_ethertype_to_item_type(spec, mask, true); 366 break; 367 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 368 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 369 ret = mlx5_nsh_proto_to_item_type(spec, mask); 370 break; 371 default: 372 ret = RTE_FLOW_ITEM_TYPE_VOID; 373 break; 374 } 375 return ret; 376 #undef MLX5_XSET_ITEM_MASK_SPEC 377 } 378 379 static const int * 380 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 381 const int *next_node) 382 { 383 const struct mlx5_flow_expand_node *node = NULL; 384 const int *next = next_node; 385 386 while (next && *next) { 387 /* 388 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 389 * flag set, because they were not found in the flow pattern. 390 */ 391 node = &graph[*next]; 392 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 393 break; 394 next = node->next; 395 } 396 return next; 397 } 398 399 #define MLX5_RSS_EXP_ELT_N 32 400 401 /** 402 * Expand RSS flows into several possible flows according to the RSS hash 403 * fields requested and the driver capabilities. 404 * 405 * @param[out] buf 406 * Buffer to store the result expansion. 407 * @param[in] size 408 * Buffer size in bytes. If 0, @p buf can be NULL. 409 * @param[in] pattern 410 * User flow pattern. 411 * @param[in] types 412 * RSS types to expand (see RTE_ETH_RSS_* definitions). 413 * @param[in] graph 414 * Input graph to expand @p pattern according to @p types. 415 * @param[in] graph_root_index 416 * Index of root node in @p graph, typically 0. 417 * 418 * @return 419 * A positive value representing the size of @p buf in bytes regardless of 420 * @p size on success, a negative errno value otherwise and rte_errno is 421 * set, the following errors are defined: 422 * 423 * -E2BIG: graph-depth @p graph is too deep. 424 * -EINVAL: @p size has not enough space for expanded pattern. 425 */ 426 static int 427 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 428 const struct rte_flow_item *pattern, uint64_t types, 429 const struct mlx5_flow_expand_node graph[], 430 int graph_root_index) 431 { 432 const struct rte_flow_item *item; 433 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 434 const int *next_node; 435 const int *stack[MLX5_RSS_EXP_ELT_N]; 436 int stack_pos = 0; 437 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 438 unsigned int i, item_idx, last_expand_item_idx = 0; 439 size_t lsize; 440 size_t user_pattern_size = 0; 441 void *addr = NULL; 442 const struct mlx5_flow_expand_node *next = NULL; 443 struct rte_flow_item missed_item; 444 int missed = 0; 445 int elt = 0; 446 const struct rte_flow_item *last_expand_item = NULL; 447 448 memset(&missed_item, 0, sizeof(missed_item)); 449 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 450 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 451 if (lsize > size) 452 return -EINVAL; 453 buf->entry[0].priority = 0; 454 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 455 buf->entries = 0; 456 addr = buf->entry[0].pattern; 457 for (item = pattern, item_idx = 0; 458 item->type != RTE_FLOW_ITEM_TYPE_END; 459 item++, item_idx++) { 460 if (!mlx5_flow_is_rss_expandable_item(item)) { 461 user_pattern_size += sizeof(*item); 462 continue; 463 } 464 last_expand_item = item; 465 last_expand_item_idx = item_idx; 466 i = 0; 467 while (node->next && node->next[i]) { 468 next = &graph[node->next[i]]; 469 if (next->type == item->type) 470 break; 471 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 472 node = next; 473 i = 0; 474 } else { 475 ++i; 476 } 477 } 478 if (next) 479 node = next; 480 user_pattern_size += sizeof(*item); 481 } 482 user_pattern_size += sizeof(*item); /* Handle END item. */ 483 lsize += user_pattern_size; 484 if (lsize > size) 485 return -EINVAL; 486 /* Copy the user pattern in the first entry of the buffer. */ 487 rte_memcpy(addr, pattern, user_pattern_size); 488 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 489 buf->entries = 1; 490 /* Start expanding. */ 491 memset(flow_items, 0, sizeof(flow_items)); 492 user_pattern_size -= sizeof(*item); 493 /* 494 * Check if the last valid item has spec set, need complete pattern, 495 * and the pattern can be used for expansion. 496 */ 497 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 498 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 499 /* Item type END indicates expansion is not required. */ 500 return lsize; 501 } 502 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 503 next = NULL; 504 missed = 1; 505 i = 0; 506 while (node->next && node->next[i]) { 507 next = &graph[node->next[i]]; 508 if (next->type == missed_item.type) { 509 flow_items[0].type = missed_item.type; 510 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 511 break; 512 } 513 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 514 node = next; 515 i = 0; 516 } else { 517 ++i; 518 } 519 next = NULL; 520 } 521 } 522 if (next && missed) { 523 elt = 2; /* missed item + item end. */ 524 node = next; 525 lsize += elt * sizeof(*item) + user_pattern_size; 526 if (lsize > size) 527 return -EINVAL; 528 if (node->rss_types & types) { 529 buf->entry[buf->entries].priority = 1; 530 buf->entry[buf->entries].pattern = addr; 531 buf->entries++; 532 rte_memcpy(addr, buf->entry[0].pattern, 533 user_pattern_size); 534 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 535 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 536 addr = (void *)(((uintptr_t)addr) + 537 elt * sizeof(*item)); 538 } 539 } else if (last_expand_item != NULL) { 540 node = mlx5_flow_expand_rss_adjust_node(pattern, 541 last_expand_item_idx, graph, node); 542 } 543 memset(flow_items, 0, sizeof(flow_items)); 544 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 545 node->next); 546 stack[stack_pos] = next_node; 547 node = next_node ? &graph[*next_node] : NULL; 548 while (node) { 549 flow_items[stack_pos].type = node->type; 550 if (node->rss_types & types) { 551 size_t n; 552 /* 553 * compute the number of items to copy from the 554 * expansion and copy it. 555 * When the stack_pos is 0, there are 1 element in it, 556 * plus the addition END item. 557 */ 558 elt = stack_pos + 2; 559 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 560 lsize += elt * sizeof(*item) + user_pattern_size; 561 if (lsize > size) 562 return -EINVAL; 563 n = elt * sizeof(*item); 564 MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); 565 buf->entry[buf->entries].priority = 566 stack_pos + 1 + missed; 567 buf->entry[buf->entries].pattern = addr; 568 buf->entries++; 569 rte_memcpy(addr, buf->entry[0].pattern, 570 user_pattern_size); 571 addr = (void *)(((uintptr_t)addr) + 572 user_pattern_size); 573 rte_memcpy(addr, &missed_item, 574 missed * sizeof(*item)); 575 addr = (void *)(((uintptr_t)addr) + 576 missed * sizeof(*item)); 577 rte_memcpy(addr, flow_items, n); 578 addr = (void *)(((uintptr_t)addr) + n); 579 } 580 /* Go deeper. */ 581 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 582 node->next) { 583 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 584 node->next); 585 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 586 rte_errno = E2BIG; 587 return -rte_errno; 588 } 589 stack[stack_pos] = next_node; 590 } else if (*(next_node + 1)) { 591 /* Follow up with the next possibility. */ 592 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 593 ++next_node); 594 } else if (!stack_pos) { 595 /* 596 * Completing the traverse over the different paths. 597 * The next_node is advanced to the terminator. 598 */ 599 ++next_node; 600 } else { 601 /* Move to the next path. */ 602 while (stack_pos) { 603 next_node = stack[--stack_pos]; 604 next_node++; 605 if (*next_node) 606 break; 607 } 608 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 609 next_node); 610 stack[stack_pos] = next_node; 611 } 612 node = next_node && *next_node ? &graph[*next_node] : NULL; 613 }; 614 return lsize; 615 } 616 617 /** 618 * Expand SQN flows into several possible flows according to the Tx queue 619 * number 620 * 621 * @param[in] buf 622 * Buffer to store the result expansion. 623 * @param[in] size 624 * Buffer size in bytes. If 0, @p buf can be NULL. 625 * @param[in] pattern 626 * User flow pattern. 627 * @param[in] sq_specs 628 * Buffer to store sq spec. 629 * 630 * @return 631 * 0 for success and negative value for failure 632 * 633 */ 634 static int 635 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 636 const struct rte_flow_item *pattern, 637 struct mlx5_rte_flow_item_sq *sq_specs) 638 { 639 const struct rte_flow_item *item; 640 bool port_representor = false; 641 size_t user_pattern_size = 0; 642 struct rte_eth_dev *dev; 643 struct mlx5_priv *priv; 644 void *addr = NULL; 645 uint16_t port_id; 646 size_t lsize; 647 int elt = 2; 648 uint16_t i; 649 650 buf->entries = 0; 651 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 652 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 653 const struct rte_flow_item_ethdev *pid_v = item->spec; 654 655 if (!pid_v) 656 return 0; 657 port_id = pid_v->port_id; 658 port_representor = true; 659 } 660 user_pattern_size += sizeof(*item); 661 } 662 if (!port_representor) 663 return 0; 664 dev = &rte_eth_devices[port_id]; 665 priv = dev->data->dev_private; 666 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 667 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 668 sizeof(buf->entry[0]) * priv->txqs_n; 669 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 670 return -EINVAL; 671 addr = buf->entry[0].pattern; 672 for (i = 0; i != priv->txqs_n; ++i) { 673 struct rte_flow_item pattern_add[] = { 674 { 675 .type = (enum rte_flow_item_type) 676 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 677 .spec = &sq_specs[i], 678 }, 679 { 680 .type = RTE_FLOW_ITEM_TYPE_END, 681 }, 682 }; 683 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 684 685 if (txq == NULL) 686 return -EINVAL; 687 buf->entry[i].pattern = addr; 688 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 689 mlx5_txq_release(dev, i); 690 rte_memcpy(addr, pattern, user_pattern_size); 691 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 692 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 693 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 694 buf->entries++; 695 } 696 return 0; 697 } 698 699 enum mlx5_expansion { 700 MLX5_EXPANSION_ROOT, 701 MLX5_EXPANSION_ROOT_OUTER, 702 MLX5_EXPANSION_OUTER_ETH, 703 MLX5_EXPANSION_OUTER_VLAN, 704 MLX5_EXPANSION_OUTER_IPV4, 705 MLX5_EXPANSION_OUTER_IPV4_UDP, 706 MLX5_EXPANSION_OUTER_IPV4_TCP, 707 MLX5_EXPANSION_OUTER_IPV4_ESP, 708 MLX5_EXPANSION_OUTER_IPV4_ICMP, 709 MLX5_EXPANSION_OUTER_IPV6, 710 MLX5_EXPANSION_OUTER_IPV6_UDP, 711 MLX5_EXPANSION_OUTER_IPV6_TCP, 712 MLX5_EXPANSION_OUTER_IPV6_ESP, 713 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 714 MLX5_EXPANSION_VXLAN, 715 MLX5_EXPANSION_STD_VXLAN, 716 MLX5_EXPANSION_L3_VXLAN, 717 MLX5_EXPANSION_VXLAN_GPE, 718 MLX5_EXPANSION_GRE, 719 MLX5_EXPANSION_NVGRE, 720 MLX5_EXPANSION_GRE_KEY, 721 MLX5_EXPANSION_MPLS, 722 MLX5_EXPANSION_ETH, 723 MLX5_EXPANSION_VLAN, 724 MLX5_EXPANSION_IPV4, 725 MLX5_EXPANSION_IPV4_UDP, 726 MLX5_EXPANSION_IPV4_TCP, 727 MLX5_EXPANSION_IPV4_ESP, 728 MLX5_EXPANSION_IPV4_ICMP, 729 MLX5_EXPANSION_IPV6, 730 MLX5_EXPANSION_IPV6_UDP, 731 MLX5_EXPANSION_IPV6_TCP, 732 MLX5_EXPANSION_IPV6_ESP, 733 MLX5_EXPANSION_IPV6_ICMP6, 734 MLX5_EXPANSION_IPV6_FRAG_EXT, 735 MLX5_EXPANSION_GTP, 736 MLX5_EXPANSION_GENEVE, 737 }; 738 739 /** Supported expansion of items. */ 740 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 741 [MLX5_EXPANSION_ROOT] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 743 MLX5_EXPANSION_IPV4, 744 MLX5_EXPANSION_IPV6), 745 .type = RTE_FLOW_ITEM_TYPE_END, 746 }, 747 [MLX5_EXPANSION_ROOT_OUTER] = { 748 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 749 MLX5_EXPANSION_OUTER_IPV4, 750 MLX5_EXPANSION_OUTER_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_END, 752 }, 753 [MLX5_EXPANSION_OUTER_ETH] = { 754 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 755 .type = RTE_FLOW_ITEM_TYPE_ETH, 756 .rss_types = 0, 757 }, 758 [MLX5_EXPANSION_OUTER_VLAN] = { 759 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 760 MLX5_EXPANSION_OUTER_IPV6), 761 .type = RTE_FLOW_ITEM_TYPE_VLAN, 762 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 763 }, 764 [MLX5_EXPANSION_OUTER_IPV4] = { 765 .next = MLX5_FLOW_EXPAND_RSS_NEXT 766 (MLX5_EXPANSION_OUTER_IPV4_UDP, 767 MLX5_EXPANSION_OUTER_IPV4_TCP, 768 MLX5_EXPANSION_OUTER_IPV4_ESP, 769 MLX5_EXPANSION_OUTER_IPV4_ICMP, 770 MLX5_EXPANSION_GRE, 771 MLX5_EXPANSION_NVGRE, 772 MLX5_EXPANSION_IPV4, 773 MLX5_EXPANSION_IPV6), 774 .type = RTE_FLOW_ITEM_TYPE_IPV4, 775 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 776 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 777 }, 778 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 779 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 780 MLX5_EXPANSION_VXLAN_GPE, 781 MLX5_EXPANSION_MPLS, 782 MLX5_EXPANSION_GENEVE, 783 MLX5_EXPANSION_GTP), 784 .type = RTE_FLOW_ITEM_TYPE_UDP, 785 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 786 }, 787 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 788 .type = RTE_FLOW_ITEM_TYPE_TCP, 789 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 790 }, 791 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 792 .type = RTE_FLOW_ITEM_TYPE_ESP, 793 .rss_types = RTE_ETH_RSS_ESP, 794 }, 795 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 796 .type = RTE_FLOW_ITEM_TYPE_ICMP, 797 }, 798 [MLX5_EXPANSION_OUTER_IPV6] = { 799 .next = MLX5_FLOW_EXPAND_RSS_NEXT 800 (MLX5_EXPANSION_OUTER_IPV6_UDP, 801 MLX5_EXPANSION_OUTER_IPV6_TCP, 802 MLX5_EXPANSION_OUTER_IPV6_ESP, 803 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 804 MLX5_EXPANSION_IPV4, 805 MLX5_EXPANSION_IPV6, 806 MLX5_EXPANSION_GRE, 807 MLX5_EXPANSION_NVGRE), 808 .type = RTE_FLOW_ITEM_TYPE_IPV6, 809 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 810 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 811 }, 812 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 813 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 814 MLX5_EXPANSION_VXLAN_GPE, 815 MLX5_EXPANSION_MPLS, 816 MLX5_EXPANSION_GENEVE, 817 MLX5_EXPANSION_GTP), 818 .type = RTE_FLOW_ITEM_TYPE_UDP, 819 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 820 }, 821 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 822 .type = RTE_FLOW_ITEM_TYPE_TCP, 823 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 824 }, 825 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 826 .type = RTE_FLOW_ITEM_TYPE_ESP, 827 .rss_types = RTE_ETH_RSS_ESP, 828 }, 829 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 830 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 831 }, 832 [MLX5_EXPANSION_VXLAN] = { 833 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 834 MLX5_EXPANSION_IPV4, 835 MLX5_EXPANSION_IPV6), 836 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 837 }, 838 [MLX5_EXPANSION_STD_VXLAN] = { 839 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 840 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 841 }, 842 [MLX5_EXPANSION_L3_VXLAN] = { 843 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 844 MLX5_EXPANSION_IPV6), 845 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 846 }, 847 [MLX5_EXPANSION_VXLAN_GPE] = { 848 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 849 MLX5_EXPANSION_IPV4, 850 MLX5_EXPANSION_IPV6), 851 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 852 }, 853 [MLX5_EXPANSION_GRE] = { 854 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 855 MLX5_EXPANSION_IPV4, 856 MLX5_EXPANSION_IPV6, 857 MLX5_EXPANSION_GRE_KEY, 858 MLX5_EXPANSION_MPLS), 859 .type = RTE_FLOW_ITEM_TYPE_GRE, 860 }, 861 [MLX5_EXPANSION_GRE_KEY] = { 862 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 863 MLX5_EXPANSION_IPV6, 864 MLX5_EXPANSION_MPLS), 865 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 866 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 867 }, 868 [MLX5_EXPANSION_NVGRE] = { 869 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 870 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 871 }, 872 [MLX5_EXPANSION_MPLS] = { 873 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 874 MLX5_EXPANSION_IPV6, 875 MLX5_EXPANSION_ETH), 876 .type = RTE_FLOW_ITEM_TYPE_MPLS, 877 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 878 }, 879 [MLX5_EXPANSION_ETH] = { 880 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 881 .type = RTE_FLOW_ITEM_TYPE_ETH, 882 }, 883 [MLX5_EXPANSION_VLAN] = { 884 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 885 MLX5_EXPANSION_IPV6), 886 .type = RTE_FLOW_ITEM_TYPE_VLAN, 887 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 888 }, 889 [MLX5_EXPANSION_IPV4] = { 890 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 891 MLX5_EXPANSION_IPV4_TCP, 892 MLX5_EXPANSION_IPV4_ESP, 893 MLX5_EXPANSION_IPV4_ICMP), 894 .type = RTE_FLOW_ITEM_TYPE_IPV4, 895 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 896 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 897 }, 898 [MLX5_EXPANSION_IPV4_UDP] = { 899 .type = RTE_FLOW_ITEM_TYPE_UDP, 900 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 901 }, 902 [MLX5_EXPANSION_IPV4_TCP] = { 903 .type = RTE_FLOW_ITEM_TYPE_TCP, 904 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 905 }, 906 [MLX5_EXPANSION_IPV4_ESP] = { 907 .type = RTE_FLOW_ITEM_TYPE_ESP, 908 .rss_types = RTE_ETH_RSS_ESP, 909 }, 910 [MLX5_EXPANSION_IPV4_ICMP] = { 911 .type = RTE_FLOW_ITEM_TYPE_ICMP, 912 }, 913 [MLX5_EXPANSION_IPV6] = { 914 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 915 MLX5_EXPANSION_IPV6_TCP, 916 MLX5_EXPANSION_IPV6_ESP, 917 MLX5_EXPANSION_IPV6_ICMP6, 918 MLX5_EXPANSION_IPV6_FRAG_EXT), 919 .type = RTE_FLOW_ITEM_TYPE_IPV6, 920 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 921 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 922 }, 923 [MLX5_EXPANSION_IPV6_UDP] = { 924 .type = RTE_FLOW_ITEM_TYPE_UDP, 925 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 926 }, 927 [MLX5_EXPANSION_IPV6_TCP] = { 928 .type = RTE_FLOW_ITEM_TYPE_TCP, 929 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 930 }, 931 [MLX5_EXPANSION_IPV6_ESP] = { 932 .type = RTE_FLOW_ITEM_TYPE_ESP, 933 .rss_types = RTE_ETH_RSS_ESP, 934 }, 935 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 936 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 937 }, 938 [MLX5_EXPANSION_IPV6_ICMP6] = { 939 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 940 }, 941 [MLX5_EXPANSION_GTP] = { 942 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 943 MLX5_EXPANSION_IPV6), 944 .type = RTE_FLOW_ITEM_TYPE_GTP, 945 }, 946 [MLX5_EXPANSION_GENEVE] = { 947 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 948 MLX5_EXPANSION_IPV4, 949 MLX5_EXPANSION_IPV6), 950 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 951 }, 952 }; 953 954 static struct rte_flow_action_handle * 955 mlx5_action_handle_create(struct rte_eth_dev *dev, 956 const struct rte_flow_indir_action_conf *conf, 957 const struct rte_flow_action *action, 958 struct rte_flow_error *error); 959 static int mlx5_action_handle_destroy 960 (struct rte_eth_dev *dev, 961 struct rte_flow_action_handle *handle, 962 struct rte_flow_error *error); 963 static int mlx5_action_handle_update 964 (struct rte_eth_dev *dev, 965 struct rte_flow_action_handle *handle, 966 const void *update, 967 struct rte_flow_error *error); 968 static int mlx5_action_handle_query 969 (struct rte_eth_dev *dev, 970 const struct rte_flow_action_handle *handle, 971 void *data, 972 struct rte_flow_error *error); 973 static int 974 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 975 struct rte_flow_tunnel *app_tunnel, 976 struct rte_flow_action **actions, 977 uint32_t *num_of_actions, 978 struct rte_flow_error *error); 979 static int 980 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 981 struct rte_flow_tunnel *app_tunnel, 982 struct rte_flow_item **items, 983 uint32_t *num_of_items, 984 struct rte_flow_error *error); 985 static int 986 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 987 struct rte_flow_item *pmd_items, 988 uint32_t num_items, struct rte_flow_error *err); 989 static int 990 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 991 struct rte_flow_action *pmd_actions, 992 uint32_t num_actions, 993 struct rte_flow_error *err); 994 static int 995 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 996 struct rte_mbuf *m, 997 struct rte_flow_restore_info *info, 998 struct rte_flow_error *err); 999 static struct rte_flow_item_flex_handle * 1000 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 1001 const struct rte_flow_item_flex_conf *conf, 1002 struct rte_flow_error *error); 1003 static int 1004 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 1005 const struct rte_flow_item_flex_handle *handle, 1006 struct rte_flow_error *error); 1007 static int 1008 mlx5_flow_info_get(struct rte_eth_dev *dev, 1009 struct rte_flow_port_info *port_info, 1010 struct rte_flow_queue_info *queue_info, 1011 struct rte_flow_error *error); 1012 static int 1013 mlx5_flow_port_configure(struct rte_eth_dev *dev, 1014 const struct rte_flow_port_attr *port_attr, 1015 uint16_t nb_queue, 1016 const struct rte_flow_queue_attr *queue_attr[], 1017 struct rte_flow_error *err); 1018 1019 static struct rte_flow_pattern_template * 1020 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 1021 const struct rte_flow_pattern_template_attr *attr, 1022 const struct rte_flow_item items[], 1023 struct rte_flow_error *error); 1024 1025 static int 1026 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1027 struct rte_flow_pattern_template *template, 1028 struct rte_flow_error *error); 1029 static struct rte_flow_actions_template * 1030 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1031 const struct rte_flow_actions_template_attr *attr, 1032 const struct rte_flow_action actions[], 1033 const struct rte_flow_action masks[], 1034 struct rte_flow_error *error); 1035 static int 1036 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1037 struct rte_flow_actions_template *template, 1038 struct rte_flow_error *error); 1039 1040 static struct rte_flow_template_table * 1041 mlx5_flow_table_create(struct rte_eth_dev *dev, 1042 const struct rte_flow_template_table_attr *attr, 1043 struct rte_flow_pattern_template *item_templates[], 1044 uint8_t nb_item_templates, 1045 struct rte_flow_actions_template *action_templates[], 1046 uint8_t nb_action_templates, 1047 struct rte_flow_error *error); 1048 static int 1049 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1050 struct rte_flow_template_table *table, 1051 struct rte_flow_error *error); 1052 static int 1053 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 1054 uint32_t group_id, 1055 const struct rte_flow_group_attr *attr, 1056 const struct rte_flow_action actions[], 1057 struct rte_flow_error *error); 1058 static struct rte_flow * 1059 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1060 uint32_t queue, 1061 const struct rte_flow_op_attr *attr, 1062 struct rte_flow_template_table *table, 1063 const struct rte_flow_item items[], 1064 uint8_t pattern_template_index, 1065 const struct rte_flow_action actions[], 1066 uint8_t action_template_index, 1067 void *user_data, 1068 struct rte_flow_error *error); 1069 static struct rte_flow * 1070 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 1071 uint32_t queue, 1072 const struct rte_flow_op_attr *attr, 1073 struct rte_flow_template_table *table, 1074 uint32_t rule_index, 1075 const struct rte_flow_action actions[], 1076 uint8_t action_template_index, 1077 void *user_data, 1078 struct rte_flow_error *error); 1079 static int 1080 mlx5_flow_async_flow_update(struct rte_eth_dev *dev, 1081 uint32_t queue, 1082 const struct rte_flow_op_attr *attr, 1083 struct rte_flow *flow, 1084 const struct rte_flow_action actions[], 1085 uint8_t action_template_index, 1086 void *user_data, 1087 struct rte_flow_error *error); 1088 static int 1089 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1090 uint32_t queue, 1091 const struct rte_flow_op_attr *attr, 1092 struct rte_flow *flow, 1093 void *user_data, 1094 struct rte_flow_error *error); 1095 static int 1096 mlx5_flow_pull(struct rte_eth_dev *dev, 1097 uint32_t queue, 1098 struct rte_flow_op_result res[], 1099 uint16_t n_res, 1100 struct rte_flow_error *error); 1101 static int 1102 mlx5_flow_push(struct rte_eth_dev *dev, 1103 uint32_t queue, 1104 struct rte_flow_error *error); 1105 1106 static struct rte_flow_action_handle * 1107 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1108 const struct rte_flow_op_attr *attr, 1109 const struct rte_flow_indir_action_conf *conf, 1110 const struct rte_flow_action *action, 1111 void *user_data, 1112 struct rte_flow_error *error); 1113 1114 static int 1115 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1116 const struct rte_flow_op_attr *attr, 1117 struct rte_flow_action_handle *handle, 1118 const void *update, 1119 void *user_data, 1120 struct rte_flow_error *error); 1121 1122 static int 1123 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1124 const struct rte_flow_op_attr *attr, 1125 struct rte_flow_action_handle *handle, 1126 void *user_data, 1127 struct rte_flow_error *error); 1128 1129 static int 1130 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1131 const struct rte_flow_op_attr *attr, 1132 const struct rte_flow_action_handle *handle, 1133 void *data, 1134 void *user_data, 1135 struct rte_flow_error *error); 1136 static int 1137 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1138 struct rte_flow_action_handle *handle, 1139 const void *update, void *query, 1140 enum rte_flow_query_update_mode qu_mode, 1141 struct rte_flow_error *error); 1142 static int 1143 mlx5_flow_async_action_handle_query_update 1144 (struct rte_eth_dev *dev, uint32_t queue_id, 1145 const struct rte_flow_op_attr *op_attr, 1146 struct rte_flow_action_handle *action_handle, 1147 const void *update, void *query, 1148 enum rte_flow_query_update_mode qu_mode, 1149 void *user_data, struct rte_flow_error *error); 1150 1151 static struct rte_flow_action_list_handle * 1152 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 1153 const struct rte_flow_indir_action_conf *conf, 1154 const struct rte_flow_action *actions, 1155 struct rte_flow_error *error); 1156 1157 static int 1158 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 1159 struct rte_flow_action_list_handle *handle, 1160 struct rte_flow_error *error); 1161 1162 static struct rte_flow_action_list_handle * 1163 mlx5_flow_async_action_list_handle_create(struct rte_eth_dev *dev, uint32_t queue_id, 1164 const struct rte_flow_op_attr *attr, 1165 const struct 1166 rte_flow_indir_action_conf *conf, 1167 const struct rte_flow_action *actions, 1168 void *user_data, 1169 struct rte_flow_error *error); 1170 static int 1171 mlx5_flow_async_action_list_handle_destroy 1172 (struct rte_eth_dev *dev, uint32_t queue_id, 1173 const struct rte_flow_op_attr *op_attr, 1174 struct rte_flow_action_list_handle *action_handle, 1175 void *user_data, struct rte_flow_error *error); 1176 static int 1177 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 1178 const 1179 struct rte_flow_action_list_handle *handle, 1180 const void **update, void **query, 1181 enum rte_flow_query_update_mode mode, 1182 struct rte_flow_error *error); 1183 static int 1184 mlx5_flow_async_action_list_handle_query_update(struct rte_eth_dev *dev, 1185 uint32_t queue_id, 1186 const struct rte_flow_op_attr *attr, 1187 const struct 1188 rte_flow_action_list_handle *handle, 1189 const void **update, 1190 void **query, 1191 enum rte_flow_query_update_mode mode, 1192 void *user_data, 1193 struct rte_flow_error *error); 1194 static const struct rte_flow_ops mlx5_flow_ops = { 1195 .validate = mlx5_flow_validate, 1196 .create = mlx5_flow_create, 1197 .destroy = mlx5_flow_destroy, 1198 .flush = mlx5_flow_flush, 1199 .isolate = mlx5_flow_isolate, 1200 .query = mlx5_flow_query, 1201 .dev_dump = mlx5_flow_dev_dump, 1202 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1203 .get_aged_flows = mlx5_flow_get_aged_flows, 1204 .action_handle_create = mlx5_action_handle_create, 1205 .action_handle_destroy = mlx5_action_handle_destroy, 1206 .action_handle_update = mlx5_action_handle_update, 1207 .action_handle_query = mlx5_action_handle_query, 1208 .action_handle_query_update = mlx5_action_handle_query_update, 1209 .action_list_handle_create = mlx5_action_list_handle_create, 1210 .action_list_handle_destroy = mlx5_action_list_handle_destroy, 1211 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1212 .tunnel_match = mlx5_flow_tunnel_match, 1213 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1214 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1215 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1216 .flex_item_create = mlx5_flow_flex_item_create, 1217 .flex_item_release = mlx5_flow_flex_item_release, 1218 .info_get = mlx5_flow_info_get, 1219 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1220 .configure = mlx5_flow_port_configure, 1221 .pattern_template_create = mlx5_flow_pattern_template_create, 1222 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1223 .actions_template_create = mlx5_flow_actions_template_create, 1224 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1225 .template_table_create = mlx5_flow_table_create, 1226 .template_table_destroy = mlx5_flow_table_destroy, 1227 .group_set_miss_actions = mlx5_flow_group_set_miss_actions, 1228 .async_create = mlx5_flow_async_flow_create, 1229 .async_create_by_index = mlx5_flow_async_flow_create_by_index, 1230 .async_destroy = mlx5_flow_async_flow_destroy, 1231 .pull = mlx5_flow_pull, 1232 .push = mlx5_flow_push, 1233 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1234 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1235 .async_action_handle_query_update = 1236 mlx5_flow_async_action_handle_query_update, 1237 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1238 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1239 .async_actions_update = mlx5_flow_async_flow_update, 1240 .async_action_list_handle_create = 1241 mlx5_flow_async_action_list_handle_create, 1242 .async_action_list_handle_destroy = 1243 mlx5_flow_async_action_list_handle_destroy, 1244 .action_list_handle_query_update = 1245 mlx5_flow_action_list_handle_query_update, 1246 .async_action_list_handle_query_update = 1247 mlx5_flow_async_action_list_handle_query_update, 1248 }; 1249 1250 /* Tunnel information. */ 1251 struct mlx5_flow_tunnel_info { 1252 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1253 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1254 }; 1255 1256 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1257 { 1258 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1259 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1260 }, 1261 { 1262 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1263 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1264 }, 1265 { 1266 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1267 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1268 }, 1269 { 1270 .tunnel = MLX5_FLOW_LAYER_GRE, 1271 .ptype = RTE_PTYPE_TUNNEL_GRE, 1272 }, 1273 { 1274 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1275 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1276 }, 1277 { 1278 .tunnel = MLX5_FLOW_LAYER_MPLS, 1279 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1280 }, 1281 { 1282 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1283 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1284 }, 1285 { 1286 .tunnel = MLX5_FLOW_LAYER_IPIP, 1287 .ptype = RTE_PTYPE_TUNNEL_IP, 1288 }, 1289 { 1290 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1291 .ptype = RTE_PTYPE_TUNNEL_IP, 1292 }, 1293 { 1294 .tunnel = MLX5_FLOW_LAYER_GTP, 1295 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1296 }, 1297 }; 1298 1299 1300 1301 /** 1302 * Translate tag ID to register. 1303 * 1304 * @param[in] dev 1305 * Pointer to the Ethernet device structure. 1306 * @param[in] feature 1307 * The feature that request the register. 1308 * @param[in] id 1309 * The request register ID. 1310 * @param[out] error 1311 * Error description in case of any. 1312 * 1313 * @return 1314 * The request register on success, a negative errno 1315 * value otherwise and rte_errno is set. 1316 */ 1317 int 1318 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1319 enum mlx5_feature_name feature, 1320 uint32_t id, 1321 struct rte_flow_error *error) 1322 { 1323 struct mlx5_priv *priv = dev->data->dev_private; 1324 struct mlx5_sh_config *config = &priv->sh->config; 1325 struct mlx5_dev_registers *reg = &priv->sh->registers; 1326 enum modify_reg start_reg; 1327 bool skip_mtr_reg = false; 1328 1329 switch (feature) { 1330 case MLX5_HAIRPIN_RX: 1331 return REG_B; 1332 case MLX5_HAIRPIN_TX: 1333 return REG_A; 1334 case MLX5_METADATA_RX: 1335 switch (config->dv_xmeta_en) { 1336 case MLX5_XMETA_MODE_LEGACY: 1337 return REG_B; 1338 case MLX5_XMETA_MODE_META16: 1339 return REG_C_0; 1340 case MLX5_XMETA_MODE_META32: 1341 return REG_C_1; 1342 case MLX5_XMETA_MODE_META32_HWS: 1343 return REG_C_1; 1344 } 1345 break; 1346 case MLX5_METADATA_TX: 1347 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1348 return REG_C_1; 1349 } else { 1350 return REG_A; 1351 } 1352 case MLX5_METADATA_FDB: 1353 switch (config->dv_xmeta_en) { 1354 case MLX5_XMETA_MODE_LEGACY: 1355 return REG_NON; 1356 case MLX5_XMETA_MODE_META16: 1357 return REG_C_0; 1358 case MLX5_XMETA_MODE_META32: 1359 return REG_C_1; 1360 case MLX5_XMETA_MODE_META32_HWS: 1361 return REG_C_1; 1362 } 1363 break; 1364 case MLX5_FLOW_MARK: 1365 switch (config->dv_xmeta_en) { 1366 case MLX5_XMETA_MODE_LEGACY: 1367 case MLX5_XMETA_MODE_META32_HWS: 1368 return REG_NON; 1369 case MLX5_XMETA_MODE_META16: 1370 return REG_C_1; 1371 case MLX5_XMETA_MODE_META32: 1372 return REG_C_0; 1373 } 1374 break; 1375 case MLX5_MTR_ID: 1376 /* 1377 * If meter color and meter id share one register, flow match 1378 * should use the meter color register for match. 1379 */ 1380 if (priv->mtr_reg_share) 1381 return reg->aso_reg; 1382 else 1383 return reg->aso_reg != REG_C_2 ? REG_C_2 : 1384 REG_C_3; 1385 case MLX5_MTR_COLOR: 1386 case MLX5_ASO_FLOW_HIT: 1387 case MLX5_ASO_CONNTRACK: 1388 case MLX5_SAMPLE_ID: 1389 /* All features use the same REG_C. */ 1390 MLX5_ASSERT(reg->aso_reg != REG_NON); 1391 return reg->aso_reg; 1392 case MLX5_COPY_MARK: 1393 /* 1394 * Metadata COPY_MARK register using is in meter suffix sub 1395 * flow while with meter. It's safe to share the same register. 1396 */ 1397 return reg->aso_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1398 case MLX5_APP_TAG: 1399 /* 1400 * If meter is enable, it will engage the register for color 1401 * match and flow match. If meter color match is not using the 1402 * REG_C_2, need to skip the REG_C_x be used by meter color 1403 * match. 1404 * If meter is disable, free to use all available registers. 1405 */ 1406 start_reg = reg->aso_reg != REG_C_2 ? REG_C_2 : 1407 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1408 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1409 if (id > (uint32_t)(REG_C_7 - start_reg)) 1410 return rte_flow_error_set(error, EINVAL, 1411 RTE_FLOW_ERROR_TYPE_ITEM, 1412 NULL, "invalid tag id"); 1413 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1414 return rte_flow_error_set(error, ENOTSUP, 1415 RTE_FLOW_ERROR_TYPE_ITEM, 1416 NULL, "unsupported tag id"); 1417 /* 1418 * This case means meter is using the REG_C_x great than 2. 1419 * Take care not to conflict with meter color REG_C_x. 1420 * If the available index REG_C_y >= REG_C_x, skip the 1421 * color register. 1422 */ 1423 if (skip_mtr_reg && priv->sh->flow_mreg_c 1424 [id + start_reg - REG_C_0] >= reg->aso_reg) { 1425 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1426 return rte_flow_error_set(error, EINVAL, 1427 RTE_FLOW_ERROR_TYPE_ITEM, 1428 NULL, "invalid tag id"); 1429 if (priv->sh->flow_mreg_c 1430 [id + 1 + start_reg - REG_C_0] != REG_NON) 1431 return priv->sh->flow_mreg_c 1432 [id + 1 + start_reg - REG_C_0]; 1433 return rte_flow_error_set(error, ENOTSUP, 1434 RTE_FLOW_ERROR_TYPE_ITEM, 1435 NULL, "unsupported tag id"); 1436 } 1437 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1438 } 1439 MLX5_ASSERT(false); 1440 return rte_flow_error_set(error, EINVAL, 1441 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1442 NULL, "invalid feature name"); 1443 } 1444 1445 /** 1446 * Check extensive flow metadata register support. 1447 * 1448 * @param dev 1449 * Pointer to rte_eth_dev structure. 1450 * 1451 * @return 1452 * True if device supports extensive flow metadata register, otherwise false. 1453 */ 1454 bool 1455 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1456 { 1457 struct mlx5_priv *priv = dev->data->dev_private; 1458 1459 /* 1460 * Having available reg_c can be regarded inclusively as supporting 1461 * extensive flow metadata register, which could mean, 1462 * - metadata register copy action by modify header. 1463 * - 16 modify header actions is supported. 1464 * - reg_c's are preserved across different domain (FDB and NIC) on 1465 * packet loopback by flow lookup miss. 1466 */ 1467 return priv->sh->flow_mreg_c[2] != REG_NON; 1468 } 1469 1470 /** 1471 * Get the lowest priority. 1472 * 1473 * @param[in] dev 1474 * Pointer to the Ethernet device structure. 1475 * @param[in] attributes 1476 * Pointer to device flow rule attributes. 1477 * 1478 * @return 1479 * The value of lowest priority of flow. 1480 */ 1481 uint32_t 1482 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1483 const struct rte_flow_attr *attr) 1484 { 1485 struct mlx5_priv *priv = dev->data->dev_private; 1486 1487 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1488 return priv->sh->flow_max_priority - 2; 1489 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1490 } 1491 1492 /** 1493 * Calculate matcher priority of the flow. 1494 * 1495 * @param[in] dev 1496 * Pointer to the Ethernet device structure. 1497 * @param[in] attr 1498 * Pointer to device flow rule attributes. 1499 * @param[in] subpriority 1500 * The priority based on the items. 1501 * @param[in] external 1502 * Flow is user flow. 1503 * @return 1504 * The matcher priority of the flow. 1505 */ 1506 uint16_t 1507 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1508 const struct rte_flow_attr *attr, 1509 uint32_t subpriority, bool external) 1510 { 1511 uint16_t priority = (uint16_t)attr->priority; 1512 struct mlx5_priv *priv = dev->data->dev_private; 1513 1514 /* NIC root rules */ 1515 if (!attr->group && !attr->transfer) { 1516 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1517 priority = priv->sh->flow_max_priority - 1; 1518 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1519 /* FDB root rules */ 1520 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1521 attr->group == 0 && 1522 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1523 return (priv->sh->flow_max_priority - 1) * 3; 1524 } 1525 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1526 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1527 return priority * 3 + subpriority; 1528 } 1529 1530 /** 1531 * Verify the @p item specifications (spec, last, mask) are compatible with the 1532 * NIC capabilities. 1533 * 1534 * @param[in] item 1535 * Item specification. 1536 * @param[in] mask 1537 * @p item->mask or flow default bit-masks. 1538 * @param[in] nic_mask 1539 * Bit-masks covering supported fields by the NIC to compare with user mask. 1540 * @param[in] size 1541 * Bit-masks size in bytes. 1542 * @param[in] range_accepted 1543 * True if range of values is accepted for specific fields, false otherwise. 1544 * @param[out] error 1545 * Pointer to error structure. 1546 * 1547 * @return 1548 * 0 on success, a negative errno value otherwise and rte_errno is set. 1549 */ 1550 int 1551 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1552 const uint8_t *mask, 1553 const uint8_t *nic_mask, 1554 unsigned int size, 1555 bool range_accepted, 1556 struct rte_flow_error *error) 1557 { 1558 unsigned int i; 1559 1560 MLX5_ASSERT(nic_mask); 1561 for (i = 0; i < size; ++i) 1562 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1563 return rte_flow_error_set(error, ENOTSUP, 1564 RTE_FLOW_ERROR_TYPE_ITEM, 1565 item, 1566 "mask enables non supported" 1567 " bits"); 1568 if (!item->spec && (item->mask || item->last)) 1569 return rte_flow_error_set(error, EINVAL, 1570 RTE_FLOW_ERROR_TYPE_ITEM, item, 1571 "mask/last without a spec is not" 1572 " supported"); 1573 if (item->spec && item->last && !range_accepted) { 1574 uint8_t spec[size]; 1575 uint8_t last[size]; 1576 unsigned int i; 1577 int ret; 1578 1579 for (i = 0; i < size; ++i) { 1580 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1581 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1582 } 1583 ret = memcmp(spec, last, size); 1584 if (ret != 0) 1585 return rte_flow_error_set(error, EINVAL, 1586 RTE_FLOW_ERROR_TYPE_ITEM, 1587 item, 1588 "range is not valid"); 1589 } 1590 return 0; 1591 } 1592 1593 /** 1594 * Adjust the hash fields according to the @p flow information. 1595 * 1596 * @param[in] dev_flow. 1597 * Pointer to the mlx5_flow. 1598 * @param[in] tunnel 1599 * 1 when the hash field is for a tunnel item. 1600 * @param[in] layer_types 1601 * RTE_ETH_RSS_* types. 1602 * @param[in] hash_fields 1603 * Item hash fields. 1604 * 1605 * @return 1606 * The hash fields that should be used. 1607 */ 1608 uint64_t 1609 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1610 int tunnel __rte_unused, uint64_t layer_types, 1611 uint64_t hash_fields) 1612 { 1613 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1614 int rss_request_inner = rss_desc->level >= 2; 1615 1616 /* Check RSS hash level for tunnel. */ 1617 if (tunnel && rss_request_inner) 1618 hash_fields |= IBV_RX_HASH_INNER; 1619 else if (tunnel || rss_request_inner) 1620 return 0; 1621 #endif 1622 /* Check if requested layer matches RSS hash fields. */ 1623 if (!(rss_desc->types & layer_types)) 1624 return 0; 1625 return hash_fields; 1626 } 1627 1628 /** 1629 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1630 * if several tunnel rules are used on this queue, the tunnel ptype will be 1631 * cleared. 1632 * 1633 * @param rxq_ctrl 1634 * Rx queue to update. 1635 */ 1636 static void 1637 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1638 { 1639 unsigned int i; 1640 uint32_t tunnel_ptype = 0; 1641 1642 /* Look up for the ptype to use. */ 1643 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1644 if (!rxq_ctrl->flow_tunnels_n[i]) 1645 continue; 1646 if (!tunnel_ptype) { 1647 tunnel_ptype = tunnels_info[i].ptype; 1648 } else { 1649 tunnel_ptype = 0; 1650 break; 1651 } 1652 } 1653 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1654 } 1655 1656 /** 1657 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1658 * flow. 1659 * 1660 * @param[in] dev 1661 * Pointer to the Ethernet device structure. 1662 * @param[in] dev_handle 1663 * Pointer to device flow handle structure. 1664 */ 1665 void 1666 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1667 struct mlx5_flow_handle *dev_handle) 1668 { 1669 struct mlx5_priv *priv = dev->data->dev_private; 1670 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1671 struct mlx5_ind_table_obj *ind_tbl = NULL; 1672 unsigned int i; 1673 1674 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1675 struct mlx5_hrxq *hrxq; 1676 1677 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1678 dev_handle->rix_hrxq); 1679 if (hrxq) 1680 ind_tbl = hrxq->ind_table; 1681 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1682 struct mlx5_shared_action_rss *shared_rss; 1683 1684 shared_rss = mlx5_ipool_get 1685 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1686 dev_handle->rix_srss); 1687 if (shared_rss) 1688 ind_tbl = shared_rss->ind_tbl; 1689 } 1690 if (!ind_tbl) 1691 return; 1692 for (i = 0; i != ind_tbl->queues_n; ++i) { 1693 int idx = ind_tbl->queues[i]; 1694 struct mlx5_rxq_ctrl *rxq_ctrl; 1695 1696 if (mlx5_is_external_rxq(dev, idx)) 1697 continue; 1698 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1699 MLX5_ASSERT(rxq_ctrl != NULL); 1700 if (rxq_ctrl == NULL) 1701 continue; 1702 /* 1703 * To support metadata register copy on Tx loopback, 1704 * this must be always enabled (metadata may arive 1705 * from other port - not from local flows only. 1706 */ 1707 if (tunnel) { 1708 unsigned int j; 1709 1710 /* Increase the counter matching the flow. */ 1711 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1712 if ((tunnels_info[j].tunnel & 1713 dev_handle->layers) == 1714 tunnels_info[j].tunnel) { 1715 rxq_ctrl->flow_tunnels_n[j]++; 1716 break; 1717 } 1718 } 1719 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1720 } 1721 } 1722 } 1723 1724 static void 1725 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1726 { 1727 struct mlx5_priv *priv = dev->data->dev_private; 1728 struct mlx5_rxq_ctrl *rxq_ctrl; 1729 uint16_t port_id; 1730 1731 if (priv->sh->shared_mark_enabled) 1732 return; 1733 if (priv->master || priv->representor) { 1734 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1735 struct mlx5_priv *opriv = 1736 rte_eth_devices[port_id].data->dev_private; 1737 1738 if (!opriv || 1739 opriv->sh != priv->sh || 1740 opriv->domain_id != priv->domain_id || 1741 opriv->mark_enabled) 1742 continue; 1743 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1744 rxq_ctrl->rxq.mark = 1; 1745 } 1746 opriv->mark_enabled = 1; 1747 } 1748 } else { 1749 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1750 rxq_ctrl->rxq.mark = 1; 1751 } 1752 priv->mark_enabled = 1; 1753 } 1754 priv->sh->shared_mark_enabled = 1; 1755 } 1756 1757 /** 1758 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1759 * 1760 * @param[in] dev 1761 * Pointer to the Ethernet device structure. 1762 * @param[in] flow 1763 * Pointer to flow structure. 1764 */ 1765 static void 1766 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1767 { 1768 struct mlx5_priv *priv = dev->data->dev_private; 1769 uint32_t handle_idx; 1770 struct mlx5_flow_handle *dev_handle; 1771 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1772 1773 MLX5_ASSERT(wks); 1774 if (wks->mark) 1775 flow_rxq_mark_flag_set(dev); 1776 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1777 handle_idx, dev_handle, next) 1778 flow_drv_rxq_flags_set(dev, dev_handle); 1779 } 1780 1781 /** 1782 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1783 * device flow if no other flow uses it with the same kind of request. 1784 * 1785 * @param dev 1786 * Pointer to Ethernet device. 1787 * @param[in] dev_handle 1788 * Pointer to the device flow handle structure. 1789 */ 1790 static void 1791 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1792 struct mlx5_flow_handle *dev_handle) 1793 { 1794 struct mlx5_priv *priv = dev->data->dev_private; 1795 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1796 struct mlx5_ind_table_obj *ind_tbl = NULL; 1797 unsigned int i; 1798 1799 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1800 struct mlx5_hrxq *hrxq; 1801 1802 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1803 dev_handle->rix_hrxq); 1804 if (hrxq) 1805 ind_tbl = hrxq->ind_table; 1806 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1807 struct mlx5_shared_action_rss *shared_rss; 1808 1809 shared_rss = mlx5_ipool_get 1810 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1811 dev_handle->rix_srss); 1812 if (shared_rss) 1813 ind_tbl = shared_rss->ind_tbl; 1814 } 1815 if (!ind_tbl) 1816 return; 1817 MLX5_ASSERT(dev->data->dev_started); 1818 for (i = 0; i != ind_tbl->queues_n; ++i) { 1819 int idx = ind_tbl->queues[i]; 1820 struct mlx5_rxq_ctrl *rxq_ctrl; 1821 1822 if (mlx5_is_external_rxq(dev, idx)) 1823 continue; 1824 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1825 MLX5_ASSERT(rxq_ctrl != NULL); 1826 if (rxq_ctrl == NULL) 1827 continue; 1828 if (tunnel) { 1829 unsigned int j; 1830 1831 /* Decrease the counter matching the flow. */ 1832 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1833 if ((tunnels_info[j].tunnel & 1834 dev_handle->layers) == 1835 tunnels_info[j].tunnel) { 1836 rxq_ctrl->flow_tunnels_n[j]--; 1837 break; 1838 } 1839 } 1840 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1841 } 1842 } 1843 } 1844 1845 /** 1846 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1847 * @p flow if no other flow uses it with the same kind of request. 1848 * 1849 * @param dev 1850 * Pointer to Ethernet device. 1851 * @param[in] flow 1852 * Pointer to the flow. 1853 */ 1854 static void 1855 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1856 { 1857 struct mlx5_priv *priv = dev->data->dev_private; 1858 uint32_t handle_idx; 1859 struct mlx5_flow_handle *dev_handle; 1860 1861 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1862 handle_idx, dev_handle, next) 1863 flow_drv_rxq_flags_trim(dev, dev_handle); 1864 } 1865 1866 /** 1867 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1868 * 1869 * @param dev 1870 * Pointer to Ethernet device. 1871 */ 1872 static void 1873 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1874 { 1875 struct mlx5_priv *priv = dev->data->dev_private; 1876 unsigned int i; 1877 1878 for (i = 0; i != priv->rxqs_n; ++i) { 1879 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1880 unsigned int j; 1881 1882 if (rxq == NULL || rxq->ctrl == NULL) 1883 continue; 1884 rxq->ctrl->rxq.mark = 0; 1885 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1886 rxq->ctrl->flow_tunnels_n[j] = 0; 1887 rxq->ctrl->rxq.tunnel = 0; 1888 } 1889 priv->mark_enabled = 0; 1890 priv->sh->shared_mark_enabled = 0; 1891 } 1892 1893 static uint64_t mlx5_restore_info_dynflag; 1894 1895 int 1896 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 1897 { 1898 struct mlx5_priv *priv = dev->data->dev_private; 1899 uint64_t supported = 0; 1900 1901 if (!is_tunnel_offload_active(dev)) { 1902 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 1903 supported |= RTE_ETH_RX_METADATA_USER_MARK; 1904 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) { 1905 DRV_LOG(DEBUG, 1906 "tunnel offload was not activated, consider setting dv_xmeta_en=%d", 1907 MLX5_XMETA_MODE_MISS_INFO); 1908 } 1909 } else { 1910 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 1911 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 && 1912 mlx5_restore_info_dynflag == 0) 1913 mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag(); 1914 } 1915 1916 if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) 1917 priv->tunnel_enabled = 1; 1918 else 1919 priv->tunnel_enabled = 0; 1920 1921 *features &= supported; 1922 return 0; 1923 } 1924 1925 /** 1926 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1927 * 1928 * @param[in] dev 1929 * Pointer to the Ethernet device structure. 1930 */ 1931 void 1932 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) 1933 { 1934 struct mlx5_priv *priv = dev->data->dev_private; 1935 uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID; 1936 unsigned int i; 1937 1938 if (priv->tunnel_enabled) 1939 mark_flag |= mlx5_restore_info_dynflag; 1940 1941 for (i = 0; i != priv->rxqs_n; ++i) { 1942 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1943 struct mlx5_rxq_data *data; 1944 1945 if (rxq == NULL || rxq->ctrl == NULL) 1946 continue; 1947 data = &rxq->ctrl->rxq; 1948 if (!rte_flow_dynf_metadata_avail()) { 1949 data->dynf_meta = 0; 1950 data->flow_meta_mask = 0; 1951 data->flow_meta_offset = -1; 1952 data->flow_meta_port_mask = 0; 1953 } else { 1954 data->dynf_meta = 1; 1955 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1956 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1957 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1958 } 1959 data->mark_flag = mark_flag; 1960 } 1961 } 1962 1963 /* 1964 * return a pointer to the desired action in the list of actions. 1965 * 1966 * @param[in] actions 1967 * The list of actions to search the action in. 1968 * @param[in] action 1969 * The action to find. 1970 * 1971 * @return 1972 * Pointer to the action in the list, if found. NULL otherwise. 1973 */ 1974 const struct rte_flow_action * 1975 mlx5_flow_find_action(const struct rte_flow_action *actions, 1976 enum rte_flow_action_type action) 1977 { 1978 if (actions == NULL) 1979 return NULL; 1980 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1981 if (actions->type == action) 1982 return actions; 1983 return NULL; 1984 } 1985 1986 /* 1987 * Validate the flag action. 1988 * 1989 * @param[in] action_flags 1990 * Bit-fields that holds the actions detected until now. 1991 * @param[in] attr 1992 * Attributes of flow that includes this action. 1993 * @param[out] error 1994 * Pointer to error structure. 1995 * 1996 * @return 1997 * 0 on success, a negative errno value otherwise and rte_errno is set. 1998 */ 1999 int 2000 mlx5_flow_validate_action_flag(uint64_t action_flags, 2001 const struct rte_flow_attr *attr, 2002 struct rte_flow_error *error) 2003 { 2004 if (action_flags & MLX5_FLOW_ACTION_MARK) 2005 return rte_flow_error_set(error, EINVAL, 2006 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2007 "can't mark and flag in same flow"); 2008 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2009 return rte_flow_error_set(error, EINVAL, 2010 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2011 "can't have 2 flag" 2012 " actions in same flow"); 2013 if (attr->egress) 2014 return rte_flow_error_set(error, ENOTSUP, 2015 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2016 "flag action not supported for " 2017 "egress"); 2018 return 0; 2019 } 2020 2021 /* 2022 * Validate the mark action. 2023 * 2024 * @param[in] action 2025 * Pointer to the queue action. 2026 * @param[in] action_flags 2027 * Bit-fields that holds the actions detected until now. 2028 * @param[in] attr 2029 * Attributes of flow that includes this action. 2030 * @param[out] error 2031 * Pointer to error structure. 2032 * 2033 * @return 2034 * 0 on success, a negative errno value otherwise and rte_errno is set. 2035 */ 2036 int 2037 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 2038 uint64_t action_flags, 2039 const struct rte_flow_attr *attr, 2040 struct rte_flow_error *error) 2041 { 2042 const struct rte_flow_action_mark *mark = action->conf; 2043 2044 if (!mark) 2045 return rte_flow_error_set(error, EINVAL, 2046 RTE_FLOW_ERROR_TYPE_ACTION, 2047 action, 2048 "configuration cannot be null"); 2049 if (mark->id >= MLX5_FLOW_MARK_MAX) 2050 return rte_flow_error_set(error, EINVAL, 2051 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2052 &mark->id, 2053 "mark id must in 0 <= id < " 2054 RTE_STR(MLX5_FLOW_MARK_MAX)); 2055 if (action_flags & MLX5_FLOW_ACTION_FLAG) 2056 return rte_flow_error_set(error, EINVAL, 2057 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2058 "can't flag and mark in same flow"); 2059 if (action_flags & MLX5_FLOW_ACTION_MARK) 2060 return rte_flow_error_set(error, EINVAL, 2061 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2062 "can't have 2 mark actions in same" 2063 " flow"); 2064 if (attr->egress) 2065 return rte_flow_error_set(error, ENOTSUP, 2066 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2067 "mark action not supported for " 2068 "egress"); 2069 return 0; 2070 } 2071 2072 /* 2073 * Validate the drop action. 2074 * 2075 * @param[in] dev 2076 * Pointer to the Ethernet device structure. 2077 * @param[in] is_root 2078 * True if flow is validated for root table. False otherwise. 2079 * @param[in] attr 2080 * Attributes of flow that includes this action. 2081 * @param[out] error 2082 * Pointer to error structure. 2083 * 2084 * @return 2085 * 0 on success, a negative errno value otherwise and rte_errno is set. 2086 */ 2087 int 2088 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 2089 bool is_root, 2090 const struct rte_flow_attr *attr, 2091 struct rte_flow_error *error) 2092 { 2093 struct mlx5_priv *priv = dev->data->dev_private; 2094 2095 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 2096 return rte_flow_error_set(error, ENOTSUP, 2097 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2098 "drop action not supported for " 2099 "egress"); 2100 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 2101 !priv->sh->dr_root_drop_action_en) { 2102 return rte_flow_error_set(error, ENOTSUP, 2103 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 2104 "drop action not supported for " 2105 "egress and transfer on group 0"); 2106 } 2107 return 0; 2108 } 2109 2110 /* 2111 * Validate the queue action. 2112 * 2113 * @param[in] action 2114 * Pointer to the queue action. 2115 * @param[in] action_flags 2116 * Bit-fields that holds the actions detected until now. 2117 * @param[in] dev 2118 * Pointer to the Ethernet device structure. 2119 * @param[in] attr 2120 * Attributes of flow that includes this action. 2121 * @param[out] error 2122 * Pointer to error structure. 2123 * 2124 * @return 2125 * 0 on success, a negative errno value otherwise and rte_errno is set. 2126 */ 2127 int 2128 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 2129 uint64_t action_flags, 2130 struct rte_eth_dev *dev, 2131 const struct rte_flow_attr *attr, 2132 struct rte_flow_error *error) 2133 { 2134 struct mlx5_priv *priv = dev->data->dev_private; 2135 const struct rte_flow_action_queue *queue = action->conf; 2136 2137 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2138 return rte_flow_error_set(error, EINVAL, 2139 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2140 "can't have 2 fate actions in" 2141 " same flow"); 2142 if (attr->egress) 2143 return rte_flow_error_set(error, ENOTSUP, 2144 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2145 "queue action not supported for egress."); 2146 if (mlx5_is_external_rxq(dev, queue->index)) 2147 return 0; 2148 if (!priv->rxqs_n) 2149 return rte_flow_error_set(error, EINVAL, 2150 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2151 NULL, "No Rx queues configured"); 2152 if (queue->index >= priv->rxqs_n) 2153 return rte_flow_error_set(error, EINVAL, 2154 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2155 &queue->index, 2156 "queue index out of range"); 2157 if (mlx5_rxq_get(dev, queue->index) == NULL) 2158 return rte_flow_error_set(error, EINVAL, 2159 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2160 &queue->index, 2161 "queue is not configured"); 2162 return 0; 2163 } 2164 2165 /** 2166 * Validate queue numbers for device RSS. 2167 * 2168 * @param[in] dev 2169 * Configured device. 2170 * @param[in] queues 2171 * Array of queue numbers. 2172 * @param[in] queues_n 2173 * Size of the @p queues array. 2174 * @param[out] error 2175 * On error, filled with a textual error description. 2176 * @param[out] queue_idx 2177 * On error, filled with an offending queue index in @p queues array. 2178 * 2179 * @return 2180 * 0 on success, a negative errno code on error. 2181 */ 2182 static int 2183 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2184 const uint16_t *queues, uint32_t queues_n, 2185 const char **error, uint32_t *queue_idx) 2186 { 2187 const struct mlx5_priv *priv = dev->data->dev_private; 2188 bool is_hairpin = false; 2189 bool is_ext_rss = false; 2190 uint32_t i; 2191 2192 for (i = 0; i != queues_n; ++i) { 2193 struct mlx5_rxq_ctrl *rxq_ctrl; 2194 2195 if (mlx5_is_external_rxq(dev, queues[0])) { 2196 is_ext_rss = true; 2197 continue; 2198 } 2199 if (is_ext_rss) { 2200 *error = "Combining external and regular RSS queues is not supported"; 2201 *queue_idx = i; 2202 return -ENOTSUP; 2203 } 2204 if (queues[i] >= priv->rxqs_n) { 2205 *error = "queue index out of range"; 2206 *queue_idx = i; 2207 return -EINVAL; 2208 } 2209 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2210 if (rxq_ctrl == NULL) { 2211 *error = "queue is not configured"; 2212 *queue_idx = i; 2213 return -EINVAL; 2214 } 2215 if (i == 0 && rxq_ctrl->is_hairpin) 2216 is_hairpin = true; 2217 if (is_hairpin != rxq_ctrl->is_hairpin) { 2218 *error = "combining hairpin and regular RSS queues is not supported"; 2219 *queue_idx = i; 2220 return -ENOTSUP; 2221 } 2222 } 2223 return 0; 2224 } 2225 2226 /* 2227 * Validate the rss action. 2228 * 2229 * @param[in] dev 2230 * Pointer to the Ethernet device structure. 2231 * @param[in] action 2232 * Pointer to the queue action. 2233 * @param[out] error 2234 * Pointer to error structure. 2235 * 2236 * @return 2237 * 0 on success, a negative errno value otherwise and rte_errno is set. 2238 */ 2239 int 2240 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2241 const struct rte_flow_action *action, 2242 struct rte_flow_error *error) 2243 { 2244 struct mlx5_priv *priv = dev->data->dev_private; 2245 const struct rte_flow_action_rss *rss = action->conf; 2246 int ret; 2247 const char *message; 2248 uint32_t queue_idx; 2249 2250 if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 2251 DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT", 2252 dev->data->port_id); 2253 } else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2254 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2255 return rte_flow_error_set(error, ENOTSUP, 2256 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2257 &rss->func, 2258 "RSS hash function not supported"); 2259 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2260 if (rss->level > 2) 2261 #else 2262 if (rss->level > 1) 2263 #endif 2264 return rte_flow_error_set(error, ENOTSUP, 2265 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2266 &rss->level, 2267 "tunnel RSS is not supported"); 2268 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2269 if (rss->key_len == 0 && rss->key != NULL) 2270 return rte_flow_error_set(error, ENOTSUP, 2271 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2272 &rss->key_len, 2273 "RSS hash key length 0"); 2274 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2275 return rte_flow_error_set(error, ENOTSUP, 2276 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2277 &rss->key_len, 2278 "RSS hash key too small"); 2279 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2280 return rte_flow_error_set(error, ENOTSUP, 2281 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2282 &rss->key_len, 2283 "RSS hash key too large"); 2284 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2285 return rte_flow_error_set(error, ENOTSUP, 2286 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2287 &rss->queue_num, 2288 "number of queues too large"); 2289 if (rss->types & MLX5_RSS_HF_MASK) 2290 return rte_flow_error_set(error, ENOTSUP, 2291 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2292 &rss->types, 2293 "some RSS protocols are not" 2294 " supported"); 2295 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2296 !(rss->types & RTE_ETH_RSS_IP)) 2297 return rte_flow_error_set(error, EINVAL, 2298 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2299 "L3 partial RSS requested but L3 RSS" 2300 " type not specified"); 2301 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2302 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2303 return rte_flow_error_set(error, EINVAL, 2304 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2305 "L4 partial RSS requested but L4 RSS" 2306 " type not specified"); 2307 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2308 return rte_flow_error_set(error, EINVAL, 2309 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2310 NULL, "No Rx queues configured"); 2311 if (!rss->queue_num) 2312 return rte_flow_error_set(error, EINVAL, 2313 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2314 NULL, "No queues configured"); 2315 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2316 &message, &queue_idx); 2317 if (ret != 0) { 2318 return rte_flow_error_set(error, -ret, 2319 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2320 &rss->queue[queue_idx], message); 2321 } 2322 return 0; 2323 } 2324 2325 /* 2326 * Validate the rss action. 2327 * 2328 * @param[in] action 2329 * Pointer to the queue action. 2330 * @param[in] action_flags 2331 * Bit-fields that holds the actions detected until now. 2332 * @param[in] dev 2333 * Pointer to the Ethernet device structure. 2334 * @param[in] attr 2335 * Attributes of flow that includes this action. 2336 * @param[in] item_flags 2337 * Items that were detected. 2338 * @param[out] error 2339 * Pointer to error structure. 2340 * 2341 * @return 2342 * 0 on success, a negative errno value otherwise and rte_errno is set. 2343 */ 2344 int 2345 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2346 uint64_t action_flags, 2347 struct rte_eth_dev *dev, 2348 const struct rte_flow_attr *attr, 2349 uint64_t item_flags, 2350 struct rte_flow_error *error) 2351 { 2352 const struct rte_flow_action_rss *rss = action->conf; 2353 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2354 int ret; 2355 2356 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2357 return rte_flow_error_set(error, EINVAL, 2358 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2359 "can't have 2 fate actions" 2360 " in same flow"); 2361 ret = mlx5_validate_action_rss(dev, action, error); 2362 if (ret) 2363 return ret; 2364 if (attr->egress) 2365 return rte_flow_error_set(error, ENOTSUP, 2366 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2367 "rss action not supported for " 2368 "egress"); 2369 if (rss->level > 1 && !tunnel) 2370 return rte_flow_error_set(error, EINVAL, 2371 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2372 "inner RSS is not supported for " 2373 "non-tunnel flows"); 2374 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2375 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2376 return rte_flow_error_set(error, EINVAL, 2377 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2378 "RSS on eCPRI is not supported now"); 2379 } 2380 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2381 !(item_flags & 2382 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2383 rss->level > 1) 2384 return rte_flow_error_set(error, EINVAL, 2385 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2386 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2387 return 0; 2388 } 2389 2390 /* 2391 * Validate the default miss action. 2392 * 2393 * @param[in] action_flags 2394 * Bit-fields that holds the actions detected until now. 2395 * @param[out] error 2396 * Pointer to error structure. 2397 * 2398 * @return 2399 * 0 on success, a negative errno value otherwise and rte_errno is set. 2400 */ 2401 int 2402 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2403 const struct rte_flow_attr *attr, 2404 struct rte_flow_error *error) 2405 { 2406 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2407 return rte_flow_error_set(error, EINVAL, 2408 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2409 "can't have 2 fate actions in" 2410 " same flow"); 2411 if (attr->egress) 2412 return rte_flow_error_set(error, ENOTSUP, 2413 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2414 "default miss action not supported " 2415 "for egress"); 2416 if (attr->group) 2417 return rte_flow_error_set(error, ENOTSUP, 2418 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2419 "only group 0 is supported"); 2420 if (attr->transfer) 2421 return rte_flow_error_set(error, ENOTSUP, 2422 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2423 NULL, "transfer is not supported"); 2424 return 0; 2425 } 2426 2427 /* 2428 * Validate the count action. 2429 * 2430 * @param[in] dev 2431 * Pointer to the Ethernet device structure. 2432 * @param[in] attr 2433 * Attributes of flow that includes this action. 2434 * @param[out] error 2435 * Pointer to error structure. 2436 * 2437 * @return 2438 * 0 on success, a negative errno value otherwise and rte_errno is set. 2439 */ 2440 int 2441 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2442 const struct rte_flow_attr *attr, 2443 struct rte_flow_error *error) 2444 { 2445 if (attr->egress) 2446 return rte_flow_error_set(error, ENOTSUP, 2447 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2448 "count action not supported for " 2449 "egress"); 2450 return 0; 2451 } 2452 2453 /* 2454 * Validate the ASO CT action. 2455 * 2456 * @param[in] dev 2457 * Pointer to the Ethernet device structure. 2458 * @param[in] conntrack 2459 * Pointer to the CT action profile. 2460 * @param[out] error 2461 * Pointer to error structure. 2462 * 2463 * @return 2464 * 0 on success, a negative errno value otherwise and rte_errno is set. 2465 */ 2466 int 2467 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2468 const struct rte_flow_action_conntrack *conntrack, 2469 struct rte_flow_error *error) 2470 { 2471 RTE_SET_USED(dev); 2472 2473 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2474 return rte_flow_error_set(error, EINVAL, 2475 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2476 "Invalid CT state"); 2477 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2478 return rte_flow_error_set(error, EINVAL, 2479 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2480 "Invalid last TCP packet flag"); 2481 return 0; 2482 } 2483 2484 /** 2485 * Validate the level value for modify field action. 2486 * 2487 * @param[in] data 2488 * Pointer to the rte_flow_action_modify_data structure either src or dst. 2489 * @param[out] error 2490 * Pointer to error structure. 2491 * 2492 * @return 2493 * 0 on success, a negative errno value otherwise and rte_errno is set. 2494 */ 2495 int 2496 flow_validate_modify_field_level(const struct rte_flow_action_modify_data *data, 2497 struct rte_flow_error *error) 2498 { 2499 if (data->level == 0) 2500 return 0; 2501 if (data->field != RTE_FLOW_FIELD_TAG && 2502 data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) 2503 return rte_flow_error_set(error, ENOTSUP, 2504 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2505 "inner header fields modification is not supported"); 2506 if (data->tag_index != 0) 2507 return rte_flow_error_set(error, EINVAL, 2508 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2509 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2510 /* 2511 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2512 * 'tag_index' field. In old API, it was provided using 'level' field 2513 * and it is still supported for backwards compatibility. 2514 */ 2515 DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field."); 2516 return 0; 2517 } 2518 2519 /** 2520 * Validate ICMP6 item. 2521 * 2522 * @param[in] item 2523 * Item specification. 2524 * @param[in] item_flags 2525 * Bit-fields that holds the items detected until now. 2526 * @param[in] ext_vlan_sup 2527 * Whether extended VLAN features are supported or not. 2528 * @param[out] error 2529 * Pointer to error structure. 2530 * 2531 * @return 2532 * 0 on success, a negative errno value otherwise and rte_errno is set. 2533 */ 2534 int 2535 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2536 uint64_t item_flags, 2537 uint8_t target_protocol, 2538 struct rte_flow_error *error) 2539 { 2540 const struct rte_flow_item_icmp6 *mask = item->mask; 2541 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2542 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2543 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2544 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2545 MLX5_FLOW_LAYER_OUTER_L4; 2546 int ret; 2547 2548 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2549 return rte_flow_error_set(error, EINVAL, 2550 RTE_FLOW_ERROR_TYPE_ITEM, item, 2551 "protocol filtering not compatible" 2552 " with ICMP6 layer"); 2553 if (!(item_flags & l3m)) 2554 return rte_flow_error_set(error, EINVAL, 2555 RTE_FLOW_ERROR_TYPE_ITEM, item, 2556 "IPv6 is mandatory to filter on" 2557 " ICMP6"); 2558 if (item_flags & l4m) 2559 return rte_flow_error_set(error, EINVAL, 2560 RTE_FLOW_ERROR_TYPE_ITEM, item, 2561 "multiple L4 layers not supported"); 2562 if (!mask) 2563 mask = &rte_flow_item_icmp6_mask; 2564 ret = mlx5_flow_item_acceptable 2565 (item, (const uint8_t *)mask, 2566 (const uint8_t *)&rte_flow_item_icmp6_mask, 2567 sizeof(struct rte_flow_item_icmp6), 2568 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2569 if (ret < 0) 2570 return ret; 2571 return 0; 2572 } 2573 2574 /** 2575 * Validate ICMP6 echo request/reply item. 2576 * 2577 * @param[in] item 2578 * Item specification. 2579 * @param[in] item_flags 2580 * Bit-fields that holds the items detected until now. 2581 * @param[in] ext_vlan_sup 2582 * Whether extended VLAN features are supported or not. 2583 * @param[out] error 2584 * Pointer to error structure. 2585 * 2586 * @return 2587 * 0 on success, a negative errno value otherwise and rte_errno is set. 2588 */ 2589 int 2590 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2591 uint64_t item_flags, 2592 uint8_t target_protocol, 2593 struct rte_flow_error *error) 2594 { 2595 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2596 const struct rte_flow_item_icmp6_echo nic_mask = { 2597 .hdr.base.type = 0xff, 2598 .hdr.base.code = 0xff, 2599 .hdr.identifier = RTE_BE16(0xffff), 2600 .hdr.sequence = RTE_BE16(0xffff), 2601 }; 2602 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2603 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2604 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2605 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2606 MLX5_FLOW_LAYER_OUTER_L4; 2607 int ret; 2608 2609 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2610 return rte_flow_error_set(error, EINVAL, 2611 RTE_FLOW_ERROR_TYPE_ITEM, item, 2612 "protocol filtering not compatible" 2613 " with ICMP6 layer"); 2614 if (!(item_flags & l3m)) 2615 return rte_flow_error_set(error, EINVAL, 2616 RTE_FLOW_ERROR_TYPE_ITEM, item, 2617 "IPv6 is mandatory to filter on" 2618 " ICMP6"); 2619 if (item_flags & l4m) 2620 return rte_flow_error_set(error, EINVAL, 2621 RTE_FLOW_ERROR_TYPE_ITEM, item, 2622 "multiple L4 layers not supported"); 2623 if (!mask) 2624 mask = &nic_mask; 2625 ret = mlx5_flow_item_acceptable 2626 (item, (const uint8_t *)mask, 2627 (const uint8_t *)&nic_mask, 2628 sizeof(struct rte_flow_item_icmp6_echo), 2629 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2630 if (ret < 0) 2631 return ret; 2632 return 0; 2633 } 2634 2635 /** 2636 * Validate ICMP item. 2637 * 2638 * @param[in] item 2639 * Item specification. 2640 * @param[in] item_flags 2641 * Bit-fields that holds the items detected until now. 2642 * @param[out] error 2643 * Pointer to error structure. 2644 * 2645 * @return 2646 * 0 on success, a negative errno value otherwise and rte_errno is set. 2647 */ 2648 int 2649 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2650 uint64_t item_flags, 2651 uint8_t target_protocol, 2652 struct rte_flow_error *error) 2653 { 2654 const struct rte_flow_item_icmp *mask = item->mask; 2655 const struct rte_flow_item_icmp nic_mask = { 2656 .hdr.icmp_type = 0xff, 2657 .hdr.icmp_code = 0xff, 2658 .hdr.icmp_ident = RTE_BE16(0xffff), 2659 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2660 }; 2661 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2662 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2663 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2664 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2665 MLX5_FLOW_LAYER_OUTER_L4; 2666 int ret; 2667 2668 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2669 return rte_flow_error_set(error, EINVAL, 2670 RTE_FLOW_ERROR_TYPE_ITEM, item, 2671 "protocol filtering not compatible" 2672 " with ICMP layer"); 2673 if (!(item_flags & l3m)) 2674 return rte_flow_error_set(error, EINVAL, 2675 RTE_FLOW_ERROR_TYPE_ITEM, item, 2676 "IPv4 is mandatory to filter" 2677 " on ICMP"); 2678 if (item_flags & l4m) 2679 return rte_flow_error_set(error, EINVAL, 2680 RTE_FLOW_ERROR_TYPE_ITEM, item, 2681 "multiple L4 layers not supported"); 2682 if (!mask) 2683 mask = &nic_mask; 2684 ret = mlx5_flow_item_acceptable 2685 (item, (const uint8_t *)mask, 2686 (const uint8_t *)&nic_mask, 2687 sizeof(struct rte_flow_item_icmp), 2688 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2689 if (ret < 0) 2690 return ret; 2691 return 0; 2692 } 2693 2694 /** 2695 * Validate Ethernet item. 2696 * 2697 * @param[in] item 2698 * Item specification. 2699 * @param[in] item_flags 2700 * Bit-fields that holds the items detected until now. 2701 * @param[out] error 2702 * Pointer to error structure. 2703 * 2704 * @return 2705 * 0 on success, a negative errno value otherwise and rte_errno is set. 2706 */ 2707 int 2708 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2709 uint64_t item_flags, bool ext_vlan_sup, 2710 struct rte_flow_error *error) 2711 { 2712 const struct rte_flow_item_eth *mask = item->mask; 2713 const struct rte_flow_item_eth nic_mask = { 2714 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2715 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2716 .hdr.ether_type = RTE_BE16(0xffff), 2717 .has_vlan = ext_vlan_sup ? 1 : 0, 2718 }; 2719 int ret; 2720 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2721 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2722 MLX5_FLOW_LAYER_OUTER_L2; 2723 2724 if (item_flags & ethm) 2725 return rte_flow_error_set(error, ENOTSUP, 2726 RTE_FLOW_ERROR_TYPE_ITEM, item, 2727 "multiple L2 layers not supported"); 2728 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2729 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2730 return rte_flow_error_set(error, EINVAL, 2731 RTE_FLOW_ERROR_TYPE_ITEM, item, 2732 "L2 layer should not follow " 2733 "L3 layers"); 2734 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2735 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2736 return rte_flow_error_set(error, EINVAL, 2737 RTE_FLOW_ERROR_TYPE_ITEM, item, 2738 "L2 layer should not follow VLAN"); 2739 if (item_flags & MLX5_FLOW_LAYER_GTP) 2740 return rte_flow_error_set(error, EINVAL, 2741 RTE_FLOW_ERROR_TYPE_ITEM, item, 2742 "L2 layer should not follow GTP"); 2743 if (!mask) 2744 mask = &rte_flow_item_eth_mask; 2745 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2746 (const uint8_t *)&nic_mask, 2747 sizeof(struct rte_flow_item_eth), 2748 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2749 return ret; 2750 } 2751 2752 /** 2753 * Validate VLAN item. 2754 * 2755 * @param[in] item 2756 * Item specification. 2757 * @param[in] item_flags 2758 * Bit-fields that holds the items detected until now. 2759 * @param[in] dev 2760 * Ethernet device flow is being created on. 2761 * @param[out] error 2762 * Pointer to error structure. 2763 * 2764 * @return 2765 * 0 on success, a negative errno value otherwise and rte_errno is set. 2766 */ 2767 int 2768 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2769 uint64_t item_flags, 2770 struct rte_eth_dev *dev, 2771 struct rte_flow_error *error) 2772 { 2773 const struct rte_flow_item_vlan *spec = item->spec; 2774 const struct rte_flow_item_vlan *mask = item->mask; 2775 const struct rte_flow_item_vlan nic_mask = { 2776 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2777 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2778 }; 2779 uint16_t vlan_tag = 0; 2780 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2781 int ret; 2782 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2783 MLX5_FLOW_LAYER_INNER_L4) : 2784 (MLX5_FLOW_LAYER_OUTER_L3 | 2785 MLX5_FLOW_LAYER_OUTER_L4); 2786 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2787 MLX5_FLOW_LAYER_OUTER_VLAN; 2788 2789 if (item_flags & vlanm) 2790 return rte_flow_error_set(error, EINVAL, 2791 RTE_FLOW_ERROR_TYPE_ITEM, item, 2792 "multiple VLAN layers not supported"); 2793 else if ((item_flags & l34m) != 0) 2794 return rte_flow_error_set(error, EINVAL, 2795 RTE_FLOW_ERROR_TYPE_ITEM, item, 2796 "VLAN cannot follow L3/L4 layer"); 2797 if (!mask) 2798 mask = &rte_flow_item_vlan_mask; 2799 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2800 (const uint8_t *)&nic_mask, 2801 sizeof(struct rte_flow_item_vlan), 2802 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2803 if (ret) 2804 return ret; 2805 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2806 struct mlx5_priv *priv = dev->data->dev_private; 2807 2808 if (priv->vmwa_context) { 2809 /* 2810 * Non-NULL context means we have a virtual machine 2811 * and SR-IOV enabled, we have to create VLAN interface 2812 * to make hypervisor to setup E-Switch vport 2813 * context correctly. We avoid creating the multiple 2814 * VLAN interfaces, so we cannot support VLAN tag mask. 2815 */ 2816 return rte_flow_error_set(error, EINVAL, 2817 RTE_FLOW_ERROR_TYPE_ITEM, 2818 item, 2819 "VLAN tag mask is not" 2820 " supported in virtual" 2821 " environment"); 2822 } 2823 } 2824 if (spec) { 2825 vlan_tag = spec->hdr.vlan_tci; 2826 vlan_tag &= mask->hdr.vlan_tci; 2827 } 2828 /* 2829 * From verbs perspective an empty VLAN is equivalent 2830 * to a packet without VLAN layer. 2831 */ 2832 if (!vlan_tag) 2833 return rte_flow_error_set(error, EINVAL, 2834 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2835 item->spec, 2836 "VLAN cannot be empty"); 2837 return 0; 2838 } 2839 2840 /** 2841 * Validate IPV4 item. 2842 * 2843 * @param[in] item 2844 * Item specification. 2845 * @param[in] item_flags 2846 * Bit-fields that holds the items detected until now. 2847 * @param[in] last_item 2848 * Previous validated item in the pattern items. 2849 * @param[in] ether_type 2850 * Type in the ethernet layer header (including dot1q). 2851 * @param[in] acc_mask 2852 * Acceptable mask, if NULL default internal default mask 2853 * will be used to check whether item fields are supported. 2854 * @param[in] range_accepted 2855 * True if range of values is accepted for specific fields, false otherwise. 2856 * @param[out] error 2857 * Pointer to error structure. 2858 * 2859 * @return 2860 * 0 on success, a negative errno value otherwise and rte_errno is set. 2861 */ 2862 int 2863 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2864 uint64_t item_flags, 2865 uint64_t last_item, 2866 uint16_t ether_type, 2867 const struct rte_flow_item_ipv4 *acc_mask, 2868 bool range_accepted, 2869 struct rte_flow_error *error) 2870 { 2871 const struct rte_flow_item_ipv4 *mask = item->mask; 2872 const struct rte_flow_item_ipv4 *spec = item->spec; 2873 const struct rte_flow_item_ipv4 nic_mask = { 2874 .hdr = { 2875 .src_addr = RTE_BE32(0xffffffff), 2876 .dst_addr = RTE_BE32(0xffffffff), 2877 .type_of_service = 0xff, 2878 .next_proto_id = 0xff, 2879 }, 2880 }; 2881 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2882 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2883 MLX5_FLOW_LAYER_OUTER_L3; 2884 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2885 MLX5_FLOW_LAYER_OUTER_L4; 2886 int ret; 2887 uint8_t next_proto = 0xFF; 2888 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2889 MLX5_FLOW_LAYER_OUTER_VLAN | 2890 MLX5_FLOW_LAYER_INNER_VLAN); 2891 2892 if ((last_item & l2_vlan) && ether_type && 2893 ether_type != RTE_ETHER_TYPE_IPV4) 2894 return rte_flow_error_set(error, EINVAL, 2895 RTE_FLOW_ERROR_TYPE_ITEM, item, 2896 "IPv4 cannot follow L2/VLAN layer " 2897 "which ether type is not IPv4"); 2898 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2899 if (mask && spec) 2900 next_proto = mask->hdr.next_proto_id & 2901 spec->hdr.next_proto_id; 2902 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2903 return rte_flow_error_set(error, EINVAL, 2904 RTE_FLOW_ERROR_TYPE_ITEM, 2905 item, 2906 "multiple tunnel " 2907 "not supported"); 2908 } 2909 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2910 return rte_flow_error_set(error, EINVAL, 2911 RTE_FLOW_ERROR_TYPE_ITEM, item, 2912 "wrong tunnel type - IPv6 specified " 2913 "but IPv4 item provided"); 2914 if (item_flags & l3m) 2915 return rte_flow_error_set(error, ENOTSUP, 2916 RTE_FLOW_ERROR_TYPE_ITEM, item, 2917 "multiple L3 layers not supported"); 2918 else if (item_flags & l4m) 2919 return rte_flow_error_set(error, EINVAL, 2920 RTE_FLOW_ERROR_TYPE_ITEM, item, 2921 "L3 cannot follow an L4 layer."); 2922 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2923 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2924 return rte_flow_error_set(error, EINVAL, 2925 RTE_FLOW_ERROR_TYPE_ITEM, item, 2926 "L3 cannot follow an NVGRE layer."); 2927 if (!mask) 2928 mask = &rte_flow_item_ipv4_mask; 2929 else if (mask->hdr.next_proto_id != 0 && 2930 mask->hdr.next_proto_id != 0xff) 2931 return rte_flow_error_set(error, EINVAL, 2932 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2933 "partial mask is not supported" 2934 " for protocol"); 2935 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2936 acc_mask ? (const uint8_t *)acc_mask 2937 : (const uint8_t *)&nic_mask, 2938 sizeof(struct rte_flow_item_ipv4), 2939 range_accepted, error); 2940 if (ret < 0) 2941 return ret; 2942 return 0; 2943 } 2944 2945 /** 2946 * Validate IPV6 item. 2947 * 2948 * @param[in] item 2949 * Item specification. 2950 * @param[in] item_flags 2951 * Bit-fields that holds the items detected until now. 2952 * @param[in] last_item 2953 * Previous validated item in the pattern items. 2954 * @param[in] ether_type 2955 * Type in the ethernet layer header (including dot1q). 2956 * @param[in] acc_mask 2957 * Acceptable mask, if NULL default internal default mask 2958 * will be used to check whether item fields are supported. 2959 * @param[out] error 2960 * Pointer to error structure. 2961 * 2962 * @return 2963 * 0 on success, a negative errno value otherwise and rte_errno is set. 2964 */ 2965 int 2966 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2967 uint64_t item_flags, 2968 uint64_t last_item, 2969 uint16_t ether_type, 2970 const struct rte_flow_item_ipv6 *acc_mask, 2971 struct rte_flow_error *error) 2972 { 2973 const struct rte_flow_item_ipv6 *mask = item->mask; 2974 const struct rte_flow_item_ipv6 *spec = item->spec; 2975 const struct rte_flow_item_ipv6 nic_mask = { 2976 .hdr = { 2977 .src_addr = 2978 "\xff\xff\xff\xff\xff\xff\xff\xff" 2979 "\xff\xff\xff\xff\xff\xff\xff\xff", 2980 .dst_addr = 2981 "\xff\xff\xff\xff\xff\xff\xff\xff" 2982 "\xff\xff\xff\xff\xff\xff\xff\xff", 2983 .vtc_flow = RTE_BE32(0xffffffff), 2984 .proto = 0xff, 2985 }, 2986 }; 2987 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2988 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2989 MLX5_FLOW_LAYER_OUTER_L3; 2990 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2991 MLX5_FLOW_LAYER_OUTER_L4; 2992 int ret; 2993 uint8_t next_proto = 0xFF; 2994 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2995 MLX5_FLOW_LAYER_OUTER_VLAN | 2996 MLX5_FLOW_LAYER_INNER_VLAN); 2997 2998 if ((last_item & l2_vlan) && ether_type && 2999 ether_type != RTE_ETHER_TYPE_IPV6) 3000 return rte_flow_error_set(error, EINVAL, 3001 RTE_FLOW_ERROR_TYPE_ITEM, item, 3002 "IPv6 cannot follow L2/VLAN layer " 3003 "which ether type is not IPv6"); 3004 if (mask && mask->hdr.proto == UINT8_MAX && spec) 3005 next_proto = spec->hdr.proto; 3006 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 3007 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 3008 return rte_flow_error_set(error, EINVAL, 3009 RTE_FLOW_ERROR_TYPE_ITEM, 3010 item, 3011 "multiple tunnel " 3012 "not supported"); 3013 } 3014 if (next_proto == IPPROTO_HOPOPTS || 3015 next_proto == IPPROTO_ROUTING || 3016 next_proto == IPPROTO_FRAGMENT || 3017 next_proto == IPPROTO_ESP || 3018 next_proto == IPPROTO_AH || 3019 next_proto == IPPROTO_DSTOPTS) 3020 return rte_flow_error_set(error, EINVAL, 3021 RTE_FLOW_ERROR_TYPE_ITEM, item, 3022 "IPv6 proto (next header) should " 3023 "not be set as extension header"); 3024 if (item_flags & MLX5_FLOW_LAYER_IPIP) 3025 return rte_flow_error_set(error, EINVAL, 3026 RTE_FLOW_ERROR_TYPE_ITEM, item, 3027 "wrong tunnel type - IPv4 specified " 3028 "but IPv6 item provided"); 3029 if (item_flags & l3m) 3030 return rte_flow_error_set(error, ENOTSUP, 3031 RTE_FLOW_ERROR_TYPE_ITEM, item, 3032 "multiple L3 layers not supported"); 3033 else if (item_flags & l4m) 3034 return rte_flow_error_set(error, EINVAL, 3035 RTE_FLOW_ERROR_TYPE_ITEM, item, 3036 "L3 cannot follow an L4 layer."); 3037 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 3038 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 3039 return rte_flow_error_set(error, EINVAL, 3040 RTE_FLOW_ERROR_TYPE_ITEM, item, 3041 "L3 cannot follow an NVGRE layer."); 3042 if (!mask) 3043 mask = &rte_flow_item_ipv6_mask; 3044 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3045 acc_mask ? (const uint8_t *)acc_mask 3046 : (const uint8_t *)&nic_mask, 3047 sizeof(struct rte_flow_item_ipv6), 3048 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3049 if (ret < 0) 3050 return ret; 3051 return 0; 3052 } 3053 3054 /** 3055 * Validate UDP item. 3056 * 3057 * @param[in] item 3058 * Item specification. 3059 * @param[in] item_flags 3060 * Bit-fields that holds the items detected until now. 3061 * @param[in] target_protocol 3062 * The next protocol in the previous item. 3063 * @param[in] flow_mask 3064 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 3065 * @param[out] error 3066 * Pointer to error structure. 3067 * 3068 * @return 3069 * 0 on success, a negative errno value otherwise and rte_errno is set. 3070 */ 3071 int 3072 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 3073 uint64_t item_flags, 3074 uint8_t target_protocol, 3075 struct rte_flow_error *error) 3076 { 3077 const struct rte_flow_item_udp *mask = item->mask; 3078 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3079 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3080 MLX5_FLOW_LAYER_OUTER_L3; 3081 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3082 MLX5_FLOW_LAYER_OUTER_L4; 3083 int ret; 3084 3085 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 3086 return rte_flow_error_set(error, EINVAL, 3087 RTE_FLOW_ERROR_TYPE_ITEM, item, 3088 "protocol filtering not compatible" 3089 " with UDP layer"); 3090 if (!(item_flags & l3m)) 3091 return rte_flow_error_set(error, EINVAL, 3092 RTE_FLOW_ERROR_TYPE_ITEM, item, 3093 "L3 is mandatory to filter on L4"); 3094 if (item_flags & l4m) 3095 return rte_flow_error_set(error, EINVAL, 3096 RTE_FLOW_ERROR_TYPE_ITEM, item, 3097 "multiple L4 layers not supported"); 3098 if (!mask) 3099 mask = &rte_flow_item_udp_mask; 3100 ret = mlx5_flow_item_acceptable 3101 (item, (const uint8_t *)mask, 3102 (const uint8_t *)&rte_flow_item_udp_mask, 3103 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3104 error); 3105 if (ret < 0) 3106 return ret; 3107 return 0; 3108 } 3109 3110 /** 3111 * Validate TCP item. 3112 * 3113 * @param[in] item 3114 * Item specification. 3115 * @param[in] item_flags 3116 * Bit-fields that holds the items detected until now. 3117 * @param[in] target_protocol 3118 * The next protocol in the previous item. 3119 * @param[out] error 3120 * Pointer to error structure. 3121 * 3122 * @return 3123 * 0 on success, a negative errno value otherwise and rte_errno is set. 3124 */ 3125 int 3126 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 3127 uint64_t item_flags, 3128 uint8_t target_protocol, 3129 const struct rte_flow_item_tcp *flow_mask, 3130 struct rte_flow_error *error) 3131 { 3132 const struct rte_flow_item_tcp *mask = item->mask; 3133 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3134 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3135 MLX5_FLOW_LAYER_OUTER_L3; 3136 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3137 MLX5_FLOW_LAYER_OUTER_L4; 3138 int ret; 3139 3140 MLX5_ASSERT(flow_mask); 3141 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3142 return rte_flow_error_set(error, EINVAL, 3143 RTE_FLOW_ERROR_TYPE_ITEM, item, 3144 "protocol filtering not compatible" 3145 " with TCP layer"); 3146 if (!(item_flags & l3m)) 3147 return rte_flow_error_set(error, EINVAL, 3148 RTE_FLOW_ERROR_TYPE_ITEM, item, 3149 "L3 is mandatory to filter on L4"); 3150 if (item_flags & l4m) 3151 return rte_flow_error_set(error, EINVAL, 3152 RTE_FLOW_ERROR_TYPE_ITEM, item, 3153 "multiple L4 layers not supported"); 3154 if (!mask) 3155 mask = &rte_flow_item_tcp_mask; 3156 ret = mlx5_flow_item_acceptable 3157 (item, (const uint8_t *)mask, 3158 (const uint8_t *)flow_mask, 3159 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3160 error); 3161 if (ret < 0) 3162 return ret; 3163 return 0; 3164 } 3165 3166 /** 3167 * Validate VXLAN item. 3168 * 3169 * @param[in] dev 3170 * Pointer to the Ethernet device structure. 3171 * @param[in] udp_dport 3172 * UDP destination port 3173 * @param[in] item 3174 * Item specification. 3175 * @param[in] item_flags 3176 * Bit-fields that holds the items detected until now. 3177 * @param root 3178 * Whether action is on root table. 3179 * @param[out] error 3180 * Pointer to error structure. 3181 * 3182 * @return 3183 * 0 on success, a negative errno value otherwise and rte_errno is set. 3184 */ 3185 int 3186 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3187 uint16_t udp_dport, 3188 const struct rte_flow_item *item, 3189 uint64_t item_flags, 3190 bool root, 3191 struct rte_flow_error *error) 3192 { 3193 const struct rte_flow_item_vxlan *spec = item->spec; 3194 const struct rte_flow_item_vxlan *mask = item->mask; 3195 int ret; 3196 struct mlx5_priv *priv = dev->data->dev_private; 3197 union vni { 3198 uint32_t vlan_id; 3199 uint8_t vni[4]; 3200 } id = { .vlan_id = 0, }; 3201 const struct rte_flow_item_vxlan nic_mask = { 3202 .hdr.vni = "\xff\xff\xff", 3203 .hdr.rsvd1 = 0xff, 3204 }; 3205 const struct rte_flow_item_vxlan *valid_mask; 3206 3207 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3208 return rte_flow_error_set(error, ENOTSUP, 3209 RTE_FLOW_ERROR_TYPE_ITEM, item, 3210 "multiple tunnel layers not" 3211 " supported"); 3212 valid_mask = &rte_flow_item_vxlan_mask; 3213 /* 3214 * Verify only UDPv4 is present as defined in 3215 * https://tools.ietf.org/html/rfc7348 3216 */ 3217 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3218 return rte_flow_error_set(error, EINVAL, 3219 RTE_FLOW_ERROR_TYPE_ITEM, item, 3220 "no outer UDP layer found"); 3221 if (!mask) 3222 mask = &rte_flow_item_vxlan_mask; 3223 3224 if (priv->sh->steering_format_version != 3225 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3226 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3227 /* non-root table */ 3228 if (!root && priv->sh->misc5_cap) 3229 valid_mask = &nic_mask; 3230 /* Group zero in NIC domain */ 3231 if (!root && priv->sh->tunnel_header_0_1) 3232 valid_mask = &nic_mask; 3233 } 3234 ret = mlx5_flow_item_acceptable 3235 (item, (const uint8_t *)mask, 3236 (const uint8_t *)valid_mask, 3237 sizeof(struct rte_flow_item_vxlan), 3238 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3239 if (ret < 0) 3240 return ret; 3241 if (spec) { 3242 memcpy(&id.vni[1], spec->hdr.vni, 3); 3243 memcpy(&id.vni[1], mask->hdr.vni, 3); 3244 } 3245 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3246 return rte_flow_error_set(error, ENOTSUP, 3247 RTE_FLOW_ERROR_TYPE_ITEM, item, 3248 "VXLAN tunnel must be fully defined"); 3249 return 0; 3250 } 3251 3252 /** 3253 * Validate VXLAN_GPE item. 3254 * 3255 * @param[in] item 3256 * Item specification. 3257 * @param[in] item_flags 3258 * Bit-fields that holds the items detected until now. 3259 * @param[in] priv 3260 * Pointer to the private data structure. 3261 * @param[in] target_protocol 3262 * The next protocol in the previous item. 3263 * @param[out] error 3264 * Pointer to error structure. 3265 * 3266 * @return 3267 * 0 on success, a negative errno value otherwise and rte_errno is set. 3268 */ 3269 int 3270 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3271 uint64_t item_flags, 3272 struct rte_eth_dev *dev, 3273 struct rte_flow_error *error) 3274 { 3275 struct mlx5_priv *priv = dev->data->dev_private; 3276 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3277 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3278 int ret; 3279 union vni { 3280 uint32_t vlan_id; 3281 uint8_t vni[4]; 3282 } id = { .vlan_id = 0, }; 3283 3284 struct rte_flow_item_vxlan_gpe nic_mask = { 3285 .vni = "\xff\xff\xff", 3286 .protocol = 0xff, 3287 }; 3288 3289 if (!priv->sh->config.l3_vxlan_en) 3290 return rte_flow_error_set(error, ENOTSUP, 3291 RTE_FLOW_ERROR_TYPE_ITEM, item, 3292 "L3 VXLAN is not enabled by device" 3293 " parameter and/or not configured in" 3294 " firmware"); 3295 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3296 return rte_flow_error_set(error, ENOTSUP, 3297 RTE_FLOW_ERROR_TYPE_ITEM, item, 3298 "multiple tunnel layers not" 3299 " supported"); 3300 /* 3301 * Verify only UDPv4 is present as defined in 3302 * https://tools.ietf.org/html/rfc7348 3303 */ 3304 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3305 return rte_flow_error_set(error, EINVAL, 3306 RTE_FLOW_ERROR_TYPE_ITEM, item, 3307 "no outer UDP layer found"); 3308 if (!mask) 3309 mask = &rte_flow_item_vxlan_gpe_mask; 3310 ret = mlx5_flow_item_acceptable 3311 (item, (const uint8_t *)mask, 3312 (const uint8_t *)&nic_mask, 3313 sizeof(struct rte_flow_item_vxlan_gpe), 3314 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3315 if (ret < 0) 3316 return ret; 3317 if (spec) { 3318 memcpy(&id.vni[1], spec->hdr.vni, 3); 3319 memcpy(&id.vni[1], mask->hdr.vni, 3); 3320 } 3321 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3322 return rte_flow_error_set(error, ENOTSUP, 3323 RTE_FLOW_ERROR_TYPE_ITEM, item, 3324 "VXLAN-GPE tunnel must be fully" 3325 " defined"); 3326 return 0; 3327 } 3328 /** 3329 * Validate GRE Key item. 3330 * 3331 * @param[in] item 3332 * Item specification. 3333 * @param[in] item_flags 3334 * Bit flags to mark detected items. 3335 * @param[in] gre_item 3336 * Pointer to gre_item 3337 * @param[out] error 3338 * Pointer to error structure. 3339 * 3340 * @return 3341 * 0 on success, a negative errno value otherwise and rte_errno is set. 3342 */ 3343 int 3344 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3345 uint64_t item_flags, 3346 const struct rte_flow_item *gre_item, 3347 struct rte_flow_error *error) 3348 { 3349 const rte_be32_t *mask = item->mask; 3350 int ret = 0; 3351 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3352 const struct rte_flow_item_gre *gre_spec; 3353 const struct rte_flow_item_gre *gre_mask; 3354 3355 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3356 return rte_flow_error_set(error, ENOTSUP, 3357 RTE_FLOW_ERROR_TYPE_ITEM, item, 3358 "Multiple GRE key not support"); 3359 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3360 return rte_flow_error_set(error, ENOTSUP, 3361 RTE_FLOW_ERROR_TYPE_ITEM, item, 3362 "No preceding GRE header"); 3363 if (item_flags & MLX5_FLOW_LAYER_INNER) 3364 return rte_flow_error_set(error, ENOTSUP, 3365 RTE_FLOW_ERROR_TYPE_ITEM, item, 3366 "GRE key following a wrong item"); 3367 gre_mask = gre_item->mask; 3368 if (!gre_mask) 3369 gre_mask = &rte_flow_item_gre_mask; 3370 gre_spec = gre_item->spec; 3371 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3372 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3373 return rte_flow_error_set(error, EINVAL, 3374 RTE_FLOW_ERROR_TYPE_ITEM, item, 3375 "Key bit must be on"); 3376 3377 if (!mask) 3378 mask = &gre_key_default_mask; 3379 ret = mlx5_flow_item_acceptable 3380 (item, (const uint8_t *)mask, 3381 (const uint8_t *)&gre_key_default_mask, 3382 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3383 return ret; 3384 } 3385 3386 /** 3387 * Validate GRE optional item. 3388 * 3389 * @param[in] dev 3390 * Pointer to the Ethernet device structure. 3391 * @param[in] item 3392 * Item specification. 3393 * @param[in] item_flags 3394 * Bit flags to mark detected items. 3395 * @param[in] attr 3396 * Flow rule attributes. 3397 * @param[in] gre_item 3398 * Pointer to gre_item 3399 * @param[out] error 3400 * Pointer to error structure. 3401 * 3402 * @return 3403 * 0 on success, a negative errno value otherwise and rte_errno is set. 3404 */ 3405 int 3406 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3407 const struct rte_flow_item *item, 3408 uint64_t item_flags, 3409 const struct rte_flow_attr *attr, 3410 const struct rte_flow_item *gre_item, 3411 struct rte_flow_error *error) 3412 { 3413 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3414 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3415 const struct rte_flow_item_gre_opt *spec = item->spec; 3416 const struct rte_flow_item_gre_opt *mask = item->mask; 3417 struct mlx5_priv *priv = dev->data->dev_private; 3418 int ret = 0; 3419 struct rte_flow_item_gre_opt nic_mask = { 3420 .checksum_rsvd = { 3421 .checksum = RTE_BE16(UINT16_MAX), 3422 .reserved1 = 0x0, 3423 }, 3424 .key = { 3425 .key = RTE_BE32(UINT32_MAX), 3426 }, 3427 .sequence = { 3428 .sequence = RTE_BE32(UINT32_MAX), 3429 }, 3430 }; 3431 3432 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3433 return rte_flow_error_set(error, ENOTSUP, 3434 RTE_FLOW_ERROR_TYPE_ITEM, item, 3435 "No preceding GRE header"); 3436 if (item_flags & MLX5_FLOW_LAYER_INNER) 3437 return rte_flow_error_set(error, ENOTSUP, 3438 RTE_FLOW_ERROR_TYPE_ITEM, item, 3439 "GRE option following a wrong item"); 3440 if (!spec || !mask) 3441 return rte_flow_error_set(error, EINVAL, 3442 RTE_FLOW_ERROR_TYPE_ITEM, item, 3443 "At least one field gre_option(checksum/key/sequence) must be specified"); 3444 if (!gre_mask) 3445 gre_mask = &rte_flow_item_gre_mask; 3446 if (mask->checksum_rsvd.checksum) 3447 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3448 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3449 return rte_flow_error_set(error, EINVAL, 3450 RTE_FLOW_ERROR_TYPE_ITEM, 3451 item, 3452 "Checksum bit must be on"); 3453 if (mask->key.key) 3454 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3455 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3456 return rte_flow_error_set(error, EINVAL, 3457 RTE_FLOW_ERROR_TYPE_ITEM, 3458 item, "Key bit must be on"); 3459 if (mask->sequence.sequence) 3460 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3461 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3462 return rte_flow_error_set(error, EINVAL, 3463 RTE_FLOW_ERROR_TYPE_ITEM, 3464 item, 3465 "Sequence bit must be on"); 3466 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3467 if (priv->sh->steering_format_version == 3468 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3469 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3470 !priv->sh->misc5_cap) || 3471 (!(priv->sh->tunnel_header_0_1 && 3472 priv->sh->tunnel_header_2_3) && 3473 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3474 return rte_flow_error_set(error, EINVAL, 3475 RTE_FLOW_ERROR_TYPE_ITEM, 3476 item, 3477 "Checksum/Sequence not supported"); 3478 } 3479 ret = mlx5_flow_item_acceptable 3480 (item, (const uint8_t *)mask, 3481 (const uint8_t *)&nic_mask, 3482 sizeof(struct rte_flow_item_gre_opt), 3483 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3484 return ret; 3485 } 3486 3487 /** 3488 * Validate GRE item. 3489 * 3490 * @param[in] item 3491 * Item specification. 3492 * @param[in] item_flags 3493 * Bit flags to mark detected items. 3494 * @param[in] target_protocol 3495 * The next protocol in the previous item. 3496 * @param[out] error 3497 * Pointer to error structure. 3498 * 3499 * @return 3500 * 0 on success, a negative errno value otherwise and rte_errno is set. 3501 */ 3502 int 3503 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3504 uint64_t item_flags, 3505 uint8_t target_protocol, 3506 struct rte_flow_error *error) 3507 { 3508 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3509 const struct rte_flow_item_gre *mask = item->mask; 3510 int ret; 3511 const struct rte_flow_item_gre nic_mask = { 3512 .c_rsvd0_ver = RTE_BE16(0xB000), 3513 .protocol = RTE_BE16(UINT16_MAX), 3514 }; 3515 3516 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3517 return rte_flow_error_set(error, EINVAL, 3518 RTE_FLOW_ERROR_TYPE_ITEM, item, 3519 "protocol filtering not compatible" 3520 " with this GRE layer"); 3521 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3522 return rte_flow_error_set(error, ENOTSUP, 3523 RTE_FLOW_ERROR_TYPE_ITEM, item, 3524 "multiple tunnel layers not" 3525 " supported"); 3526 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3527 return rte_flow_error_set(error, ENOTSUP, 3528 RTE_FLOW_ERROR_TYPE_ITEM, item, 3529 "L3 Layer is missing"); 3530 if (!mask) 3531 mask = &rte_flow_item_gre_mask; 3532 ret = mlx5_flow_item_acceptable 3533 (item, (const uint8_t *)mask, 3534 (const uint8_t *)&nic_mask, 3535 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3536 error); 3537 if (ret < 0) 3538 return ret; 3539 #ifndef HAVE_MLX5DV_DR 3540 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3541 if (spec && (spec->protocol & mask->protocol)) 3542 return rte_flow_error_set(error, ENOTSUP, 3543 RTE_FLOW_ERROR_TYPE_ITEM, item, 3544 "without MPLS support the" 3545 " specification cannot be used for" 3546 " filtering"); 3547 #endif 3548 #endif 3549 return 0; 3550 } 3551 3552 /** 3553 * Validate Geneve item. 3554 * 3555 * @param[in] item 3556 * Item specification. 3557 * @param[in] itemFlags 3558 * Bit-fields that holds the items detected until now. 3559 * @param[in] enPriv 3560 * Pointer to the private data structure. 3561 * @param[out] error 3562 * Pointer to error structure. 3563 * 3564 * @return 3565 * 0 on success, a negative errno value otherwise and rte_errno is set. 3566 */ 3567 3568 int 3569 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3570 uint64_t item_flags, 3571 struct rte_eth_dev *dev, 3572 struct rte_flow_error *error) 3573 { 3574 struct mlx5_priv *priv = dev->data->dev_private; 3575 const struct rte_flow_item_geneve *spec = item->spec; 3576 const struct rte_flow_item_geneve *mask = item->mask; 3577 int ret; 3578 uint16_t gbhdr; 3579 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3580 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3581 const struct rte_flow_item_geneve nic_mask = { 3582 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3583 .vni = "\xff\xff\xff", 3584 .protocol = RTE_BE16(UINT16_MAX), 3585 }; 3586 3587 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3588 return rte_flow_error_set(error, ENOTSUP, 3589 RTE_FLOW_ERROR_TYPE_ITEM, item, 3590 "L3 Geneve is not enabled by device" 3591 " parameter and/or not configured in" 3592 " firmware"); 3593 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3594 return rte_flow_error_set(error, ENOTSUP, 3595 RTE_FLOW_ERROR_TYPE_ITEM, item, 3596 "multiple tunnel layers not" 3597 " supported"); 3598 /* 3599 * Verify only UDPv4 is present as defined in 3600 * https://tools.ietf.org/html/rfc7348 3601 */ 3602 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3603 return rte_flow_error_set(error, EINVAL, 3604 RTE_FLOW_ERROR_TYPE_ITEM, item, 3605 "no outer UDP layer found"); 3606 if (!mask) 3607 mask = &rte_flow_item_geneve_mask; 3608 ret = mlx5_flow_item_acceptable 3609 (item, (const uint8_t *)mask, 3610 (const uint8_t *)&nic_mask, 3611 sizeof(struct rte_flow_item_geneve), 3612 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3613 if (ret) 3614 return ret; 3615 if (spec) { 3616 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3617 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3618 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3619 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3620 return rte_flow_error_set(error, ENOTSUP, 3621 RTE_FLOW_ERROR_TYPE_ITEM, 3622 item, 3623 "Geneve protocol unsupported" 3624 " fields are being used"); 3625 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3626 return rte_flow_error_set 3627 (error, ENOTSUP, 3628 RTE_FLOW_ERROR_TYPE_ITEM, 3629 item, 3630 "Unsupported Geneve options length"); 3631 } 3632 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3633 return rte_flow_error_set 3634 (error, ENOTSUP, 3635 RTE_FLOW_ERROR_TYPE_ITEM, item, 3636 "Geneve tunnel must be fully defined"); 3637 return 0; 3638 } 3639 3640 /** 3641 * Validate Geneve TLV option item. 3642 * 3643 * @param[in] item 3644 * Item specification. 3645 * @param[in] last_item 3646 * Previous validated item in the pattern items. 3647 * @param[in] geneve_item 3648 * Previous GENEVE item specification. 3649 * @param[in] dev 3650 * Pointer to the rte_eth_dev structure. 3651 * @param[out] error 3652 * Pointer to error structure. 3653 * 3654 * @return 3655 * 0 on success, a negative errno value otherwise and rte_errno is set. 3656 */ 3657 int 3658 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3659 uint64_t last_item, 3660 const struct rte_flow_item *geneve_item, 3661 struct rte_eth_dev *dev, 3662 struct rte_flow_error *error) 3663 { 3664 struct mlx5_priv *priv = dev->data->dev_private; 3665 struct mlx5_dev_ctx_shared *sh = priv->sh; 3666 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3667 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3668 uint8_t data_max_supported = 3669 hca_attr->max_geneve_tlv_option_data_len * 4; 3670 const struct rte_flow_item_geneve *geneve_spec; 3671 const struct rte_flow_item_geneve *geneve_mask; 3672 const struct rte_flow_item_geneve_opt *spec = item->spec; 3673 const struct rte_flow_item_geneve_opt *mask = item->mask; 3674 unsigned int i; 3675 unsigned int data_len; 3676 uint8_t tlv_option_len; 3677 uint16_t optlen_m, optlen_v; 3678 const struct rte_flow_item_geneve_opt full_mask = { 3679 .option_class = RTE_BE16(0xffff), 3680 .option_type = 0xff, 3681 .option_len = 0x1f, 3682 }; 3683 3684 if (!mask) 3685 mask = &rte_flow_item_geneve_opt_mask; 3686 if (!spec) 3687 return rte_flow_error_set 3688 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3689 "Geneve TLV opt class/type/length must be specified"); 3690 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3691 return rte_flow_error_set 3692 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3693 "Geneve TLV opt length exceeds the limit (31)"); 3694 /* Check if class type and length masks are full. */ 3695 if (full_mask.option_class != mask->option_class || 3696 full_mask.option_type != mask->option_type || 3697 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3698 return rte_flow_error_set 3699 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3700 "Geneve TLV opt class/type/length masks must be full"); 3701 /* Check if length is supported */ 3702 if ((uint32_t)spec->option_len > 3703 hca_attr->max_geneve_tlv_option_data_len) 3704 return rte_flow_error_set 3705 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3706 "Geneve TLV opt length not supported"); 3707 if (hca_attr->max_geneve_tlv_options > 1) 3708 DRV_LOG(DEBUG, 3709 "max_geneve_tlv_options supports more than 1 option"); 3710 /* Check GENEVE item preceding. */ 3711 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3712 return rte_flow_error_set 3713 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3714 "Geneve opt item must be preceded with Geneve item"); 3715 geneve_spec = geneve_item->spec; 3716 geneve_mask = geneve_item->mask ? geneve_item->mask : 3717 &rte_flow_item_geneve_mask; 3718 /* Check if GENEVE TLV option size doesn't exceed option length */ 3719 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3720 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3721 tlv_option_len = spec->option_len & mask->option_len; 3722 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3723 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3724 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3725 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3726 if ((optlen_v & optlen_m) <= tlv_option_len) 3727 return rte_flow_error_set 3728 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3729 "GENEVE TLV option length exceeds optlen"); 3730 } 3731 /* Check if length is 0 or data is 0. */ 3732 if (spec->data == NULL || spec->option_len == 0) 3733 return rte_flow_error_set 3734 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3735 "Geneve TLV opt with zero data/length not supported"); 3736 /* Check not all data & mask are 0. */ 3737 data_len = spec->option_len * 4; 3738 if (mask->data == NULL) { 3739 for (i = 0; i < data_len; i++) 3740 if (spec->data[i]) 3741 break; 3742 if (i == data_len) 3743 return rte_flow_error_set(error, ENOTSUP, 3744 RTE_FLOW_ERROR_TYPE_ITEM, item, 3745 "Can't match on Geneve option data 0"); 3746 } else { 3747 for (i = 0; i < data_len; i++) 3748 if (spec->data[i] & mask->data[i]) 3749 break; 3750 if (i == data_len) 3751 return rte_flow_error_set(error, ENOTSUP, 3752 RTE_FLOW_ERROR_TYPE_ITEM, item, 3753 "Can't match on Geneve option data and mask 0"); 3754 /* Check data mask supported. */ 3755 for (i = data_max_supported; i < data_len ; i++) 3756 if (mask->data[i]) 3757 return rte_flow_error_set(error, ENOTSUP, 3758 RTE_FLOW_ERROR_TYPE_ITEM, item, 3759 "Data mask is of unsupported size"); 3760 } 3761 /* Check GENEVE option is supported in NIC. */ 3762 if (!hca_attr->geneve_tlv_opt) 3763 return rte_flow_error_set 3764 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3765 "Geneve TLV opt not supported"); 3766 /* Check if we already have geneve option with different type/class. */ 3767 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3768 geneve_opt_resource = sh->geneve_tlv_option_resource; 3769 if (geneve_opt_resource != NULL) 3770 if (geneve_opt_resource->option_class != spec->option_class || 3771 geneve_opt_resource->option_type != spec->option_type || 3772 geneve_opt_resource->length != spec->option_len) { 3773 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3774 return rte_flow_error_set(error, ENOTSUP, 3775 RTE_FLOW_ERROR_TYPE_ITEM, item, 3776 "Only one Geneve TLV option supported"); 3777 } 3778 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3779 return 0; 3780 } 3781 3782 /** 3783 * Validate MPLS item. 3784 * 3785 * @param[in] dev 3786 * Pointer to the rte_eth_dev structure. 3787 * @param[in] item 3788 * Item specification. 3789 * @param[in] item_flags 3790 * Bit-fields that holds the items detected until now. 3791 * @param[in] prev_layer 3792 * The protocol layer indicated in previous item. 3793 * @param[out] error 3794 * Pointer to error structure. 3795 * 3796 * @return 3797 * 0 on success, a negative errno value otherwise and rte_errno is set. 3798 */ 3799 int 3800 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3801 const struct rte_flow_item *item __rte_unused, 3802 uint64_t item_flags __rte_unused, 3803 uint64_t prev_layer __rte_unused, 3804 struct rte_flow_error *error) 3805 { 3806 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3807 const struct rte_flow_item_mpls *mask = item->mask; 3808 struct mlx5_priv *priv = dev->data->dev_private; 3809 int ret; 3810 3811 if (!priv->sh->dev_cap.mpls_en) 3812 return rte_flow_error_set(error, ENOTSUP, 3813 RTE_FLOW_ERROR_TYPE_ITEM, item, 3814 "MPLS not supported or" 3815 " disabled in firmware" 3816 " configuration."); 3817 /* MPLS over UDP, GRE is allowed */ 3818 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3819 MLX5_FLOW_LAYER_GRE | 3820 MLX5_FLOW_LAYER_GRE_KEY))) 3821 return rte_flow_error_set(error, EINVAL, 3822 RTE_FLOW_ERROR_TYPE_ITEM, item, 3823 "protocol filtering not compatible" 3824 " with MPLS layer"); 3825 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3826 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3827 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3828 return rte_flow_error_set(error, ENOTSUP, 3829 RTE_FLOW_ERROR_TYPE_ITEM, item, 3830 "multiple tunnel layers not" 3831 " supported"); 3832 if (!mask) 3833 mask = &rte_flow_item_mpls_mask; 3834 ret = mlx5_flow_item_acceptable 3835 (item, (const uint8_t *)mask, 3836 (const uint8_t *)&rte_flow_item_mpls_mask, 3837 sizeof(struct rte_flow_item_mpls), 3838 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3839 if (ret < 0) 3840 return ret; 3841 return 0; 3842 #else 3843 return rte_flow_error_set(error, ENOTSUP, 3844 RTE_FLOW_ERROR_TYPE_ITEM, item, 3845 "MPLS is not supported by Verbs, please" 3846 " update."); 3847 #endif 3848 } 3849 3850 /** 3851 * Validate NVGRE item. 3852 * 3853 * @param[in] item 3854 * Item specification. 3855 * @param[in] item_flags 3856 * Bit flags to mark detected items. 3857 * @param[in] target_protocol 3858 * The next protocol in the previous item. 3859 * @param[out] error 3860 * Pointer to error structure. 3861 * 3862 * @return 3863 * 0 on success, a negative errno value otherwise and rte_errno is set. 3864 */ 3865 int 3866 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3867 uint64_t item_flags, 3868 uint8_t target_protocol, 3869 struct rte_flow_error *error) 3870 { 3871 const struct rte_flow_item_nvgre *mask = item->mask; 3872 int ret; 3873 3874 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3875 return rte_flow_error_set(error, EINVAL, 3876 RTE_FLOW_ERROR_TYPE_ITEM, item, 3877 "protocol filtering not compatible" 3878 " with this GRE layer"); 3879 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3880 return rte_flow_error_set(error, ENOTSUP, 3881 RTE_FLOW_ERROR_TYPE_ITEM, item, 3882 "multiple tunnel layers not" 3883 " supported"); 3884 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3885 return rte_flow_error_set(error, ENOTSUP, 3886 RTE_FLOW_ERROR_TYPE_ITEM, item, 3887 "L3 Layer is missing"); 3888 if (!mask) 3889 mask = &rte_flow_item_nvgre_mask; 3890 ret = mlx5_flow_item_acceptable 3891 (item, (const uint8_t *)mask, 3892 (const uint8_t *)&rte_flow_item_nvgre_mask, 3893 sizeof(struct rte_flow_item_nvgre), 3894 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3895 if (ret < 0) 3896 return ret; 3897 return 0; 3898 } 3899 3900 /** 3901 * Validate eCPRI item. 3902 * 3903 * @param[in] item 3904 * Item specification. 3905 * @param[in] item_flags 3906 * Bit-fields that holds the items detected until now. 3907 * @param[in] last_item 3908 * Previous validated item in the pattern items. 3909 * @param[in] ether_type 3910 * Type in the ethernet layer header (including dot1q). 3911 * @param[in] acc_mask 3912 * Acceptable mask, if NULL default internal default mask 3913 * will be used to check whether item fields are supported. 3914 * @param[out] error 3915 * Pointer to error structure. 3916 * 3917 * @return 3918 * 0 on success, a negative errno value otherwise and rte_errno is set. 3919 */ 3920 int 3921 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3922 uint64_t item_flags, 3923 uint64_t last_item, 3924 uint16_t ether_type, 3925 const struct rte_flow_item_ecpri *acc_mask, 3926 struct rte_flow_error *error) 3927 { 3928 const struct rte_flow_item_ecpri *mask = item->mask; 3929 const struct rte_flow_item_ecpri nic_mask = { 3930 .hdr = { 3931 .common = { 3932 .u32 = 3933 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3934 .type = 0xFF, 3935 }).u32), 3936 }, 3937 .dummy[0] = 0xFFFFFFFF, 3938 }, 3939 }; 3940 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3941 MLX5_FLOW_LAYER_OUTER_VLAN); 3942 struct rte_flow_item_ecpri mask_lo; 3943 3944 if (!(last_item & outer_l2_vlan) && 3945 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3946 return rte_flow_error_set(error, EINVAL, 3947 RTE_FLOW_ERROR_TYPE_ITEM, item, 3948 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3949 if ((last_item & outer_l2_vlan) && ether_type && 3950 ether_type != RTE_ETHER_TYPE_ECPRI) 3951 return rte_flow_error_set(error, EINVAL, 3952 RTE_FLOW_ERROR_TYPE_ITEM, item, 3953 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3954 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3955 return rte_flow_error_set(error, EINVAL, 3956 RTE_FLOW_ERROR_TYPE_ITEM, item, 3957 "eCPRI with tunnel is not supported right now"); 3958 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3959 return rte_flow_error_set(error, ENOTSUP, 3960 RTE_FLOW_ERROR_TYPE_ITEM, item, 3961 "multiple L3 layers not supported"); 3962 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3963 return rte_flow_error_set(error, EINVAL, 3964 RTE_FLOW_ERROR_TYPE_ITEM, item, 3965 "eCPRI cannot coexist with a TCP layer"); 3966 /* In specification, eCPRI could be over UDP layer. */ 3967 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3968 return rte_flow_error_set(error, EINVAL, 3969 RTE_FLOW_ERROR_TYPE_ITEM, item, 3970 "eCPRI over UDP layer is not yet supported right now"); 3971 /* Mask for type field in common header could be zero. */ 3972 if (!mask) 3973 mask = &rte_flow_item_ecpri_mask; 3974 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3975 /* Input mask is in big-endian format. */ 3976 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3977 return rte_flow_error_set(error, EINVAL, 3978 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3979 "partial mask is not supported for protocol"); 3980 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3981 return rte_flow_error_set(error, EINVAL, 3982 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3983 "message header mask must be after a type mask"); 3984 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3985 acc_mask ? (const uint8_t *)acc_mask 3986 : (const uint8_t *)&nic_mask, 3987 sizeof(struct rte_flow_item_ecpri), 3988 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3989 } 3990 3991 /** 3992 * Validate the NSH item. 3993 * 3994 * @param[in] dev 3995 * Pointer to Ethernet device on which flow rule is being created on. 3996 * @param[out] error 3997 * Pointer to error structure. 3998 * 3999 * @return 4000 * 0 on success, a negative errno value otherwise and rte_errno is set. 4001 */ 4002 int 4003 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev, 4004 const struct rte_flow_item *item, 4005 struct rte_flow_error *error) 4006 { 4007 struct mlx5_priv *priv = dev->data->dev_private; 4008 4009 if (item->mask) { 4010 return rte_flow_error_set(error, ENOTSUP, 4011 RTE_FLOW_ERROR_TYPE_ITEM, item, 4012 "NSH fields matching is not supported"); 4013 } 4014 4015 if (!priv->sh->config.dv_flow_en) { 4016 return rte_flow_error_set(error, ENOTSUP, 4017 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 4018 NULL, "NSH support requires DV flow interface"); 4019 } 4020 4021 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) { 4022 return rte_flow_error_set(error, ENOTSUP, 4023 RTE_FLOW_ERROR_TYPE_ITEM, item, 4024 "Current FW does not support matching on NSH"); 4025 } 4026 4027 return 0; 4028 } 4029 4030 static int 4031 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 4032 const struct rte_flow_attr *attr __rte_unused, 4033 const struct rte_flow_item items[] __rte_unused, 4034 const struct rte_flow_action actions[] __rte_unused, 4035 bool external __rte_unused, 4036 int hairpin __rte_unused, 4037 struct rte_flow_error *error) 4038 { 4039 return rte_flow_error_set(error, ENOTSUP, 4040 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4041 } 4042 4043 static struct mlx5_flow * 4044 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 4045 const struct rte_flow_attr *attr __rte_unused, 4046 const struct rte_flow_item items[] __rte_unused, 4047 const struct rte_flow_action actions[] __rte_unused, 4048 struct rte_flow_error *error) 4049 { 4050 rte_flow_error_set(error, ENOTSUP, 4051 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4052 return NULL; 4053 } 4054 4055 static int 4056 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 4057 struct mlx5_flow *dev_flow __rte_unused, 4058 const struct rte_flow_attr *attr __rte_unused, 4059 const struct rte_flow_item items[] __rte_unused, 4060 const struct rte_flow_action actions[] __rte_unused, 4061 struct rte_flow_error *error) 4062 { 4063 return rte_flow_error_set(error, ENOTSUP, 4064 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4065 } 4066 4067 static int 4068 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 4069 struct rte_flow *flow __rte_unused, 4070 struct rte_flow_error *error) 4071 { 4072 return rte_flow_error_set(error, ENOTSUP, 4073 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4074 } 4075 4076 static void 4077 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 4078 struct rte_flow *flow __rte_unused) 4079 { 4080 } 4081 4082 static void 4083 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 4084 struct rte_flow *flow __rte_unused) 4085 { 4086 } 4087 4088 static int 4089 flow_null_query(struct rte_eth_dev *dev __rte_unused, 4090 struct rte_flow *flow __rte_unused, 4091 const struct rte_flow_action *actions __rte_unused, 4092 void *data __rte_unused, 4093 struct rte_flow_error *error) 4094 { 4095 return rte_flow_error_set(error, ENOTSUP, 4096 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4097 } 4098 4099 static int 4100 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 4101 uint32_t domains __rte_unused, 4102 uint32_t flags __rte_unused) 4103 { 4104 return 0; 4105 } 4106 4107 int 4108 flow_null_get_aged_flows(struct rte_eth_dev *dev, 4109 void **context __rte_unused, 4110 uint32_t nb_contexts __rte_unused, 4111 struct rte_flow_error *error __rte_unused) 4112 { 4113 DRV_LOG(ERR, "port %u get aged flows is not supported.", 4114 dev->data->port_id); 4115 return -ENOTSUP; 4116 } 4117 4118 uint32_t 4119 flow_null_counter_allocate(struct rte_eth_dev *dev) 4120 { 4121 DRV_LOG(ERR, "port %u counter allocate is not supported.", 4122 dev->data->port_id); 4123 return 0; 4124 } 4125 4126 void 4127 flow_null_counter_free(struct rte_eth_dev *dev, 4128 uint32_t counter __rte_unused) 4129 { 4130 DRV_LOG(ERR, "port %u counter free is not supported.", 4131 dev->data->port_id); 4132 } 4133 4134 int 4135 flow_null_counter_query(struct rte_eth_dev *dev, 4136 uint32_t counter __rte_unused, 4137 bool clear __rte_unused, 4138 uint64_t *pkts __rte_unused, 4139 uint64_t *bytes __rte_unused, 4140 void **action __rte_unused) 4141 { 4142 DRV_LOG(ERR, "port %u counter query is not supported.", 4143 dev->data->port_id); 4144 return -ENOTSUP; 4145 } 4146 4147 /* Void driver to protect from null pointer reference. */ 4148 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 4149 .validate = flow_null_validate, 4150 .prepare = flow_null_prepare, 4151 .translate = flow_null_translate, 4152 .apply = flow_null_apply, 4153 .remove = flow_null_remove, 4154 .destroy = flow_null_destroy, 4155 .query = flow_null_query, 4156 .sync_domain = flow_null_sync_domain, 4157 .get_aged_flows = flow_null_get_aged_flows, 4158 .counter_alloc = flow_null_counter_allocate, 4159 .counter_free = flow_null_counter_free, 4160 .counter_query = flow_null_counter_query 4161 }; 4162 4163 /** 4164 * Select flow driver type according to flow attributes and device 4165 * configuration. 4166 * 4167 * @param[in] dev 4168 * Pointer to the dev structure. 4169 * @param[in] attr 4170 * Pointer to the flow attributes. 4171 * 4172 * @return 4173 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4174 */ 4175 static enum mlx5_flow_drv_type 4176 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4177 { 4178 struct mlx5_priv *priv = dev->data->dev_private; 4179 /* The OS can determine first a specific flow type (DV, VERBS) */ 4180 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4181 4182 if (type != MLX5_FLOW_TYPE_MAX) 4183 return type; 4184 /* 4185 * Currently when dv_flow_en == 2, only HW steering engine is 4186 * supported. New engines can also be chosen here if ready. 4187 */ 4188 if (priv->sh->config.dv_flow_en == 2) 4189 return MLX5_FLOW_TYPE_HW; 4190 if (!attr) 4191 return MLX5_FLOW_TYPE_MIN; 4192 /* If no OS specific type - continue with DV/VERBS selection */ 4193 if (attr->transfer && priv->sh->config.dv_esw_en) 4194 type = MLX5_FLOW_TYPE_DV; 4195 if (!attr->transfer) 4196 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4197 MLX5_FLOW_TYPE_VERBS; 4198 return type; 4199 } 4200 4201 #define flow_get_drv_ops(type) flow_drv_ops[type] 4202 4203 /** 4204 * Flow driver validation API. This abstracts calling driver specific functions. 4205 * The type of flow driver is determined according to flow attributes. 4206 * 4207 * @param[in] dev 4208 * Pointer to the dev structure. 4209 * @param[in] attr 4210 * Pointer to the flow attributes. 4211 * @param[in] items 4212 * Pointer to the list of items. 4213 * @param[in] actions 4214 * Pointer to the list of actions. 4215 * @param[in] external 4216 * This flow rule is created by request external to PMD. 4217 * @param[in] hairpin 4218 * Number of hairpin TX actions, 0 means classic flow. 4219 * @param[out] error 4220 * Pointer to the error structure. 4221 * 4222 * @return 4223 * 0 on success, a negative errno value otherwise and rte_errno is set. 4224 */ 4225 static inline int 4226 flow_drv_validate(struct rte_eth_dev *dev, 4227 const struct rte_flow_attr *attr, 4228 const struct rte_flow_item items[], 4229 const struct rte_flow_action actions[], 4230 bool external, int hairpin, struct rte_flow_error *error) 4231 { 4232 const struct mlx5_flow_driver_ops *fops; 4233 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4234 4235 fops = flow_get_drv_ops(type); 4236 return fops->validate(dev, attr, items, actions, external, 4237 hairpin, error); 4238 } 4239 4240 /** 4241 * Flow driver preparation API. This abstracts calling driver specific 4242 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4243 * calculates the size of memory required for device flow, allocates the memory, 4244 * initializes the device flow and returns the pointer. 4245 * 4246 * @note 4247 * This function initializes device flow structure such as dv or verbs in 4248 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4249 * rest. For example, adding returning device flow to flow->dev_flow list and 4250 * setting backward reference to the flow should be done out of this function. 4251 * layers field is not filled either. 4252 * 4253 * @param[in] dev 4254 * Pointer to the dev structure. 4255 * @param[in] attr 4256 * Pointer to the flow attributes. 4257 * @param[in] items 4258 * Pointer to the list of items. 4259 * @param[in] actions 4260 * Pointer to the list of actions. 4261 * @param[in] flow_idx 4262 * This memory pool index to the flow. 4263 * @param[out] error 4264 * Pointer to the error structure. 4265 * 4266 * @return 4267 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4268 */ 4269 static inline struct mlx5_flow * 4270 flow_drv_prepare(struct rte_eth_dev *dev, 4271 const struct rte_flow *flow, 4272 const struct rte_flow_attr *attr, 4273 const struct rte_flow_item items[], 4274 const struct rte_flow_action actions[], 4275 uint32_t flow_idx, 4276 struct rte_flow_error *error) 4277 { 4278 const struct mlx5_flow_driver_ops *fops; 4279 enum mlx5_flow_drv_type type = flow->drv_type; 4280 struct mlx5_flow *mlx5_flow = NULL; 4281 4282 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4283 fops = flow_get_drv_ops(type); 4284 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4285 if (mlx5_flow) 4286 mlx5_flow->flow_idx = flow_idx; 4287 return mlx5_flow; 4288 } 4289 4290 /** 4291 * Flow driver translation API. This abstracts calling driver specific 4292 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4293 * translates a generic flow into a driver flow. flow_drv_prepare() must 4294 * precede. 4295 * 4296 * @note 4297 * dev_flow->layers could be filled as a result of parsing during translation 4298 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4299 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4300 * flow->actions could be overwritten even though all the expanded dev_flows 4301 * have the same actions. 4302 * 4303 * @param[in] dev 4304 * Pointer to the rte dev structure. 4305 * @param[in, out] dev_flow 4306 * Pointer to the mlx5 flow. 4307 * @param[in] attr 4308 * Pointer to the flow attributes. 4309 * @param[in] items 4310 * Pointer to the list of items. 4311 * @param[in] actions 4312 * Pointer to the list of actions. 4313 * @param[out] error 4314 * Pointer to the error structure. 4315 * 4316 * @return 4317 * 0 on success, a negative errno value otherwise and rte_errno is set. 4318 */ 4319 static inline int 4320 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4321 const struct rte_flow_attr *attr, 4322 const struct rte_flow_item items[], 4323 const struct rte_flow_action actions[], 4324 struct rte_flow_error *error) 4325 { 4326 const struct mlx5_flow_driver_ops *fops; 4327 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4328 4329 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4330 fops = flow_get_drv_ops(type); 4331 return fops->translate(dev, dev_flow, attr, items, actions, error); 4332 } 4333 4334 /** 4335 * Flow driver apply API. This abstracts calling driver specific functions. 4336 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4337 * translated driver flows on to device. flow_drv_translate() must precede. 4338 * 4339 * @param[in] dev 4340 * Pointer to Ethernet device structure. 4341 * @param[in, out] flow 4342 * Pointer to flow structure. 4343 * @param[out] error 4344 * Pointer to error structure. 4345 * 4346 * @return 4347 * 0 on success, a negative errno value otherwise and rte_errno is set. 4348 */ 4349 static inline int 4350 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4351 struct rte_flow_error *error) 4352 { 4353 const struct mlx5_flow_driver_ops *fops; 4354 enum mlx5_flow_drv_type type = flow->drv_type; 4355 4356 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4357 fops = flow_get_drv_ops(type); 4358 return fops->apply(dev, flow, error); 4359 } 4360 4361 /** 4362 * Flow driver destroy API. This abstracts calling driver specific functions. 4363 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4364 * on device and releases resources of the flow. 4365 * 4366 * @param[in] dev 4367 * Pointer to Ethernet device. 4368 * @param[in, out] flow 4369 * Pointer to flow structure. 4370 */ 4371 static inline void 4372 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4373 { 4374 const struct mlx5_flow_driver_ops *fops; 4375 enum mlx5_flow_drv_type type = flow->drv_type; 4376 4377 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4378 fops = flow_get_drv_ops(type); 4379 fops->destroy(dev, flow); 4380 } 4381 4382 /** 4383 * Flow driver find RSS policy tbl API. This abstracts calling driver 4384 * specific functions. Parent flow (rte_flow) should have driver 4385 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4386 * 4387 * @param[in] dev 4388 * Pointer to Ethernet device. 4389 * @param[in, out] flow 4390 * Pointer to flow structure. 4391 * @param[in] policy 4392 * Pointer to meter policy table. 4393 * @param[in] rss_desc 4394 * Pointer to rss_desc 4395 */ 4396 static struct mlx5_flow_meter_sub_policy * 4397 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4398 struct rte_flow *flow, 4399 struct mlx5_flow_meter_policy *policy, 4400 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4401 { 4402 const struct mlx5_flow_driver_ops *fops; 4403 enum mlx5_flow_drv_type type = flow->drv_type; 4404 4405 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4406 fops = flow_get_drv_ops(type); 4407 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4408 } 4409 4410 /** 4411 * Flow driver color tag rule API. This abstracts calling driver 4412 * specific functions. Parent flow (rte_flow) should have driver 4413 * type (drv_type). It will create the color tag rules in hierarchy meter. 4414 * 4415 * @param[in] dev 4416 * Pointer to Ethernet device. 4417 * @param[in, out] flow 4418 * Pointer to flow structure. 4419 * @param[in] fm 4420 * Pointer to flow meter structure. 4421 * @param[in] src_port 4422 * The src port this extra rule should use. 4423 * @param[in] item 4424 * The src port id match item. 4425 * @param[out] error 4426 * Pointer to error structure. 4427 */ 4428 static int 4429 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4430 struct rte_flow *flow, 4431 struct mlx5_flow_meter_info *fm, 4432 int32_t src_port, 4433 const struct rte_flow_item *item, 4434 struct rte_flow_error *error) 4435 { 4436 const struct mlx5_flow_driver_ops *fops; 4437 enum mlx5_flow_drv_type type = flow->drv_type; 4438 4439 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4440 fops = flow_get_drv_ops(type); 4441 return fops->meter_hierarchy_rule_create(dev, fm, 4442 src_port, item, error); 4443 } 4444 4445 /** 4446 * Get RSS action from the action list. 4447 * 4448 * @param[in] dev 4449 * Pointer to Ethernet device. 4450 * @param[in] actions 4451 * Pointer to the list of actions. 4452 * @param[in] flow 4453 * Parent flow structure pointer. 4454 * 4455 * @return 4456 * Pointer to the RSS action if exist, else return NULL. 4457 */ 4458 static const struct rte_flow_action_rss* 4459 flow_get_rss_action(struct rte_eth_dev *dev, 4460 const struct rte_flow_action actions[]) 4461 { 4462 struct mlx5_priv *priv = dev->data->dev_private; 4463 const struct rte_flow_action_rss *rss = NULL; 4464 struct mlx5_meter_policy_action_container *acg; 4465 struct mlx5_meter_policy_action_container *acy; 4466 4467 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4468 switch (actions->type) { 4469 case RTE_FLOW_ACTION_TYPE_RSS: 4470 rss = actions->conf; 4471 break; 4472 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4473 { 4474 const struct rte_flow_action_sample *sample = 4475 actions->conf; 4476 const struct rte_flow_action *act = sample->actions; 4477 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4478 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4479 rss = act->conf; 4480 break; 4481 } 4482 case RTE_FLOW_ACTION_TYPE_METER: 4483 { 4484 uint32_t mtr_idx; 4485 struct mlx5_flow_meter_info *fm; 4486 struct mlx5_flow_meter_policy *policy; 4487 const struct rte_flow_action_meter *mtr = actions->conf; 4488 4489 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4490 if (fm && !fm->def_policy) { 4491 policy = mlx5_flow_meter_policy_find(dev, 4492 fm->policy_id, NULL); 4493 MLX5_ASSERT(policy); 4494 if (policy->is_hierarchy) { 4495 policy = 4496 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4497 policy); 4498 if (!policy) 4499 return NULL; 4500 } 4501 if (policy->is_rss) { 4502 acg = 4503 &policy->act_cnt[RTE_COLOR_GREEN]; 4504 acy = 4505 &policy->act_cnt[RTE_COLOR_YELLOW]; 4506 if (acg->fate_action == 4507 MLX5_FLOW_FATE_SHARED_RSS) 4508 rss = acg->rss->conf; 4509 else if (acy->fate_action == 4510 MLX5_FLOW_FATE_SHARED_RSS) 4511 rss = acy->rss->conf; 4512 } 4513 } 4514 break; 4515 } 4516 default: 4517 break; 4518 } 4519 } 4520 return rss; 4521 } 4522 4523 /** 4524 * Get ASO age action by index. 4525 * 4526 * @param[in] dev 4527 * Pointer to the Ethernet device structure. 4528 * @param[in] age_idx 4529 * Index to the ASO age action. 4530 * 4531 * @return 4532 * The specified ASO age action. 4533 */ 4534 struct mlx5_aso_age_action* 4535 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4536 { 4537 uint16_t pool_idx = age_idx & UINT16_MAX; 4538 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4539 struct mlx5_priv *priv = dev->data->dev_private; 4540 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4541 struct mlx5_aso_age_pool *pool; 4542 4543 rte_rwlock_read_lock(&mng->resize_rwl); 4544 pool = mng->pools[pool_idx]; 4545 rte_rwlock_read_unlock(&mng->resize_rwl); 4546 return &pool->actions[offset - 1]; 4547 } 4548 4549 /* maps indirect action to translated direct in some actions array */ 4550 struct mlx5_translated_action_handle { 4551 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4552 int index; /**< Index in related array of rte_flow_action. */ 4553 }; 4554 4555 /** 4556 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4557 * direct action if translation possible. 4558 * This functionality used to run same execution path for both direct and 4559 * indirect actions on flow create. All necessary preparations for indirect 4560 * action handling should be performed on *handle* actions list returned 4561 * from this call. 4562 * 4563 * @param[in] dev 4564 * Pointer to Ethernet device. 4565 * @param[in] actions 4566 * List of actions to translate. 4567 * @param[out] handle 4568 * List to store translated indirect action object handles. 4569 * @param[in, out] indir_n 4570 * Size of *handle* array. On return should be updated with number of 4571 * indirect actions retrieved from the *actions* list. 4572 * @param[out] translated_actions 4573 * List of actions where all indirect actions were translated to direct 4574 * if possible. NULL if no translation took place. 4575 * @param[out] error 4576 * Pointer to the error structure. 4577 * 4578 * @return 4579 * 0 on success, a negative errno value otherwise and rte_errno is set. 4580 */ 4581 static int 4582 flow_action_handles_translate(struct rte_eth_dev *dev, 4583 const struct rte_flow_action actions[], 4584 struct mlx5_translated_action_handle *handle, 4585 int *indir_n, 4586 struct rte_flow_action **translated_actions, 4587 struct rte_flow_error *error) 4588 { 4589 struct mlx5_priv *priv = dev->data->dev_private; 4590 struct rte_flow_action *translated = NULL; 4591 size_t actions_size; 4592 int n; 4593 int copied_n = 0; 4594 struct mlx5_translated_action_handle *handle_end = NULL; 4595 4596 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4597 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4598 continue; 4599 if (copied_n == *indir_n) { 4600 return rte_flow_error_set 4601 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4602 NULL, "too many shared actions"); 4603 } 4604 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4605 sizeof(actions[n].conf)); 4606 handle[copied_n].index = n; 4607 copied_n++; 4608 } 4609 n++; 4610 *indir_n = copied_n; 4611 if (!copied_n) 4612 return 0; 4613 actions_size = sizeof(struct rte_flow_action) * n; 4614 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4615 if (!translated) { 4616 rte_errno = ENOMEM; 4617 return -ENOMEM; 4618 } 4619 memcpy(translated, actions, actions_size); 4620 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4621 struct mlx5_shared_action_rss *shared_rss; 4622 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4623 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4624 uint32_t idx = act_idx & 4625 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4626 4627 switch (type) { 4628 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4629 shared_rss = mlx5_ipool_get 4630 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4631 translated[handle->index].type = 4632 RTE_FLOW_ACTION_TYPE_RSS; 4633 translated[handle->index].conf = 4634 &shared_rss->origin; 4635 break; 4636 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4637 translated[handle->index].type = 4638 (enum rte_flow_action_type) 4639 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4640 translated[handle->index].conf = (void *)(uintptr_t)idx; 4641 break; 4642 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4643 translated[handle->index].type = 4644 (enum rte_flow_action_type) 4645 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4646 translated[handle->index].conf = (void *)(uintptr_t)idx; 4647 break; 4648 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4649 if (priv->sh->flow_hit_aso_en) { 4650 translated[handle->index].type = 4651 (enum rte_flow_action_type) 4652 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4653 translated[handle->index].conf = 4654 (void *)(uintptr_t)idx; 4655 break; 4656 } 4657 /* Fall-through */ 4658 case MLX5_INDIRECT_ACTION_TYPE_CT: 4659 if (priv->sh->ct_aso_en) { 4660 translated[handle->index].type = 4661 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4662 translated[handle->index].conf = 4663 (void *)(uintptr_t)idx; 4664 break; 4665 } 4666 /* Fall-through */ 4667 default: 4668 mlx5_free(translated); 4669 return rte_flow_error_set 4670 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4671 NULL, "invalid indirect action type"); 4672 } 4673 } 4674 *translated_actions = translated; 4675 return 0; 4676 } 4677 4678 /** 4679 * Get Shared RSS action from the action list. 4680 * 4681 * @param[in] dev 4682 * Pointer to Ethernet device. 4683 * @param[in] shared 4684 * Pointer to the list of actions. 4685 * @param[in] shared_n 4686 * Actions list length. 4687 * 4688 * @return 4689 * The MLX5 RSS action ID if exists, otherwise return 0. 4690 */ 4691 static uint32_t 4692 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4693 struct mlx5_translated_action_handle *handle, 4694 int shared_n) 4695 { 4696 struct mlx5_translated_action_handle *handle_end; 4697 struct mlx5_priv *priv = dev->data->dev_private; 4698 struct mlx5_shared_action_rss *shared_rss; 4699 4700 4701 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4702 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4703 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4704 uint32_t idx = act_idx & 4705 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4706 switch (type) { 4707 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4708 shared_rss = mlx5_ipool_get 4709 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4710 idx); 4711 __atomic_fetch_add(&shared_rss->refcnt, 1, 4712 __ATOMIC_RELAXED); 4713 return idx; 4714 default: 4715 break; 4716 } 4717 } 4718 return 0; 4719 } 4720 4721 static unsigned int 4722 find_graph_root(uint32_t rss_level) 4723 { 4724 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4725 MLX5_EXPANSION_ROOT_OUTER; 4726 } 4727 4728 /** 4729 * Get layer flags from the prefix flow. 4730 * 4731 * Some flows may be split to several subflows, the prefix subflow gets the 4732 * match items and the suffix sub flow gets the actions. 4733 * Some actions need the user defined match item flags to get the detail for 4734 * the action. 4735 * This function helps the suffix flow to get the item layer flags from prefix 4736 * subflow. 4737 * 4738 * @param[in] dev_flow 4739 * Pointer the created prefix subflow. 4740 * 4741 * @return 4742 * The layers get from prefix subflow. 4743 */ 4744 static inline uint64_t 4745 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4746 { 4747 uint64_t layers = 0; 4748 4749 /* 4750 * Layers bits could be localization, but usually the compiler will 4751 * help to do the optimization work for source code. 4752 * If no decap actions, use the layers directly. 4753 */ 4754 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4755 return dev_flow->handle->layers; 4756 /* Convert L3 layers with decap action. */ 4757 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4758 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4759 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4760 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4761 /* Convert L4 layers with decap action. */ 4762 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4763 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4764 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4765 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4766 return layers; 4767 } 4768 4769 /** 4770 * Get metadata split action information. 4771 * 4772 * @param[in] actions 4773 * Pointer to the list of actions. 4774 * @param[out] qrss 4775 * Pointer to the return pointer. 4776 * @param[out] qrss_type 4777 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4778 * if no QUEUE/RSS is found. 4779 * @param[out] encap_idx 4780 * Pointer to the index of the encap action if exists, otherwise the last 4781 * action index. 4782 * 4783 * @return 4784 * Total number of actions. 4785 */ 4786 static int 4787 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4788 const struct rte_flow_action **qrss, 4789 int *encap_idx) 4790 { 4791 const struct rte_flow_action_raw_encap *raw_encap; 4792 int actions_n = 0; 4793 int raw_decap_idx = -1; 4794 4795 *encap_idx = -1; 4796 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4797 switch (actions->type) { 4798 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4799 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4800 *encap_idx = actions_n; 4801 break; 4802 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4803 raw_decap_idx = actions_n; 4804 break; 4805 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4806 raw_encap = actions->conf; 4807 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4808 *encap_idx = raw_decap_idx != -1 ? 4809 raw_decap_idx : actions_n; 4810 break; 4811 case RTE_FLOW_ACTION_TYPE_QUEUE: 4812 case RTE_FLOW_ACTION_TYPE_RSS: 4813 *qrss = actions; 4814 break; 4815 default: 4816 break; 4817 } 4818 actions_n++; 4819 } 4820 if (*encap_idx == -1) 4821 *encap_idx = actions_n; 4822 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4823 return actions_n + 1; 4824 } 4825 4826 /** 4827 * Check if the action will change packet. 4828 * 4829 * @param dev 4830 * Pointer to Ethernet device. 4831 * @param[in] type 4832 * action type. 4833 * 4834 * @return 4835 * true if action will change packet, false otherwise. 4836 */ 4837 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4838 enum rte_flow_action_type type) 4839 { 4840 struct mlx5_priv *priv = dev->data->dev_private; 4841 4842 switch (type) { 4843 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4844 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4845 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4846 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4847 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4848 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4849 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4850 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4851 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4852 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4853 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4854 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4855 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4856 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4857 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4858 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4859 case RTE_FLOW_ACTION_TYPE_SET_META: 4860 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4861 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4862 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4863 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4864 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4865 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4866 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4867 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4868 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4869 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4870 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4871 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4872 return true; 4873 case RTE_FLOW_ACTION_TYPE_FLAG: 4874 case RTE_FLOW_ACTION_TYPE_MARK: 4875 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4876 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4877 return true; 4878 else 4879 return false; 4880 default: 4881 return false; 4882 } 4883 } 4884 4885 /** 4886 * Check meter action from the action list. 4887 * 4888 * @param dev 4889 * Pointer to Ethernet device. 4890 * @param[in] actions 4891 * Pointer to the list of actions. 4892 * @param[out] has_mtr 4893 * Pointer to the meter exist flag. 4894 * @param[out] has_modify 4895 * Pointer to the flag showing there's packet change action. 4896 * @param[out] meter_id 4897 * Pointer to the meter id. 4898 * 4899 * @return 4900 * Total number of actions. 4901 */ 4902 static int 4903 flow_check_meter_action(struct rte_eth_dev *dev, 4904 const struct rte_flow_action actions[], 4905 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4906 { 4907 const struct rte_flow_action_meter *mtr = NULL; 4908 int actions_n = 0; 4909 4910 MLX5_ASSERT(has_mtr); 4911 *has_mtr = false; 4912 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4913 switch (actions->type) { 4914 case RTE_FLOW_ACTION_TYPE_METER: 4915 mtr = actions->conf; 4916 *meter_id = mtr->mtr_id; 4917 *has_mtr = true; 4918 break; 4919 default: 4920 break; 4921 } 4922 if (!*has_mtr) 4923 *has_modify |= flow_check_modify_action_type(dev, 4924 actions->type); 4925 actions_n++; 4926 } 4927 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4928 return actions_n + 1; 4929 } 4930 4931 /** 4932 * Check if the flow should be split due to hairpin. 4933 * The reason for the split is that in current HW we can't 4934 * support encap and push-vlan on Rx, so if a flow contains 4935 * these actions we move it to Tx. 4936 * 4937 * @param dev 4938 * Pointer to Ethernet device. 4939 * @param[in] attr 4940 * Flow rule attributes. 4941 * @param[in] actions 4942 * Associated actions (list terminated by the END action). 4943 * 4944 * @return 4945 * > 0 the number of actions and the flow should be split, 4946 * 0 when no split required. 4947 */ 4948 static int 4949 flow_check_hairpin_split(struct rte_eth_dev *dev, 4950 const struct rte_flow_attr *attr, 4951 const struct rte_flow_action actions[]) 4952 { 4953 int queue_action = 0; 4954 int action_n = 0; 4955 int split = 0; 4956 int push_vlan = 0; 4957 const struct rte_flow_action_queue *queue; 4958 const struct rte_flow_action_rss *rss; 4959 const struct rte_flow_action_raw_encap *raw_encap; 4960 const struct rte_eth_hairpin_conf *conf; 4961 4962 if (!attr->ingress) 4963 return 0; 4964 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4965 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4966 push_vlan = 1; 4967 switch (actions->type) { 4968 case RTE_FLOW_ACTION_TYPE_QUEUE: 4969 queue = actions->conf; 4970 if (queue == NULL) 4971 return 0; 4972 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4973 if (conf == NULL || conf->tx_explicit != 0) 4974 return 0; 4975 queue_action = 1; 4976 action_n++; 4977 break; 4978 case RTE_FLOW_ACTION_TYPE_RSS: 4979 rss = actions->conf; 4980 if (rss == NULL || rss->queue_num == 0) 4981 return 0; 4982 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4983 if (conf == NULL || conf->tx_explicit != 0) 4984 return 0; 4985 queue_action = 1; 4986 action_n++; 4987 break; 4988 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4989 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4990 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4991 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4992 split++; 4993 action_n++; 4994 break; 4995 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4996 if (push_vlan) 4997 split++; 4998 action_n++; 4999 break; 5000 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5001 raw_encap = actions->conf; 5002 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5003 split++; 5004 action_n++; 5005 break; 5006 default: 5007 action_n++; 5008 break; 5009 } 5010 } 5011 if (split && queue_action) 5012 return action_n; 5013 return 0; 5014 } 5015 5016 /* Declare flow create/destroy prototype in advance. */ 5017 static uint32_t 5018 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 5019 const struct rte_flow_attr *attr, 5020 const struct rte_flow_item items[], 5021 const struct rte_flow_action actions[], 5022 bool external, struct rte_flow_error *error); 5023 5024 static void 5025 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 5026 uint32_t flow_idx); 5027 5028 int 5029 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 5030 struct mlx5_list_entry *entry, void *cb_ctx) 5031 { 5032 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 5033 struct mlx5_flow_mreg_copy_resource *mcp_res = 5034 container_of(entry, typeof(*mcp_res), hlist_ent); 5035 5036 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 5037 } 5038 5039 struct mlx5_list_entry * 5040 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 5041 { 5042 struct rte_eth_dev *dev = tool_ctx; 5043 struct mlx5_priv *priv = dev->data->dev_private; 5044 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 5045 struct mlx5_flow_mreg_copy_resource *mcp_res; 5046 struct rte_flow_error *error = ctx->error; 5047 uint32_t idx = 0; 5048 int ret; 5049 uint32_t mark_id = *(uint32_t *)(ctx->data); 5050 struct rte_flow_attr attr = { 5051 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5052 .ingress = 1, 5053 }; 5054 struct mlx5_rte_flow_item_tag tag_spec = { 5055 .data = mark_id, 5056 }; 5057 struct rte_flow_item items[] = { 5058 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 5059 }; 5060 struct rte_flow_action_mark ftag = { 5061 .id = mark_id, 5062 }; 5063 struct mlx5_flow_action_copy_mreg cp_mreg = { 5064 .dst = REG_B, 5065 .src = REG_NON, 5066 }; 5067 struct rte_flow_action_jump jump = { 5068 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 5069 }; 5070 struct rte_flow_action actions[] = { 5071 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 5072 }; 5073 5074 /* Fill the register fields in the flow. */ 5075 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 5076 if (ret < 0) 5077 return NULL; 5078 tag_spec.id = ret; 5079 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5080 if (ret < 0) 5081 return NULL; 5082 cp_mreg.src = ret; 5083 /* Provide the full width of FLAG specific value. */ 5084 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 5085 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 5086 /* Build a new flow. */ 5087 if (mark_id != MLX5_DEFAULT_COPY_ID) { 5088 items[0] = (struct rte_flow_item){ 5089 .type = (enum rte_flow_item_type) 5090 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5091 .spec = &tag_spec, 5092 }; 5093 items[1] = (struct rte_flow_item){ 5094 .type = RTE_FLOW_ITEM_TYPE_END, 5095 }; 5096 actions[0] = (struct rte_flow_action){ 5097 .type = (enum rte_flow_action_type) 5098 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 5099 .conf = &ftag, 5100 }; 5101 actions[1] = (struct rte_flow_action){ 5102 .type = (enum rte_flow_action_type) 5103 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5104 .conf = &cp_mreg, 5105 }; 5106 actions[2] = (struct rte_flow_action){ 5107 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5108 .conf = &jump, 5109 }; 5110 actions[3] = (struct rte_flow_action){ 5111 .type = RTE_FLOW_ACTION_TYPE_END, 5112 }; 5113 } else { 5114 /* Default rule, wildcard match. */ 5115 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 5116 items[0] = (struct rte_flow_item){ 5117 .type = RTE_FLOW_ITEM_TYPE_END, 5118 }; 5119 actions[0] = (struct rte_flow_action){ 5120 .type = (enum rte_flow_action_type) 5121 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5122 .conf = &cp_mreg, 5123 }; 5124 actions[1] = (struct rte_flow_action){ 5125 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5126 .conf = &jump, 5127 }; 5128 actions[2] = (struct rte_flow_action){ 5129 .type = RTE_FLOW_ACTION_TYPE_END, 5130 }; 5131 } 5132 /* Build a new entry. */ 5133 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5134 if (!mcp_res) { 5135 rte_errno = ENOMEM; 5136 return NULL; 5137 } 5138 mcp_res->idx = idx; 5139 mcp_res->mark_id = mark_id; 5140 /* 5141 * The copy Flows are not included in any list. There 5142 * ones are referenced from other Flows and can not 5143 * be applied, removed, deleted in arbitrary order 5144 * by list traversing. 5145 */ 5146 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 5147 &attr, items, actions, false, error); 5148 if (!mcp_res->rix_flow) { 5149 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 5150 return NULL; 5151 } 5152 return &mcp_res->hlist_ent; 5153 } 5154 5155 struct mlx5_list_entry * 5156 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5157 void *cb_ctx __rte_unused) 5158 { 5159 struct rte_eth_dev *dev = tool_ctx; 5160 struct mlx5_priv *priv = dev->data->dev_private; 5161 struct mlx5_flow_mreg_copy_resource *mcp_res; 5162 uint32_t idx = 0; 5163 5164 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5165 if (!mcp_res) { 5166 rte_errno = ENOMEM; 5167 return NULL; 5168 } 5169 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 5170 mcp_res->idx = idx; 5171 return &mcp_res->hlist_ent; 5172 } 5173 5174 void 5175 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5176 { 5177 struct mlx5_flow_mreg_copy_resource *mcp_res = 5178 container_of(entry, typeof(*mcp_res), hlist_ent); 5179 struct rte_eth_dev *dev = tool_ctx; 5180 struct mlx5_priv *priv = dev->data->dev_private; 5181 5182 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5183 } 5184 5185 /** 5186 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5187 * 5188 * As mark_id is unique, if there's already a registered flow for the mark_id, 5189 * return by increasing the reference counter of the resource. Otherwise, create 5190 * the resource (mcp_res) and flow. 5191 * 5192 * Flow looks like, 5193 * - If ingress port is ANY and reg_c[1] is mark_id, 5194 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5195 * 5196 * For default flow (zero mark_id), flow is like, 5197 * - If ingress port is ANY, 5198 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5199 * 5200 * @param dev 5201 * Pointer to Ethernet device. 5202 * @param mark_id 5203 * ID of MARK action, zero means default flow for META. 5204 * @param[out] error 5205 * Perform verbose error reporting if not NULL. 5206 * 5207 * @return 5208 * Associated resource on success, NULL otherwise and rte_errno is set. 5209 */ 5210 static struct mlx5_flow_mreg_copy_resource * 5211 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5212 struct rte_flow_error *error) 5213 { 5214 struct mlx5_priv *priv = dev->data->dev_private; 5215 struct mlx5_list_entry *entry; 5216 struct mlx5_flow_cb_ctx ctx = { 5217 .dev = dev, 5218 .error = error, 5219 .data = &mark_id, 5220 }; 5221 5222 /* Check if already registered. */ 5223 MLX5_ASSERT(priv->mreg_cp_tbl); 5224 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5225 if (!entry) 5226 return NULL; 5227 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5228 hlist_ent); 5229 } 5230 5231 void 5232 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5233 { 5234 struct mlx5_flow_mreg_copy_resource *mcp_res = 5235 container_of(entry, typeof(*mcp_res), hlist_ent); 5236 struct rte_eth_dev *dev = tool_ctx; 5237 struct mlx5_priv *priv = dev->data->dev_private; 5238 5239 MLX5_ASSERT(mcp_res->rix_flow); 5240 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5241 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5242 } 5243 5244 /** 5245 * Release flow in RX_CP_TBL. 5246 * 5247 * @param dev 5248 * Pointer to Ethernet device. 5249 * @flow 5250 * Parent flow for wich copying is provided. 5251 */ 5252 static void 5253 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5254 struct rte_flow *flow) 5255 { 5256 struct mlx5_flow_mreg_copy_resource *mcp_res; 5257 struct mlx5_priv *priv = dev->data->dev_private; 5258 5259 if (!flow->rix_mreg_copy) 5260 return; 5261 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5262 flow->rix_mreg_copy); 5263 if (!mcp_res || !priv->mreg_cp_tbl) 5264 return; 5265 MLX5_ASSERT(mcp_res->rix_flow); 5266 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5267 flow->rix_mreg_copy = 0; 5268 } 5269 5270 /** 5271 * Remove the default copy action from RX_CP_TBL. 5272 * 5273 * This functions is called in the mlx5_dev_start(). No thread safe 5274 * is guaranteed. 5275 * 5276 * @param dev 5277 * Pointer to Ethernet device. 5278 */ 5279 static void 5280 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5281 { 5282 struct mlx5_list_entry *entry; 5283 struct mlx5_priv *priv = dev->data->dev_private; 5284 struct mlx5_flow_cb_ctx ctx; 5285 uint32_t mark_id; 5286 5287 /* Check if default flow is registered. */ 5288 if (!priv->mreg_cp_tbl) 5289 return; 5290 mark_id = MLX5_DEFAULT_COPY_ID; 5291 ctx.data = &mark_id; 5292 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5293 if (!entry) 5294 return; 5295 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5296 } 5297 5298 /** 5299 * Add the default copy action in RX_CP_TBL. 5300 * 5301 * This functions is called in the mlx5_dev_start(). No thread safe 5302 * is guaranteed. 5303 * 5304 * @param dev 5305 * Pointer to Ethernet device. 5306 * @param[out] error 5307 * Perform verbose error reporting if not NULL. 5308 * 5309 * @return 5310 * 0 for success, negative value otherwise and rte_errno is set. 5311 */ 5312 static int 5313 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5314 struct rte_flow_error *error) 5315 { 5316 struct mlx5_priv *priv = dev->data->dev_private; 5317 struct mlx5_flow_mreg_copy_resource *mcp_res; 5318 struct mlx5_flow_cb_ctx ctx; 5319 uint32_t mark_id; 5320 5321 /* Check whether extensive metadata feature is engaged. */ 5322 if (!priv->sh->config.dv_flow_en || 5323 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5324 !mlx5_flow_ext_mreg_supported(dev) || 5325 !priv->sh->dv_regc0_mask) 5326 return 0; 5327 /* 5328 * Add default mreg copy flow may be called multiple time, but 5329 * only be called once in stop. Avoid register it twice. 5330 */ 5331 mark_id = MLX5_DEFAULT_COPY_ID; 5332 ctx.data = &mark_id; 5333 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5334 return 0; 5335 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5336 if (!mcp_res) 5337 return -rte_errno; 5338 return 0; 5339 } 5340 5341 /** 5342 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5343 * 5344 * All the flow having Q/RSS action should be split by 5345 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5346 * performs the following, 5347 * - CQE->flow_tag := reg_c[1] (MARK) 5348 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5349 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5350 * but there should be a flow per each MARK ID set by MARK action. 5351 * 5352 * For the aforementioned reason, if there's a MARK action in flow's action 5353 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5354 * the MARK ID to CQE's flow_tag like, 5355 * - If reg_c[1] is mark_id, 5356 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5357 * 5358 * For SET_META action which stores value in reg_c[0], as the destination is 5359 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5360 * MARK ID means the default flow. The default flow looks like, 5361 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5362 * 5363 * @param dev 5364 * Pointer to Ethernet device. 5365 * @param flow 5366 * Pointer to flow structure. 5367 * @param[in] actions 5368 * Pointer to the list of actions. 5369 * @param[out] error 5370 * Perform verbose error reporting if not NULL. 5371 * 5372 * @return 5373 * 0 on success, negative value otherwise and rte_errno is set. 5374 */ 5375 static int 5376 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5377 struct rte_flow *flow, 5378 const struct rte_flow_action *actions, 5379 struct rte_flow_error *error) 5380 { 5381 struct mlx5_priv *priv = dev->data->dev_private; 5382 struct mlx5_sh_config *config = &priv->sh->config; 5383 struct mlx5_flow_mreg_copy_resource *mcp_res; 5384 const struct rte_flow_action_mark *mark; 5385 5386 /* Check whether extensive metadata feature is engaged. */ 5387 if (!config->dv_flow_en || 5388 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5389 !mlx5_flow_ext_mreg_supported(dev) || 5390 !priv->sh->dv_regc0_mask) 5391 return 0; 5392 /* Find MARK action. */ 5393 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5394 switch (actions->type) { 5395 case RTE_FLOW_ACTION_TYPE_FLAG: 5396 mcp_res = flow_mreg_add_copy_action 5397 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5398 if (!mcp_res) 5399 return -rte_errno; 5400 flow->rix_mreg_copy = mcp_res->idx; 5401 return 0; 5402 case RTE_FLOW_ACTION_TYPE_MARK: 5403 mark = (const struct rte_flow_action_mark *) 5404 actions->conf; 5405 mcp_res = 5406 flow_mreg_add_copy_action(dev, mark->id, error); 5407 if (!mcp_res) 5408 return -rte_errno; 5409 flow->rix_mreg_copy = mcp_res->idx; 5410 return 0; 5411 default: 5412 break; 5413 } 5414 } 5415 return 0; 5416 } 5417 5418 #define MLX5_MAX_SPLIT_ACTIONS 24 5419 #define MLX5_MAX_SPLIT_ITEMS 24 5420 5421 /** 5422 * Split the hairpin flow. 5423 * Since HW can't support encap and push-vlan on Rx, we move these 5424 * actions to Tx. 5425 * If the count action is after the encap then we also 5426 * move the count action. in this case the count will also measure 5427 * the outer bytes. 5428 * 5429 * @param dev 5430 * Pointer to Ethernet device. 5431 * @param[in] actions 5432 * Associated actions (list terminated by the END action). 5433 * @param[out] actions_rx 5434 * Rx flow actions. 5435 * @param[out] actions_tx 5436 * Tx flow actions.. 5437 * @param[out] pattern_tx 5438 * The pattern items for the Tx flow. 5439 * @param[out] flow_id 5440 * The flow ID connected to this flow. 5441 * 5442 * @return 5443 * 0 on success. 5444 */ 5445 static int 5446 flow_hairpin_split(struct rte_eth_dev *dev, 5447 const struct rte_flow_action actions[], 5448 struct rte_flow_action actions_rx[], 5449 struct rte_flow_action actions_tx[], 5450 struct rte_flow_item pattern_tx[], 5451 uint32_t flow_id) 5452 { 5453 const struct rte_flow_action_raw_encap *raw_encap; 5454 const struct rte_flow_action_raw_decap *raw_decap; 5455 struct mlx5_rte_flow_action_set_tag *set_tag; 5456 struct rte_flow_action *tag_action; 5457 struct mlx5_rte_flow_item_tag *tag_item; 5458 struct rte_flow_item *item; 5459 char *addr; 5460 int push_vlan = 0; 5461 int encap = 0; 5462 5463 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5464 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5465 push_vlan = 1; 5466 switch (actions->type) { 5467 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5468 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5469 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5470 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5471 rte_memcpy(actions_tx, actions, 5472 sizeof(struct rte_flow_action)); 5473 actions_tx++; 5474 break; 5475 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5476 if (push_vlan) { 5477 rte_memcpy(actions_tx, actions, 5478 sizeof(struct rte_flow_action)); 5479 actions_tx++; 5480 } else { 5481 rte_memcpy(actions_rx, actions, 5482 sizeof(struct rte_flow_action)); 5483 actions_rx++; 5484 } 5485 break; 5486 case RTE_FLOW_ACTION_TYPE_COUNT: 5487 if (encap) { 5488 rte_memcpy(actions_tx, actions, 5489 sizeof(struct rte_flow_action)); 5490 actions_tx++; 5491 } else { 5492 rte_memcpy(actions_rx, actions, 5493 sizeof(struct rte_flow_action)); 5494 actions_rx++; 5495 } 5496 break; 5497 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5498 raw_encap = actions->conf; 5499 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5500 memcpy(actions_tx, actions, 5501 sizeof(struct rte_flow_action)); 5502 actions_tx++; 5503 encap = 1; 5504 } else { 5505 rte_memcpy(actions_rx, actions, 5506 sizeof(struct rte_flow_action)); 5507 actions_rx++; 5508 } 5509 break; 5510 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5511 raw_decap = actions->conf; 5512 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5513 memcpy(actions_tx, actions, 5514 sizeof(struct rte_flow_action)); 5515 actions_tx++; 5516 } else { 5517 rte_memcpy(actions_rx, actions, 5518 sizeof(struct rte_flow_action)); 5519 actions_rx++; 5520 } 5521 break; 5522 default: 5523 rte_memcpy(actions_rx, actions, 5524 sizeof(struct rte_flow_action)); 5525 actions_rx++; 5526 break; 5527 } 5528 } 5529 /* Add set meta action and end action for the Rx flow. */ 5530 tag_action = actions_rx; 5531 tag_action->type = (enum rte_flow_action_type) 5532 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5533 actions_rx++; 5534 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5535 actions_rx++; 5536 set_tag = (void *)actions_rx; 5537 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5538 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5539 .data = flow_id, 5540 }; 5541 MLX5_ASSERT(set_tag->id > REG_NON); 5542 tag_action->conf = set_tag; 5543 /* Create Tx item list. */ 5544 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5545 addr = (void *)&pattern_tx[2]; 5546 item = pattern_tx; 5547 item->type = (enum rte_flow_item_type) 5548 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5549 tag_item = (void *)addr; 5550 tag_item->data = flow_id; 5551 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5552 MLX5_ASSERT(set_tag->id > REG_NON); 5553 item->spec = tag_item; 5554 addr += sizeof(struct mlx5_rte_flow_item_tag); 5555 tag_item = (void *)addr; 5556 tag_item->data = UINT32_MAX; 5557 tag_item->id = UINT16_MAX; 5558 item->mask = tag_item; 5559 item->last = NULL; 5560 item++; 5561 item->type = RTE_FLOW_ITEM_TYPE_END; 5562 return 0; 5563 } 5564 5565 /** 5566 * The last stage of splitting chain, just creates the subflow 5567 * without any modification. 5568 * 5569 * @param[in] dev 5570 * Pointer to Ethernet device. 5571 * @param[in] flow 5572 * Parent flow structure pointer. 5573 * @param[in, out] sub_flow 5574 * Pointer to return the created subflow, may be NULL. 5575 * @param[in] attr 5576 * Flow rule attributes. 5577 * @param[in] items 5578 * Pattern specification (list terminated by the END pattern item). 5579 * @param[in] actions 5580 * Associated actions (list terminated by the END action). 5581 * @param[in] flow_split_info 5582 * Pointer to flow split info structure. 5583 * @param[out] error 5584 * Perform verbose error reporting if not NULL. 5585 * @return 5586 * 0 on success, negative value otherwise 5587 */ 5588 static int 5589 flow_create_split_inner(struct rte_eth_dev *dev, 5590 struct rte_flow *flow, 5591 struct mlx5_flow **sub_flow, 5592 const struct rte_flow_attr *attr, 5593 const struct rte_flow_item items[], 5594 const struct rte_flow_action actions[], 5595 struct mlx5_flow_split_info *flow_split_info, 5596 struct rte_flow_error *error) 5597 { 5598 struct mlx5_flow *dev_flow; 5599 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5600 5601 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5602 flow_split_info->flow_idx, error); 5603 if (!dev_flow) 5604 return -rte_errno; 5605 dev_flow->flow = flow; 5606 dev_flow->external = flow_split_info->external; 5607 dev_flow->skip_scale = flow_split_info->skip_scale; 5608 /* Subflow object was created, we must include one in the list. */ 5609 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5610 dev_flow->handle, next); 5611 /* 5612 * If dev_flow is as one of the suffix flow, some actions in suffix 5613 * flow may need some user defined item layer flags, and pass the 5614 * Metadata rxq mark flag to suffix flow as well. 5615 */ 5616 if (flow_split_info->prefix_layers) 5617 dev_flow->handle->layers = flow_split_info->prefix_layers; 5618 if (flow_split_info->prefix_mark) { 5619 MLX5_ASSERT(wks); 5620 wks->mark = 1; 5621 } 5622 if (sub_flow) 5623 *sub_flow = dev_flow; 5624 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5625 dev_flow->dv.table_id = flow_split_info->table_id; 5626 #endif 5627 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5628 } 5629 5630 /** 5631 * Get the sub policy of a meter. 5632 * 5633 * @param[in] dev 5634 * Pointer to Ethernet device. 5635 * @param[in] flow 5636 * Parent flow structure pointer. 5637 * @param wks 5638 * Pointer to thread flow work space. 5639 * @param[in] attr 5640 * Flow rule attributes. 5641 * @param[in] items 5642 * Pattern specification (list terminated by the END pattern item). 5643 * @param[out] error 5644 * Perform verbose error reporting if not NULL. 5645 * 5646 * @return 5647 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5648 */ 5649 static struct mlx5_flow_meter_sub_policy * 5650 get_meter_sub_policy(struct rte_eth_dev *dev, 5651 struct rte_flow *flow, 5652 struct mlx5_flow_workspace *wks, 5653 const struct rte_flow_attr *attr, 5654 const struct rte_flow_item items[], 5655 struct rte_flow_error *error) 5656 { 5657 struct mlx5_flow_meter_policy *policy; 5658 struct mlx5_flow_meter_policy *final_policy; 5659 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5660 5661 policy = wks->policy; 5662 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5663 if (final_policy->is_rss || final_policy->is_queue) { 5664 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5665 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5666 uint32_t i; 5667 5668 /* 5669 * This is a tmp dev_flow, 5670 * no need to register any matcher for it in translate. 5671 */ 5672 wks->skip_matcher_reg = 1; 5673 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5674 struct mlx5_flow dev_flow = {0}; 5675 struct mlx5_flow_handle dev_handle = { {0} }; 5676 uint8_t fate = final_policy->act_cnt[i].fate_action; 5677 5678 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5679 const struct rte_flow_action_rss *rss_act = 5680 final_policy->act_cnt[i].rss->conf; 5681 struct rte_flow_action rss_actions[2] = { 5682 [0] = { 5683 .type = RTE_FLOW_ACTION_TYPE_RSS, 5684 .conf = rss_act, 5685 }, 5686 [1] = { 5687 .type = RTE_FLOW_ACTION_TYPE_END, 5688 .conf = NULL, 5689 } 5690 }; 5691 5692 dev_flow.handle = &dev_handle; 5693 dev_flow.ingress = attr->ingress; 5694 dev_flow.flow = flow; 5695 dev_flow.external = 0; 5696 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5697 dev_flow.dv.transfer = attr->transfer; 5698 #endif 5699 /** 5700 * Translate RSS action to get rss hash fields. 5701 */ 5702 if (flow_drv_translate(dev, &dev_flow, attr, 5703 items, rss_actions, error)) 5704 goto exit; 5705 rss_desc_v[i] = wks->rss_desc; 5706 rss_desc_v[i].symmetric_hash_function = 5707 dev_flow.symmetric_hash_function; 5708 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5709 rss_desc_v[i].hash_fields = 5710 dev_flow.hash_fields; 5711 rss_desc_v[i].queue_num = 5712 rss_desc_v[i].hash_fields ? 5713 rss_desc_v[i].queue_num : 1; 5714 rss_desc_v[i].tunnel = 5715 !!(dev_flow.handle->layers & 5716 MLX5_FLOW_LAYER_TUNNEL); 5717 /* Use the RSS queues in the containers. */ 5718 rss_desc_v[i].queue = 5719 (uint16_t *)(uintptr_t)rss_act->queue; 5720 rss_desc[i] = &rss_desc_v[i]; 5721 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5722 /* This is queue action. */ 5723 rss_desc_v[i] = wks->rss_desc; 5724 rss_desc_v[i].key_len = 0; 5725 rss_desc_v[i].hash_fields = 0; 5726 rss_desc_v[i].queue = 5727 &final_policy->act_cnt[i].queue; 5728 rss_desc_v[i].queue_num = 1; 5729 rss_desc[i] = &rss_desc_v[i]; 5730 } else { 5731 rss_desc[i] = NULL; 5732 } 5733 } 5734 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5735 flow, policy, rss_desc); 5736 } else { 5737 enum mlx5_meter_domain mtr_domain = 5738 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5739 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5740 MLX5_MTR_DOMAIN_INGRESS); 5741 sub_policy = policy->sub_policys[mtr_domain][0]; 5742 } 5743 if (!sub_policy) 5744 rte_flow_error_set(error, EINVAL, 5745 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5746 "Failed to get meter sub-policy."); 5747 exit: 5748 return sub_policy; 5749 } 5750 5751 /** 5752 * Split the meter flow. 5753 * 5754 * As meter flow will split to three sub flow, other than meter 5755 * action, the other actions make sense to only meter accepts 5756 * the packet. If it need to be dropped, no other additional 5757 * actions should be take. 5758 * 5759 * One kind of special action which decapsulates the L3 tunnel 5760 * header will be in the prefix sub flow, as not to take the 5761 * L3 tunnel header into account. 5762 * 5763 * @param[in] dev 5764 * Pointer to Ethernet device. 5765 * @param[in] flow 5766 * Parent flow structure pointer. 5767 * @param wks 5768 * Pointer to thread flow work space. 5769 * @param[in] attr 5770 * Flow rule attributes. 5771 * @param[in] items 5772 * Pattern specification (list terminated by the END pattern item). 5773 * @param[out] sfx_items 5774 * Suffix flow match items (list terminated by the END pattern item). 5775 * @param[in] actions 5776 * Associated actions (list terminated by the END action). 5777 * @param[out] actions_sfx 5778 * Suffix flow actions. 5779 * @param[out] actions_pre 5780 * Prefix flow actions. 5781 * @param[out] mtr_flow_id 5782 * Pointer to meter flow id. 5783 * @param[out] error 5784 * Perform verbose error reporting if not NULL. 5785 * 5786 * @return 5787 * 0 on success, a negative errno value otherwise and rte_errno is set. 5788 */ 5789 static int 5790 flow_meter_split_prep(struct rte_eth_dev *dev, 5791 struct rte_flow *flow, 5792 struct mlx5_flow_workspace *wks, 5793 const struct rte_flow_attr *attr, 5794 const struct rte_flow_item items[], 5795 struct rte_flow_item sfx_items[], 5796 const struct rte_flow_action actions[], 5797 struct rte_flow_action actions_sfx[], 5798 struct rte_flow_action actions_pre[], 5799 uint32_t *mtr_flow_id, 5800 struct rte_flow_error *error) 5801 { 5802 struct mlx5_priv *priv = dev->data->dev_private; 5803 struct mlx5_flow_meter_info *fm = wks->fm; 5804 struct rte_flow_action *tag_action = NULL; 5805 struct rte_flow_item *tag_item; 5806 struct mlx5_rte_flow_action_set_tag *set_tag; 5807 const struct rte_flow_action_raw_encap *raw_encap; 5808 const struct rte_flow_action_raw_decap *raw_decap; 5809 struct mlx5_rte_flow_item_tag *tag_item_spec; 5810 struct mlx5_rte_flow_item_tag *tag_item_mask; 5811 uint32_t tag_id = 0; 5812 struct rte_flow_item *vlan_item_dst = NULL; 5813 const struct rte_flow_item *vlan_item_src = NULL; 5814 const struct rte_flow_item *orig_items = items; 5815 struct rte_flow_action *hw_mtr_action; 5816 struct rte_flow_action *action_pre_head = NULL; 5817 uint16_t flow_src_port = priv->representor_id; 5818 bool mtr_first; 5819 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5820 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5821 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5822 uint32_t flow_id = 0; 5823 uint32_t flow_id_reversed = 0; 5824 uint8_t flow_id_bits = 0; 5825 bool after_meter = false; 5826 int shift; 5827 5828 /* Prepare the suffix subflow items. */ 5829 tag_item = sfx_items++; 5830 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5831 int item_type = items->type; 5832 5833 switch (item_type) { 5834 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5835 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5836 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5837 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5838 return -rte_errno; 5839 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5840 flow_src_port != priv->representor_id) { 5841 if (flow_drv_mtr_hierarchy_rule_create(dev, 5842 flow, fm, 5843 flow_src_port, 5844 items, 5845 error)) 5846 return -rte_errno; 5847 } 5848 memcpy(sfx_items, items, sizeof(*sfx_items)); 5849 sfx_items++; 5850 break; 5851 case RTE_FLOW_ITEM_TYPE_VLAN: 5852 /* Determine if copy vlan item below. */ 5853 vlan_item_src = items; 5854 vlan_item_dst = sfx_items++; 5855 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5856 break; 5857 default: 5858 break; 5859 } 5860 } 5861 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5862 sfx_items++; 5863 mtr_first = priv->sh->meter_aso_en && 5864 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5865 /* For ASO meter, meter must be before tag in TX direction. */ 5866 if (mtr_first) { 5867 action_pre_head = actions_pre++; 5868 /* Leave space for tag action. */ 5869 tag_action = actions_pre++; 5870 } 5871 /* Prepare the actions for prefix and suffix flow. */ 5872 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5873 struct rte_flow_action *action_cur = NULL; 5874 5875 switch (actions->type) { 5876 case RTE_FLOW_ACTION_TYPE_METER: 5877 if (mtr_first) { 5878 action_cur = action_pre_head; 5879 } else { 5880 /* Leave space for tag action. */ 5881 tag_action = actions_pre++; 5882 action_cur = actions_pre++; 5883 } 5884 after_meter = true; 5885 break; 5886 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5887 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5888 action_cur = actions_pre++; 5889 break; 5890 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5891 raw_encap = actions->conf; 5892 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5893 action_cur = actions_pre++; 5894 break; 5895 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5896 raw_decap = actions->conf; 5897 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5898 action_cur = actions_pre++; 5899 break; 5900 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5901 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5902 if (vlan_item_dst && vlan_item_src) { 5903 memcpy(vlan_item_dst, vlan_item_src, 5904 sizeof(*vlan_item_dst)); 5905 /* 5906 * Convert to internal match item, it is used 5907 * for vlan push and set vid. 5908 */ 5909 vlan_item_dst->type = (enum rte_flow_item_type) 5910 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5911 } 5912 break; 5913 case RTE_FLOW_ACTION_TYPE_COUNT: 5914 if (fm->def_policy) 5915 action_cur = after_meter ? 5916 actions_sfx++ : actions_pre++; 5917 break; 5918 default: 5919 break; 5920 } 5921 if (!action_cur) 5922 action_cur = (fm->def_policy) ? 5923 actions_sfx++ : actions_pre++; 5924 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5925 } 5926 /* Add end action to the actions. */ 5927 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5928 if (priv->sh->meter_aso_en) { 5929 /** 5930 * For ASO meter, need to add an extra jump action explicitly, 5931 * to jump from meter to policer table. 5932 */ 5933 struct mlx5_flow_meter_sub_policy *sub_policy; 5934 struct mlx5_flow_tbl_data_entry *tbl_data; 5935 5936 if (!fm->def_policy) { 5937 sub_policy = get_meter_sub_policy(dev, flow, wks, 5938 attr, orig_items, 5939 error); 5940 if (!sub_policy) 5941 return -rte_errno; 5942 } else { 5943 enum mlx5_meter_domain mtr_domain = 5944 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5945 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5946 MLX5_MTR_DOMAIN_INGRESS); 5947 5948 sub_policy = 5949 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5950 } 5951 tbl_data = container_of(sub_policy->tbl_rsc, 5952 struct mlx5_flow_tbl_data_entry, tbl); 5953 hw_mtr_action = actions_pre++; 5954 hw_mtr_action->type = (enum rte_flow_action_type) 5955 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5956 hw_mtr_action->conf = tbl_data->jump.action; 5957 } 5958 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5959 actions_pre++; 5960 if (!tag_action) 5961 return rte_flow_error_set(error, ENOMEM, 5962 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5963 NULL, "No tag action space."); 5964 if (!mtr_flow_id) { 5965 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5966 goto exit; 5967 } 5968 /* Only default-policy Meter creates mtr flow id. */ 5969 if (fm->def_policy) { 5970 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5971 if (!tag_id) 5972 return rte_flow_error_set(error, ENOMEM, 5973 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5974 "Failed to allocate meter flow id."); 5975 flow_id = tag_id - 1; 5976 flow_id_bits = (!flow_id) ? 1 : 5977 (MLX5_REG_BITS - rte_clz32(flow_id)); 5978 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5979 mtr_reg_bits) { 5980 mlx5_ipool_free(fm->flow_ipool, tag_id); 5981 return rte_flow_error_set(error, EINVAL, 5982 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5983 "Meter flow id exceeds max limit."); 5984 } 5985 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5986 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5987 } 5988 /* Build tag actions and items for meter_id/meter flow_id. */ 5989 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5990 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5991 tag_item_mask = tag_item_spec + 1; 5992 /* Both flow_id and meter_id share the same register. */ 5993 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5994 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5995 0, error), 5996 .offset = mtr_id_offset, 5997 .length = mtr_reg_bits, 5998 .data = flow->meter, 5999 }; 6000 /* 6001 * The color Reg bits used by flow_id are growing from 6002 * msb to lsb, so must do bit reverse for flow_id val in RegC. 6003 */ 6004 for (shift = 0; shift < flow_id_bits; shift++) 6005 flow_id_reversed = (flow_id_reversed << 1) | 6006 ((flow_id >> shift) & 0x1); 6007 set_tag->data |= 6008 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 6009 tag_item_spec->id = set_tag->id; 6010 tag_item_spec->data = set_tag->data << mtr_id_offset; 6011 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 6012 tag_action->type = (enum rte_flow_action_type) 6013 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 6014 tag_action->conf = set_tag; 6015 tag_item->type = (enum rte_flow_item_type) 6016 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 6017 tag_item->spec = tag_item_spec; 6018 tag_item->last = NULL; 6019 tag_item->mask = tag_item_mask; 6020 exit: 6021 if (mtr_flow_id) 6022 *mtr_flow_id = tag_id; 6023 return 0; 6024 } 6025 6026 /** 6027 * Split action list having QUEUE/RSS for metadata register copy. 6028 * 6029 * Once Q/RSS action is detected in user's action list, the flow action 6030 * should be split in order to copy metadata registers, which will happen in 6031 * RX_CP_TBL like, 6032 * - CQE->flow_tag := reg_c[1] (MARK) 6033 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 6034 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 6035 * This is because the last action of each flow must be a terminal action 6036 * (QUEUE, RSS or DROP). 6037 * 6038 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 6039 * stored and kept in the mlx5_flow structure per each sub_flow. 6040 * 6041 * The Q/RSS action is replaced with, 6042 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 6043 * And the following JUMP action is added at the end, 6044 * - JUMP, to RX_CP_TBL. 6045 * 6046 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 6047 * flow_create_split_metadata() routine. The flow will look like, 6048 * - If flow ID matches (reg_c[2]), perform Q/RSS. 6049 * 6050 * @param dev 6051 * Pointer to Ethernet device. 6052 * @param[out] split_actions 6053 * Pointer to store split actions to jump to CP_TBL. 6054 * @param[in] actions 6055 * Pointer to the list of original flow actions. 6056 * @param[in] qrss 6057 * Pointer to the Q/RSS action. 6058 * @param[in] actions_n 6059 * Number of original actions. 6060 * @param[in] mtr_sfx 6061 * Check if it is in meter suffix table. 6062 * @param[out] error 6063 * Perform verbose error reporting if not NULL. 6064 * 6065 * @return 6066 * non-zero unique flow_id on success, otherwise 0 and 6067 * error/rte_error are set. 6068 */ 6069 static uint32_t 6070 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 6071 struct rte_flow_action *split_actions, 6072 const struct rte_flow_action *actions, 6073 const struct rte_flow_action *qrss, 6074 int actions_n, int mtr_sfx, 6075 struct rte_flow_error *error) 6076 { 6077 struct mlx5_priv *priv = dev->data->dev_private; 6078 struct mlx5_rte_flow_action_set_tag *set_tag; 6079 struct rte_flow_action_jump *jump; 6080 const int qrss_idx = qrss - actions; 6081 uint32_t flow_id = 0; 6082 int ret = 0; 6083 6084 /* 6085 * Given actions will be split 6086 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 6087 * - Add jump to mreg CP_TBL. 6088 * As a result, there will be one more action. 6089 */ 6090 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 6091 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 6092 ++actions_n; 6093 set_tag = (void *)(split_actions + actions_n); 6094 /* 6095 * If we are not the meter suffix flow, add the tag action. 6096 * Since meter suffix flow already has the tag added. 6097 */ 6098 if (!mtr_sfx) { 6099 /* 6100 * Allocate the new subflow ID. This one is unique within 6101 * device and not shared with representors. Otherwise, 6102 * we would have to resolve multi-thread access synch 6103 * issue. Each flow on the shared device is appended 6104 * with source vport identifier, so the resulting 6105 * flows will be unique in the shared (by master and 6106 * representors) domain even if they have coinciding 6107 * IDs. 6108 */ 6109 mlx5_ipool_malloc(priv->sh->ipool 6110 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 6111 if (!flow_id) 6112 return rte_flow_error_set(error, ENOMEM, 6113 RTE_FLOW_ERROR_TYPE_ACTION, 6114 NULL, "can't allocate id " 6115 "for split Q/RSS subflow"); 6116 /* Internal SET_TAG action to set flow ID. */ 6117 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 6118 .data = flow_id, 6119 }; 6120 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 6121 if (ret < 0) 6122 return ret; 6123 set_tag->id = ret; 6124 /* Construct new actions array. */ 6125 /* Replace QUEUE/RSS action. */ 6126 split_actions[qrss_idx] = (struct rte_flow_action){ 6127 .type = (enum rte_flow_action_type) 6128 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6129 .conf = set_tag, 6130 }; 6131 } else { 6132 /* 6133 * If we are the suffix flow of meter, tag already exist. 6134 * Set the QUEUE/RSS action to void. 6135 */ 6136 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 6137 } 6138 /* JUMP action to jump to mreg copy table (CP_TBL). */ 6139 jump = (void *)(set_tag + 1); 6140 *jump = (struct rte_flow_action_jump){ 6141 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6142 }; 6143 split_actions[actions_n - 2] = (struct rte_flow_action){ 6144 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6145 .conf = jump, 6146 }; 6147 split_actions[actions_n - 1] = (struct rte_flow_action){ 6148 .type = RTE_FLOW_ACTION_TYPE_END, 6149 }; 6150 return flow_id; 6151 } 6152 6153 /** 6154 * Extend the given action list for Tx metadata copy. 6155 * 6156 * Copy the given action list to the ext_actions and add flow metadata register 6157 * copy action in order to copy reg_a set by WQE to reg_c[0]. 6158 * 6159 * @param[out] ext_actions 6160 * Pointer to the extended action list. 6161 * @param[in] actions 6162 * Pointer to the list of actions. 6163 * @param[in] actions_n 6164 * Number of actions in the list. 6165 * @param[out] error 6166 * Perform verbose error reporting if not NULL. 6167 * @param[in] encap_idx 6168 * The encap action index. 6169 * 6170 * @return 6171 * 0 on success, negative value otherwise 6172 */ 6173 static int 6174 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6175 struct rte_flow_action *ext_actions, 6176 const struct rte_flow_action *actions, 6177 int actions_n, struct rte_flow_error *error, 6178 int encap_idx) 6179 { 6180 struct mlx5_flow_action_copy_mreg *cp_mreg = 6181 (struct mlx5_flow_action_copy_mreg *) 6182 (ext_actions + actions_n + 1); 6183 int ret; 6184 6185 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6186 if (ret < 0) 6187 return ret; 6188 cp_mreg->dst = ret; 6189 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6190 if (ret < 0) 6191 return ret; 6192 cp_mreg->src = ret; 6193 if (encap_idx != 0) 6194 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6195 if (encap_idx == actions_n - 1) { 6196 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6197 .type = (enum rte_flow_action_type) 6198 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6199 .conf = cp_mreg, 6200 }; 6201 ext_actions[actions_n] = (struct rte_flow_action){ 6202 .type = RTE_FLOW_ACTION_TYPE_END, 6203 }; 6204 } else { 6205 ext_actions[encap_idx] = (struct rte_flow_action){ 6206 .type = (enum rte_flow_action_type) 6207 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6208 .conf = cp_mreg, 6209 }; 6210 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6211 sizeof(*ext_actions) * (actions_n - encap_idx)); 6212 } 6213 return 0; 6214 } 6215 6216 /** 6217 * Check the match action from the action list. 6218 * 6219 * @param[in] actions 6220 * Pointer to the list of actions. 6221 * @param[in] attr 6222 * Flow rule attributes. 6223 * @param[in] action 6224 * The action to be check if exist. 6225 * @param[out] match_action_pos 6226 * Pointer to the position of the matched action if exists, otherwise is -1. 6227 * @param[out] qrss_action_pos 6228 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6229 * @param[out] modify_after_mirror 6230 * Pointer to the flag of modify action after FDB mirroring. 6231 * 6232 * @return 6233 * > 0 the total number of actions. 6234 * 0 if not found match action in action list. 6235 */ 6236 static int 6237 flow_check_match_action(const struct rte_flow_action actions[], 6238 const struct rte_flow_attr *attr, 6239 enum rte_flow_action_type action, 6240 int *match_action_pos, int *qrss_action_pos, 6241 int *modify_after_mirror) 6242 { 6243 const struct rte_flow_action_sample *sample; 6244 const struct rte_flow_action_raw_decap *decap; 6245 const struct rte_flow_action *action_cur = NULL; 6246 int actions_n = 0; 6247 uint32_t ratio = 0; 6248 int sub_type = 0; 6249 int flag = 0; 6250 int fdb_mirror = 0; 6251 6252 *match_action_pos = -1; 6253 *qrss_action_pos = -1; 6254 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6255 if (actions->type == action) { 6256 flag = 1; 6257 *match_action_pos = actions_n; 6258 } 6259 switch (actions->type) { 6260 case RTE_FLOW_ACTION_TYPE_QUEUE: 6261 case RTE_FLOW_ACTION_TYPE_RSS: 6262 *qrss_action_pos = actions_n; 6263 break; 6264 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6265 sample = actions->conf; 6266 ratio = sample->ratio; 6267 sub_type = ((const struct rte_flow_action *) 6268 (sample->actions))->type; 6269 if (ratio == 1 && attr->transfer && 6270 sub_type != RTE_FLOW_ACTION_TYPE_END) 6271 fdb_mirror = 1; 6272 break; 6273 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6274 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6275 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6276 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6277 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6278 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6279 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6280 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6281 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6282 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6283 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6284 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6285 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6286 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6287 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6288 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6289 case RTE_FLOW_ACTION_TYPE_FLAG: 6290 case RTE_FLOW_ACTION_TYPE_MARK: 6291 case RTE_FLOW_ACTION_TYPE_SET_META: 6292 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6293 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6294 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6295 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6296 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6297 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6298 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6299 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6300 case RTE_FLOW_ACTION_TYPE_METER: 6301 if (fdb_mirror) 6302 *modify_after_mirror = 1; 6303 break; 6304 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6305 decap = actions->conf; 6306 action_cur = actions; 6307 while ((++action_cur)->type == RTE_FLOW_ACTION_TYPE_VOID) 6308 ; 6309 if (action_cur->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6310 const struct rte_flow_action_raw_encap *encap = 6311 action_cur->conf; 6312 if (decap->size <= 6313 MLX5_ENCAPSULATION_DECISION_SIZE && 6314 encap->size > 6315 MLX5_ENCAPSULATION_DECISION_SIZE) 6316 /* L3 encap. */ 6317 break; 6318 } 6319 if (fdb_mirror) 6320 *modify_after_mirror = 1; 6321 break; 6322 default: 6323 break; 6324 } 6325 actions_n++; 6326 } 6327 if (flag && fdb_mirror && !*modify_after_mirror) { 6328 /* FDB mirroring uses the destination array to implement 6329 * instead of FLOW_SAMPLER object. 6330 */ 6331 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6332 flag = 0; 6333 } 6334 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6335 return flag ? actions_n + 1 : 0; 6336 } 6337 6338 #define SAMPLE_SUFFIX_ITEM 3 6339 6340 /** 6341 * Split the sample flow. 6342 * 6343 * As sample flow will split to two sub flow, sample flow with 6344 * sample action, the other actions will move to new suffix flow. 6345 * 6346 * Also add unique tag id with tag action in the sample flow, 6347 * the same tag id will be as match in the suffix flow. 6348 * 6349 * @param dev 6350 * Pointer to Ethernet device. 6351 * @param[in] add_tag 6352 * Add extra tag action flag. 6353 * @param[out] sfx_items 6354 * Suffix flow match items (list terminated by the END pattern item). 6355 * @param[in] actions 6356 * Associated actions (list terminated by the END action). 6357 * @param[out] actions_sfx 6358 * Suffix flow actions. 6359 * @param[out] actions_pre 6360 * Prefix flow actions. 6361 * @param[in] actions_n 6362 * The total number of actions. 6363 * @param[in] sample_action_pos 6364 * The sample action position. 6365 * @param[in] qrss_action_pos 6366 * The Queue/RSS action position. 6367 * @param[in] jump_table 6368 * Add extra jump action flag. 6369 * @param[out] error 6370 * Perform verbose error reporting if not NULL. 6371 * 6372 * @return 6373 * 0 on success, or unique flow_id, a negative errno value 6374 * otherwise and rte_errno is set. 6375 */ 6376 static int 6377 flow_sample_split_prep(struct rte_eth_dev *dev, 6378 int add_tag, 6379 const struct rte_flow_item items[], 6380 struct rte_flow_item sfx_items[], 6381 const struct rte_flow_action actions[], 6382 struct rte_flow_action actions_sfx[], 6383 struct rte_flow_action actions_pre[], 6384 int actions_n, 6385 int sample_action_pos, 6386 int qrss_action_pos, 6387 int jump_table, 6388 struct rte_flow_error *error) 6389 { 6390 struct mlx5_priv *priv = dev->data->dev_private; 6391 struct mlx5_rte_flow_action_set_tag *set_tag; 6392 struct mlx5_rte_flow_item_tag *tag_spec; 6393 struct mlx5_rte_flow_item_tag *tag_mask; 6394 struct rte_flow_action_jump *jump_action; 6395 uint32_t tag_id = 0; 6396 int append_index = 0; 6397 int set_tag_idx = -1; 6398 int index; 6399 int ret; 6400 6401 if (sample_action_pos < 0) 6402 return rte_flow_error_set(error, EINVAL, 6403 RTE_FLOW_ERROR_TYPE_ACTION, 6404 NULL, "invalid position of sample " 6405 "action in list"); 6406 /* Prepare the actions for prefix and suffix flow. */ 6407 if (add_tag) { 6408 /* Update the new added tag action index preceding 6409 * the PUSH_VLAN or ENCAP action. 6410 */ 6411 const struct rte_flow_action_raw_encap *raw_encap; 6412 const struct rte_flow_action *action = actions; 6413 int encap_idx; 6414 int action_idx = 0; 6415 int raw_decap_idx = -1; 6416 int push_vlan_idx = -1; 6417 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6418 switch (action->type) { 6419 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6420 raw_decap_idx = action_idx; 6421 break; 6422 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6423 raw_encap = action->conf; 6424 if (raw_encap->size > 6425 MLX5_ENCAPSULATION_DECISION_SIZE) { 6426 encap_idx = raw_decap_idx != -1 ? 6427 raw_decap_idx : action_idx; 6428 if (encap_idx < sample_action_pos && 6429 push_vlan_idx == -1) 6430 set_tag_idx = encap_idx; 6431 } 6432 break; 6433 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6434 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6435 encap_idx = action_idx; 6436 if (encap_idx < sample_action_pos && 6437 push_vlan_idx == -1) 6438 set_tag_idx = encap_idx; 6439 break; 6440 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6441 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6442 if (action_idx < sample_action_pos && 6443 push_vlan_idx == -1) { 6444 set_tag_idx = action_idx; 6445 push_vlan_idx = action_idx; 6446 } 6447 break; 6448 default: 6449 break; 6450 } 6451 action_idx++; 6452 } 6453 } 6454 /* Prepare the actions for prefix and suffix flow. */ 6455 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6456 index = qrss_action_pos; 6457 /* Put the preceding the Queue/RSS action into prefix flow. */ 6458 if (index != 0) 6459 memcpy(actions_pre, actions, 6460 sizeof(struct rte_flow_action) * index); 6461 /* Put others preceding the sample action into prefix flow. */ 6462 if (sample_action_pos > index + 1) 6463 memcpy(actions_pre + index, actions + index + 1, 6464 sizeof(struct rte_flow_action) * 6465 (sample_action_pos - index - 1)); 6466 index = sample_action_pos - 1; 6467 /* Put Queue/RSS action into Suffix flow. */ 6468 memcpy(actions_sfx, actions + qrss_action_pos, 6469 sizeof(struct rte_flow_action)); 6470 actions_sfx++; 6471 } else if (add_tag && set_tag_idx >= 0) { 6472 if (set_tag_idx > 0) 6473 memcpy(actions_pre, actions, 6474 sizeof(struct rte_flow_action) * set_tag_idx); 6475 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6476 sizeof(struct rte_flow_action) * 6477 (sample_action_pos - set_tag_idx)); 6478 index = sample_action_pos; 6479 } else { 6480 index = sample_action_pos; 6481 if (index != 0) 6482 memcpy(actions_pre, actions, 6483 sizeof(struct rte_flow_action) * index); 6484 } 6485 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6486 * For CX6DX and above, metadata registers Cx preserve their value, 6487 * add an extra tag action for NIC-RX and E-Switch Domain. 6488 */ 6489 if (add_tag) { 6490 /* Prepare the prefix tag action. */ 6491 append_index++; 6492 set_tag = (void *)(actions_pre + actions_n + append_index); 6493 /* Trust VF/SF on CX5 not supported meter so that the reserved 6494 * metadata regC is REG_NON, back to use application tag 6495 * index 0. 6496 */ 6497 if (unlikely(priv->sh->registers.aso_reg == REG_NON)) 6498 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6499 else 6500 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6501 if (ret < 0) 6502 return ret; 6503 mlx5_ipool_malloc(priv->sh->ipool 6504 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6505 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6506 .id = ret, 6507 .data = tag_id, 6508 }; 6509 /* Prepare the suffix subflow items. */ 6510 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6511 tag_spec->data = tag_id; 6512 tag_spec->id = set_tag->id; 6513 tag_mask = tag_spec + 1; 6514 tag_mask->data = UINT32_MAX; 6515 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6516 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6517 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6518 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6519 memcpy(sfx_items, items, sizeof(*sfx_items)); 6520 sfx_items++; 6521 break; 6522 } 6523 } 6524 sfx_items[0] = (struct rte_flow_item){ 6525 .type = (enum rte_flow_item_type) 6526 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6527 .spec = tag_spec, 6528 .last = NULL, 6529 .mask = tag_mask, 6530 }; 6531 sfx_items[1] = (struct rte_flow_item){ 6532 .type = (enum rte_flow_item_type) 6533 RTE_FLOW_ITEM_TYPE_END, 6534 }; 6535 /* Prepare the tag action in prefix subflow. */ 6536 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6537 actions_pre[set_tag_idx] = 6538 (struct rte_flow_action){ 6539 .type = (enum rte_flow_action_type) 6540 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6541 .conf = set_tag, 6542 }; 6543 /* Update next sample position due to add one tag action */ 6544 index += 1; 6545 } 6546 /* Copy the sample action into prefix flow. */ 6547 memcpy(actions_pre + index, actions + sample_action_pos, 6548 sizeof(struct rte_flow_action)); 6549 index += 1; 6550 /* For the modify action after the sample action in E-Switch mirroring, 6551 * Add the extra jump action in prefix subflow and jump into the next 6552 * table, then do the modify action in the new table. 6553 */ 6554 if (jump_table) { 6555 /* Prepare the prefix jump action. */ 6556 append_index++; 6557 jump_action = (void *)(actions_pre + actions_n + append_index); 6558 jump_action->group = jump_table; 6559 actions_pre[index++] = 6560 (struct rte_flow_action){ 6561 .type = (enum rte_flow_action_type) 6562 RTE_FLOW_ACTION_TYPE_JUMP, 6563 .conf = jump_action, 6564 }; 6565 } 6566 actions_pre[index] = (struct rte_flow_action){ 6567 .type = (enum rte_flow_action_type) 6568 RTE_FLOW_ACTION_TYPE_END, 6569 }; 6570 /* Put the actions after sample into Suffix flow. */ 6571 memcpy(actions_sfx, actions + sample_action_pos + 1, 6572 sizeof(struct rte_flow_action) * 6573 (actions_n - sample_action_pos - 1)); 6574 return tag_id; 6575 } 6576 6577 /** 6578 * The splitting for metadata feature. 6579 * 6580 * - Q/RSS action on NIC Rx should be split in order to pass by 6581 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6582 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6583 * 6584 * - All the actions on NIC Tx should have a mreg copy action to 6585 * copy reg_a from WQE to reg_c[0]. 6586 * 6587 * @param dev 6588 * Pointer to Ethernet device. 6589 * @param[in] flow 6590 * Parent flow structure pointer. 6591 * @param[in] attr 6592 * Flow rule attributes. 6593 * @param[in] items 6594 * Pattern specification (list terminated by the END pattern item). 6595 * @param[in] actions 6596 * Associated actions (list terminated by the END action). 6597 * @param[in] flow_split_info 6598 * Pointer to flow split info structure. 6599 * @param[out] error 6600 * Perform verbose error reporting if not NULL. 6601 * @return 6602 * 0 on success, negative value otherwise 6603 */ 6604 static int 6605 flow_create_split_metadata(struct rte_eth_dev *dev, 6606 struct rte_flow *flow, 6607 const struct rte_flow_attr *attr, 6608 const struct rte_flow_item items[], 6609 const struct rte_flow_action actions[], 6610 struct mlx5_flow_split_info *flow_split_info, 6611 struct rte_flow_error *error) 6612 { 6613 struct mlx5_priv *priv = dev->data->dev_private; 6614 struct mlx5_sh_config *config = &priv->sh->config; 6615 const struct rte_flow_action *qrss = NULL; 6616 struct rte_flow_action *ext_actions = NULL; 6617 struct mlx5_flow *dev_flow = NULL; 6618 uint32_t qrss_id = 0; 6619 int mtr_sfx = 0; 6620 size_t act_size; 6621 int actions_n; 6622 int encap_idx; 6623 int ret; 6624 6625 /* Check whether extensive metadata feature is engaged. */ 6626 if (!config->dv_flow_en || 6627 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6628 !mlx5_flow_ext_mreg_supported(dev)) 6629 return flow_create_split_inner(dev, flow, NULL, attr, items, 6630 actions, flow_split_info, error); 6631 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6632 &encap_idx); 6633 if (qrss) { 6634 /* Exclude hairpin flows from splitting. */ 6635 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6636 const struct rte_flow_action_queue *queue; 6637 6638 queue = qrss->conf; 6639 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6640 qrss = NULL; 6641 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6642 const struct rte_flow_action_rss *rss; 6643 6644 rss = qrss->conf; 6645 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6646 qrss = NULL; 6647 } 6648 } 6649 if (qrss) { 6650 /* Check if it is in meter suffix table. */ 6651 mtr_sfx = attr->group == 6652 ((attr->transfer && priv->fdb_def_rule) ? 6653 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6654 MLX5_FLOW_TABLE_LEVEL_METER); 6655 /* 6656 * Q/RSS action on NIC Rx should be split in order to pass by 6657 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6658 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6659 */ 6660 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6661 sizeof(struct rte_flow_action_set_tag) + 6662 sizeof(struct rte_flow_action_jump); 6663 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6664 SOCKET_ID_ANY); 6665 if (!ext_actions) 6666 return rte_flow_error_set(error, ENOMEM, 6667 RTE_FLOW_ERROR_TYPE_ACTION, 6668 NULL, "no memory to split " 6669 "metadata flow"); 6670 /* 6671 * Create the new actions list with removed Q/RSS action 6672 * and appended set tag and jump to register copy table 6673 * (RX_CP_TBL). We should preallocate unique tag ID here 6674 * in advance, because it is needed for set tag action. 6675 */ 6676 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6677 qrss, actions_n, 6678 mtr_sfx, error); 6679 if (!mtr_sfx && !qrss_id) { 6680 ret = -rte_errno; 6681 goto exit; 6682 } 6683 } else if (attr->egress) { 6684 /* 6685 * All the actions on NIC Tx should have a metadata register 6686 * copy action to copy reg_a from WQE to reg_c[meta] 6687 */ 6688 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6689 sizeof(struct mlx5_flow_action_copy_mreg); 6690 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6691 SOCKET_ID_ANY); 6692 if (!ext_actions) 6693 return rte_flow_error_set(error, ENOMEM, 6694 RTE_FLOW_ERROR_TYPE_ACTION, 6695 NULL, "no memory to split " 6696 "metadata flow"); 6697 /* Create the action list appended with copy register. */ 6698 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6699 actions_n, error, encap_idx); 6700 if (ret < 0) 6701 goto exit; 6702 } 6703 /* Add the unmodified original or prefix subflow. */ 6704 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6705 items, ext_actions ? ext_actions : 6706 actions, flow_split_info, error); 6707 if (ret < 0) 6708 goto exit; 6709 MLX5_ASSERT(dev_flow); 6710 if (qrss) { 6711 const struct rte_flow_attr q_attr = { 6712 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6713 .ingress = 1, 6714 }; 6715 /* Internal PMD action to set register. */ 6716 struct mlx5_rte_flow_item_tag q_tag_spec = { 6717 .data = qrss_id, 6718 .id = REG_NON, 6719 }; 6720 struct rte_flow_item q_items[] = { 6721 { 6722 .type = (enum rte_flow_item_type) 6723 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6724 .spec = &q_tag_spec, 6725 .last = NULL, 6726 .mask = NULL, 6727 }, 6728 { 6729 .type = RTE_FLOW_ITEM_TYPE_END, 6730 }, 6731 }; 6732 struct rte_flow_action q_actions[] = { 6733 { 6734 .type = qrss->type, 6735 .conf = qrss->conf, 6736 }, 6737 { 6738 .type = RTE_FLOW_ACTION_TYPE_END, 6739 }, 6740 }; 6741 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6742 6743 /* 6744 * Configure the tag item only if there is no meter subflow. 6745 * Since tag is already marked in the meter suffix subflow 6746 * we can just use the meter suffix items as is. 6747 */ 6748 if (qrss_id) { 6749 /* Not meter subflow. */ 6750 MLX5_ASSERT(!mtr_sfx); 6751 /* 6752 * Put unique id in prefix flow due to it is destroyed 6753 * after suffix flow and id will be freed after there 6754 * is no actual flows with this id and identifier 6755 * reallocation becomes possible (for example, for 6756 * other flows in other threads). 6757 */ 6758 dev_flow->handle->split_flow_id = qrss_id; 6759 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6760 error); 6761 if (ret < 0) 6762 goto exit; 6763 q_tag_spec.id = ret; 6764 } 6765 dev_flow = NULL; 6766 /* Add suffix subflow to execute Q/RSS. */ 6767 flow_split_info->prefix_layers = layers; 6768 flow_split_info->prefix_mark = 0; 6769 flow_split_info->table_id = 0; 6770 ret = flow_create_split_inner(dev, flow, &dev_flow, 6771 &q_attr, mtr_sfx ? items : 6772 q_items, q_actions, 6773 flow_split_info, error); 6774 if (ret < 0) 6775 goto exit; 6776 /* qrss ID should be freed if failed. */ 6777 qrss_id = 0; 6778 MLX5_ASSERT(dev_flow); 6779 } 6780 6781 exit: 6782 /* 6783 * We do not destroy the partially created sub_flows in case of error. 6784 * These ones are included into parent flow list and will be destroyed 6785 * by flow_drv_destroy. 6786 */ 6787 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6788 qrss_id); 6789 mlx5_free(ext_actions); 6790 return ret; 6791 } 6792 6793 /** 6794 * Create meter internal drop flow with the original pattern. 6795 * 6796 * @param dev 6797 * Pointer to Ethernet device. 6798 * @param[in] flow 6799 * Parent flow structure pointer. 6800 * @param[in] attr 6801 * Flow rule attributes. 6802 * @param[in] items 6803 * Pattern specification (list terminated by the END pattern item). 6804 * @param[in] flow_split_info 6805 * Pointer to flow split info structure. 6806 * @param[in] fm 6807 * Pointer to flow meter structure. 6808 * @param[out] error 6809 * Perform verbose error reporting if not NULL. 6810 * @return 6811 * 0 on success, negative value otherwise 6812 */ 6813 static uint32_t 6814 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6815 struct rte_flow *flow, 6816 const struct rte_flow_attr *attr, 6817 const struct rte_flow_item items[], 6818 struct mlx5_flow_split_info *flow_split_info, 6819 struct mlx5_flow_meter_info *fm, 6820 struct rte_flow_error *error) 6821 { 6822 struct mlx5_flow *dev_flow = NULL; 6823 struct rte_flow_attr drop_attr = *attr; 6824 struct rte_flow_action drop_actions[3]; 6825 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6826 6827 MLX5_ASSERT(fm->drop_cnt); 6828 drop_actions[0].type = 6829 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6830 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6831 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6832 drop_actions[1].conf = NULL; 6833 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6834 drop_actions[2].conf = NULL; 6835 drop_split_info.external = false; 6836 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6837 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6838 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6839 return flow_create_split_inner(dev, flow, &dev_flow, 6840 &drop_attr, items, drop_actions, 6841 &drop_split_info, error); 6842 } 6843 6844 /** 6845 * The splitting for meter feature. 6846 * 6847 * - The meter flow will be split to two flows as prefix and 6848 * suffix flow. The packets make sense only it pass the prefix 6849 * meter action. 6850 * 6851 * - Reg_C_5 is used for the packet to match betweend prefix and 6852 * suffix flow. 6853 * 6854 * @param dev 6855 * Pointer to Ethernet device. 6856 * @param[in] flow 6857 * Parent flow structure pointer. 6858 * @param[in] attr 6859 * Flow rule attributes. 6860 * @param[in] items 6861 * Pattern specification (list terminated by the END pattern item). 6862 * @param[in] actions 6863 * Associated actions (list terminated by the END action). 6864 * @param[in] flow_split_info 6865 * Pointer to flow split info structure. 6866 * @param[out] error 6867 * Perform verbose error reporting if not NULL. 6868 * @return 6869 * 0 on success, negative value otherwise 6870 */ 6871 static int 6872 flow_create_split_meter(struct rte_eth_dev *dev, 6873 struct rte_flow *flow, 6874 const struct rte_flow_attr *attr, 6875 const struct rte_flow_item items[], 6876 const struct rte_flow_action actions[], 6877 struct mlx5_flow_split_info *flow_split_info, 6878 struct rte_flow_error *error) 6879 { 6880 struct mlx5_priv *priv = dev->data->dev_private; 6881 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6882 struct rte_flow_action *sfx_actions = NULL; 6883 struct rte_flow_action *pre_actions = NULL; 6884 struct rte_flow_item *sfx_items = NULL; 6885 struct mlx5_flow *dev_flow = NULL; 6886 struct rte_flow_attr sfx_attr = *attr; 6887 struct mlx5_flow_meter_info *fm = NULL; 6888 uint8_t skip_scale_restore; 6889 bool has_mtr = false; 6890 bool has_modify = false; 6891 bool set_mtr_reg = true; 6892 bool is_mtr_hierarchy = false; 6893 uint32_t meter_id = 0; 6894 uint32_t mtr_idx = 0; 6895 uint32_t mtr_flow_id = 0; 6896 size_t act_size; 6897 size_t item_size; 6898 int actions_n = 0; 6899 int ret = 0; 6900 6901 if (priv->mtr_en) 6902 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6903 &has_modify, &meter_id); 6904 if (has_mtr) { 6905 if (flow->meter) { 6906 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6907 if (!fm) 6908 return rte_flow_error_set(error, EINVAL, 6909 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6910 NULL, "Meter not found."); 6911 } else { 6912 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6913 if (!fm) 6914 return rte_flow_error_set(error, EINVAL, 6915 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6916 NULL, "Meter not found."); 6917 ret = mlx5_flow_meter_attach(priv, fm, 6918 &sfx_attr, error); 6919 if (ret) 6920 return -rte_errno; 6921 flow->meter = mtr_idx; 6922 } 6923 MLX5_ASSERT(wks); 6924 wks->fm = fm; 6925 if (!fm->def_policy) { 6926 wks->policy = mlx5_flow_meter_policy_find(dev, 6927 fm->policy_id, 6928 NULL); 6929 MLX5_ASSERT(wks->policy); 6930 if (wks->policy->mark) 6931 wks->mark = 1; 6932 if (wks->policy->is_hierarchy) { 6933 wks->final_policy = 6934 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6935 wks->policy); 6936 if (!wks->final_policy) 6937 return rte_flow_error_set(error, 6938 EINVAL, 6939 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6940 "Failed to find terminal policy of hierarchy."); 6941 is_mtr_hierarchy = true; 6942 } 6943 } 6944 /* 6945 * If it isn't default-policy Meter, and 6946 * 1. Not meter hierarchy and there's no action in flow to change 6947 * packet (modify/encap/decap etc.), OR 6948 * 2. No drop count needed for this meter. 6949 * Then no need to use regC to save meter id anymore. 6950 */ 6951 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6952 set_mtr_reg = false; 6953 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6954 #define METER_PREFIX_ACTION 7 6955 act_size = (sizeof(struct rte_flow_action) * 6956 (actions_n + METER_PREFIX_ACTION)) + 6957 sizeof(struct mlx5_rte_flow_action_set_tag); 6958 /* Suffix items: tag, vlan, port id, end. */ 6959 #define METER_SUFFIX_ITEM 4 6960 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6961 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6962 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6963 0, SOCKET_ID_ANY); 6964 if (!sfx_actions) 6965 return rte_flow_error_set(error, ENOMEM, 6966 RTE_FLOW_ERROR_TYPE_ACTION, 6967 NULL, "no memory to split " 6968 "meter flow"); 6969 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6970 act_size); 6971 /* There's no suffix flow for meter of non-default policy. */ 6972 if (!fm->def_policy) 6973 pre_actions = sfx_actions + 1; 6974 else 6975 pre_actions = sfx_actions + actions_n; 6976 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6977 items, sfx_items, actions, 6978 sfx_actions, pre_actions, 6979 (set_mtr_reg ? &mtr_flow_id : NULL), 6980 error); 6981 if (ret) { 6982 ret = -rte_errno; 6983 goto exit; 6984 } 6985 /* Add the prefix subflow. */ 6986 skip_scale_restore = flow_split_info->skip_scale; 6987 flow_split_info->skip_scale |= 6988 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6989 ret = flow_create_split_inner(dev, flow, &dev_flow, 6990 attr, items, pre_actions, 6991 flow_split_info, error); 6992 flow_split_info->skip_scale = skip_scale_restore; 6993 if (ret) { 6994 if (mtr_flow_id) 6995 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6996 ret = -rte_errno; 6997 goto exit; 6998 } 6999 if (mtr_flow_id) { 7000 dev_flow->handle->split_flow_id = mtr_flow_id; 7001 dev_flow->handle->is_meter_flow_id = 1; 7002 } 7003 if (!fm->def_policy) { 7004 if (!set_mtr_reg && fm->drop_cnt) 7005 ret = 7006 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 7007 &sfx_attr, items, 7008 flow_split_info, 7009 fm, error); 7010 goto exit; 7011 } 7012 /* Setting the sfx group atrr. */ 7013 sfx_attr.group = sfx_attr.transfer ? 7014 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 7015 MLX5_FLOW_TABLE_LEVEL_METER; 7016 flow_split_info->prefix_layers = 7017 flow_get_prefix_layer_flags(dev_flow); 7018 flow_split_info->prefix_mark |= wks->mark; 7019 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 7020 } 7021 /* Add the prefix subflow. */ 7022 ret = flow_create_split_metadata(dev, flow, 7023 &sfx_attr, sfx_items ? 7024 sfx_items : items, 7025 sfx_actions ? sfx_actions : actions, 7026 flow_split_info, error); 7027 exit: 7028 if (sfx_actions) 7029 mlx5_free(sfx_actions); 7030 return ret; 7031 } 7032 7033 /** 7034 * The splitting for sample feature. 7035 * 7036 * Once Sample action is detected in the action list, the flow actions should 7037 * be split into prefix sub flow and suffix sub flow. 7038 * 7039 * The original items remain in the prefix sub flow, all actions preceding the 7040 * sample action and the sample action itself will be copied to the prefix 7041 * sub flow, the actions following the sample action will be copied to the 7042 * suffix sub flow, Queue action always be located in the suffix sub flow. 7043 * 7044 * In order to make the packet from prefix sub flow matches with suffix sub 7045 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 7046 * flow uses tag item with the unique flow id. 7047 * 7048 * @param dev 7049 * Pointer to Ethernet device. 7050 * @param[in] flow 7051 * Parent flow structure pointer. 7052 * @param[in] attr 7053 * Flow rule attributes. 7054 * @param[in] items 7055 * Pattern specification (list terminated by the END pattern item). 7056 * @param[in] actions 7057 * Associated actions (list terminated by the END action). 7058 * @param[in] flow_split_info 7059 * Pointer to flow split info structure. 7060 * @param[out] error 7061 * Perform verbose error reporting if not NULL. 7062 * @return 7063 * 0 on success, negative value otherwise 7064 */ 7065 static int 7066 flow_create_split_sample(struct rte_eth_dev *dev, 7067 struct rte_flow *flow, 7068 const struct rte_flow_attr *attr, 7069 const struct rte_flow_item items[], 7070 const struct rte_flow_action actions[], 7071 struct mlx5_flow_split_info *flow_split_info, 7072 struct rte_flow_error *error) 7073 { 7074 struct mlx5_priv *priv = dev->data->dev_private; 7075 struct rte_flow_action *sfx_actions = NULL; 7076 struct rte_flow_action *pre_actions = NULL; 7077 struct rte_flow_item *sfx_items = NULL; 7078 struct mlx5_flow *dev_flow = NULL; 7079 struct rte_flow_attr sfx_attr = *attr; 7080 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7081 struct mlx5_flow_dv_sample_resource *sample_res; 7082 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 7083 struct mlx5_flow_tbl_resource *sfx_tbl; 7084 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 7085 #endif 7086 size_t act_size; 7087 size_t item_size; 7088 uint32_t fdb_tx = 0; 7089 int32_t tag_id = 0; 7090 int actions_n = 0; 7091 int sample_action_pos; 7092 int qrss_action_pos; 7093 int add_tag = 0; 7094 int modify_after_mirror = 0; 7095 uint16_t jump_table = 0; 7096 const uint32_t next_ft_step = 1; 7097 int ret = 0; 7098 struct mlx5_priv *item_port_priv = NULL; 7099 const struct rte_flow_item *item; 7100 7101 if (priv->sampler_en) 7102 actions_n = flow_check_match_action(actions, attr, 7103 RTE_FLOW_ACTION_TYPE_SAMPLE, 7104 &sample_action_pos, &qrss_action_pos, 7105 &modify_after_mirror); 7106 if (actions_n) { 7107 /* The prefix actions must includes sample, tag, end. */ 7108 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 7109 + sizeof(struct mlx5_rte_flow_action_set_tag); 7110 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 7111 sizeof(struct mlx5_rte_flow_item_tag) * 2; 7112 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 7113 item_size), 0, SOCKET_ID_ANY); 7114 if (!sfx_actions) 7115 return rte_flow_error_set(error, ENOMEM, 7116 RTE_FLOW_ERROR_TYPE_ACTION, 7117 NULL, "no memory to split " 7118 "sample flow"); 7119 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 7120 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 7121 const struct rte_flow_item_port_id *spec; 7122 7123 spec = (const struct rte_flow_item_port_id *)item->spec; 7124 if (spec) 7125 item_port_priv = 7126 mlx5_port_to_eswitch_info(spec->id, true); 7127 break; 7128 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 7129 const struct rte_flow_item_ethdev *spec; 7130 7131 spec = (const struct rte_flow_item_ethdev *)item->spec; 7132 if (spec) 7133 item_port_priv = 7134 mlx5_port_to_eswitch_info(spec->port_id, true); 7135 break; 7136 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 7137 const struct rte_flow_item_ethdev *spec; 7138 7139 spec = (const struct rte_flow_item_ethdev *)item->spec; 7140 if (spec) 7141 item_port_priv = 7142 mlx5_port_to_eswitch_info(spec->port_id, true); 7143 break; 7144 } 7145 } 7146 /* The representor_id is UINT16_MAX for uplink. */ 7147 fdb_tx = (attr->transfer && 7148 flow_source_vport_representor(priv, item_port_priv)); 7149 /* 7150 * When reg_c_preserve is set, metadata registers Cx preserve 7151 * their value even through packet duplication. 7152 */ 7153 add_tag = (!fdb_tx || 7154 priv->sh->cdev->config.hca_attr.reg_c_preserve); 7155 if (add_tag) 7156 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 7157 + act_size); 7158 if (modify_after_mirror) 7159 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 7160 next_ft_step; 7161 pre_actions = sfx_actions + actions_n; 7162 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 7163 actions, sfx_actions, 7164 pre_actions, actions_n, 7165 sample_action_pos, 7166 qrss_action_pos, jump_table, 7167 error); 7168 if (tag_id < 0 || (add_tag && !tag_id)) { 7169 ret = -rte_errno; 7170 goto exit; 7171 } 7172 if (modify_after_mirror) 7173 flow_split_info->skip_scale = 7174 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 7175 /* Add the prefix subflow. */ 7176 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7177 items, pre_actions, 7178 flow_split_info, error); 7179 if (ret) { 7180 ret = -rte_errno; 7181 goto exit; 7182 } 7183 dev_flow->handle->split_flow_id = tag_id; 7184 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7185 if (!modify_after_mirror) { 7186 /* Set the sfx group attr. */ 7187 sample_res = (struct mlx5_flow_dv_sample_resource *) 7188 dev_flow->dv.sample_res; 7189 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7190 sample_res->normal_path_tbl; 7191 sfx_tbl_data = container_of(sfx_tbl, 7192 struct mlx5_flow_tbl_data_entry, 7193 tbl); 7194 sfx_attr.group = sfx_attr.transfer ? 7195 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7196 } else { 7197 MLX5_ASSERT(attr->transfer); 7198 sfx_attr.group = jump_table; 7199 } 7200 flow_split_info->prefix_layers = 7201 flow_get_prefix_layer_flags(dev_flow); 7202 MLX5_ASSERT(wks); 7203 flow_split_info->prefix_mark |= wks->mark; 7204 /* Suffix group level already be scaled with factor, set 7205 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7206 * again in translation. 7207 */ 7208 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7209 #endif 7210 } 7211 /* Add the suffix subflow. */ 7212 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7213 sfx_items ? sfx_items : items, 7214 sfx_actions ? sfx_actions : actions, 7215 flow_split_info, error); 7216 exit: 7217 if (sfx_actions) 7218 mlx5_free(sfx_actions); 7219 return ret; 7220 } 7221 7222 /** 7223 * Split the flow to subflow set. The splitters might be linked 7224 * in the chain, like this: 7225 * flow_create_split_outer() calls: 7226 * flow_create_split_meter() calls: 7227 * flow_create_split_metadata(meter_subflow_0) calls: 7228 * flow_create_split_inner(metadata_subflow_0) 7229 * flow_create_split_inner(metadata_subflow_1) 7230 * flow_create_split_inner(metadata_subflow_2) 7231 * flow_create_split_metadata(meter_subflow_1) calls: 7232 * flow_create_split_inner(metadata_subflow_0) 7233 * flow_create_split_inner(metadata_subflow_1) 7234 * flow_create_split_inner(metadata_subflow_2) 7235 * 7236 * This provide flexible way to add new levels of flow splitting. 7237 * The all of successfully created subflows are included to the 7238 * parent flow dev_flow list. 7239 * 7240 * @param dev 7241 * Pointer to Ethernet device. 7242 * @param[in] flow 7243 * Parent flow structure pointer. 7244 * @param[in] attr 7245 * Flow rule attributes. 7246 * @param[in] items 7247 * Pattern specification (list terminated by the END pattern item). 7248 * @param[in] actions 7249 * Associated actions (list terminated by the END action). 7250 * @param[in] flow_split_info 7251 * Pointer to flow split info structure. 7252 * @param[out] error 7253 * Perform verbose error reporting if not NULL. 7254 * @return 7255 * 0 on success, negative value otherwise 7256 */ 7257 static int 7258 flow_create_split_outer(struct rte_eth_dev *dev, 7259 struct rte_flow *flow, 7260 const struct rte_flow_attr *attr, 7261 const struct rte_flow_item items[], 7262 const struct rte_flow_action actions[], 7263 struct mlx5_flow_split_info *flow_split_info, 7264 struct rte_flow_error *error) 7265 { 7266 int ret; 7267 7268 ret = flow_create_split_sample(dev, flow, attr, items, 7269 actions, flow_split_info, error); 7270 MLX5_ASSERT(ret <= 0); 7271 return ret; 7272 } 7273 7274 static inline struct mlx5_flow_tunnel * 7275 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7276 { 7277 struct mlx5_flow_tunnel *tunnel; 7278 7279 #pragma GCC diagnostic push 7280 #pragma GCC diagnostic ignored "-Wcast-qual" 7281 tunnel = (typeof(tunnel))flow->tunnel; 7282 #pragma GCC diagnostic pop 7283 7284 return tunnel; 7285 } 7286 7287 /** 7288 * Create a flow and add it to @p list. 7289 * 7290 * @param dev 7291 * Pointer to Ethernet device. 7292 * @param list 7293 * Pointer to a TAILQ flow list. If this parameter NULL, 7294 * no list insertion occurred, flow is just created, 7295 * this is caller's responsibility to track the 7296 * created flow. 7297 * @param[in] attr 7298 * Flow rule attributes. 7299 * @param[in] items 7300 * Pattern specification (list terminated by the END pattern item). 7301 * @param[in] actions 7302 * Associated actions (list terminated by the END action). 7303 * @param[in] external 7304 * This flow rule is created by request external to PMD. 7305 * @param[out] error 7306 * Perform verbose error reporting if not NULL. 7307 * 7308 * @return 7309 * A flow index on success, 0 otherwise and rte_errno is set. 7310 */ 7311 static uint32_t 7312 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7313 const struct rte_flow_attr *attr, 7314 const struct rte_flow_item items[], 7315 const struct rte_flow_action original_actions[], 7316 bool external, struct rte_flow_error *error) 7317 { 7318 struct mlx5_priv *priv = dev->data->dev_private; 7319 struct rte_flow *flow = NULL; 7320 struct mlx5_flow *dev_flow; 7321 const struct rte_flow_action_rss *rss = NULL; 7322 struct mlx5_translated_action_handle 7323 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7324 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7325 union { 7326 struct mlx5_flow_expand_rss buf; 7327 uint8_t buffer[8192]; 7328 } expand_buffer; 7329 union { 7330 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7331 uint8_t buffer[2048]; 7332 } actions_rx; 7333 union { 7334 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7335 uint8_t buffer[2048]; 7336 } actions_hairpin_tx; 7337 union { 7338 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7339 uint8_t buffer[2048]; 7340 } items_tx; 7341 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7342 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7343 struct mlx5_flow_rss_desc *rss_desc; 7344 const struct rte_flow_action *p_actions_rx; 7345 uint32_t i; 7346 uint32_t idx = 0; 7347 int hairpin_flow; 7348 struct rte_flow_attr attr_tx = { .priority = 0 }; 7349 const struct rte_flow_action *actions; 7350 struct rte_flow_action *translated_actions = NULL; 7351 struct mlx5_flow_tunnel *tunnel; 7352 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7353 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7354 struct mlx5_flow_split_info flow_split_info = { 7355 .external = !!external, 7356 .skip_scale = 0, 7357 .flow_idx = 0, 7358 .prefix_mark = 0, 7359 .prefix_layers = 0, 7360 .table_id = 0 7361 }; 7362 int ret; 7363 7364 MLX5_ASSERT(wks); 7365 rss_desc = &wks->rss_desc; 7366 ret = flow_action_handles_translate(dev, original_actions, 7367 indir_actions, 7368 &indir_actions_n, 7369 &translated_actions, error); 7370 if (ret < 0) { 7371 MLX5_ASSERT(translated_actions == NULL); 7372 return 0; 7373 } 7374 actions = translated_actions ? translated_actions : original_actions; 7375 p_actions_rx = actions; 7376 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7377 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7378 external, hairpin_flow, error); 7379 if (ret < 0) 7380 goto error_before_hairpin_split; 7381 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7382 if (!flow) { 7383 rte_errno = ENOMEM; 7384 goto error_before_hairpin_split; 7385 } 7386 if (hairpin_flow > 0) { 7387 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7388 rte_errno = EINVAL; 7389 goto error_before_hairpin_split; 7390 } 7391 flow_hairpin_split(dev, actions, actions_rx.actions, 7392 actions_hairpin_tx.actions, items_tx.items, 7393 idx); 7394 p_actions_rx = actions_rx.actions; 7395 } 7396 flow_split_info.flow_idx = idx; 7397 flow->drv_type = flow_get_drv_type(dev, attr); 7398 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7399 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7400 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7401 /* RSS Action only works on NIC RX domain */ 7402 if (attr->ingress) 7403 rss = flow_get_rss_action(dev, p_actions_rx); 7404 if (rss) { 7405 MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); 7406 rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func); 7407 /* 7408 * The following information is required by 7409 * mlx5_flow_hashfields_adjust() in advance. 7410 */ 7411 rss_desc->level = rss->level; 7412 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7413 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7414 } 7415 flow->dev_handles = 0; 7416 if (rss && rss->types) { 7417 unsigned int graph_root; 7418 7419 graph_root = find_graph_root(rss->level); 7420 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7421 items, rss->types, 7422 mlx5_support_expansion, graph_root); 7423 MLX5_ASSERT(ret > 0 && 7424 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7425 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7426 for (i = 0; i < buf->entries; ++i) 7427 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7428 } 7429 } else { 7430 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7431 sizeof(expand_buffer.buffer), 7432 items, sq_specs); 7433 if (ret) { 7434 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7435 NULL, "not enough memory for rte_flow"); 7436 goto error; 7437 } 7438 if (buf->entries == 0) { 7439 buf->entries = 1; 7440 buf->entry[0].pattern = (void *)(uintptr_t)items; 7441 } 7442 } 7443 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7444 indir_actions_n); 7445 for (i = 0; i < buf->entries; ++i) { 7446 /* Initialize flow split data. */ 7447 flow_split_info.prefix_layers = 0; 7448 flow_split_info.prefix_mark = 0; 7449 flow_split_info.skip_scale = 0; 7450 /* 7451 * The splitter may create multiple dev_flows, 7452 * depending on configuration. In the simplest 7453 * case it just creates unmodified original flow. 7454 */ 7455 ret = flow_create_split_outer(dev, flow, attr, 7456 buf->entry[i].pattern, 7457 p_actions_rx, &flow_split_info, 7458 error); 7459 if (ret < 0) 7460 goto error; 7461 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7462 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7463 p_actions_rx, 7464 idx, 7465 wks->flows[0].tunnel, 7466 &default_miss_ctx, 7467 error); 7468 if (ret < 0) { 7469 mlx5_free(default_miss_ctx.queue); 7470 goto error; 7471 } 7472 } 7473 } 7474 /* Create the tx flow. */ 7475 if (hairpin_flow) { 7476 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7477 attr_tx.ingress = 0; 7478 attr_tx.egress = 1; 7479 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7480 actions_hairpin_tx.actions, 7481 idx, error); 7482 if (!dev_flow) 7483 goto error; 7484 dev_flow->flow = flow; 7485 dev_flow->external = 0; 7486 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7487 dev_flow->handle, next); 7488 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7489 items_tx.items, 7490 actions_hairpin_tx.actions, error); 7491 if (ret < 0) 7492 goto error; 7493 } 7494 /* 7495 * Update the metadata register copy table. If extensive 7496 * metadata feature is enabled and registers are supported 7497 * we might create the extra rte_flow for each unique 7498 * MARK/FLAG action ID. 7499 * 7500 * The table is updated for ingress and transfer flows only, because 7501 * the egress Flows belong to the different device and 7502 * copy table should be updated in peer NIC Rx domain. 7503 */ 7504 if ((attr->ingress || attr->transfer) && 7505 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7506 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7507 if (ret) 7508 goto error; 7509 } 7510 /* 7511 * If the flow is external (from application) OR device is started, 7512 * OR mreg discover, then apply immediately. 7513 */ 7514 if (external || dev->data->dev_started || 7515 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7516 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7517 ret = flow_drv_apply(dev, flow, error); 7518 if (ret < 0) 7519 goto error; 7520 } 7521 flow->type = type; 7522 flow_rxq_flags_set(dev, flow); 7523 rte_free(translated_actions); 7524 tunnel = flow_tunnel_from_rule(wks->flows); 7525 if (tunnel) { 7526 flow->tunnel = 1; 7527 flow->tunnel_id = tunnel->tunnel_id; 7528 __atomic_fetch_add(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7529 mlx5_free(default_miss_ctx.queue); 7530 } 7531 mlx5_flow_pop_thread_workspace(); 7532 return idx; 7533 error: 7534 MLX5_ASSERT(flow); 7535 ret = rte_errno; /* Save rte_errno before cleanup. */ 7536 flow_mreg_del_copy_action(dev, flow); 7537 flow_drv_destroy(dev, flow); 7538 if (rss_desc->shared_rss) 7539 __atomic_fetch_sub(&((struct mlx5_shared_action_rss *) 7540 mlx5_ipool_get 7541 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7542 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7543 mlx5_ipool_free(priv->flows[type], idx); 7544 rte_errno = ret; /* Restore rte_errno. */ 7545 ret = rte_errno; 7546 rte_errno = ret; 7547 error_before_hairpin_split: 7548 mlx5_flow_pop_thread_workspace(); 7549 rte_free(translated_actions); 7550 return 0; 7551 } 7552 7553 /** 7554 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7555 * incoming packets to table 1. 7556 * 7557 * Other flow rules, requested for group n, will be created in 7558 * e-switch table n+1. 7559 * Jump action to e-switch group n will be created to group n+1. 7560 * 7561 * Used when working in switchdev mode, to utilise advantages of table 1 7562 * and above. 7563 * 7564 * @param dev 7565 * Pointer to Ethernet device. 7566 * 7567 * @return 7568 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7569 */ 7570 struct rte_flow * 7571 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7572 { 7573 const struct rte_flow_attr attr = { 7574 .group = 0, 7575 .priority = 0, 7576 .ingress = 0, 7577 .egress = 0, 7578 .transfer = 1, 7579 }; 7580 const struct rte_flow_item pattern = { 7581 .type = RTE_FLOW_ITEM_TYPE_END, 7582 }; 7583 struct rte_flow_action_jump jump = { 7584 .group = 1, 7585 }; 7586 const struct rte_flow_action actions[] = { 7587 { 7588 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7589 .conf = &jump, 7590 }, 7591 { 7592 .type = RTE_FLOW_ACTION_TYPE_END, 7593 }, 7594 }; 7595 struct rte_flow_error error; 7596 7597 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7598 &attr, &pattern, 7599 actions, false, &error); 7600 } 7601 7602 /** 7603 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7604 * and sq number, directs all packets to peer vport. 7605 * 7606 * @param dev 7607 * Pointer to Ethernet device. 7608 * @param sq_num 7609 * SQ number. 7610 * 7611 * @return 7612 * Flow ID on success, 0 otherwise and rte_errno is set. 7613 */ 7614 uint32_t 7615 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7616 { 7617 struct rte_flow_attr attr = { 7618 .group = 0, 7619 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7620 .ingress = 0, 7621 .egress = 0, 7622 .transfer = 1, 7623 }; 7624 struct rte_flow_item_port_id port_spec = { 7625 .id = MLX5_PORT_ESW_MGR, 7626 }; 7627 struct mlx5_rte_flow_item_sq sq_spec = { 7628 .queue = sq_num, 7629 }; 7630 struct rte_flow_item pattern[] = { 7631 { 7632 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7633 .spec = &port_spec, 7634 }, 7635 { 7636 .type = (enum rte_flow_item_type) 7637 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7638 .spec = &sq_spec, 7639 }, 7640 { 7641 .type = RTE_FLOW_ITEM_TYPE_END, 7642 }, 7643 }; 7644 struct rte_flow_action_jump jump = { 7645 .group = 1, 7646 }; 7647 struct rte_flow_action_port_id port = { 7648 .id = dev->data->port_id, 7649 }; 7650 struct rte_flow_action actions[] = { 7651 { 7652 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7653 .conf = &jump, 7654 }, 7655 { 7656 .type = RTE_FLOW_ACTION_TYPE_END, 7657 }, 7658 }; 7659 struct rte_flow_error error; 7660 7661 /* 7662 * Creates group 0, highest priority jump flow. 7663 * Matches txq to bypass kernel packets. 7664 */ 7665 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7666 false, &error) == 0) 7667 return 0; 7668 /* Create group 1, lowest priority redirect flow for txq. */ 7669 attr.group = 1; 7670 actions[0].conf = &port; 7671 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7672 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7673 actions, false, &error); 7674 } 7675 7676 /** 7677 * Validate a flow supported by the NIC. 7678 * 7679 * @see rte_flow_validate() 7680 * @see rte_flow_ops 7681 */ 7682 int 7683 mlx5_flow_validate(struct rte_eth_dev *dev, 7684 const struct rte_flow_attr *attr, 7685 const struct rte_flow_item items[], 7686 const struct rte_flow_action original_actions[], 7687 struct rte_flow_error *error) 7688 { 7689 int hairpin_flow; 7690 struct mlx5_translated_action_handle 7691 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7692 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7693 const struct rte_flow_action *actions; 7694 struct rte_flow_action *translated_actions = NULL; 7695 int ret = flow_action_handles_translate(dev, original_actions, 7696 indir_actions, 7697 &indir_actions_n, 7698 &translated_actions, error); 7699 7700 if (ret) 7701 return ret; 7702 actions = translated_actions ? translated_actions : original_actions; 7703 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7704 ret = flow_drv_validate(dev, attr, items, actions, 7705 true, hairpin_flow, error); 7706 rte_free(translated_actions); 7707 return ret; 7708 } 7709 7710 static int 7711 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7712 const struct rte_flow_attr *attr, 7713 const uint32_t orig_prio, 7714 const struct rte_flow_item *items, 7715 const struct rte_flow_action *actions, 7716 uint32_t flow_idx) 7717 { 7718 struct mlx5_priv *priv = dev->data->dev_private; 7719 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7720 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7721 struct rte_flow_error error; 7722 int len, ret; 7723 7724 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7725 if (!flow_info) { 7726 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7727 return -1; 7728 } 7729 flow_info->orig_prio = orig_prio; 7730 flow_info->attr = *attr; 7731 /* Standby mode rule awlays saves it in low priority entry. */ 7732 flow_info->flow_idx_low_prio = flow_idx; 7733 7734 /* Store matching items. */ 7735 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7736 if (ret <= 0) { 7737 DRV_LOG(ERR, "Can't get items length."); 7738 goto end; 7739 } 7740 len = RTE_ALIGN(ret, 16); 7741 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7742 if (!flow_info->items) { 7743 DRV_LOG(ERR, "No enough memory for items caching."); 7744 goto end; 7745 } 7746 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7747 if (ret <= 0) { 7748 DRV_LOG(ERR, "Can't duplicate items."); 7749 goto end; 7750 } 7751 7752 /* Store flow actions. */ 7753 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7754 if (ret <= 0) { 7755 DRV_LOG(ERR, "Can't get actions length."); 7756 goto end; 7757 } 7758 len = RTE_ALIGN(ret, 16); 7759 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7760 if (!flow_info->actions) { 7761 DRV_LOG(ERR, "No enough memory for actions caching."); 7762 goto end; 7763 } 7764 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7765 if (ret <= 0) { 7766 DRV_LOG(ERR, "Can't duplicate actions."); 7767 goto end; 7768 } 7769 7770 /* Insert to the list end. */ 7771 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7772 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7773 } else { 7774 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7775 while (LIST_NEXT(tmp_info, next)) 7776 tmp_info = LIST_NEXT(tmp_info, next); 7777 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7778 } 7779 return 0; 7780 end: 7781 if (flow_info->items) 7782 mlx5_free(flow_info->items); 7783 if (flow_info->actions) 7784 mlx5_free(flow_info->actions); 7785 mlx5_free(flow_info); 7786 return -1; 7787 } 7788 7789 static int 7790 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7791 { 7792 struct mlx5_priv *priv = dev->data->dev_private; 7793 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7794 struct mlx5_dv_flow_info *flow_info; 7795 struct rte_flow_attr attr; 7796 struct rte_flow_error error; 7797 struct rte_flow *high, *low; 7798 7799 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7800 while (flow_info) { 7801 /* DUP flow may have the same priority. */ 7802 if (flow_info->orig_prio != flow_info->attr.priority) { 7803 attr = flow_info->attr; 7804 if (orig_prio) 7805 attr.priority = flow_info->orig_prio; 7806 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7807 &attr, flow_info->items, flow_info->actions, 7808 true, &error); 7809 if (!flow_info->flow_idx_high_prio) { 7810 DRV_LOG(ERR, "Priority toggle failed internally."); 7811 goto err; 7812 } 7813 } 7814 flow_info = LIST_NEXT(flow_info, next); 7815 } 7816 /* Delete the low priority rules and swap the flow handle. */ 7817 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7818 while (flow_info) { 7819 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7820 if (flow_info->orig_prio != flow_info->attr.priority) { 7821 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7822 flow_info->flow_idx_high_prio); 7823 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7824 flow_info->flow_idx_low_prio); 7825 if (high && low) { 7826 RTE_SWAP(*low, *high); 7827 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7828 flow_info->flow_idx_low_prio); 7829 flow_info->flow_idx_high_prio = 0; 7830 } 7831 } 7832 flow_info = LIST_NEXT(flow_info, next); 7833 } 7834 return 0; 7835 err: 7836 /* Destroy preceding successful high priority rules. */ 7837 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7838 while (flow_info) { 7839 if (flow_info->orig_prio != flow_info->attr.priority) { 7840 if (flow_info->flow_idx_high_prio) 7841 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7842 flow_info->flow_idx_high_prio); 7843 else 7844 break; 7845 flow_info->flow_idx_high_prio = 0; 7846 } 7847 flow_info = LIST_NEXT(flow_info, next); 7848 } 7849 return -1; 7850 } 7851 7852 /** 7853 * Set the mode of the flow engine of a process to active or standby during live migration. 7854 * 7855 * @param[in] mode 7856 * MLX5 flow engine mode, @see `enum mlx5_flow_engine_mode`. 7857 * @param[in] flags 7858 * Flow engine mode specific flags. 7859 * 7860 * @return 7861 * Negative value on error, positive on success. 7862 */ 7863 int 7864 rte_pmd_mlx5_flow_engine_set_mode(enum mlx5_flow_engine_mode mode, uint32_t flags) 7865 { 7866 struct mlx5_priv *priv; 7867 struct mlx5_flow_engine_mode_info *mode_info; 7868 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7869 uint16_t port, port_id; 7870 uint16_t toggle_num = 0; 7871 struct rte_eth_dev *dev; 7872 enum mlx5_flow_engine_mode orig_mode; 7873 uint32_t orig_flags; 7874 bool need_toggle = false; 7875 7876 /* Check if flags combinations are supported. */ 7877 if (flags && flags != MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7878 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7879 return -1; 7880 } 7881 MLX5_ETH_FOREACH_DEV(port, NULL) { 7882 dev = &rte_eth_devices[port]; 7883 priv = dev->data->dev_private; 7884 mode_info = &priv->mode_info; 7885 /* No mode change. Assume all devices hold the same mode. */ 7886 if (mode_info->mode == mode) { 7887 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7888 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7889 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7890 port, mode_info->mode_flag); 7891 orig_mode = mode_info->mode; 7892 orig_flags = mode_info->mode_flag; 7893 goto err; 7894 } 7895 mode_info->mode_flag = flags; 7896 toggle_num++; 7897 continue; 7898 } 7899 /* Active -> standby. */ 7900 if (mode == MLX5_FLOW_ENGINE_MODE_STANDBY) { 7901 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7902 DRV_LOG(ERR, "Cached rule existed"); 7903 orig_mode = mode_info->mode; 7904 orig_flags = mode_info->mode_flag; 7905 goto err; 7906 } 7907 mode_info->mode_flag = flags; 7908 mode_info->mode = mode; 7909 toggle_num++; 7910 /* Standby -> active. */ 7911 } else if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7912 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7913 DRV_LOG(INFO, "No cached rule existed"); 7914 } else { 7915 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7916 orig_mode = mode_info->mode; 7917 orig_flags = mode_info->mode_flag; 7918 need_toggle = true; 7919 goto err; 7920 } 7921 } 7922 toggle_num++; 7923 } 7924 } 7925 if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7926 /* Clear cache flow rules. */ 7927 MLX5_ETH_FOREACH_DEV(port, NULL) { 7928 priv = rte_eth_devices[port].data->dev_private; 7929 mode_info = &priv->mode_info; 7930 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7931 while (flow_info) { 7932 tmp_info = LIST_NEXT(flow_info, next); 7933 LIST_REMOVE(flow_info, next); 7934 mlx5_free(flow_info->actions); 7935 mlx5_free(flow_info->items); 7936 mlx5_free(flow_info); 7937 flow_info = tmp_info; 7938 } 7939 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7940 } 7941 } 7942 return toggle_num; 7943 err: 7944 /* Rollback all preceding successful ports. */ 7945 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7946 if (port_id == port) 7947 break; 7948 priv = rte_eth_devices[port_id].data->dev_private; 7949 mode_info = &priv->mode_info; 7950 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7951 mlx5_flow_cache_flow_toggle(dev, false)) 7952 return -EPERM; 7953 mode_info->mode = orig_mode; 7954 mode_info->mode_flag = orig_flags; 7955 } 7956 return -EINVAL; 7957 } 7958 /** 7959 * Create a flow. 7960 * 7961 * @see rte_flow_create() 7962 * @see rte_flow_ops 7963 */ 7964 struct rte_flow * 7965 mlx5_flow_create(struct rte_eth_dev *dev, 7966 const struct rte_flow_attr *attr, 7967 const struct rte_flow_item items[], 7968 const struct rte_flow_action actions[], 7969 struct rte_flow_error *error) 7970 { 7971 struct mlx5_priv *priv = dev->data->dev_private; 7972 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7973 uint32_t prio = attr->priority; 7974 uint32_t flow_idx; 7975 7976 if (priv->sh->config.dv_flow_en == 2) { 7977 rte_flow_error_set(error, ENOTSUP, 7978 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7979 NULL, 7980 "Flow non-Q creation not supported"); 7981 return NULL; 7982 } 7983 /* 7984 * If the device is not started yet, it is not allowed to created a 7985 * flow from application. PMD default flows and traffic control flows 7986 * are not affected. 7987 */ 7988 if (unlikely(!dev->data->dev_started)) { 7989 DRV_LOG(DEBUG, "port %u is not started when " 7990 "inserting a flow", dev->data->port_id); 7991 rte_flow_error_set(error, ENODEV, 7992 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7993 NULL, 7994 "port not started"); 7995 return NULL; 7996 } 7997 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7998 if (attr->transfer || 7999 (attr->ingress && 8000 !(priv->mode_info.mode_flag & MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 8001 new_attr->priority += 1; 8002 } 8003 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 8004 if (!flow_idx) 8005 return NULL; 8006 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 8007 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 8008 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 8009 flow_idx = 0; 8010 } 8011 } 8012 return (void *)(uintptr_t)flow_idx; 8013 } 8014 8015 /** 8016 * Destroy a flow in a list. 8017 * 8018 * @param dev 8019 * Pointer to Ethernet device. 8020 * @param[in] flow_idx 8021 * Index of flow to destroy. 8022 */ 8023 static void 8024 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 8025 uint32_t flow_idx) 8026 { 8027 struct mlx5_priv *priv = dev->data->dev_private; 8028 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 8029 8030 if (!flow) 8031 return; 8032 MLX5_ASSERT(flow->type == type); 8033 /* 8034 * Update RX queue flags only if port is started, otherwise it is 8035 * already clean. 8036 */ 8037 if (dev->data->dev_started) 8038 flow_rxq_flags_trim(dev, flow); 8039 flow_drv_destroy(dev, flow); 8040 if (flow->tunnel) { 8041 struct mlx5_flow_tunnel *tunnel; 8042 8043 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 8044 RTE_VERIFY(tunnel); 8045 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 8046 mlx5_flow_tunnel_free(dev, tunnel); 8047 } 8048 flow_mreg_del_copy_action(dev, flow); 8049 mlx5_ipool_free(priv->flows[type], flow_idx); 8050 } 8051 8052 /** 8053 * Destroy all flows. 8054 * 8055 * @param dev 8056 * Pointer to Ethernet device. 8057 * @param type 8058 * Flow type to be flushed. 8059 * @param active 8060 * If flushing is called actively. 8061 */ 8062 void 8063 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 8064 bool active) 8065 { 8066 struct mlx5_priv *priv = dev->data->dev_private; 8067 uint32_t num_flushed = 0, fidx = 1; 8068 struct rte_flow *flow; 8069 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8070 struct mlx5_dv_flow_info *flow_info; 8071 8072 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 8073 if (priv->sh->config.dv_flow_en == 2 && 8074 type == MLX5_FLOW_TYPE_GEN) { 8075 flow_hw_q_flow_flush(dev, NULL); 8076 return; 8077 } 8078 #endif 8079 8080 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 8081 flow_list_destroy(dev, type, fidx); 8082 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 8083 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8084 while (flow_info) { 8085 /* Romove the cache flow info. */ 8086 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 8087 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8088 LIST_REMOVE(flow_info, next); 8089 mlx5_free(flow_info->items); 8090 mlx5_free(flow_info->actions); 8091 mlx5_free(flow_info); 8092 break; 8093 } 8094 flow_info = LIST_NEXT(flow_info, next); 8095 } 8096 } 8097 num_flushed++; 8098 } 8099 if (active) { 8100 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 8101 dev->data->port_id, num_flushed); 8102 } 8103 } 8104 8105 /** 8106 * Stop all default actions for flows. 8107 * 8108 * @param dev 8109 * Pointer to Ethernet device. 8110 */ 8111 void 8112 mlx5_flow_stop_default(struct rte_eth_dev *dev) 8113 { 8114 flow_mreg_del_default_copy_action(dev); 8115 flow_rxq_flags_clear(dev); 8116 } 8117 8118 /** 8119 * Set rxq flag. 8120 * 8121 * @param[in] dev 8122 * Pointer to the rte_eth_dev structure. 8123 * @param[in] enable 8124 * Flag to enable or not. 8125 */ 8126 void 8127 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 8128 { 8129 struct mlx5_priv *priv = dev->data->dev_private; 8130 unsigned int i; 8131 8132 if ((!priv->mark_enabled && !enable) || 8133 (priv->mark_enabled && enable)) 8134 return; 8135 for (i = 0; i < priv->rxqs_n; ++i) { 8136 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 8137 8138 /* With RXQ start/stop feature, RXQ might be stopped. */ 8139 if (!rxq_ctrl) 8140 continue; 8141 rxq_ctrl->rxq.mark = enable; 8142 } 8143 priv->mark_enabled = enable; 8144 } 8145 8146 /** 8147 * Start all default actions for flows. 8148 * 8149 * @param dev 8150 * Pointer to Ethernet device. 8151 * @return 8152 * 0 on success, a negative errno value otherwise and rte_errno is set. 8153 */ 8154 int 8155 mlx5_flow_start_default(struct rte_eth_dev *dev) 8156 { 8157 struct rte_flow_error error; 8158 8159 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8160 return flow_mreg_add_default_copy_action(dev, &error); 8161 } 8162 8163 /** 8164 * Release key of thread specific flow workspace data. 8165 */ 8166 void 8167 flow_release_workspace(void *data) 8168 { 8169 struct mlx5_flow_workspace *wks = data; 8170 struct mlx5_flow_workspace *next; 8171 8172 while (wks) { 8173 next = wks->next; 8174 free(wks); 8175 wks = next; 8176 } 8177 } 8178 8179 static struct mlx5_flow_workspace *gc_head; 8180 static rte_spinlock_t mlx5_flow_workspace_lock = RTE_SPINLOCK_INITIALIZER; 8181 8182 static void 8183 mlx5_flow_workspace_gc_add(struct mlx5_flow_workspace *ws) 8184 { 8185 rte_spinlock_lock(&mlx5_flow_workspace_lock); 8186 ws->gc = gc_head; 8187 gc_head = ws; 8188 rte_spinlock_unlock(&mlx5_flow_workspace_lock); 8189 } 8190 8191 void 8192 mlx5_flow_workspace_gc_release(void) 8193 { 8194 while (gc_head) { 8195 struct mlx5_flow_workspace *wks = gc_head; 8196 8197 gc_head = wks->gc; 8198 flow_release_workspace(wks); 8199 } 8200 } 8201 8202 /** 8203 * Get thread specific current flow workspace. 8204 * 8205 * @return pointer to thread specific flow workspace data, NULL on error. 8206 */ 8207 struct mlx5_flow_workspace* 8208 mlx5_flow_get_thread_workspace(void) 8209 { 8210 struct mlx5_flow_workspace *data; 8211 8212 data = mlx5_flow_os_get_specific_workspace(); 8213 MLX5_ASSERT(data && data->inuse); 8214 if (!data || !data->inuse) 8215 DRV_LOG(ERR, "flow workspace not initialized."); 8216 return data; 8217 } 8218 8219 /** 8220 * Allocate and init new flow workspace. 8221 * 8222 * @return pointer to flow workspace data, NULL on error. 8223 */ 8224 static struct mlx5_flow_workspace* 8225 flow_alloc_thread_workspace(void) 8226 { 8227 size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); 8228 size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; 8229 struct mlx5_flow_workspace *data = calloc(1, data_size + 8230 rss_queue_array_size); 8231 8232 if (!data) { 8233 DRV_LOG(ERR, "Failed to allocate flow workspace memory."); 8234 return NULL; 8235 } 8236 data->rss_desc.queue = RTE_PTR_ADD(data, data_size); 8237 return data; 8238 } 8239 8240 /** 8241 * Get new thread specific flow workspace. 8242 * 8243 * If current workspace inuse, create new one and set as current. 8244 * 8245 * @return pointer to thread specific flow workspace data, NULL on error. 8246 */ 8247 struct mlx5_flow_workspace* 8248 mlx5_flow_push_thread_workspace(void) 8249 { 8250 struct mlx5_flow_workspace *curr; 8251 struct mlx5_flow_workspace *data; 8252 8253 curr = mlx5_flow_os_get_specific_workspace(); 8254 if (!curr) { 8255 data = flow_alloc_thread_workspace(); 8256 if (!data) 8257 return NULL; 8258 mlx5_flow_workspace_gc_add(data); 8259 } else if (!curr->inuse) { 8260 data = curr; 8261 } else if (curr->next) { 8262 data = curr->next; 8263 } else { 8264 data = flow_alloc_thread_workspace(); 8265 if (!data) 8266 return NULL; 8267 curr->next = data; 8268 data->prev = curr; 8269 } 8270 data->inuse = 1; 8271 data->flow_idx = 0; 8272 /* Set as current workspace */ 8273 if (mlx5_flow_os_set_specific_workspace(data)) 8274 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8275 return data; 8276 } 8277 8278 /** 8279 * Close current thread specific flow workspace. 8280 * 8281 * If previous workspace available, set it as current. 8282 * 8283 * @return pointer to thread specific flow workspace data, NULL on error. 8284 */ 8285 void 8286 mlx5_flow_pop_thread_workspace(void) 8287 { 8288 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8289 8290 if (!data) 8291 return; 8292 if (!data->inuse) { 8293 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8294 return; 8295 } 8296 data->inuse = 0; 8297 if (!data->prev) 8298 return; 8299 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8300 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8301 } 8302 8303 /** 8304 * Verify the flow list is empty 8305 * 8306 * @param dev 8307 * Pointer to Ethernet device. 8308 * 8309 * @return the number of flows not released. 8310 */ 8311 int 8312 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8313 { 8314 struct mlx5_priv *priv = dev->data->dev_private; 8315 struct rte_flow *flow; 8316 uint32_t idx = 0; 8317 int ret = 0, i; 8318 8319 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8320 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8321 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8322 dev->data->port_id, (void *)flow); 8323 ret++; 8324 } 8325 } 8326 return ret; 8327 } 8328 8329 /** 8330 * Enable default hairpin egress flow. 8331 * 8332 * @param dev 8333 * Pointer to Ethernet device. 8334 * @param sq_num 8335 * The SQ hw number. 8336 * 8337 * @return 8338 * 0 on success, a negative errno value otherwise and rte_errno is set. 8339 */ 8340 int 8341 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8342 uint32_t sq_num) 8343 { 8344 const struct rte_flow_attr attr = { 8345 .egress = 1, 8346 .priority = 0, 8347 }; 8348 struct mlx5_rte_flow_item_sq queue_spec = { 8349 .queue = sq_num, 8350 }; 8351 struct mlx5_rte_flow_item_sq queue_mask = { 8352 .queue = UINT32_MAX, 8353 }; 8354 struct rte_flow_item items[] = { 8355 { 8356 .type = (enum rte_flow_item_type) 8357 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8358 .spec = &queue_spec, 8359 .last = NULL, 8360 .mask = &queue_mask, 8361 }, 8362 { 8363 .type = RTE_FLOW_ITEM_TYPE_END, 8364 }, 8365 }; 8366 struct rte_flow_action_jump jump = { 8367 .group = MLX5_HAIRPIN_TX_TABLE, 8368 }; 8369 struct rte_flow_action actions[2]; 8370 uint32_t flow_idx; 8371 struct rte_flow_error error; 8372 8373 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8374 actions[0].conf = &jump; 8375 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8376 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8377 &attr, items, actions, false, &error); 8378 if (!flow_idx) { 8379 DRV_LOG(DEBUG, 8380 "Failed to create ctrl flow: rte_errno(%d)," 8381 " type(%d), message(%s)", 8382 rte_errno, error.type, 8383 error.message ? error.message : " (no stated reason)"); 8384 return -rte_errno; 8385 } 8386 return 0; 8387 } 8388 8389 /** 8390 * Enable a control flow configured from the control plane. 8391 * 8392 * @param dev 8393 * Pointer to Ethernet device. 8394 * @param eth_spec 8395 * An Ethernet flow spec to apply. 8396 * @param eth_mask 8397 * An Ethernet flow mask to apply. 8398 * @param vlan_spec 8399 * A VLAN flow spec to apply. 8400 * @param vlan_mask 8401 * A VLAN flow mask to apply. 8402 * 8403 * @return 8404 * 0 on success, a negative errno value otherwise and rte_errno is set. 8405 */ 8406 int 8407 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8408 struct rte_flow_item_eth *eth_spec, 8409 struct rte_flow_item_eth *eth_mask, 8410 struct rte_flow_item_vlan *vlan_spec, 8411 struct rte_flow_item_vlan *vlan_mask) 8412 { 8413 struct mlx5_priv *priv = dev->data->dev_private; 8414 const struct rte_flow_attr attr = { 8415 .ingress = 1, 8416 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8417 }; 8418 struct rte_flow_item items[] = { 8419 { 8420 .type = RTE_FLOW_ITEM_TYPE_ETH, 8421 .spec = eth_spec, 8422 .last = NULL, 8423 .mask = eth_mask, 8424 }, 8425 { 8426 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8427 RTE_FLOW_ITEM_TYPE_END, 8428 .spec = vlan_spec, 8429 .last = NULL, 8430 .mask = vlan_mask, 8431 }, 8432 { 8433 .type = RTE_FLOW_ITEM_TYPE_END, 8434 }, 8435 }; 8436 uint16_t queue[priv->reta_idx_n]; 8437 struct rte_flow_action_rss action_rss = { 8438 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8439 .level = 0, 8440 .types = priv->rss_conf.rss_hf, 8441 .key_len = priv->rss_conf.rss_key_len, 8442 .queue_num = priv->reta_idx_n, 8443 .key = priv->rss_conf.rss_key, 8444 .queue = queue, 8445 }; 8446 struct rte_flow_action actions[] = { 8447 { 8448 .type = RTE_FLOW_ACTION_TYPE_RSS, 8449 .conf = &action_rss, 8450 }, 8451 { 8452 .type = RTE_FLOW_ACTION_TYPE_END, 8453 }, 8454 }; 8455 uint32_t flow_idx; 8456 struct rte_flow_error error; 8457 unsigned int i; 8458 8459 if (!priv->reta_idx_n || !priv->rxqs_n) { 8460 return 0; 8461 } 8462 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8463 action_rss.types = 0; 8464 for (i = 0; i != priv->reta_idx_n; ++i) 8465 queue[i] = (*priv->reta_idx)[i]; 8466 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8467 &attr, items, actions, false, &error); 8468 if (!flow_idx) 8469 return -rte_errno; 8470 return 0; 8471 } 8472 8473 /** 8474 * Enable a flow control configured from the control plane. 8475 * 8476 * @param dev 8477 * Pointer to Ethernet device. 8478 * @param eth_spec 8479 * An Ethernet flow spec to apply. 8480 * @param eth_mask 8481 * An Ethernet flow mask to apply. 8482 * 8483 * @return 8484 * 0 on success, a negative errno value otherwise and rte_errno is set. 8485 */ 8486 int 8487 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8488 struct rte_flow_item_eth *eth_spec, 8489 struct rte_flow_item_eth *eth_mask) 8490 { 8491 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8492 } 8493 8494 /** 8495 * Create default miss flow rule matching lacp traffic 8496 * 8497 * @param dev 8498 * Pointer to Ethernet device. 8499 * @param eth_spec 8500 * An Ethernet flow spec to apply. 8501 * 8502 * @return 8503 * 0 on success, a negative errno value otherwise and rte_errno is set. 8504 */ 8505 int 8506 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8507 { 8508 /* 8509 * The LACP matching is done by only using ether type since using 8510 * a multicast dst mac causes kernel to give low priority to this flow. 8511 */ 8512 static const struct rte_flow_item_eth lacp_spec = { 8513 .hdr.ether_type = RTE_BE16(0x8809), 8514 }; 8515 static const struct rte_flow_item_eth lacp_mask = { 8516 .hdr.ether_type = 0xffff, 8517 }; 8518 const struct rte_flow_attr attr = { 8519 .ingress = 1, 8520 }; 8521 struct rte_flow_item items[] = { 8522 { 8523 .type = RTE_FLOW_ITEM_TYPE_ETH, 8524 .spec = &lacp_spec, 8525 .mask = &lacp_mask, 8526 }, 8527 { 8528 .type = RTE_FLOW_ITEM_TYPE_END, 8529 }, 8530 }; 8531 struct rte_flow_action actions[] = { 8532 { 8533 .type = (enum rte_flow_action_type) 8534 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8535 }, 8536 { 8537 .type = RTE_FLOW_ACTION_TYPE_END, 8538 }, 8539 }; 8540 struct rte_flow_error error; 8541 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8542 &attr, items, actions, 8543 false, &error); 8544 8545 if (!flow_idx) 8546 return -rte_errno; 8547 return 0; 8548 } 8549 8550 /** 8551 * Destroy a flow. 8552 * 8553 * @see rte_flow_destroy() 8554 * @see rte_flow_ops 8555 */ 8556 int 8557 mlx5_flow_destroy(struct rte_eth_dev *dev, 8558 struct rte_flow *flow, 8559 struct rte_flow_error *error __rte_unused) 8560 { 8561 struct mlx5_priv *priv = dev->data->dev_private; 8562 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8563 struct mlx5_dv_flow_info *flow_info; 8564 8565 if (priv->sh->config.dv_flow_en == 2) 8566 return rte_flow_error_set(error, ENOTSUP, 8567 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8568 NULL, 8569 "Flow non-Q destruction not supported"); 8570 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8571 (uintptr_t)(void *)flow); 8572 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8573 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8574 while (flow_info) { 8575 /* Romove the cache flow info. */ 8576 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8577 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8578 LIST_REMOVE(flow_info, next); 8579 mlx5_free(flow_info->items); 8580 mlx5_free(flow_info->actions); 8581 mlx5_free(flow_info); 8582 break; 8583 } 8584 flow_info = LIST_NEXT(flow_info, next); 8585 } 8586 } 8587 return 0; 8588 } 8589 8590 /** 8591 * Destroy all flows. 8592 * 8593 * @see rte_flow_flush() 8594 * @see rte_flow_ops 8595 */ 8596 int 8597 mlx5_flow_flush(struct rte_eth_dev *dev, 8598 struct rte_flow_error *error __rte_unused) 8599 { 8600 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8601 return 0; 8602 } 8603 8604 /** 8605 * Isolated mode. 8606 * 8607 * @see rte_flow_isolate() 8608 * @see rte_flow_ops 8609 */ 8610 int 8611 mlx5_flow_isolate(struct rte_eth_dev *dev, 8612 int enable, 8613 struct rte_flow_error *error) 8614 { 8615 struct mlx5_priv *priv = dev->data->dev_private; 8616 8617 if (dev->data->dev_started) { 8618 rte_flow_error_set(error, EBUSY, 8619 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8620 NULL, 8621 "port must be stopped first"); 8622 return -rte_errno; 8623 } 8624 if (!enable && !priv->sh->config.repr_matching) 8625 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8626 "isolated mode cannot be disabled when " 8627 "representor matching is disabled"); 8628 priv->isolated = !!enable; 8629 if (enable) 8630 dev->dev_ops = &mlx5_dev_ops_isolate; 8631 else 8632 dev->dev_ops = &mlx5_dev_ops; 8633 8634 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8635 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8636 8637 return 0; 8638 } 8639 8640 /** 8641 * Query a flow. 8642 * 8643 * @see rte_flow_query() 8644 * @see rte_flow_ops 8645 */ 8646 static int 8647 flow_drv_query(struct rte_eth_dev *dev, 8648 struct rte_flow *eflow, 8649 const struct rte_flow_action *actions, 8650 void *data, 8651 struct rte_flow_error *error) 8652 { 8653 struct mlx5_priv *priv = dev->data->dev_private; 8654 const struct mlx5_flow_driver_ops *fops; 8655 struct rte_flow *flow = NULL; 8656 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8657 8658 if (priv->sh->config.dv_flow_en == 2) { 8659 #ifdef HAVE_MLX5_HWS_SUPPORT 8660 flow = eflow; 8661 ftype = MLX5_FLOW_TYPE_HW; 8662 #endif 8663 } else { 8664 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8665 (uintptr_t)(void *)eflow); 8666 } 8667 if (!flow) { 8668 return rte_flow_error_set(error, ENOENT, 8669 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8670 NULL, 8671 "invalid flow handle"); 8672 } 8673 if (ftype == MLX5_FLOW_TYPE_MIN) 8674 ftype = flow->drv_type; 8675 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8676 fops = flow_get_drv_ops(ftype); 8677 8678 return fops->query(dev, flow, actions, data, error); 8679 } 8680 8681 /** 8682 * Query a flow. 8683 * 8684 * @see rte_flow_query() 8685 * @see rte_flow_ops 8686 */ 8687 int 8688 mlx5_flow_query(struct rte_eth_dev *dev, 8689 struct rte_flow *flow, 8690 const struct rte_flow_action *actions, 8691 void *data, 8692 struct rte_flow_error *error) 8693 { 8694 int ret; 8695 8696 ret = flow_drv_query(dev, flow, actions, data, 8697 error); 8698 if (ret < 0) 8699 return ret; 8700 return 0; 8701 } 8702 8703 /** 8704 * Get rte_flow callbacks. 8705 * 8706 * @param dev 8707 * Pointer to Ethernet device structure. 8708 * @param ops 8709 * Pointer to operation-specific structure. 8710 * 8711 * @return 0 8712 */ 8713 int 8714 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8715 const struct rte_flow_ops **ops) 8716 { 8717 *ops = &mlx5_flow_ops; 8718 return 0; 8719 } 8720 8721 /** 8722 * Validate meter policy actions. 8723 * Dispatcher for action type specific validation. 8724 * 8725 * @param[in] dev 8726 * Pointer to the Ethernet device structure. 8727 * @param[in] action 8728 * The meter policy action object to validate. 8729 * @param[in] attr 8730 * Attributes of flow to determine steering domain. 8731 * @param[out] is_rss 8732 * Is RSS or not. 8733 * @param[out] domain_bitmap 8734 * Domain bitmap. 8735 * @param[out] is_def_policy 8736 * Is default policy or not. 8737 * @param[out] error 8738 * Perform verbose error reporting if not NULL. Initialized in case of 8739 * error only. 8740 * 8741 * @return 8742 * 0 on success, otherwise negative errno value. 8743 */ 8744 int 8745 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8746 const struct rte_flow_action *actions[RTE_COLORS], 8747 struct rte_flow_attr *attr, 8748 bool *is_rss, 8749 uint8_t *domain_bitmap, 8750 uint8_t *policy_mode, 8751 struct rte_mtr_error *error) 8752 { 8753 const struct mlx5_flow_driver_ops *fops; 8754 8755 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8756 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8757 domain_bitmap, policy_mode, error); 8758 } 8759 8760 /** 8761 * Destroy the meter table set. 8762 * 8763 * @param[in] dev 8764 * Pointer to Ethernet device. 8765 * @param[in] mtr_policy 8766 * Meter policy struct. 8767 */ 8768 void 8769 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8770 struct mlx5_flow_meter_policy *mtr_policy) 8771 { 8772 const struct mlx5_flow_driver_ops *fops; 8773 8774 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8775 fops->destroy_mtr_acts(dev, mtr_policy); 8776 } 8777 8778 /** 8779 * Create policy action, lock free, 8780 * (mutex should be acquired by caller). 8781 * Dispatcher for action type specific call. 8782 * 8783 * @param[in] dev 8784 * Pointer to the Ethernet device structure. 8785 * @param[in] mtr_policy 8786 * Meter policy struct. 8787 * @param[in] action 8788 * Action specification used to create meter actions. 8789 * @param[in] attr 8790 * Flow rule attributes. 8791 * @param[out] error 8792 * Perform verbose error reporting if not NULL. Initialized in case of 8793 * error only. 8794 * 8795 * @return 8796 * 0 on success, otherwise negative errno value. 8797 */ 8798 int 8799 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8800 struct mlx5_flow_meter_policy *mtr_policy, 8801 const struct rte_flow_action *actions[RTE_COLORS], 8802 struct rte_flow_attr *attr, 8803 struct rte_mtr_error *error) 8804 { 8805 const struct mlx5_flow_driver_ops *fops; 8806 8807 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8808 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8809 } 8810 8811 /** 8812 * Create policy rules, lock free, 8813 * (mutex should be acquired by caller). 8814 * Dispatcher for action type specific call. 8815 * 8816 * @param[in] dev 8817 * Pointer to the Ethernet device structure. 8818 * @param[in] mtr_policy 8819 * Meter policy struct. 8820 * 8821 * @return 8822 * 0 on success, -1 otherwise. 8823 */ 8824 int 8825 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8826 struct mlx5_flow_meter_policy *mtr_policy) 8827 { 8828 const struct mlx5_flow_driver_ops *fops; 8829 8830 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8831 return fops->create_policy_rules(dev, mtr_policy); 8832 } 8833 8834 /** 8835 * Destroy policy rules, lock free, 8836 * (mutex should be acquired by caller). 8837 * Dispatcher for action type specific call. 8838 * 8839 * @param[in] dev 8840 * Pointer to the Ethernet device structure. 8841 * @param[in] mtr_policy 8842 * Meter policy struct. 8843 */ 8844 void 8845 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8846 struct mlx5_flow_meter_policy *mtr_policy) 8847 { 8848 const struct mlx5_flow_driver_ops *fops; 8849 8850 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8851 fops->destroy_policy_rules(dev, mtr_policy); 8852 } 8853 8854 /** 8855 * Destroy the default policy table set. 8856 * 8857 * @param[in] dev 8858 * Pointer to Ethernet device. 8859 */ 8860 void 8861 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8862 { 8863 const struct mlx5_flow_driver_ops *fops; 8864 8865 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8866 fops->destroy_def_policy(dev); 8867 } 8868 8869 /** 8870 * Destroy the default policy table set. 8871 * 8872 * @param[in] dev 8873 * Pointer to Ethernet device. 8874 * 8875 * @return 8876 * 0 on success, -1 otherwise. 8877 */ 8878 int 8879 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8880 { 8881 const struct mlx5_flow_driver_ops *fops; 8882 8883 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8884 return fops->create_def_policy(dev); 8885 } 8886 8887 /** 8888 * Create the needed meter and suffix tables. 8889 * 8890 * @param[in] dev 8891 * Pointer to Ethernet device. 8892 * 8893 * @return 8894 * 0 on success, -1 otherwise. 8895 */ 8896 int 8897 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8898 struct mlx5_flow_meter_info *fm, 8899 uint32_t mtr_idx, 8900 uint8_t domain_bitmap) 8901 { 8902 const struct mlx5_flow_driver_ops *fops; 8903 8904 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8905 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8906 } 8907 8908 /** 8909 * Destroy the meter table set. 8910 * 8911 * @param[in] dev 8912 * Pointer to Ethernet device. 8913 * @param[in] tbl 8914 * Pointer to the meter table set. 8915 */ 8916 void 8917 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8918 struct mlx5_flow_meter_info *fm) 8919 { 8920 const struct mlx5_flow_driver_ops *fops; 8921 8922 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8923 fops->destroy_mtr_tbls(dev, fm); 8924 } 8925 8926 /** 8927 * Destroy the global meter drop table. 8928 * 8929 * @param[in] dev 8930 * Pointer to Ethernet device. 8931 */ 8932 void 8933 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8934 { 8935 const struct mlx5_flow_driver_ops *fops; 8936 8937 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8938 fops->destroy_mtr_drop_tbls(dev); 8939 } 8940 8941 /** 8942 * Destroy the sub policy table with RX queue. 8943 * 8944 * @param[in] dev 8945 * Pointer to Ethernet device. 8946 * @param[in] mtr_policy 8947 * Pointer to meter policy table. 8948 */ 8949 void 8950 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8951 struct mlx5_flow_meter_policy *mtr_policy) 8952 { 8953 const struct mlx5_flow_driver_ops *fops; 8954 8955 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8956 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8957 } 8958 8959 /** 8960 * Allocate the needed aso flow meter id. 8961 * 8962 * @param[in] dev 8963 * Pointer to Ethernet device. 8964 * 8965 * @return 8966 * Index to aso flow meter on success, NULL otherwise. 8967 */ 8968 uint32_t 8969 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8970 { 8971 const struct mlx5_flow_driver_ops *fops; 8972 8973 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8974 return fops->create_meter(dev); 8975 } 8976 8977 /** 8978 * Free the aso flow meter id. 8979 * 8980 * @param[in] dev 8981 * Pointer to Ethernet device. 8982 * @param[in] mtr_idx 8983 * Index to aso flow meter to be free. 8984 * 8985 * @return 8986 * 0 on success. 8987 */ 8988 void 8989 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8990 { 8991 const struct mlx5_flow_driver_ops *fops; 8992 8993 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8994 fops->free_meter(dev, mtr_idx); 8995 } 8996 8997 /** 8998 * Allocate a counter. 8999 * 9000 * @param[in] dev 9001 * Pointer to Ethernet device structure. 9002 * 9003 * @return 9004 * Index to allocated counter on success, 0 otherwise. 9005 */ 9006 uint32_t 9007 mlx5_counter_alloc(struct rte_eth_dev *dev) 9008 { 9009 struct rte_flow_attr attr = { .transfer = 0 }; 9010 9011 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 9012 (dev); 9013 } 9014 9015 /** 9016 * Free a counter. 9017 * 9018 * @param[in] dev 9019 * Pointer to Ethernet device structure. 9020 * @param[in] cnt 9021 * Index to counter to be free. 9022 */ 9023 void 9024 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 9025 { 9026 struct rte_flow_attr attr = { .transfer = 0 }; 9027 9028 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 9029 } 9030 9031 /** 9032 * Query counter statistics. 9033 * 9034 * @param[in] dev 9035 * Pointer to Ethernet device structure. 9036 * @param[in] cnt 9037 * Index to counter to query. 9038 * @param[in] clear 9039 * Set to clear counter statistics. 9040 * @param[out] pkts 9041 * The counter hits packets number to save. 9042 * @param[out] bytes 9043 * The counter hits bytes number to save. 9044 * 9045 * @return 9046 * 0 on success, a negative errno value otherwise. 9047 */ 9048 int 9049 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 9050 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 9051 { 9052 struct rte_flow_attr attr = { .transfer = 0 }; 9053 9054 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 9055 (dev, cnt, clear, pkts, bytes, action); 9056 } 9057 9058 /** 9059 * Get information about HWS pre-configurable resources. 9060 * 9061 * @param[in] dev 9062 * Pointer to the rte_eth_dev structure. 9063 * @param[out] port_info 9064 * Pointer to port information. 9065 * @param[out] queue_info 9066 * Pointer to queue information. 9067 * @param[out] error 9068 * Pointer to error structure. 9069 * 9070 * @return 9071 * 0 on success, a negative errno value otherwise and rte_errno is set. 9072 */ 9073 static int 9074 mlx5_flow_info_get(struct rte_eth_dev *dev, 9075 struct rte_flow_port_info *port_info, 9076 struct rte_flow_queue_info *queue_info, 9077 struct rte_flow_error *error) 9078 { 9079 const struct mlx5_flow_driver_ops *fops; 9080 struct rte_flow_attr attr = {0}; 9081 9082 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9083 return rte_flow_error_set(error, ENOTSUP, 9084 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9085 NULL, 9086 "info get with incorrect steering mode"); 9087 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9088 return fops->info_get(dev, port_info, queue_info, error); 9089 } 9090 9091 /** 9092 * Configure port HWS resources. 9093 * 9094 * @param[in] dev 9095 * Pointer to the rte_eth_dev structure. 9096 * @param[in] port_attr 9097 * Port configuration attributes. 9098 * @param[in] nb_queue 9099 * Number of queue. 9100 * @param[in] queue_attr 9101 * Array that holds attributes for each flow queue. 9102 * @param[out] error 9103 * Pointer to error structure. 9104 * 9105 * @return 9106 * 0 on success, a negative errno value otherwise and rte_errno is set. 9107 */ 9108 static int 9109 mlx5_flow_port_configure(struct rte_eth_dev *dev, 9110 const struct rte_flow_port_attr *port_attr, 9111 uint16_t nb_queue, 9112 const struct rte_flow_queue_attr *queue_attr[], 9113 struct rte_flow_error *error) 9114 { 9115 const struct mlx5_flow_driver_ops *fops; 9116 struct rte_flow_attr attr = {0}; 9117 9118 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9119 return rte_flow_error_set(error, ENOTSUP, 9120 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9121 NULL, 9122 "port configure with incorrect steering mode"); 9123 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9124 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 9125 } 9126 9127 /** 9128 * Validate item template. 9129 * 9130 * @param[in] dev 9131 * Pointer to the rte_eth_dev structure. 9132 * @param[in] attr 9133 * Pointer to the item template attributes. 9134 * @param[in] items 9135 * The template item pattern. 9136 * @param[out] error 9137 * Pointer to error structure. 9138 * 9139 * @return 9140 * 0 on success, a negative errno value otherwise and rte_errno is set. 9141 */ 9142 int 9143 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 9144 const struct rte_flow_pattern_template_attr *attr, 9145 const struct rte_flow_item items[], 9146 struct rte_flow_error *error) 9147 { 9148 const struct mlx5_flow_driver_ops *fops; 9149 struct rte_flow_attr fattr = {0}; 9150 9151 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9152 rte_flow_error_set(error, ENOTSUP, 9153 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9154 "pattern validate with incorrect steering mode"); 9155 return -ENOTSUP; 9156 } 9157 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9158 return fops->pattern_validate(dev, attr, items, error); 9159 } 9160 9161 /** 9162 * Create flow item template. 9163 * 9164 * @param[in] dev 9165 * Pointer to the rte_eth_dev structure. 9166 * @param[in] attr 9167 * Pointer to the item template attributes. 9168 * @param[in] items 9169 * The template item pattern. 9170 * @param[out] error 9171 * Pointer to error structure. 9172 * 9173 * @return 9174 * 0 on success, a negative errno value otherwise and rte_errno is set. 9175 */ 9176 static struct rte_flow_pattern_template * 9177 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9178 const struct rte_flow_pattern_template_attr *attr, 9179 const struct rte_flow_item items[], 9180 struct rte_flow_error *error) 9181 { 9182 const struct mlx5_flow_driver_ops *fops; 9183 struct rte_flow_attr fattr = {0}; 9184 9185 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9186 rte_flow_error_set(error, ENOTSUP, 9187 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9188 NULL, 9189 "pattern create with incorrect steering mode"); 9190 return NULL; 9191 } 9192 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9193 return fops->pattern_template_create(dev, attr, items, error); 9194 } 9195 9196 /** 9197 * Destroy flow item template. 9198 * 9199 * @param[in] dev 9200 * Pointer to the rte_eth_dev structure. 9201 * @param[in] template 9202 * Pointer to the item template to be destroyed. 9203 * @param[out] error 9204 * Pointer to error structure. 9205 * 9206 * @return 9207 * 0 on success, a negative errno value otherwise and rte_errno is set. 9208 */ 9209 static int 9210 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9211 struct rte_flow_pattern_template *template, 9212 struct rte_flow_error *error) 9213 { 9214 const struct mlx5_flow_driver_ops *fops; 9215 struct rte_flow_attr attr = {0}; 9216 9217 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9218 return rte_flow_error_set(error, ENOTSUP, 9219 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9220 NULL, 9221 "pattern destroy with incorrect steering mode"); 9222 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9223 return fops->pattern_template_destroy(dev, template, error); 9224 } 9225 9226 /** 9227 * Validate flow actions template. 9228 * 9229 * @param[in] dev 9230 * Pointer to the rte_eth_dev structure. 9231 * @param[in] attr 9232 * Pointer to the action template attributes. 9233 * @param[in] actions 9234 * Associated actions (list terminated by the END action). 9235 * @param[in] masks 9236 * List of actions that marks which of the action's member is constant. 9237 * @param[out] error 9238 * Pointer to error structure. 9239 * 9240 * @return 9241 * 0 on success, a negative errno value otherwise and rte_errno is set. 9242 */ 9243 int 9244 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9245 const struct rte_flow_actions_template_attr *attr, 9246 const struct rte_flow_action actions[], 9247 const struct rte_flow_action masks[], 9248 struct rte_flow_error *error) 9249 { 9250 const struct mlx5_flow_driver_ops *fops; 9251 struct rte_flow_attr fattr = {0}; 9252 9253 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9254 rte_flow_error_set(error, ENOTSUP, 9255 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9256 "actions validate with incorrect steering mode"); 9257 return -ENOTSUP; 9258 } 9259 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9260 return fops->actions_validate(dev, attr, actions, masks, error); 9261 } 9262 9263 /** 9264 * Create flow item template. 9265 * 9266 * @param[in] dev 9267 * Pointer to the rte_eth_dev structure. 9268 * @param[in] attr 9269 * Pointer to the action template attributes. 9270 * @param[in] actions 9271 * Associated actions (list terminated by the END action). 9272 * @param[in] masks 9273 * List of actions that marks which of the action's member is constant. 9274 * @param[out] error 9275 * Pointer to error structure. 9276 * 9277 * @return 9278 * 0 on success, a negative errno value otherwise and rte_errno is set. 9279 */ 9280 static struct rte_flow_actions_template * 9281 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9282 const struct rte_flow_actions_template_attr *attr, 9283 const struct rte_flow_action actions[], 9284 const struct rte_flow_action masks[], 9285 struct rte_flow_error *error) 9286 { 9287 const struct mlx5_flow_driver_ops *fops; 9288 struct rte_flow_attr fattr = {0}; 9289 9290 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9291 rte_flow_error_set(error, ENOTSUP, 9292 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9293 NULL, 9294 "action create with incorrect steering mode"); 9295 return NULL; 9296 } 9297 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9298 return fops->actions_template_create(dev, attr, actions, masks, error); 9299 } 9300 9301 /** 9302 * Destroy flow action template. 9303 * 9304 * @param[in] dev 9305 * Pointer to the rte_eth_dev structure. 9306 * @param[in] template 9307 * Pointer to the action template to be destroyed. 9308 * @param[out] error 9309 * Pointer to error structure. 9310 * 9311 * @return 9312 * 0 on success, a negative errno value otherwise and rte_errno is set. 9313 */ 9314 static int 9315 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9316 struct rte_flow_actions_template *template, 9317 struct rte_flow_error *error) 9318 { 9319 const struct mlx5_flow_driver_ops *fops; 9320 struct rte_flow_attr attr = {0}; 9321 9322 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9323 return rte_flow_error_set(error, ENOTSUP, 9324 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9325 NULL, 9326 "action destroy with incorrect steering mode"); 9327 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9328 return fops->actions_template_destroy(dev, template, error); 9329 } 9330 9331 /** 9332 * Create flow table. 9333 * 9334 * @param[in] dev 9335 * Pointer to the rte_eth_dev structure. 9336 * @param[in] attr 9337 * Pointer to the table attributes. 9338 * @param[in] item_templates 9339 * Item template array to be binded to the table. 9340 * @param[in] nb_item_templates 9341 * Number of item template. 9342 * @param[in] action_templates 9343 * Action template array to be binded to the table. 9344 * @param[in] nb_action_templates 9345 * Number of action template. 9346 * @param[out] error 9347 * Pointer to error structure. 9348 * 9349 * @return 9350 * Table on success, NULL otherwise and rte_errno is set. 9351 */ 9352 static struct rte_flow_template_table * 9353 mlx5_flow_table_create(struct rte_eth_dev *dev, 9354 const struct rte_flow_template_table_attr *attr, 9355 struct rte_flow_pattern_template *item_templates[], 9356 uint8_t nb_item_templates, 9357 struct rte_flow_actions_template *action_templates[], 9358 uint8_t nb_action_templates, 9359 struct rte_flow_error *error) 9360 { 9361 const struct mlx5_flow_driver_ops *fops; 9362 struct rte_flow_attr fattr = {0}; 9363 9364 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9365 rte_flow_error_set(error, ENOTSUP, 9366 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9367 NULL, 9368 "table create with incorrect steering mode"); 9369 return NULL; 9370 } 9371 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9372 return fops->template_table_create(dev, 9373 attr, 9374 item_templates, 9375 nb_item_templates, 9376 action_templates, 9377 nb_action_templates, 9378 error); 9379 } 9380 9381 /** 9382 * PMD destroy flow table. 9383 * 9384 * @param[in] dev 9385 * Pointer to the rte_eth_dev structure. 9386 * @param[in] table 9387 * Pointer to the table to be destroyed. 9388 * @param[out] error 9389 * Pointer to error structure. 9390 * 9391 * @return 9392 * 0 on success, a negative errno value otherwise and rte_errno is set. 9393 */ 9394 static int 9395 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9396 struct rte_flow_template_table *table, 9397 struct rte_flow_error *error) 9398 { 9399 const struct mlx5_flow_driver_ops *fops; 9400 struct rte_flow_attr attr = {0}; 9401 9402 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9403 return rte_flow_error_set(error, ENOTSUP, 9404 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9405 NULL, 9406 "table destroy with incorrect steering mode"); 9407 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9408 return fops->template_table_destroy(dev, table, error); 9409 } 9410 9411 /** 9412 * PMD group set miss actions. 9413 * 9414 * @param[in] dev 9415 * Pointer to the rte_eth_dev structure. 9416 * @param[in] attr 9417 * Pointer to group attributes 9418 * @param[in] actions 9419 * Array of actions 9420 * @param[out] error 9421 * Pointer to error structure. 9422 * 9423 * @return 9424 * 0 on success, a negative errno value otherwise and rte_errno is set. 9425 */ 9426 static int 9427 mlx5_flow_group_set_miss_actions(struct rte_eth_dev *dev, 9428 uint32_t group_id, 9429 const struct rte_flow_group_attr *attr, 9430 const struct rte_flow_action actions[], 9431 struct rte_flow_error *error) 9432 { 9433 const struct mlx5_flow_driver_ops *fops; 9434 struct rte_flow_attr fattr = {0}; 9435 9436 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9437 return rte_flow_error_set(error, ENOTSUP, 9438 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9439 NULL, 9440 "group set miss actions with incorrect steering mode"); 9441 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9442 return fops->group_set_miss_actions(dev, group_id, attr, actions, error); 9443 } 9444 9445 /** 9446 * Enqueue flow creation. 9447 * 9448 * @param[in] dev 9449 * Pointer to the rte_eth_dev structure. 9450 * @param[in] queue_id 9451 * The queue to create the flow. 9452 * @param[in] attr 9453 * Pointer to the flow operation attributes. 9454 * @param[in] items 9455 * Items with flow spec value. 9456 * @param[in] pattern_template_index 9457 * The item pattern flow follows from the table. 9458 * @param[in] actions 9459 * Action with flow spec value. 9460 * @param[in] action_template_index 9461 * The action pattern flow follows from the table. 9462 * @param[in] user_data 9463 * Pointer to the user_data. 9464 * @param[out] error 9465 * Pointer to error structure. 9466 * 9467 * @return 9468 * Flow pointer on success, NULL otherwise and rte_errno is set. 9469 */ 9470 static struct rte_flow * 9471 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 9472 uint32_t queue_id, 9473 const struct rte_flow_op_attr *attr, 9474 struct rte_flow_template_table *table, 9475 const struct rte_flow_item items[], 9476 uint8_t pattern_template_index, 9477 const struct rte_flow_action actions[], 9478 uint8_t action_template_index, 9479 void *user_data, 9480 struct rte_flow_error *error) 9481 { 9482 const struct mlx5_flow_driver_ops *fops; 9483 struct rte_flow_attr fattr = {0}; 9484 9485 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9486 rte_flow_error_set(error, ENOTSUP, 9487 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9488 NULL, 9489 "flow_q create with incorrect steering mode"); 9490 return NULL; 9491 } 9492 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9493 return fops->async_flow_create(dev, queue_id, attr, table, 9494 items, pattern_template_index, 9495 actions, action_template_index, 9496 user_data, error); 9497 } 9498 9499 /** 9500 * Enqueue flow creation by index. 9501 * 9502 * @param[in] dev 9503 * Pointer to the rte_eth_dev structure. 9504 * @param[in] queue_id 9505 * The queue to create the flow. 9506 * @param[in] attr 9507 * Pointer to the flow operation attributes. 9508 * @param[in] rule_index 9509 * The item pattern flow follows from the table. 9510 * @param[in] actions 9511 * Action with flow spec value. 9512 * @param[in] action_template_index 9513 * The action pattern flow follows from the table. 9514 * @param[in] user_data 9515 * Pointer to the user_data. 9516 * @param[out] error 9517 * Pointer to error structure. 9518 * 9519 * @return 9520 * Flow pointer on success, NULL otherwise and rte_errno is set. 9521 */ 9522 static struct rte_flow * 9523 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 9524 uint32_t queue_id, 9525 const struct rte_flow_op_attr *attr, 9526 struct rte_flow_template_table *table, 9527 uint32_t rule_index, 9528 const struct rte_flow_action actions[], 9529 uint8_t action_template_index, 9530 void *user_data, 9531 struct rte_flow_error *error) 9532 { 9533 const struct mlx5_flow_driver_ops *fops; 9534 struct rte_flow_attr fattr = {0}; 9535 9536 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9537 rte_flow_error_set(error, ENOTSUP, 9538 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9539 NULL, 9540 "flow_q create with incorrect steering mode"); 9541 return NULL; 9542 } 9543 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9544 return fops->async_flow_create_by_index(dev, queue_id, attr, table, 9545 rule_index, actions, action_template_index, 9546 user_data, error); 9547 } 9548 9549 /** 9550 * Enqueue flow update. 9551 * 9552 * @param[in] dev 9553 * Pointer to the rte_eth_dev structure. 9554 * @param[in] queue 9555 * The queue to destroy the flow. 9556 * @param[in] attr 9557 * Pointer to the flow operation attributes. 9558 * @param[in] flow 9559 * Pointer to the flow to be destroyed. 9560 * @param[in] actions 9561 * Action with flow spec value. 9562 * @param[in] action_template_index 9563 * The action pattern flow follows from the table. 9564 * @param[in] user_data 9565 * Pointer to the user_data. 9566 * @param[out] error 9567 * Pointer to error structure. 9568 * 9569 * @return 9570 * 0 on success, negative value otherwise and rte_errno is set. 9571 */ 9572 static int 9573 mlx5_flow_async_flow_update(struct rte_eth_dev *dev, 9574 uint32_t queue, 9575 const struct rte_flow_op_attr *attr, 9576 struct rte_flow *flow, 9577 const struct rte_flow_action actions[], 9578 uint8_t action_template_index, 9579 void *user_data, 9580 struct rte_flow_error *error) 9581 { 9582 const struct mlx5_flow_driver_ops *fops; 9583 struct rte_flow_attr fattr = {0}; 9584 9585 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9586 return rte_flow_error_set(error, ENOTSUP, 9587 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9588 NULL, 9589 "flow_q update with incorrect steering mode"); 9590 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9591 return fops->async_flow_update(dev, queue, attr, flow, 9592 actions, action_template_index, user_data, error); 9593 } 9594 9595 /** 9596 * Enqueue flow destruction. 9597 * 9598 * @param[in] dev 9599 * Pointer to the rte_eth_dev structure. 9600 * @param[in] queue 9601 * The queue to destroy the flow. 9602 * @param[in] attr 9603 * Pointer to the flow operation attributes. 9604 * @param[in] flow 9605 * Pointer to the flow to be destroyed. 9606 * @param[in] user_data 9607 * Pointer to the user_data. 9608 * @param[out] error 9609 * Pointer to error structure. 9610 * 9611 * @return 9612 * 0 on success, negative value otherwise and rte_errno is set. 9613 */ 9614 static int 9615 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 9616 uint32_t queue, 9617 const struct rte_flow_op_attr *attr, 9618 struct rte_flow *flow, 9619 void *user_data, 9620 struct rte_flow_error *error) 9621 { 9622 const struct mlx5_flow_driver_ops *fops; 9623 struct rte_flow_attr fattr = {0}; 9624 9625 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9626 return rte_flow_error_set(error, ENOTSUP, 9627 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9628 NULL, 9629 "flow_q destroy with incorrect steering mode"); 9630 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9631 return fops->async_flow_destroy(dev, queue, attr, flow, 9632 user_data, error); 9633 } 9634 9635 /** 9636 * Pull the enqueued flows. 9637 * 9638 * @param[in] dev 9639 * Pointer to the rte_eth_dev structure. 9640 * @param[in] queue 9641 * The queue to pull the result. 9642 * @param[in/out] res 9643 * Array to save the results. 9644 * @param[in] n_res 9645 * Available result with the array. 9646 * @param[out] error 9647 * Pointer to error structure. 9648 * 9649 * @return 9650 * Result number on success, negative value otherwise and rte_errno is set. 9651 */ 9652 static int 9653 mlx5_flow_pull(struct rte_eth_dev *dev, 9654 uint32_t queue, 9655 struct rte_flow_op_result res[], 9656 uint16_t n_res, 9657 struct rte_flow_error *error) 9658 { 9659 const struct mlx5_flow_driver_ops *fops; 9660 struct rte_flow_attr attr = {0}; 9661 9662 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9663 return rte_flow_error_set(error, ENOTSUP, 9664 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9665 NULL, 9666 "flow_q pull with incorrect steering mode"); 9667 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9668 return fops->pull(dev, queue, res, n_res, error); 9669 } 9670 9671 /** 9672 * Push the enqueued flows. 9673 * 9674 * @param[in] dev 9675 * Pointer to the rte_eth_dev structure. 9676 * @param[in] queue 9677 * The queue to push the flows. 9678 * @param[out] error 9679 * Pointer to error structure. 9680 * 9681 * @return 9682 * 0 on success, negative value otherwise and rte_errno is set. 9683 */ 9684 static int 9685 mlx5_flow_push(struct rte_eth_dev *dev, 9686 uint32_t queue, 9687 struct rte_flow_error *error) 9688 { 9689 const struct mlx5_flow_driver_ops *fops; 9690 struct rte_flow_attr attr = {0}; 9691 9692 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9693 return rte_flow_error_set(error, ENOTSUP, 9694 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9695 NULL, 9696 "flow_q push with incorrect steering mode"); 9697 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9698 return fops->push(dev, queue, error); 9699 } 9700 9701 /** 9702 * Create shared action. 9703 * 9704 * @param[in] dev 9705 * Pointer to the rte_eth_dev structure. 9706 * @param[in] queue 9707 * Which queue to be used.. 9708 * @param[in] attr 9709 * Operation attribute. 9710 * @param[in] conf 9711 * Indirect action configuration. 9712 * @param[in] action 9713 * rte_flow action detail. 9714 * @param[in] user_data 9715 * Pointer to the user_data. 9716 * @param[out] error 9717 * Pointer to error structure. 9718 * 9719 * @return 9720 * Action handle on success, NULL otherwise and rte_errno is set. 9721 */ 9722 static struct rte_flow_action_handle * 9723 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 9724 const struct rte_flow_op_attr *attr, 9725 const struct rte_flow_indir_action_conf *conf, 9726 const struct rte_flow_action *action, 9727 void *user_data, 9728 struct rte_flow_error *error) 9729 { 9730 const struct mlx5_flow_driver_ops *fops = 9731 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9732 9733 return fops->async_action_create(dev, queue, attr, conf, action, 9734 user_data, error); 9735 } 9736 9737 /** 9738 * Update shared action. 9739 * 9740 * @param[in] dev 9741 * Pointer to the rte_eth_dev structure. 9742 * @param[in] queue 9743 * Which queue to be used.. 9744 * @param[in] attr 9745 * Operation attribute. 9746 * @param[in] handle 9747 * Action handle to be updated. 9748 * @param[in] update 9749 * Update value. 9750 * @param[in] user_data 9751 * Pointer to the user_data. 9752 * @param[out] error 9753 * Pointer to error structure. 9754 * 9755 * @return 9756 * 0 on success, negative value otherwise and rte_errno is set. 9757 */ 9758 static int 9759 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 9760 const struct rte_flow_op_attr *attr, 9761 struct rte_flow_action_handle *handle, 9762 const void *update, 9763 void *user_data, 9764 struct rte_flow_error *error) 9765 { 9766 const struct mlx5_flow_driver_ops *fops = 9767 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9768 9769 return fops->async_action_update(dev, queue, attr, handle, 9770 update, user_data, error); 9771 } 9772 9773 static int 9774 mlx5_flow_async_action_handle_query_update 9775 (struct rte_eth_dev *dev, uint32_t queue_id, 9776 const struct rte_flow_op_attr *op_attr, 9777 struct rte_flow_action_handle *action_handle, 9778 const void *update, void *query, 9779 enum rte_flow_query_update_mode qu_mode, 9780 void *user_data, struct rte_flow_error *error) 9781 { 9782 const struct mlx5_flow_driver_ops *fops = 9783 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9784 9785 if (!fops || !fops->async_action_query_update) 9786 return rte_flow_error_set(error, ENOTSUP, 9787 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 9788 "async query_update not supported"); 9789 return fops->async_action_query_update 9790 (dev, queue_id, op_attr, action_handle, 9791 update, query, qu_mode, user_data, error); 9792 } 9793 9794 /** 9795 * Query shared action. 9796 * 9797 * @param[in] dev 9798 * Pointer to the rte_eth_dev structure. 9799 * @param[in] queue 9800 * Which queue to be used.. 9801 * @param[in] attr 9802 * Operation attribute. 9803 * @param[in] handle 9804 * Action handle to be updated. 9805 * @param[in] data 9806 * Pointer query result data. 9807 * @param[in] user_data 9808 * Pointer to the user_data. 9809 * @param[out] error 9810 * Pointer to error structure. 9811 * 9812 * @return 9813 * 0 on success, negative value otherwise and rte_errno is set. 9814 */ 9815 static int 9816 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9817 const struct rte_flow_op_attr *attr, 9818 const struct rte_flow_action_handle *handle, 9819 void *data, 9820 void *user_data, 9821 struct rte_flow_error *error) 9822 { 9823 const struct mlx5_flow_driver_ops *fops = 9824 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9825 9826 return fops->async_action_query(dev, queue, attr, handle, 9827 data, user_data, error); 9828 } 9829 9830 /** 9831 * Destroy shared action. 9832 * 9833 * @param[in] dev 9834 * Pointer to the rte_eth_dev structure. 9835 * @param[in] queue 9836 * Which queue to be used.. 9837 * @param[in] attr 9838 * Operation attribute. 9839 * @param[in] handle 9840 * Action handle to be destroyed. 9841 * @param[in] user_data 9842 * Pointer to the user_data. 9843 * @param[out] error 9844 * Pointer to error structure. 9845 * 9846 * @return 9847 * 0 on success, negative value otherwise and rte_errno is set. 9848 */ 9849 static int 9850 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9851 const struct rte_flow_op_attr *attr, 9852 struct rte_flow_action_handle *handle, 9853 void *user_data, 9854 struct rte_flow_error *error) 9855 { 9856 const struct mlx5_flow_driver_ops *fops = 9857 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9858 9859 return fops->async_action_destroy(dev, queue, attr, handle, 9860 user_data, error); 9861 } 9862 9863 /** 9864 * Allocate a new memory for the counter values wrapped by all the needed 9865 * management. 9866 * 9867 * @param[in] sh 9868 * Pointer to mlx5_dev_ctx_shared object. 9869 * 9870 * @return 9871 * 0 on success, a negative errno value otherwise. 9872 */ 9873 static int 9874 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9875 { 9876 struct mlx5_counter_stats_mem_mng *mem_mng; 9877 volatile struct flow_counter_stats *raw_data; 9878 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9879 int size = (sizeof(struct flow_counter_stats) * 9880 MLX5_COUNTERS_PER_POOL + 9881 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9882 sizeof(struct mlx5_counter_stats_mem_mng); 9883 size_t pgsize = rte_mem_page_size(); 9884 uint8_t *mem; 9885 int ret; 9886 int i; 9887 9888 if (pgsize == (size_t)-1) { 9889 DRV_LOG(ERR, "Failed to get mem page size"); 9890 rte_errno = ENOMEM; 9891 return -ENOMEM; 9892 } 9893 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9894 if (!mem) { 9895 rte_errno = ENOMEM; 9896 return -ENOMEM; 9897 } 9898 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9899 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9900 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9901 sh->cdev->pdn, mem, size, 9902 &mem_mng->wm); 9903 if (ret) { 9904 rte_errno = errno; 9905 mlx5_free(mem); 9906 return -rte_errno; 9907 } 9908 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9909 raw_data = (volatile struct flow_counter_stats *)mem; 9910 for (i = 0; i < raws_n; ++i) { 9911 mem_mng->raws[i].mem_mng = mem_mng; 9912 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9913 } 9914 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9915 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9916 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9917 next); 9918 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9919 sh->sws_cmng.mem_mng = mem_mng; 9920 return 0; 9921 } 9922 9923 /** 9924 * Set the statistic memory to the new counter pool. 9925 * 9926 * @param[in] sh 9927 * Pointer to mlx5_dev_ctx_shared object. 9928 * @param[in] pool 9929 * Pointer to the pool to set the statistic memory. 9930 * 9931 * @return 9932 * 0 on success, a negative errno value otherwise. 9933 */ 9934 static int 9935 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9936 struct mlx5_flow_counter_pool *pool) 9937 { 9938 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9939 /* Resize statistic memory once used out. */ 9940 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9941 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9942 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9943 return -1; 9944 } 9945 rte_spinlock_lock(&pool->sl); 9946 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9947 rte_spinlock_unlock(&pool->sl); 9948 pool->raw_hw = NULL; 9949 return 0; 9950 } 9951 9952 #define MLX5_POOL_QUERY_FREQ_US 1000000 9953 9954 /** 9955 * Set the periodic procedure for triggering asynchronous batch queries for all 9956 * the counter pools. 9957 * 9958 * @param[in] sh 9959 * Pointer to mlx5_dev_ctx_shared object. 9960 */ 9961 void 9962 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9963 { 9964 uint32_t pools_n, us; 9965 9966 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9967 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9968 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9969 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9970 sh->sws_cmng.query_thread_on = 0; 9971 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9972 } else { 9973 sh->sws_cmng.query_thread_on = 1; 9974 } 9975 } 9976 9977 /** 9978 * The periodic procedure for triggering asynchronous batch queries for all the 9979 * counter pools. This function is probably called by the host thread. 9980 * 9981 * @param[in] arg 9982 * The parameter for the alarm process. 9983 */ 9984 void 9985 mlx5_flow_query_alarm(void *arg) 9986 { 9987 struct mlx5_dev_ctx_shared *sh = arg; 9988 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9989 uint16_t pool_index = cmng->pool_index; 9990 struct mlx5_flow_counter_pool *pool; 9991 uint16_t n_valid; 9992 int ret; 9993 9994 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9995 goto set_alarm; 9996 rte_spinlock_lock(&cmng->pool_update_sl); 9997 pool = cmng->pools[pool_index]; 9998 n_valid = cmng->n_valid; 9999 rte_spinlock_unlock(&cmng->pool_update_sl); 10000 /* Set the statistic memory to the new created pool. */ 10001 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 10002 goto set_alarm; 10003 if (pool->raw_hw) 10004 /* There is a pool query in progress. */ 10005 goto set_alarm; 10006 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 10007 if (!pool->raw_hw) 10008 /* No free counter statistics raw memory. */ 10009 goto set_alarm; 10010 /* 10011 * Identify the counters released between query trigger and query 10012 * handle more efficiently. The counter released in this gap period 10013 * should wait for a new round of query as the new arrived packets 10014 * will not be taken into account. 10015 */ 10016 pool->query_gen++; 10017 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 10018 MLX5_COUNTERS_PER_POOL, 10019 NULL, NULL, 10020 pool->raw_hw->mem_mng->wm.lkey, 10021 (void *)(uintptr_t) 10022 pool->raw_hw->data, 10023 sh->devx_comp, 10024 (uint64_t)(uintptr_t)pool); 10025 if (ret) { 10026 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 10027 " %d", pool->min_dcs->id); 10028 pool->raw_hw = NULL; 10029 goto set_alarm; 10030 } 10031 LIST_REMOVE(pool->raw_hw, next); 10032 cmng->pending_queries++; 10033 pool_index++; 10034 if (pool_index >= n_valid) 10035 pool_index = 0; 10036 set_alarm: 10037 cmng->pool_index = pool_index; 10038 mlx5_set_query_alarm(sh); 10039 } 10040 10041 /** 10042 * Check and callback event for new aged flow in the counter pool 10043 * 10044 * @param[in] sh 10045 * Pointer to mlx5_dev_ctx_shared object. 10046 * @param[in] pool 10047 * Pointer to Current counter pool. 10048 */ 10049 static void 10050 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 10051 struct mlx5_flow_counter_pool *pool) 10052 { 10053 struct mlx5_priv *priv; 10054 struct mlx5_flow_counter *cnt; 10055 struct mlx5_age_info *age_info; 10056 struct mlx5_age_param *age_param; 10057 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 10058 struct mlx5_counter_stats_raw *prev = pool->raw; 10059 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 10060 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 10061 uint16_t expected = AGE_CANDIDATE; 10062 uint32_t i; 10063 10064 pool->time_of_last_age_check = curr_time; 10065 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 10066 cnt = MLX5_POOL_GET_CNT(pool, i); 10067 age_param = MLX5_CNT_TO_AGE(cnt); 10068 if (__atomic_load_n(&age_param->state, 10069 __ATOMIC_RELAXED) != AGE_CANDIDATE) 10070 continue; 10071 if (cur->data[i].hits != prev->data[i].hits) { 10072 __atomic_store_n(&age_param->sec_since_last_hit, 0, 10073 __ATOMIC_RELAXED); 10074 continue; 10075 } 10076 if (__atomic_fetch_add(&age_param->sec_since_last_hit, 10077 time_delta, 10078 __ATOMIC_RELAXED) + time_delta <= age_param->timeout) 10079 continue; 10080 /** 10081 * Hold the lock first, or if between the 10082 * state AGE_TMOUT and tailq operation the 10083 * release happened, the release procedure 10084 * may delete a non-existent tailq node. 10085 */ 10086 priv = rte_eth_devices[age_param->port_id].data->dev_private; 10087 age_info = GET_PORT_AGE_INFO(priv); 10088 rte_spinlock_lock(&age_info->aged_sl); 10089 if (__atomic_compare_exchange_n(&age_param->state, &expected, 10090 AGE_TMOUT, false, 10091 __ATOMIC_RELAXED, 10092 __ATOMIC_RELAXED)) { 10093 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 10094 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 10095 } 10096 rte_spinlock_unlock(&age_info->aged_sl); 10097 } 10098 mlx5_age_event_prepare(sh); 10099 } 10100 10101 /** 10102 * Handler for the HW respond about ready values from an asynchronous batch 10103 * query. This function is probably called by the host thread. 10104 * 10105 * @param[in] sh 10106 * The pointer to the shared device context. 10107 * @param[in] async_id 10108 * The Devx async ID. 10109 * @param[in] status 10110 * The status of the completion. 10111 */ 10112 void 10113 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 10114 uint64_t async_id, int status) 10115 { 10116 struct mlx5_flow_counter_pool *pool = 10117 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 10118 struct mlx5_counter_stats_raw *raw_to_free; 10119 uint8_t query_gen = pool->query_gen ^ 1; 10120 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 10121 enum mlx5_counter_type cnt_type = 10122 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 10123 MLX5_COUNTER_TYPE_ORIGIN; 10124 10125 if (unlikely(status)) { 10126 raw_to_free = pool->raw_hw; 10127 } else { 10128 raw_to_free = pool->raw; 10129 if (pool->is_aged) 10130 mlx5_flow_aging_check(sh, pool); 10131 rte_spinlock_lock(&pool->sl); 10132 pool->raw = pool->raw_hw; 10133 rte_spinlock_unlock(&pool->sl); 10134 /* Be sure the new raw counters data is updated in memory. */ 10135 rte_io_wmb(); 10136 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 10137 rte_spinlock_lock(&cmng->csl[cnt_type]); 10138 TAILQ_CONCAT(&cmng->counters[cnt_type], 10139 &pool->counters[query_gen], next); 10140 rte_spinlock_unlock(&cmng->csl[cnt_type]); 10141 } 10142 } 10143 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 10144 pool->raw_hw = NULL; 10145 sh->sws_cmng.pending_queries--; 10146 } 10147 10148 static int 10149 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 10150 const struct flow_grp_info *grp_info, 10151 struct rte_flow_error *error) 10152 { 10153 if (grp_info->transfer && grp_info->external && 10154 grp_info->fdb_def_rule) { 10155 if (group == UINT32_MAX) 10156 return rte_flow_error_set 10157 (error, EINVAL, 10158 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10159 NULL, 10160 "group index not supported"); 10161 *table = group + 1; 10162 } else { 10163 *table = group; 10164 } 10165 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 10166 return 0; 10167 } 10168 10169 /** 10170 * Translate the rte_flow group index to HW table value. 10171 * 10172 * If tunnel offload is disabled, all group ids converted to flow table 10173 * id using the standard method. 10174 * If tunnel offload is enabled, group id can be converted using the 10175 * standard or tunnel conversion method. Group conversion method 10176 * selection depends on flags in `grp_info` parameter: 10177 * - Internal (grp_info.external == 0) groups conversion uses the 10178 * standard method. 10179 * - Group ids in JUMP action converted with the tunnel conversion. 10180 * - Group id in rule attribute conversion depends on a rule type and 10181 * group id value: 10182 * ** non zero group attributes converted with the tunnel method 10183 * ** zero group attribute in non-tunnel rule is converted using the 10184 * standard method - there's only one root table 10185 * ** zero group attribute in steer tunnel rule is converted with the 10186 * standard method - single root table 10187 * ** zero group attribute in match tunnel rule is a special OvS 10188 * case: that value is used for portability reasons. That group 10189 * id is converted with the tunnel conversion method. 10190 * 10191 * @param[in] dev 10192 * Port device 10193 * @param[in] tunnel 10194 * PMD tunnel offload object 10195 * @param[in] group 10196 * rte_flow group index value. 10197 * @param[out] table 10198 * HW table value. 10199 * @param[in] grp_info 10200 * flags used for conversion 10201 * @param[out] error 10202 * Pointer to error structure. 10203 * 10204 * @return 10205 * 0 on success, a negative errno value otherwise and rte_errno is set. 10206 */ 10207 int 10208 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 10209 const struct mlx5_flow_tunnel *tunnel, 10210 uint32_t group, uint32_t *table, 10211 const struct flow_grp_info *grp_info, 10212 struct rte_flow_error *error) 10213 { 10214 int ret; 10215 bool standard_translation; 10216 10217 if (!grp_info->skip_scale && grp_info->external && 10218 group < MLX5_MAX_TABLES_EXTERNAL) 10219 group *= MLX5_FLOW_TABLE_FACTOR; 10220 if (is_tunnel_offload_active(dev)) { 10221 standard_translation = !grp_info->external || 10222 grp_info->std_tbl_fix; 10223 } else { 10224 standard_translation = true; 10225 } 10226 DRV_LOG(DEBUG, 10227 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 10228 dev->data->port_id, group, grp_info->transfer, 10229 grp_info->external, grp_info->fdb_def_rule, 10230 standard_translation ? "STANDARD" : "TUNNEL"); 10231 if (standard_translation) 10232 ret = flow_group_to_table(dev->data->port_id, group, table, 10233 grp_info, error); 10234 else 10235 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 10236 table, error); 10237 10238 return ret; 10239 } 10240 10241 /** 10242 * Discover availability of metadata reg_c's. 10243 * 10244 * Iteratively use test flows to check availability. 10245 * 10246 * @param[in] dev 10247 * Pointer to the Ethernet device structure. 10248 * 10249 * @return 10250 * 0 on success, a negative errno value otherwise and rte_errno is set. 10251 */ 10252 int 10253 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 10254 { 10255 struct mlx5_priv *priv = dev->data->dev_private; 10256 enum modify_reg idx; 10257 int n = 0; 10258 10259 /* reg_c[0] and reg_c[1] are reserved. */ 10260 priv->sh->flow_mreg_c[n++] = REG_C_0; 10261 priv->sh->flow_mreg_c[n++] = REG_C_1; 10262 /* Discover availability of other reg_c's. */ 10263 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 10264 struct rte_flow_attr attr = { 10265 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 10266 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 10267 .ingress = 1, 10268 }; 10269 struct rte_flow_item items[] = { 10270 [0] = { 10271 .type = RTE_FLOW_ITEM_TYPE_END, 10272 }, 10273 }; 10274 struct rte_flow_action actions[] = { 10275 [0] = { 10276 .type = (enum rte_flow_action_type) 10277 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 10278 .conf = &(struct mlx5_flow_action_copy_mreg){ 10279 .src = REG_C_1, 10280 .dst = idx, 10281 }, 10282 }, 10283 [1] = { 10284 .type = RTE_FLOW_ACTION_TYPE_JUMP, 10285 .conf = &(struct rte_flow_action_jump){ 10286 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 10287 }, 10288 }, 10289 [2] = { 10290 .type = RTE_FLOW_ACTION_TYPE_END, 10291 }, 10292 }; 10293 uint32_t flow_idx; 10294 struct rte_flow *flow; 10295 struct rte_flow_error error; 10296 10297 if (!priv->sh->config.dv_flow_en) 10298 break; 10299 /* Create internal flow, validation skips copy action. */ 10300 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 10301 items, actions, false, &error); 10302 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10303 flow_idx); 10304 if (!flow) 10305 continue; 10306 priv->sh->flow_mreg_c[n++] = idx; 10307 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 10308 } 10309 for (; n < MLX5_MREG_C_NUM; ++n) 10310 priv->sh->flow_mreg_c[n] = REG_NON; 10311 priv->sh->metadata_regc_check_flag = 1; 10312 return 0; 10313 } 10314 10315 int 10316 save_dump_file(const uint8_t *data, uint32_t size, 10317 uint32_t type, uint64_t id, void *arg, FILE *file) 10318 { 10319 char line[BUF_SIZE]; 10320 uint32_t out = 0; 10321 uint32_t k; 10322 uint32_t actions_num; 10323 struct rte_flow_query_count *count; 10324 10325 memset(line, 0, BUF_SIZE); 10326 switch (type) { 10327 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 10328 actions_num = *(uint32_t *)(arg); 10329 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 10330 type, id, actions_num); 10331 break; 10332 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 10333 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 10334 type, id); 10335 break; 10336 case DR_DUMP_REC_TYPE_PMD_COUNTER: 10337 count = (struct rte_flow_query_count *)arg; 10338 fprintf(file, 10339 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 10340 type, id, count->hits, count->bytes); 10341 return 0; 10342 default: 10343 return -1; 10344 } 10345 10346 for (k = 0; k < size; k++) { 10347 /* Make sure we do not overrun the line buffer length. */ 10348 if (out >= BUF_SIZE - 4) { 10349 line[out] = '\0'; 10350 break; 10351 } 10352 out += snprintf(line + out, BUF_SIZE - out, "%02x", 10353 (data[k]) & 0xff); 10354 } 10355 fprintf(file, "%s\n", line); 10356 return 0; 10357 } 10358 10359 int 10360 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 10361 struct rte_flow_query_count *count, struct rte_flow_error *error) 10362 { 10363 struct rte_flow_action action[2]; 10364 enum mlx5_flow_drv_type ftype; 10365 const struct mlx5_flow_driver_ops *fops; 10366 10367 if (!flow) { 10368 return rte_flow_error_set(error, ENOENT, 10369 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10370 NULL, 10371 "invalid flow handle"); 10372 } 10373 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 10374 action[1].type = RTE_FLOW_ACTION_TYPE_END; 10375 if (flow->counter) { 10376 memset(count, 0, sizeof(struct rte_flow_query_count)); 10377 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 10378 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 10379 ftype < MLX5_FLOW_TYPE_MAX); 10380 fops = flow_get_drv_ops(ftype); 10381 return fops->query(dev, flow, action, count, error); 10382 } 10383 return -1; 10384 } 10385 10386 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10387 /** 10388 * Dump flow ipool data to file 10389 * 10390 * @param[in] dev 10391 * The pointer to Ethernet device. 10392 * @param[in] file 10393 * A pointer to a file for output. 10394 * @param[out] error 10395 * Perform verbose error reporting if not NULL. PMDs initialize this 10396 * structure in case of error only. 10397 * @return 10398 * 0 on success, a negative value otherwise. 10399 */ 10400 int 10401 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 10402 struct rte_flow *flow, FILE *file, 10403 struct rte_flow_error *error) 10404 { 10405 struct mlx5_priv *priv = dev->data->dev_private; 10406 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10407 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10408 uint32_t handle_idx; 10409 struct mlx5_flow_handle *dh; 10410 struct rte_flow_query_count count; 10411 uint32_t actions_num; 10412 const uint8_t *data; 10413 size_t size; 10414 uint64_t id; 10415 uint32_t type; 10416 void *action = NULL; 10417 10418 if (!flow) { 10419 return rte_flow_error_set(error, ENOENT, 10420 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10421 NULL, 10422 "invalid flow handle"); 10423 } 10424 handle_idx = flow->dev_handles; 10425 /* query counter */ 10426 if (flow->counter && 10427 (!mlx5_counter_query(dev, flow->counter, false, 10428 &count.hits, &count.bytes, &action)) && action) { 10429 id = (uint64_t)(uintptr_t)action; 10430 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10431 save_dump_file(NULL, 0, type, 10432 id, (void *)&count, file); 10433 } 10434 10435 while (handle_idx) { 10436 dh = mlx5_ipool_get(priv->sh->ipool 10437 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 10438 if (!dh) 10439 continue; 10440 handle_idx = dh->next.next; 10441 10442 /* Get modify_hdr and encap_decap buf from ipools. */ 10443 encap_decap = NULL; 10444 modify_hdr = dh->dvh.modify_hdr; 10445 10446 if (dh->dvh.rix_encap_decap) { 10447 encap_decap = mlx5_ipool_get(priv->sh->ipool 10448 [MLX5_IPOOL_DECAP_ENCAP], 10449 dh->dvh.rix_encap_decap); 10450 } 10451 if (modify_hdr) { 10452 data = (const uint8_t *)modify_hdr->actions; 10453 size = (size_t)(modify_hdr->actions_num) * 8; 10454 id = (uint64_t)(uintptr_t)modify_hdr->action; 10455 actions_num = modify_hdr->actions_num; 10456 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10457 save_dump_file(data, size, type, id, 10458 (void *)(&actions_num), file); 10459 } 10460 if (encap_decap) { 10461 data = encap_decap->buf; 10462 size = encap_decap->size; 10463 id = (uint64_t)(uintptr_t)encap_decap->action; 10464 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10465 save_dump_file(data, size, type, 10466 id, NULL, file); 10467 } 10468 } 10469 return 0; 10470 } 10471 10472 /** 10473 * Dump all flow's encap_decap/modify_hdr/counter data to file 10474 * 10475 * @param[in] dev 10476 * The pointer to Ethernet device. 10477 * @param[in] file 10478 * A pointer to a file for output. 10479 * @param[out] error 10480 * Perform verbose error reporting if not NULL. PMDs initialize this 10481 * structure in case of error only. 10482 * @return 10483 * 0 on success, a negative value otherwise. 10484 */ 10485 static int 10486 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 10487 FILE *file, struct rte_flow_error *error __rte_unused) 10488 { 10489 struct mlx5_priv *priv = dev->data->dev_private; 10490 struct mlx5_dev_ctx_shared *sh = priv->sh; 10491 struct mlx5_hlist *h; 10492 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10493 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10494 struct rte_flow_query_count count; 10495 uint32_t actions_num; 10496 const uint8_t *data; 10497 size_t size; 10498 uint64_t id; 10499 uint32_t type; 10500 uint32_t i; 10501 uint32_t j; 10502 struct mlx5_list_inconst *l_inconst; 10503 struct mlx5_list_entry *e; 10504 int lcore_index; 10505 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 10506 uint32_t max; 10507 void *action; 10508 10509 /* encap_decap hlist is lcore_share, get global core cache. */ 10510 i = MLX5_LIST_GLOBAL; 10511 h = sh->encaps_decaps; 10512 if (h) { 10513 for (j = 0; j <= h->mask; j++) { 10514 l_inconst = &h->buckets[j].l; 10515 if (!l_inconst || !l_inconst->cache[i]) 10516 continue; 10517 10518 e = LIST_FIRST(&l_inconst->cache[i]->h); 10519 while (e) { 10520 encap_decap = 10521 (struct mlx5_flow_dv_encap_decap_resource *)e; 10522 data = encap_decap->buf; 10523 size = encap_decap->size; 10524 id = (uint64_t)(uintptr_t)encap_decap->action; 10525 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10526 save_dump_file(data, size, type, 10527 id, NULL, file); 10528 e = LIST_NEXT(e, next); 10529 } 10530 } 10531 } 10532 10533 /* get modify_hdr */ 10534 h = sh->modify_cmds; 10535 if (h) { 10536 lcore_index = rte_lcore_index(rte_lcore_id()); 10537 if (unlikely(lcore_index == -1)) { 10538 lcore_index = MLX5_LIST_NLCORE; 10539 rte_spinlock_lock(&h->l_const.lcore_lock); 10540 } 10541 i = lcore_index; 10542 10543 if (lcore_index == MLX5_LIST_NLCORE) { 10544 for (i = 0; i <= (uint32_t)lcore_index; i++) { 10545 for (j = 0; j <= h->mask; j++) { 10546 l_inconst = &h->buckets[j].l; 10547 if (!l_inconst || !l_inconst->cache[i]) 10548 continue; 10549 10550 e = LIST_FIRST(&l_inconst->cache[i]->h); 10551 while (e) { 10552 modify_hdr = 10553 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10554 data = (const uint8_t *)modify_hdr->actions; 10555 size = (size_t)(modify_hdr->actions_num) * 8; 10556 actions_num = modify_hdr->actions_num; 10557 id = (uint64_t)(uintptr_t)modify_hdr->action; 10558 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10559 save_dump_file(data, size, type, id, 10560 (void *)(&actions_num), file); 10561 e = LIST_NEXT(e, next); 10562 } 10563 } 10564 } 10565 } else { 10566 for (j = 0; j <= h->mask; j++) { 10567 l_inconst = &h->buckets[j].l; 10568 if (!l_inconst || !l_inconst->cache[i]) 10569 continue; 10570 10571 e = LIST_FIRST(&l_inconst->cache[i]->h); 10572 while (e) { 10573 modify_hdr = 10574 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10575 data = (const uint8_t *)modify_hdr->actions; 10576 size = (size_t)(modify_hdr->actions_num) * 8; 10577 actions_num = modify_hdr->actions_num; 10578 id = (uint64_t)(uintptr_t)modify_hdr->action; 10579 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10580 save_dump_file(data, size, type, id, 10581 (void *)(&actions_num), file); 10582 e = LIST_NEXT(e, next); 10583 } 10584 } 10585 } 10586 10587 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10588 rte_spinlock_unlock(&h->l_const.lcore_lock); 10589 } 10590 10591 /* get counter */ 10592 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10593 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10594 for (j = 1; j <= max; j++) { 10595 action = NULL; 10596 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10597 &count.bytes, &action)) && action) { 10598 id = (uint64_t)(uintptr_t)action; 10599 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10600 save_dump_file(NULL, 0, type, 10601 id, (void *)&count, file); 10602 } 10603 } 10604 return 0; 10605 } 10606 #endif 10607 10608 /** 10609 * Dump flow raw hw data to file 10610 * 10611 * @param[in] dev 10612 * The pointer to Ethernet device. 10613 * @param[in] file 10614 * A pointer to a file for output. 10615 * @param[out] error 10616 * Perform verbose error reporting if not NULL. PMDs initialize this 10617 * structure in case of error only. 10618 * @return 10619 * 0 on success, a negative value otherwise. 10620 */ 10621 int 10622 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10623 FILE *file, 10624 struct rte_flow_error *error __rte_unused) 10625 { 10626 struct mlx5_priv *priv = dev->data->dev_private; 10627 struct mlx5_dev_ctx_shared *sh = priv->sh; 10628 uint32_t handle_idx; 10629 int ret; 10630 struct mlx5_flow_handle *dh; 10631 struct rte_flow *flow; 10632 10633 if (!sh->config.dv_flow_en) { 10634 if (fputs("device dv flow disabled\n", file) <= 0) 10635 return -errno; 10636 return -ENOTSUP; 10637 } 10638 10639 /* dump all */ 10640 if (!flow_idx) { 10641 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10642 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10643 return -EINVAL; 10644 10645 if (sh->config.dv_flow_en == 2) 10646 return mlx5dr_debug_dump(priv->dr_ctx, file); 10647 #endif 10648 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10649 sh->rx_domain, 10650 sh->tx_domain, file); 10651 } 10652 /* dump one */ 10653 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10654 (uintptr_t)(void *)flow_idx); 10655 if (!flow) 10656 return -EINVAL; 10657 10658 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10659 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10660 #endif 10661 handle_idx = flow->dev_handles; 10662 while (handle_idx) { 10663 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10664 handle_idx); 10665 if (!dh) 10666 return -ENOENT; 10667 if (dh->drv_flow) { 10668 if (sh->config.dv_flow_en == 2) 10669 return -ENOTSUP; 10670 10671 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10672 file); 10673 if (ret) 10674 return -ENOENT; 10675 } 10676 handle_idx = dh->next.next; 10677 } 10678 return 0; 10679 } 10680 10681 /** 10682 * Get aged-out flows. 10683 * 10684 * @param[in] dev 10685 * Pointer to the Ethernet device structure. 10686 * @param[in] context 10687 * The address of an array of pointers to the aged-out flows contexts. 10688 * @param[in] nb_countexts 10689 * The length of context array pointers. 10690 * @param[out] error 10691 * Perform verbose error reporting if not NULL. Initialized in case of 10692 * error only. 10693 * 10694 * @return 10695 * how many contexts get in success, otherwise negative errno value. 10696 * if nb_contexts is 0, return the amount of all aged contexts. 10697 * if nb_contexts is not 0 , return the amount of aged flows reported 10698 * in the context array. 10699 */ 10700 int 10701 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10702 uint32_t nb_contexts, struct rte_flow_error *error) 10703 { 10704 struct rte_flow_attr attr = { .transfer = 0 }; 10705 10706 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10707 (dev, contexts, nb_contexts, error); 10708 } 10709 10710 /** 10711 * Get aged-out flows per HWS queue. 10712 * 10713 * @param[in] dev 10714 * Pointer to the Ethernet device structure. 10715 * @param[in] queue_id 10716 * Flow queue to query. 10717 * @param[in] context 10718 * The address of an array of pointers to the aged-out flows contexts. 10719 * @param[in] nb_countexts 10720 * The length of context array pointers. 10721 * @param[out] error 10722 * Perform verbose error reporting if not NULL. Initialized in case of 10723 * error only. 10724 * 10725 * @return 10726 * how many contexts get in success, otherwise negative errno value. 10727 * if nb_contexts is 0, return the amount of all aged contexts. 10728 * if nb_contexts is not 0 , return the amount of aged flows reported 10729 * in the context array. 10730 */ 10731 int 10732 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10733 void **contexts, uint32_t nb_contexts, 10734 struct rte_flow_error *error) 10735 { 10736 const struct mlx5_flow_driver_ops *fops; 10737 struct rte_flow_attr attr = { 0 }; 10738 10739 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10740 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10741 return fops->get_q_aged_flows(dev, queue_id, contexts, 10742 nb_contexts, error); 10743 } 10744 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10745 dev->data->port_id, queue_id); 10746 return rte_flow_error_set(error, ENOTSUP, 10747 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10748 "get Q aged flows with incorrect steering mode"); 10749 } 10750 10751 /* Wrapper for driver action_validate op callback */ 10752 static int 10753 flow_drv_action_validate(struct rte_eth_dev *dev, 10754 const struct rte_flow_indir_action_conf *conf, 10755 const struct rte_flow_action *action, 10756 const struct mlx5_flow_driver_ops *fops, 10757 struct rte_flow_error *error) 10758 { 10759 static const char err_msg[] = "indirect action validation unsupported"; 10760 10761 if (!fops->action_validate) { 10762 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10763 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10764 NULL, err_msg); 10765 return -rte_errno; 10766 } 10767 return fops->action_validate(dev, conf, action, error); 10768 } 10769 10770 /** 10771 * Destroys the shared action by handle. 10772 * 10773 * @param dev 10774 * Pointer to Ethernet device structure. 10775 * @param[in] handle 10776 * Handle for the indirect action object to be destroyed. 10777 * @param[out] error 10778 * Perform verbose error reporting if not NULL. PMDs initialize this 10779 * structure in case of error only. 10780 * 10781 * @return 10782 * 0 on success, a negative errno value otherwise and rte_errno is set. 10783 * 10784 * @note: wrapper for driver action_create op callback. 10785 */ 10786 static int 10787 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10788 struct rte_flow_action_handle *handle, 10789 struct rte_flow_error *error) 10790 { 10791 static const char err_msg[] = "indirect action destruction unsupported"; 10792 struct rte_flow_attr attr = { .transfer = 0 }; 10793 const struct mlx5_flow_driver_ops *fops = 10794 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10795 10796 if (!fops->action_destroy) { 10797 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10798 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10799 NULL, err_msg); 10800 return -rte_errno; 10801 } 10802 return fops->action_destroy(dev, handle, error); 10803 } 10804 10805 /* Wrapper for driver action_destroy op callback */ 10806 static int 10807 flow_drv_action_update(struct rte_eth_dev *dev, 10808 struct rte_flow_action_handle *handle, 10809 const void *update, 10810 const struct mlx5_flow_driver_ops *fops, 10811 struct rte_flow_error *error) 10812 { 10813 static const char err_msg[] = "indirect action update unsupported"; 10814 10815 if (!fops->action_update) { 10816 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10817 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10818 NULL, err_msg); 10819 return -rte_errno; 10820 } 10821 return fops->action_update(dev, handle, update, error); 10822 } 10823 10824 /* Wrapper for driver action_destroy op callback */ 10825 static int 10826 flow_drv_action_query(struct rte_eth_dev *dev, 10827 const struct rte_flow_action_handle *handle, 10828 void *data, 10829 const struct mlx5_flow_driver_ops *fops, 10830 struct rte_flow_error *error) 10831 { 10832 static const char err_msg[] = "indirect action query unsupported"; 10833 10834 if (!fops->action_query) { 10835 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10836 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10837 NULL, err_msg); 10838 return -rte_errno; 10839 } 10840 return fops->action_query(dev, handle, data, error); 10841 } 10842 10843 /** 10844 * Create indirect action for reuse in multiple flow rules. 10845 * 10846 * @param dev 10847 * Pointer to Ethernet device structure. 10848 * @param conf 10849 * Pointer to indirect action object configuration. 10850 * @param[in] action 10851 * Action configuration for indirect action object creation. 10852 * @param[out] error 10853 * Perform verbose error reporting if not NULL. PMDs initialize this 10854 * structure in case of error only. 10855 * @return 10856 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10857 */ 10858 static struct rte_flow_action_handle * 10859 mlx5_action_handle_create(struct rte_eth_dev *dev, 10860 const struct rte_flow_indir_action_conf *conf, 10861 const struct rte_flow_action *action, 10862 struct rte_flow_error *error) 10863 { 10864 static const char err_msg[] = "indirect action creation unsupported"; 10865 struct rte_flow_attr attr = { .transfer = 0 }; 10866 const struct mlx5_flow_driver_ops *fops = 10867 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10868 10869 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10870 return NULL; 10871 if (!fops->action_create) { 10872 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10873 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10874 NULL, err_msg); 10875 return NULL; 10876 } 10877 return fops->action_create(dev, conf, action, error); 10878 } 10879 10880 /** 10881 * Updates inplace the indirect action configuration pointed by *handle* 10882 * with the configuration provided as *update* argument. 10883 * The update of the indirect action configuration effects all flow rules 10884 * reusing the action via handle. 10885 * 10886 * @param dev 10887 * Pointer to Ethernet device structure. 10888 * @param[in] handle 10889 * Handle for the indirect action to be updated. 10890 * @param[in] update 10891 * Action specification used to modify the action pointed by handle. 10892 * *update* could be of same type with the action pointed by the *handle* 10893 * handle argument, or some other structures like a wrapper, depending on 10894 * the indirect action type. 10895 * @param[out] error 10896 * Perform verbose error reporting if not NULL. PMDs initialize this 10897 * structure in case of error only. 10898 * 10899 * @return 10900 * 0 on success, a negative errno value otherwise and rte_errno is set. 10901 */ 10902 static int 10903 mlx5_action_handle_update(struct rte_eth_dev *dev, 10904 struct rte_flow_action_handle *handle, 10905 const void *update, 10906 struct rte_flow_error *error) 10907 { 10908 struct rte_flow_attr attr = { .transfer = 0 }; 10909 const struct mlx5_flow_driver_ops *fops = 10910 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10911 int ret; 10912 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 10913 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 10914 10915 switch (type) { 10916 case MLX5_INDIRECT_ACTION_TYPE_CT: 10917 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 10918 ret = 0; 10919 break; 10920 default: 10921 ret = flow_drv_action_validate(dev, NULL, 10922 (const struct rte_flow_action *)update, 10923 fops, error); 10924 } 10925 if (ret) 10926 return ret; 10927 return flow_drv_action_update(dev, handle, update, fops, 10928 error); 10929 } 10930 10931 /** 10932 * Query the indirect action by handle. 10933 * 10934 * This function allows retrieving action-specific data such as counters. 10935 * Data is gathered by special action which may be present/referenced in 10936 * more than one flow rule definition. 10937 * 10938 * see @RTE_FLOW_ACTION_TYPE_COUNT 10939 * 10940 * @param dev 10941 * Pointer to Ethernet device structure. 10942 * @param[in] handle 10943 * Handle for the indirect action to query. 10944 * @param[in, out] data 10945 * Pointer to storage for the associated query data type. 10946 * @param[out] error 10947 * Perform verbose error reporting if not NULL. PMDs initialize this 10948 * structure in case of error only. 10949 * 10950 * @return 10951 * 0 on success, a negative errno value otherwise and rte_errno is set. 10952 */ 10953 static int 10954 mlx5_action_handle_query(struct rte_eth_dev *dev, 10955 const struct rte_flow_action_handle *handle, 10956 void *data, 10957 struct rte_flow_error *error) 10958 { 10959 struct rte_flow_attr attr = { .transfer = 0 }; 10960 const struct mlx5_flow_driver_ops *fops = 10961 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10962 10963 return flow_drv_action_query(dev, handle, data, fops, error); 10964 } 10965 10966 static int 10967 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10968 struct rte_flow_action_handle *handle, 10969 const void *update, void *query, 10970 enum rte_flow_query_update_mode qu_mode, 10971 struct rte_flow_error *error) 10972 { 10973 struct rte_flow_attr attr = { .transfer = 0 }; 10974 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10975 const struct mlx5_flow_driver_ops *fops; 10976 10977 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10978 return rte_flow_error_set(error, ENOTSUP, 10979 RTE_FLOW_ERROR_TYPE_ACTION, 10980 NULL, "invalid driver type"); 10981 fops = flow_get_drv_ops(drv_type); 10982 if (!fops || !fops->action_query_update) 10983 return rte_flow_error_set(error, ENOTSUP, 10984 RTE_FLOW_ERROR_TYPE_ACTION, 10985 NULL, "no query_update handler"); 10986 return fops->action_query_update(dev, handle, update, 10987 query, qu_mode, error); 10988 } 10989 10990 10991 #define MLX5_DRV_FOPS_OR_ERR(dev, fops, drv_cb, ret) \ 10992 { \ 10993 struct rte_flow_attr attr = { .transfer = 0 }; \ 10994 enum mlx5_flow_drv_type drv_type = flow_get_drv_type((dev), &attr); \ 10995 if (drv_type == MLX5_FLOW_TYPE_MIN || \ 10996 drv_type == MLX5_FLOW_TYPE_MAX) { \ 10997 rte_flow_error_set(error, ENOTSUP, \ 10998 RTE_FLOW_ERROR_TYPE_ACTION, \ 10999 NULL, "invalid driver type"); \ 11000 return ret; \ 11001 } \ 11002 (fops) = flow_get_drv_ops(drv_type); \ 11003 if (!(fops) || !(fops)->drv_cb) { \ 11004 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, \ 11005 NULL, "no action_list handler"); \ 11006 return ret; \ 11007 } \ 11008 } 11009 11010 static struct rte_flow_action_list_handle * 11011 mlx5_action_list_handle_create(struct rte_eth_dev *dev, 11012 const struct rte_flow_indir_action_conf *conf, 11013 const struct rte_flow_action *actions, 11014 struct rte_flow_error *error) 11015 { 11016 const struct mlx5_flow_driver_ops *fops; 11017 11018 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_create, NULL); 11019 return fops->action_list_handle_create(dev, conf, actions, error); 11020 } 11021 11022 static int 11023 mlx5_action_list_handle_destroy(struct rte_eth_dev *dev, 11024 struct rte_flow_action_list_handle *handle, 11025 struct rte_flow_error *error) 11026 { 11027 const struct mlx5_flow_driver_ops *fops; 11028 11029 MLX5_DRV_FOPS_OR_ERR(dev, fops, action_list_handle_destroy, ENOTSUP); 11030 return fops->action_list_handle_destroy(dev, handle, error); 11031 } 11032 11033 static struct rte_flow_action_list_handle * 11034 mlx5_flow_async_action_list_handle_create(struct rte_eth_dev *dev, 11035 uint32_t queue_id, 11036 const struct 11037 rte_flow_op_attr *op_attr, 11038 const struct 11039 rte_flow_indir_action_conf *conf, 11040 const struct rte_flow_action *actions, 11041 void *user_data, 11042 struct rte_flow_error *error) 11043 { 11044 const struct mlx5_flow_driver_ops *fops; 11045 11046 MLX5_DRV_FOPS_OR_ERR(dev, fops, async_action_list_handle_create, NULL); 11047 return fops->async_action_list_handle_create(dev, queue_id, op_attr, 11048 conf, actions, user_data, 11049 error); 11050 } 11051 11052 static int 11053 mlx5_flow_async_action_list_handle_destroy 11054 (struct rte_eth_dev *dev, uint32_t queue_id, 11055 const struct rte_flow_op_attr *op_attr, 11056 struct rte_flow_action_list_handle *action_handle, 11057 void *user_data, struct rte_flow_error *error) 11058 { 11059 const struct mlx5_flow_driver_ops *fops; 11060 11061 MLX5_DRV_FOPS_OR_ERR(dev, fops, 11062 async_action_list_handle_destroy, ENOTSUP); 11063 return fops->async_action_list_handle_destroy(dev, queue_id, op_attr, 11064 action_handle, user_data, 11065 error); 11066 } 11067 11068 static int 11069 mlx5_flow_action_list_handle_query_update(struct rte_eth_dev *dev, 11070 const 11071 struct rte_flow_action_list_handle *handle, 11072 const void **update, void **query, 11073 enum rte_flow_query_update_mode mode, 11074 struct rte_flow_error *error) 11075 { 11076 const struct mlx5_flow_driver_ops *fops; 11077 11078 MLX5_DRV_FOPS_OR_ERR(dev, fops, 11079 action_list_handle_query_update, ENOTSUP); 11080 return fops->action_list_handle_query_update(dev, handle, update, query, 11081 mode, error); 11082 } 11083 11084 static int 11085 mlx5_flow_async_action_list_handle_query_update(struct rte_eth_dev *dev, 11086 uint32_t queue_id, 11087 const 11088 struct rte_flow_op_attr *op_attr, 11089 const struct 11090 rte_flow_action_list_handle *handle, 11091 const void **update, 11092 void **query, 11093 enum 11094 rte_flow_query_update_mode mode, 11095 void *user_data, 11096 struct rte_flow_error *error) 11097 { 11098 const struct mlx5_flow_driver_ops *fops; 11099 11100 MLX5_DRV_FOPS_OR_ERR(dev, fops, 11101 async_action_list_handle_query_update, ENOTSUP); 11102 return fops->async_action_list_handle_query_update(dev, queue_id, op_attr, 11103 handle, update, 11104 query, mode, 11105 user_data, error); 11106 } 11107 11108 11109 /** 11110 * Destroy all indirect actions (shared RSS). 11111 * 11112 * @param dev 11113 * Pointer to Ethernet device. 11114 * 11115 * @return 11116 * 0 on success, a negative errno value otherwise and rte_errno is set. 11117 */ 11118 int 11119 mlx5_action_handle_flush(struct rte_eth_dev *dev) 11120 { 11121 struct rte_flow_error error; 11122 struct mlx5_priv *priv = dev->data->dev_private; 11123 struct mlx5_shared_action_rss *shared_rss; 11124 int ret = 0; 11125 uint32_t idx; 11126 11127 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 11128 priv->rss_shared_actions, idx, shared_rss, next) { 11129 ret |= mlx5_action_handle_destroy(dev, 11130 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 11131 } 11132 return ret; 11133 } 11134 11135 /** 11136 * Validate existing indirect actions against current device configuration 11137 * and attach them to device resources. 11138 * 11139 * @param dev 11140 * Pointer to Ethernet device. 11141 * 11142 * @return 11143 * 0 on success, a negative errno value otherwise and rte_errno is set. 11144 */ 11145 int 11146 mlx5_action_handle_attach(struct rte_eth_dev *dev) 11147 { 11148 struct mlx5_priv *priv = dev->data->dev_private; 11149 int ret = 0; 11150 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 11151 11152 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11153 const char *message; 11154 uint32_t queue_idx; 11155 11156 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 11157 ind_tbl->queues_n, 11158 &message, &queue_idx); 11159 if (ret != 0) { 11160 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 11161 dev->data->port_id, ind_tbl->queues[queue_idx], 11162 message); 11163 break; 11164 } 11165 } 11166 if (ret != 0) 11167 return ret; 11168 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11169 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 11170 if (ret != 0) { 11171 DRV_LOG(ERR, "Port %u could not attach " 11172 "indirection table obj %p", 11173 dev->data->port_id, (void *)ind_tbl); 11174 goto error; 11175 } 11176 } 11177 11178 return 0; 11179 error: 11180 ind_tbl_last = ind_tbl; 11181 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11182 if (ind_tbl == ind_tbl_last) 11183 break; 11184 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 11185 DRV_LOG(CRIT, "Port %u could not detach " 11186 "indirection table obj %p on rollback", 11187 dev->data->port_id, (void *)ind_tbl); 11188 } 11189 return ret; 11190 } 11191 11192 /** 11193 * Detach indirect actions of the device from its resources. 11194 * 11195 * @param dev 11196 * Pointer to Ethernet device. 11197 * 11198 * @return 11199 * 0 on success, a negative errno value otherwise and rte_errno is set. 11200 */ 11201 int 11202 mlx5_action_handle_detach(struct rte_eth_dev *dev) 11203 { 11204 struct mlx5_priv *priv = dev->data->dev_private; 11205 int ret = 0; 11206 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 11207 11208 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11209 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 11210 if (ret != 0) { 11211 DRV_LOG(ERR, "Port %u could not detach " 11212 "indirection table obj %p", 11213 dev->data->port_id, (void *)ind_tbl); 11214 goto error; 11215 } 11216 } 11217 return 0; 11218 error: 11219 ind_tbl_last = ind_tbl; 11220 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 11221 if (ind_tbl == ind_tbl_last) 11222 break; 11223 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 11224 DRV_LOG(CRIT, "Port %u could not attach " 11225 "indirection table obj %p on rollback", 11226 dev->data->port_id, (void *)ind_tbl); 11227 } 11228 return ret; 11229 } 11230 11231 #ifndef HAVE_MLX5DV_DR 11232 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 11233 #else 11234 #define MLX5_DOMAIN_SYNC_FLOW \ 11235 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 11236 #endif 11237 11238 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 11239 { 11240 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 11241 const struct mlx5_flow_driver_ops *fops; 11242 int ret; 11243 struct rte_flow_attr attr = { .transfer = 0 }; 11244 11245 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11246 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 11247 if (ret > 0) 11248 ret = -ret; 11249 return ret; 11250 } 11251 11252 const struct mlx5_flow_tunnel * 11253 mlx5_get_tof(const struct rte_flow_item *item, 11254 const struct rte_flow_action *action, 11255 enum mlx5_tof_rule_type *rule_type) 11256 { 11257 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11258 if (item->type == (typeof(item->type)) 11259 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 11260 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 11261 return flow_items_to_tunnel(item); 11262 } 11263 } 11264 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 11265 if (action->type == (typeof(action->type)) 11266 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 11267 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 11268 return flow_actions_to_tunnel(action); 11269 } 11270 } 11271 return NULL; 11272 } 11273 11274 /** 11275 * tunnel offload functionality is defined for DV environment only 11276 */ 11277 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 11278 __extension__ 11279 union tunnel_offload_mark { 11280 uint32_t val; 11281 struct { 11282 uint32_t app_reserve:8; 11283 uint32_t table_id:15; 11284 uint32_t transfer:1; 11285 uint32_t _unused_:8; 11286 }; 11287 }; 11288 11289 static bool 11290 mlx5_access_tunnel_offload_db 11291 (struct rte_eth_dev *dev, 11292 bool (*match)(struct rte_eth_dev *, 11293 struct mlx5_flow_tunnel *, const void *), 11294 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11295 void (*miss)(struct rte_eth_dev *, void *), 11296 void *ctx, bool lock_op); 11297 11298 static int 11299 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 11300 struct rte_flow *flow, 11301 const struct rte_flow_attr *attr, 11302 const struct rte_flow_action *app_actions, 11303 uint32_t flow_idx, 11304 const struct mlx5_flow_tunnel *tunnel, 11305 struct tunnel_default_miss_ctx *ctx, 11306 struct rte_flow_error *error) 11307 { 11308 struct mlx5_priv *priv = dev->data->dev_private; 11309 struct mlx5_flow *dev_flow; 11310 struct rte_flow_attr miss_attr = *attr; 11311 const struct rte_flow_item miss_items[2] = { 11312 { 11313 .type = RTE_FLOW_ITEM_TYPE_ETH, 11314 .spec = NULL, 11315 .last = NULL, 11316 .mask = NULL 11317 }, 11318 { 11319 .type = RTE_FLOW_ITEM_TYPE_END, 11320 .spec = NULL, 11321 .last = NULL, 11322 .mask = NULL 11323 } 11324 }; 11325 union tunnel_offload_mark mark_id; 11326 struct rte_flow_action_mark miss_mark; 11327 struct rte_flow_action miss_actions[3] = { 11328 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 11329 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 11330 }; 11331 const struct rte_flow_action_jump *jump_data; 11332 uint32_t i, flow_table = 0; /* prevent compilation warning */ 11333 struct flow_grp_info grp_info = { 11334 .external = 1, 11335 .transfer = attr->transfer, 11336 .fdb_def_rule = !!priv->fdb_def_rule, 11337 .std_tbl_fix = 0, 11338 }; 11339 int ret; 11340 11341 if (!attr->transfer) { 11342 uint32_t q_size; 11343 11344 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 11345 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 11346 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 11347 0, SOCKET_ID_ANY); 11348 if (!ctx->queue) 11349 return rte_flow_error_set 11350 (error, ENOMEM, 11351 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11352 NULL, "invalid default miss RSS"); 11353 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 11354 ctx->action_rss.level = 0, 11355 ctx->action_rss.types = priv->rss_conf.rss_hf, 11356 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 11357 ctx->action_rss.queue_num = priv->reta_idx_n, 11358 ctx->action_rss.key = priv->rss_conf.rss_key, 11359 ctx->action_rss.queue = ctx->queue; 11360 if (!priv->reta_idx_n || !priv->rxqs_n) 11361 return rte_flow_error_set 11362 (error, EINVAL, 11363 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11364 NULL, "invalid port configuration"); 11365 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 11366 ctx->action_rss.types = 0; 11367 for (i = 0; i != priv->reta_idx_n; ++i) 11368 ctx->queue[i] = (*priv->reta_idx)[i]; 11369 } else { 11370 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 11371 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 11372 } 11373 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 11374 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 11375 jump_data = app_actions->conf; 11376 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 11377 miss_attr.group = jump_data->group; 11378 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 11379 &flow_table, &grp_info, error); 11380 if (ret) 11381 return rte_flow_error_set(error, EINVAL, 11382 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11383 NULL, "invalid tunnel id"); 11384 mark_id.app_reserve = 0; 11385 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 11386 mark_id.transfer = !!attr->transfer; 11387 mark_id._unused_ = 0; 11388 miss_mark.id = mark_id.val; 11389 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 11390 miss_items, miss_actions, flow_idx, error); 11391 if (!dev_flow) 11392 return -rte_errno; 11393 dev_flow->flow = flow; 11394 dev_flow->external = true; 11395 dev_flow->tunnel = tunnel; 11396 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 11397 /* Subflow object was created, we must include one in the list. */ 11398 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 11399 dev_flow->handle, next); 11400 DRV_LOG(DEBUG, 11401 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 11402 dev->data->port_id, tunnel->app_tunnel.type, 11403 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 11404 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 11405 miss_actions, error); 11406 if (!ret) 11407 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 11408 error); 11409 11410 return ret; 11411 } 11412 11413 static const struct mlx5_flow_tbl_data_entry * 11414 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 11415 { 11416 struct mlx5_priv *priv = dev->data->dev_private; 11417 struct mlx5_dev_ctx_shared *sh = priv->sh; 11418 struct mlx5_list_entry *he; 11419 union tunnel_offload_mark mbits = { .val = mark }; 11420 union mlx5_flow_tbl_key table_key = { 11421 { 11422 .level = tunnel_id_to_flow_tbl(mbits.table_id), 11423 .id = 0, 11424 .reserved = 0, 11425 .dummy = 0, 11426 .is_fdb = !!mbits.transfer, 11427 .is_egress = 0, 11428 } 11429 }; 11430 struct mlx5_flow_cb_ctx ctx = { 11431 .data = &table_key.v64, 11432 }; 11433 11434 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 11435 return he ? 11436 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 11437 } 11438 11439 static void 11440 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 11441 struct mlx5_list_entry *entry) 11442 { 11443 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11444 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11445 11446 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11447 tunnel_flow_tbl_to_id(tte->flow_table)); 11448 mlx5_free(tte); 11449 } 11450 11451 static int 11452 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 11453 struct mlx5_list_entry *entry, void *cb_ctx) 11454 { 11455 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11456 union tunnel_tbl_key tbl = { 11457 .val = *(uint64_t *)(ctx->data), 11458 }; 11459 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11460 11461 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 11462 } 11463 11464 static struct mlx5_list_entry * 11465 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 11466 { 11467 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11468 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11469 struct tunnel_tbl_entry *tte; 11470 union tunnel_tbl_key tbl = { 11471 .val = *(uint64_t *)(ctx->data), 11472 }; 11473 11474 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 11475 sizeof(*tte), 0, 11476 SOCKET_ID_ANY); 11477 if (!tte) 11478 goto err; 11479 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11480 &tte->flow_table); 11481 if (tte->flow_table >= MLX5_MAX_TABLES) { 11482 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 11483 tte->flow_table); 11484 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11485 tte->flow_table); 11486 goto err; 11487 } else if (!tte->flow_table) { 11488 goto err; 11489 } 11490 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 11491 tte->tunnel_id = tbl.tunnel_id; 11492 tte->group = tbl.group; 11493 return &tte->hash; 11494 err: 11495 if (tte) 11496 mlx5_free(tte); 11497 return NULL; 11498 } 11499 11500 static struct mlx5_list_entry * 11501 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 11502 struct mlx5_list_entry *oentry, 11503 void *cb_ctx __rte_unused) 11504 { 11505 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 11506 0, SOCKET_ID_ANY); 11507 11508 if (!tte) 11509 return NULL; 11510 memcpy(tte, oentry, sizeof(*tte)); 11511 return &tte->hash; 11512 } 11513 11514 static void 11515 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 11516 struct mlx5_list_entry *entry) 11517 { 11518 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11519 11520 mlx5_free(tte); 11521 } 11522 11523 static uint32_t 11524 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 11525 const struct mlx5_flow_tunnel *tunnel, 11526 uint32_t group, uint32_t *table, 11527 struct rte_flow_error *error) 11528 { 11529 struct mlx5_list_entry *he; 11530 struct tunnel_tbl_entry *tte; 11531 union tunnel_tbl_key key = { 11532 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 11533 .group = group 11534 }; 11535 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11536 struct mlx5_hlist *group_hash; 11537 struct mlx5_flow_cb_ctx ctx = { 11538 .data = &key.val, 11539 }; 11540 11541 group_hash = tunnel ? tunnel->groups : thub->groups; 11542 he = mlx5_hlist_register(group_hash, key.val, &ctx); 11543 if (!he) 11544 return rte_flow_error_set(error, EINVAL, 11545 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 11546 NULL, 11547 "tunnel group index not supported"); 11548 tte = container_of(he, typeof(*tte), hash); 11549 *table = tte->flow_table; 11550 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 11551 dev->data->port_id, key.tunnel_id, group, *table); 11552 return 0; 11553 } 11554 11555 static void 11556 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 11557 struct mlx5_flow_tunnel *tunnel) 11558 { 11559 struct mlx5_priv *priv = dev->data->dev_private; 11560 struct mlx5_indexed_pool *ipool; 11561 11562 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11563 dev->data->port_id, tunnel->tunnel_id); 11564 LIST_REMOVE(tunnel, chain); 11565 mlx5_hlist_destroy(tunnel->groups); 11566 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11567 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11568 } 11569 11570 static bool 11571 mlx5_access_tunnel_offload_db 11572 (struct rte_eth_dev *dev, 11573 bool (*match)(struct rte_eth_dev *, 11574 struct mlx5_flow_tunnel *, const void *), 11575 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11576 void (*miss)(struct rte_eth_dev *, void *), 11577 void *ctx, bool lock_op) 11578 { 11579 bool verdict = false; 11580 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11581 struct mlx5_flow_tunnel *tunnel; 11582 11583 rte_spinlock_lock(&thub->sl); 11584 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11585 verdict = match(dev, tunnel, (const void *)ctx); 11586 if (verdict) 11587 break; 11588 } 11589 if (!lock_op) 11590 rte_spinlock_unlock(&thub->sl); 11591 if (verdict && hit) 11592 hit(dev, tunnel, ctx); 11593 if (!verdict && miss) 11594 miss(dev, ctx); 11595 if (lock_op) 11596 rte_spinlock_unlock(&thub->sl); 11597 11598 return verdict; 11599 } 11600 11601 struct tunnel_db_find_tunnel_id_ctx { 11602 uint32_t tunnel_id; 11603 struct mlx5_flow_tunnel *tunnel; 11604 }; 11605 11606 static bool 11607 find_tunnel_id_match(struct rte_eth_dev *dev, 11608 struct mlx5_flow_tunnel *tunnel, const void *x) 11609 { 11610 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11611 11612 RTE_SET_USED(dev); 11613 return tunnel->tunnel_id == ctx->tunnel_id; 11614 } 11615 11616 static void 11617 find_tunnel_id_hit(struct rte_eth_dev *dev, 11618 struct mlx5_flow_tunnel *tunnel, void *x) 11619 { 11620 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11621 RTE_SET_USED(dev); 11622 ctx->tunnel = tunnel; 11623 } 11624 11625 static struct mlx5_flow_tunnel * 11626 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11627 { 11628 struct tunnel_db_find_tunnel_id_ctx ctx = { 11629 .tunnel_id = id, 11630 }; 11631 11632 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11633 find_tunnel_id_hit, NULL, &ctx, true); 11634 11635 return ctx.tunnel; 11636 } 11637 11638 static struct mlx5_flow_tunnel * 11639 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11640 const struct rte_flow_tunnel *app_tunnel) 11641 { 11642 struct mlx5_priv *priv = dev->data->dev_private; 11643 struct mlx5_indexed_pool *ipool; 11644 struct mlx5_flow_tunnel *tunnel; 11645 uint32_t id; 11646 11647 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11648 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11649 if (!tunnel) 11650 return NULL; 11651 if (id >= MLX5_MAX_TUNNELS) { 11652 mlx5_ipool_free(ipool, id); 11653 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11654 return NULL; 11655 } 11656 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11657 priv->sh, 11658 mlx5_flow_tunnel_grp2tbl_create_cb, 11659 mlx5_flow_tunnel_grp2tbl_match_cb, 11660 mlx5_flow_tunnel_grp2tbl_remove_cb, 11661 mlx5_flow_tunnel_grp2tbl_clone_cb, 11662 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11663 if (!tunnel->groups) { 11664 mlx5_ipool_free(ipool, id); 11665 return NULL; 11666 } 11667 /* initiate new PMD tunnel */ 11668 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11669 tunnel->tunnel_id = id; 11670 tunnel->action.type = (typeof(tunnel->action.type)) 11671 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11672 tunnel->action.conf = tunnel; 11673 tunnel->item.type = (typeof(tunnel->item.type)) 11674 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11675 tunnel->item.spec = tunnel; 11676 tunnel->item.last = NULL; 11677 tunnel->item.mask = NULL; 11678 11679 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11680 dev->data->port_id, tunnel->tunnel_id); 11681 11682 return tunnel; 11683 } 11684 11685 struct tunnel_db_get_tunnel_ctx { 11686 const struct rte_flow_tunnel *app_tunnel; 11687 struct mlx5_flow_tunnel *tunnel; 11688 }; 11689 11690 static bool get_tunnel_match(struct rte_eth_dev *dev, 11691 struct mlx5_flow_tunnel *tunnel, const void *x) 11692 { 11693 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11694 11695 RTE_SET_USED(dev); 11696 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11697 sizeof(*ctx->app_tunnel)); 11698 } 11699 11700 static void get_tunnel_hit(struct rte_eth_dev *dev, 11701 struct mlx5_flow_tunnel *tunnel, void *x) 11702 { 11703 /* called under tunnel spinlock protection */ 11704 struct tunnel_db_get_tunnel_ctx *ctx = x; 11705 11706 RTE_SET_USED(dev); 11707 tunnel->refctn++; 11708 ctx->tunnel = tunnel; 11709 } 11710 11711 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11712 { 11713 /* called under tunnel spinlock protection */ 11714 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11715 struct tunnel_db_get_tunnel_ctx *ctx = x; 11716 11717 rte_spinlock_unlock(&thub->sl); 11718 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11719 rte_spinlock_lock(&thub->sl); 11720 if (ctx->tunnel) { 11721 ctx->tunnel->refctn = 1; 11722 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11723 } 11724 } 11725 11726 11727 static int 11728 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11729 const struct rte_flow_tunnel *app_tunnel, 11730 struct mlx5_flow_tunnel **tunnel) 11731 { 11732 struct tunnel_db_get_tunnel_ctx ctx = { 11733 .app_tunnel = app_tunnel, 11734 }; 11735 11736 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11737 get_tunnel_miss, &ctx, true); 11738 *tunnel = ctx.tunnel; 11739 return ctx.tunnel ? 0 : -ENOMEM; 11740 } 11741 11742 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11743 { 11744 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11745 11746 if (!thub) 11747 return; 11748 if (!LIST_EMPTY(&thub->tunnels)) 11749 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11750 mlx5_hlist_destroy(thub->groups); 11751 mlx5_free(thub); 11752 } 11753 11754 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11755 { 11756 int err; 11757 struct mlx5_flow_tunnel_hub *thub; 11758 11759 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11760 0, SOCKET_ID_ANY); 11761 if (!thub) 11762 return -ENOMEM; 11763 LIST_INIT(&thub->tunnels); 11764 rte_spinlock_init(&thub->sl); 11765 thub->groups = mlx5_hlist_create("flow groups", 64, 11766 false, true, sh, 11767 mlx5_flow_tunnel_grp2tbl_create_cb, 11768 mlx5_flow_tunnel_grp2tbl_match_cb, 11769 mlx5_flow_tunnel_grp2tbl_remove_cb, 11770 mlx5_flow_tunnel_grp2tbl_clone_cb, 11771 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11772 if (!thub->groups) { 11773 err = -rte_errno; 11774 goto err; 11775 } 11776 sh->tunnel_hub = thub; 11777 11778 return 0; 11779 11780 err: 11781 if (thub->groups) 11782 mlx5_hlist_destroy(thub->groups); 11783 if (thub) 11784 mlx5_free(thub); 11785 return err; 11786 } 11787 11788 static inline int 11789 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11790 struct rte_flow_tunnel *tunnel, 11791 struct rte_flow_error *error) 11792 { 11793 struct mlx5_priv *priv = dev->data->dev_private; 11794 11795 if (!priv->sh->config.dv_flow_en) 11796 return rte_flow_error_set(error, ENOTSUP, 11797 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11798 "flow DV interface is off"); 11799 if (!is_tunnel_offload_active(dev)) 11800 return rte_flow_error_set(error, ENOTSUP, 11801 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11802 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11803 if (!tunnel) 11804 return rte_flow_error_set(error, EINVAL, 11805 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11806 "no application tunnel"); 11807 switch (tunnel->type) { 11808 default: 11809 return rte_flow_error_set(error, EINVAL, 11810 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11811 "unsupported tunnel type"); 11812 case RTE_FLOW_ITEM_TYPE_VXLAN: 11813 case RTE_FLOW_ITEM_TYPE_GRE: 11814 case RTE_FLOW_ITEM_TYPE_NVGRE: 11815 case RTE_FLOW_ITEM_TYPE_GENEVE: 11816 break; 11817 } 11818 return 0; 11819 } 11820 11821 static int 11822 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11823 struct rte_flow_tunnel *app_tunnel, 11824 struct rte_flow_action **actions, 11825 uint32_t *num_of_actions, 11826 struct rte_flow_error *error) 11827 { 11828 struct mlx5_flow_tunnel *tunnel; 11829 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11830 11831 if (ret) 11832 return ret; 11833 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11834 if (ret < 0) { 11835 return rte_flow_error_set(error, ret, 11836 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11837 "failed to initialize pmd tunnel"); 11838 } 11839 *actions = &tunnel->action; 11840 *num_of_actions = 1; 11841 return 0; 11842 } 11843 11844 static int 11845 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11846 struct rte_flow_tunnel *app_tunnel, 11847 struct rte_flow_item **items, 11848 uint32_t *num_of_items, 11849 struct rte_flow_error *error) 11850 { 11851 struct mlx5_flow_tunnel *tunnel; 11852 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11853 11854 if (ret) 11855 return ret; 11856 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11857 if (ret < 0) { 11858 return rte_flow_error_set(error, ret, 11859 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11860 "failed to initialize pmd tunnel"); 11861 } 11862 *items = &tunnel->item; 11863 *num_of_items = 1; 11864 return 0; 11865 } 11866 11867 struct tunnel_db_element_release_ctx { 11868 struct rte_flow_item *items; 11869 struct rte_flow_action *actions; 11870 uint32_t num_elements; 11871 struct rte_flow_error *error; 11872 int ret; 11873 }; 11874 11875 static bool 11876 tunnel_element_release_match(struct rte_eth_dev *dev, 11877 struct mlx5_flow_tunnel *tunnel, const void *x) 11878 { 11879 const struct tunnel_db_element_release_ctx *ctx = x; 11880 11881 RTE_SET_USED(dev); 11882 if (ctx->num_elements != 1) 11883 return false; 11884 else if (ctx->items) 11885 return ctx->items == &tunnel->item; 11886 else if (ctx->actions) 11887 return ctx->actions == &tunnel->action; 11888 11889 return false; 11890 } 11891 11892 static void 11893 tunnel_element_release_hit(struct rte_eth_dev *dev, 11894 struct mlx5_flow_tunnel *tunnel, void *x) 11895 { 11896 struct tunnel_db_element_release_ctx *ctx = x; 11897 ctx->ret = 0; 11898 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 11899 mlx5_flow_tunnel_free(dev, tunnel); 11900 } 11901 11902 static void 11903 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11904 { 11905 struct tunnel_db_element_release_ctx *ctx = x; 11906 RTE_SET_USED(dev); 11907 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11908 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11909 "invalid argument"); 11910 } 11911 11912 static int 11913 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11914 struct rte_flow_item *pmd_items, 11915 uint32_t num_items, struct rte_flow_error *err) 11916 { 11917 struct tunnel_db_element_release_ctx ctx = { 11918 .items = pmd_items, 11919 .actions = NULL, 11920 .num_elements = num_items, 11921 .error = err, 11922 }; 11923 11924 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11925 tunnel_element_release_hit, 11926 tunnel_element_release_miss, &ctx, false); 11927 11928 return ctx.ret; 11929 } 11930 11931 static int 11932 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11933 struct rte_flow_action *pmd_actions, 11934 uint32_t num_actions, struct rte_flow_error *err) 11935 { 11936 struct tunnel_db_element_release_ctx ctx = { 11937 .items = NULL, 11938 .actions = pmd_actions, 11939 .num_elements = num_actions, 11940 .error = err, 11941 }; 11942 11943 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11944 tunnel_element_release_hit, 11945 tunnel_element_release_miss, &ctx, false); 11946 11947 return ctx.ret; 11948 } 11949 11950 static int 11951 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11952 struct rte_mbuf *m, 11953 struct rte_flow_restore_info *info, 11954 struct rte_flow_error *err) 11955 { 11956 uint64_t ol_flags = m->ol_flags; 11957 const struct mlx5_flow_tbl_data_entry *tble; 11958 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11959 struct mlx5_priv *priv = dev->data->dev_private; 11960 11961 if (priv->tunnel_enabled == 0) 11962 goto err; 11963 if ((ol_flags & mask) != mask) 11964 goto err; 11965 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11966 if (!tble) { 11967 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11968 dev->data->port_id, m->hash.fdir.hi); 11969 goto err; 11970 } 11971 MLX5_ASSERT(tble->tunnel); 11972 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11973 info->group_id = tble->group_id; 11974 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11975 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11976 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11977 11978 return 0; 11979 11980 err: 11981 return rte_flow_error_set(err, EINVAL, 11982 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11983 "failed to get restore info"); 11984 } 11985 11986 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11987 static int 11988 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11989 __rte_unused struct rte_flow_tunnel *app_tunnel, 11990 __rte_unused struct rte_flow_action **actions, 11991 __rte_unused uint32_t *num_of_actions, 11992 __rte_unused struct rte_flow_error *error) 11993 { 11994 return -ENOTSUP; 11995 } 11996 11997 static int 11998 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11999 __rte_unused struct rte_flow_tunnel *app_tunnel, 12000 __rte_unused struct rte_flow_item **items, 12001 __rte_unused uint32_t *num_of_items, 12002 __rte_unused struct rte_flow_error *error) 12003 { 12004 return -ENOTSUP; 12005 } 12006 12007 static int 12008 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 12009 __rte_unused struct rte_flow_item *pmd_items, 12010 __rte_unused uint32_t num_items, 12011 __rte_unused struct rte_flow_error *err) 12012 { 12013 return -ENOTSUP; 12014 } 12015 12016 static int 12017 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 12018 __rte_unused struct rte_flow_action *pmd_action, 12019 __rte_unused uint32_t num_actions, 12020 __rte_unused struct rte_flow_error *err) 12021 { 12022 return -ENOTSUP; 12023 } 12024 12025 static int 12026 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 12027 __rte_unused struct rte_mbuf *m, 12028 __rte_unused struct rte_flow_restore_info *i, 12029 __rte_unused struct rte_flow_error *err) 12030 { 12031 return -ENOTSUP; 12032 } 12033 12034 static int 12035 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 12036 __rte_unused struct rte_flow *flow, 12037 __rte_unused const struct rte_flow_attr *attr, 12038 __rte_unused const struct rte_flow_action *actions, 12039 __rte_unused uint32_t flow_idx, 12040 __rte_unused const struct mlx5_flow_tunnel *tunnel, 12041 __rte_unused struct tunnel_default_miss_ctx *ctx, 12042 __rte_unused struct rte_flow_error *error) 12043 { 12044 return -ENOTSUP; 12045 } 12046 12047 static struct mlx5_flow_tunnel * 12048 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 12049 __rte_unused uint32_t id) 12050 { 12051 return NULL; 12052 } 12053 12054 static void 12055 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 12056 __rte_unused struct mlx5_flow_tunnel *tunnel) 12057 { 12058 } 12059 12060 static uint32_t 12061 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 12062 __rte_unused const struct mlx5_flow_tunnel *t, 12063 __rte_unused uint32_t group, 12064 __rte_unused uint32_t *table, 12065 struct rte_flow_error *error) 12066 { 12067 return rte_flow_error_set(error, ENOTSUP, 12068 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 12069 "tunnel offload requires DV support"); 12070 } 12071 12072 void 12073 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 12074 __rte_unused uint16_t port_id) 12075 { 12076 } 12077 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 12078 12079 /* Flex flow item API */ 12080 static struct rte_flow_item_flex_handle * 12081 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 12082 const struct rte_flow_item_flex_conf *conf, 12083 struct rte_flow_error *error) 12084 { 12085 static const char err_msg[] = "flex item creation unsupported"; 12086 struct mlx5_priv *priv = dev->data->dev_private; 12087 struct rte_flow_attr attr = { .transfer = 0 }; 12088 const struct mlx5_flow_driver_ops *fops = 12089 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 12090 12091 if (!priv->pci_dev) { 12092 rte_flow_error_set(error, ENOTSUP, 12093 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 12094 "create flex item on PF only"); 12095 return NULL; 12096 } 12097 switch (priv->pci_dev->id.device_id) { 12098 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 12099 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 12100 break; 12101 default: 12102 rte_flow_error_set(error, ENOTSUP, 12103 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 12104 "flex item available on BlueField ports only"); 12105 return NULL; 12106 } 12107 if (!fops->item_create) { 12108 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 12109 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 12110 NULL, err_msg); 12111 return NULL; 12112 } 12113 return fops->item_create(dev, conf, error); 12114 } 12115 12116 static int 12117 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 12118 const struct rte_flow_item_flex_handle *handle, 12119 struct rte_flow_error *error) 12120 { 12121 static const char err_msg[] = "flex item release unsupported"; 12122 struct rte_flow_attr attr = { .transfer = 0 }; 12123 const struct mlx5_flow_driver_ops *fops = 12124 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 12125 12126 if (!fops->item_release) { 12127 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 12128 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 12129 NULL, err_msg); 12130 return -rte_errno; 12131 } 12132 return fops->item_release(dev, handle, error); 12133 } 12134 12135 static void 12136 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 12137 { 12138 int ret; 12139 struct rte_flow_error error; 12140 12141 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 12142 char *item_name; 12143 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 12144 sizeof(item_name), 12145 (void *)(uintptr_t)item->type, &error); 12146 if (ret > 0) 12147 printf("%s ", item_name); 12148 else 12149 printf("%d\n", (int)item->type); 12150 } 12151 printf("END\n"); 12152 } 12153 12154 static int 12155 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 12156 { 12157 const struct rte_flow_item_udp *spec = udp_item->spec; 12158 const struct rte_flow_item_udp *mask = udp_item->mask; 12159 uint16_t udp_dport = 0; 12160 12161 if (spec != NULL) { 12162 if (!mask) 12163 mask = &rte_flow_item_udp_mask; 12164 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 12165 mask->hdr.dst_port); 12166 } 12167 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 12168 } 12169 12170 static const struct mlx5_flow_expand_node * 12171 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 12172 unsigned int item_idx, 12173 const struct mlx5_flow_expand_node graph[], 12174 const struct mlx5_flow_expand_node *node) 12175 { 12176 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 12177 12178 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 12179 node != NULL && 12180 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 12181 /* 12182 * The expansion node is VXLAN and it is also the last 12183 * expandable item in the pattern, so need to continue 12184 * expansion of the inner tunnel. 12185 */ 12186 MLX5_ASSERT(item_idx > 0); 12187 prev_item = pattern + item_idx - 1; 12188 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 12189 if (mlx5_flow_is_std_vxlan_port(prev_item)) 12190 return &graph[MLX5_EXPANSION_STD_VXLAN]; 12191 return &graph[MLX5_EXPANSION_L3_VXLAN]; 12192 } 12193 return node; 12194 } 12195 12196 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 12197 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 12198 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 12199 }; 12200 12201 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 12202 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 12203 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 12204 { 9, 10, 11 }, { 12, 13, 14 }, 12205 }; 12206 12207 /** 12208 * Discover the number of available flow priorities. 12209 * 12210 * @param dev 12211 * Ethernet device. 12212 * 12213 * @return 12214 * On success, number of available flow priorities. 12215 * On failure, a negative errno-style code and rte_errno is set. 12216 */ 12217 int 12218 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 12219 { 12220 static const uint16_t vprio[] = {8, 16}; 12221 const struct mlx5_priv *priv = dev->data->dev_private; 12222 const struct mlx5_flow_driver_ops *fops; 12223 enum mlx5_flow_drv_type type; 12224 int ret; 12225 12226 type = mlx5_flow_os_get_type(); 12227 if (type == MLX5_FLOW_TYPE_MAX) { 12228 type = MLX5_FLOW_TYPE_VERBS; 12229 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 12230 type = MLX5_FLOW_TYPE_DV; 12231 } 12232 fops = flow_get_drv_ops(type); 12233 if (fops->discover_priorities == NULL) { 12234 DRV_LOG(ERR, "Priority discovery not supported"); 12235 rte_errno = ENOTSUP; 12236 return -rte_errno; 12237 } 12238 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 12239 if (ret < 0) 12240 return ret; 12241 switch (ret) { 12242 case 8: 12243 ret = RTE_DIM(priority_map_3); 12244 break; 12245 case 16: 12246 ret = RTE_DIM(priority_map_5); 12247 break; 12248 default: 12249 rte_errno = ENOTSUP; 12250 DRV_LOG(ERR, 12251 "port %u maximum priority: %d expected 8/16", 12252 dev->data->port_id, ret); 12253 return -rte_errno; 12254 } 12255 DRV_LOG(INFO, "port %u supported flow priorities:" 12256 " 0-%d for ingress or egress root table," 12257 " 0-%d for non-root table or transfer root table.", 12258 dev->data->port_id, ret - 2, 12259 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 12260 return ret; 12261 } 12262 12263 /** 12264 * Adjust flow priority based on the highest layer and the request priority. 12265 * 12266 * @param[in] dev 12267 * Pointer to the Ethernet device structure. 12268 * @param[in] priority 12269 * The rule base priority. 12270 * @param[in] subpriority 12271 * The priority based on the items. 12272 * 12273 * @return 12274 * The new priority. 12275 */ 12276 uint32_t 12277 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 12278 uint32_t subpriority) 12279 { 12280 uint32_t res = 0; 12281 struct mlx5_priv *priv = dev->data->dev_private; 12282 12283 switch (priv->sh->flow_max_priority) { 12284 case RTE_DIM(priority_map_3): 12285 res = priority_map_3[priority][subpriority]; 12286 break; 12287 case RTE_DIM(priority_map_5): 12288 res = priority_map_5[priority][subpriority]; 12289 break; 12290 } 12291 return res; 12292 } 12293 12294 /** 12295 * Get the priority for sending traffic to kernel table. 12296 * 12297 * @param[in] dev 12298 * Pointer to the Ethernet device structure. 12299 * 12300 * @return 12301 * On success: the value of priority for sending traffic to kernel table 12302 * On failure: -1 12303 */ 12304 uint32_t 12305 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 12306 { 12307 struct mlx5_priv *priv = dev->data->dev_private; 12308 uint32_t res; 12309 12310 switch (priv->sh->flow_max_priority) { 12311 case RTE_DIM(priority_map_5): 12312 res = 15; 12313 break; 12314 case RTE_DIM(priority_map_3): 12315 res = 7; 12316 break; 12317 default: 12318 DRV_LOG(ERR, 12319 "port %u maximum priority: %d expected 8/16", 12320 dev->data->port_id, priv->sh->flow_max_priority); 12321 res = (uint32_t)-1; 12322 } 12323 return res; 12324 } 12325 12326 /** 12327 * Get the E-Switch Manager vport id. 12328 * 12329 * @param[in] dev 12330 * Pointer to the Ethernet device structure. 12331 * 12332 * @return 12333 * The vport id. 12334 */ 12335 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 12336 { 12337 struct mlx5_priv *priv = dev->data->dev_private; 12338 struct mlx5_common_device *cdev = priv->sh->cdev; 12339 12340 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 12341 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 12342 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 12343 12344 if (priv->pci_dev == NULL) 12345 return 0; 12346 switch (priv->pci_dev->id.device_id) { 12347 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 12348 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 12349 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 12350 /* 12351 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 12352 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 12353 * ECPF vport ID is selected and not the host port (0) in any BF case. 12354 */ 12355 return (int16_t)MLX5_ECPF_VPORT_ID; 12356 default: 12357 return MLX5_PF_VPORT_ID; 12358 } 12359 } 12360 12361 /** 12362 * Parse item to get the vport id. 12363 * 12364 * @param[in] dev 12365 * Pointer to the Ethernet device structure. 12366 * @param[in] item 12367 * The src port id match item. 12368 * @param[out] vport_id 12369 * Pointer to put the vport id. 12370 * @param[out] all_ports 12371 * Indicate if the item matches all ports. 12372 * @param[out] error 12373 * Pointer to error structure. 12374 * 12375 * @return 12376 * 0 on success, a negative errno value otherwise and rte_errno is set. 12377 */ 12378 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 12379 const struct rte_flow_item *item, 12380 uint16_t *vport_id, 12381 bool *all_ports, 12382 struct rte_flow_error *error) 12383 { 12384 struct mlx5_priv *port_priv; 12385 const struct rte_flow_item_port_id *pid_v = NULL; 12386 const struct rte_flow_item_ethdev *dev_v = NULL; 12387 uint32_t esw_mgr_port; 12388 uint32_t src_port; 12389 12390 if (all_ports) 12391 *all_ports = false; 12392 switch (item->type) { 12393 case RTE_FLOW_ITEM_TYPE_PORT_ID: 12394 pid_v = item->spec; 12395 if (!pid_v) 12396 return 0; 12397 src_port = pid_v->id; 12398 esw_mgr_port = MLX5_PORT_ESW_MGR; 12399 break; 12400 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 12401 dev_v = item->spec; 12402 if (!dev_v) { 12403 if (all_ports) 12404 *all_ports = true; 12405 return 0; 12406 } 12407 src_port = dev_v->port_id; 12408 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12409 break; 12410 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 12411 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12412 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12413 break; 12414 default: 12415 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12416 NULL, "Incorrect item type."); 12417 } 12418 if (src_port == esw_mgr_port) { 12419 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 12420 } else { 12421 port_priv = mlx5_port_to_eswitch_info(src_port, false); 12422 if (!port_priv) 12423 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12424 NULL, "Failed to get port info."); 12425 *vport_id = port_priv->representor_id; 12426 } 12427 12428 return 0; 12429 } 12430 12431 int 12432 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 12433 uint16_t *proxy_port_id, 12434 struct rte_flow_error *error) 12435 { 12436 const struct mlx5_priv *priv = dev->data->dev_private; 12437 uint16_t port_id; 12438 12439 if (!priv->sh->config.dv_esw_en) 12440 return rte_flow_error_set(error, EINVAL, 12441 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12442 NULL, 12443 "unable to provide a proxy port" 12444 " without E-Switch configured"); 12445 if (!priv->master && !priv->representor) 12446 return rte_flow_error_set(error, EINVAL, 12447 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12448 NULL, 12449 "unable to provide a proxy port" 12450 " for port which is not a master" 12451 " or a representor port"); 12452 if (priv->master) { 12453 *proxy_port_id = dev->data->port_id; 12454 return 0; 12455 } 12456 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 12457 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 12458 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 12459 12460 if (port_priv->master && 12461 port_priv->domain_id == priv->domain_id) { 12462 *proxy_port_id = port_id; 12463 return 0; 12464 } 12465 } 12466 return rte_flow_error_set(error, ENODEV, 12467 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12468 NULL, "unable to find a proxy port"); 12469 } 12470