1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static __rte_always_inline int 168 mlx5_need_cache_flow(const struct mlx5_priv *priv, 169 const struct rte_flow_attr *attr) 170 { 171 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 172 (attr ? !attr->group : true) && 173 priv->mode_info.mode == MLX5_FLOW_ENGINE_MODE_STANDBY && 174 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 175 } 176 177 static bool 178 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 179 { 180 switch (item->type) { 181 case RTE_FLOW_ITEM_TYPE_ETH: 182 case RTE_FLOW_ITEM_TYPE_VLAN: 183 case RTE_FLOW_ITEM_TYPE_IPV4: 184 case RTE_FLOW_ITEM_TYPE_IPV6: 185 case RTE_FLOW_ITEM_TYPE_UDP: 186 case RTE_FLOW_ITEM_TYPE_TCP: 187 case RTE_FLOW_ITEM_TYPE_ESP: 188 case RTE_FLOW_ITEM_TYPE_ICMP: 189 case RTE_FLOW_ITEM_TYPE_ICMP6: 190 case RTE_FLOW_ITEM_TYPE_VXLAN: 191 case RTE_FLOW_ITEM_TYPE_NVGRE: 192 case RTE_FLOW_ITEM_TYPE_GRE: 193 case RTE_FLOW_ITEM_TYPE_GENEVE: 194 case RTE_FLOW_ITEM_TYPE_MPLS: 195 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 196 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 197 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 198 case RTE_FLOW_ITEM_TYPE_GTP: 199 return true; 200 default: 201 break; 202 } 203 return false; 204 } 205 206 /** 207 * Network Service Header (NSH) and its next protocol values 208 * are described in RFC-8393. 209 */ 210 static enum rte_flow_item_type 211 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 212 { 213 enum rte_flow_item_type type; 214 215 switch (proto_mask & proto_spec) { 216 case 0: 217 type = RTE_FLOW_ITEM_TYPE_VOID; 218 break; 219 case RTE_VXLAN_GPE_TYPE_IPV4: 220 type = RTE_FLOW_ITEM_TYPE_IPV4; 221 break; 222 case RTE_VXLAN_GPE_TYPE_IPV6: 223 type = RTE_VXLAN_GPE_TYPE_IPV6; 224 break; 225 case RTE_VXLAN_GPE_TYPE_ETH: 226 type = RTE_FLOW_ITEM_TYPE_ETH; 227 break; 228 default: 229 type = RTE_FLOW_ITEM_TYPE_END; 230 } 231 return type; 232 } 233 234 static enum rte_flow_item_type 235 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 236 { 237 enum rte_flow_item_type type; 238 239 switch (proto_mask & proto_spec) { 240 case 0: 241 type = RTE_FLOW_ITEM_TYPE_VOID; 242 break; 243 case IPPROTO_UDP: 244 type = RTE_FLOW_ITEM_TYPE_UDP; 245 break; 246 case IPPROTO_TCP: 247 type = RTE_FLOW_ITEM_TYPE_TCP; 248 break; 249 case IPPROTO_IPIP: 250 type = RTE_FLOW_ITEM_TYPE_IPV4; 251 break; 252 case IPPROTO_IPV6: 253 type = RTE_FLOW_ITEM_TYPE_IPV6; 254 break; 255 case IPPROTO_ESP: 256 type = RTE_FLOW_ITEM_TYPE_ESP; 257 break; 258 default: 259 type = RTE_FLOW_ITEM_TYPE_END; 260 } 261 return type; 262 } 263 264 static enum rte_flow_item_type 265 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 266 rte_be16_t type_mask, bool is_tunnel) 267 { 268 enum rte_flow_item_type type; 269 270 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 271 case 0: 272 type = RTE_FLOW_ITEM_TYPE_VOID; 273 break; 274 case RTE_ETHER_TYPE_TEB: 275 type = is_tunnel ? 276 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 277 break; 278 case RTE_ETHER_TYPE_VLAN: 279 type = !is_tunnel ? 280 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 281 break; 282 case RTE_ETHER_TYPE_IPV4: 283 type = RTE_FLOW_ITEM_TYPE_IPV4; 284 break; 285 case RTE_ETHER_TYPE_IPV6: 286 type = RTE_FLOW_ITEM_TYPE_IPV6; 287 break; 288 default: 289 type = RTE_FLOW_ITEM_TYPE_END; 290 } 291 return type; 292 } 293 294 static enum rte_flow_item_type 295 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 296 { 297 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 298 do { \ 299 const void *m = item->mask; \ 300 const void *s = item->spec; \ 301 mask = m ? \ 302 ((const struct rte_flow_item_##type *)m)->fld : \ 303 rte_flow_item_##type##_mask.fld; \ 304 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 305 } while (0) 306 307 enum rte_flow_item_type ret; 308 uint16_t spec, mask; 309 310 if (item == NULL || item->spec == NULL) 311 return RTE_FLOW_ITEM_TYPE_VOID; 312 switch (item->type) { 313 case RTE_FLOW_ITEM_TYPE_ETH: 314 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 315 if (!mask) 316 return RTE_FLOW_ITEM_TYPE_VOID; 317 ret = mlx5_ethertype_to_item_type(spec, mask, false); 318 break; 319 case RTE_FLOW_ITEM_TYPE_VLAN: 320 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 321 if (!mask) 322 return RTE_FLOW_ITEM_TYPE_VOID; 323 ret = mlx5_ethertype_to_item_type(spec, mask, false); 324 break; 325 case RTE_FLOW_ITEM_TYPE_IPV4: 326 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 327 if (!mask) 328 return RTE_FLOW_ITEM_TYPE_VOID; 329 ret = mlx5_inet_proto_to_item_type(spec, mask); 330 break; 331 case RTE_FLOW_ITEM_TYPE_IPV6: 332 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 333 if (!mask) 334 return RTE_FLOW_ITEM_TYPE_VOID; 335 ret = mlx5_inet_proto_to_item_type(spec, mask); 336 break; 337 case RTE_FLOW_ITEM_TYPE_GENEVE: 338 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 339 ret = mlx5_ethertype_to_item_type(spec, mask, true); 340 break; 341 case RTE_FLOW_ITEM_TYPE_GRE: 342 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 343 ret = mlx5_ethertype_to_item_type(spec, mask, true); 344 break; 345 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 346 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 347 ret = mlx5_nsh_proto_to_item_type(spec, mask); 348 break; 349 default: 350 ret = RTE_FLOW_ITEM_TYPE_VOID; 351 break; 352 } 353 return ret; 354 #undef MLX5_XSET_ITEM_MASK_SPEC 355 } 356 357 static const int * 358 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 359 const int *next_node) 360 { 361 const struct mlx5_flow_expand_node *node = NULL; 362 const int *next = next_node; 363 364 while (next && *next) { 365 /* 366 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 367 * flag set, because they were not found in the flow pattern. 368 */ 369 node = &graph[*next]; 370 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 371 break; 372 next = node->next; 373 } 374 return next; 375 } 376 377 #define MLX5_RSS_EXP_ELT_N 16 378 379 /** 380 * Expand RSS flows into several possible flows according to the RSS hash 381 * fields requested and the driver capabilities. 382 * 383 * @param[out] buf 384 * Buffer to store the result expansion. 385 * @param[in] size 386 * Buffer size in bytes. If 0, @p buf can be NULL. 387 * @param[in] pattern 388 * User flow pattern. 389 * @param[in] types 390 * RSS types to expand (see RTE_ETH_RSS_* definitions). 391 * @param[in] graph 392 * Input graph to expand @p pattern according to @p types. 393 * @param[in] graph_root_index 394 * Index of root node in @p graph, typically 0. 395 * 396 * @return 397 * A positive value representing the size of @p buf in bytes regardless of 398 * @p size on success, a negative errno value otherwise and rte_errno is 399 * set, the following errors are defined: 400 * 401 * -E2BIG: graph-depth @p graph is too deep. 402 * -EINVAL: @p size has not enough space for expanded pattern. 403 */ 404 static int 405 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 406 const struct rte_flow_item *pattern, uint64_t types, 407 const struct mlx5_flow_expand_node graph[], 408 int graph_root_index) 409 { 410 const struct rte_flow_item *item; 411 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 412 const int *next_node; 413 const int *stack[MLX5_RSS_EXP_ELT_N]; 414 int stack_pos = 0; 415 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 416 unsigned int i, item_idx, last_expand_item_idx = 0; 417 size_t lsize; 418 size_t user_pattern_size = 0; 419 void *addr = NULL; 420 const struct mlx5_flow_expand_node *next = NULL; 421 struct rte_flow_item missed_item; 422 int missed = 0; 423 int elt = 0; 424 const struct rte_flow_item *last_expand_item = NULL; 425 426 memset(&missed_item, 0, sizeof(missed_item)); 427 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 428 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 429 if (lsize > size) 430 return -EINVAL; 431 buf->entry[0].priority = 0; 432 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 433 buf->entries = 0; 434 addr = buf->entry[0].pattern; 435 for (item = pattern, item_idx = 0; 436 item->type != RTE_FLOW_ITEM_TYPE_END; 437 item++, item_idx++) { 438 if (!mlx5_flow_is_rss_expandable_item(item)) { 439 user_pattern_size += sizeof(*item); 440 continue; 441 } 442 last_expand_item = item; 443 last_expand_item_idx = item_idx; 444 i = 0; 445 while (node->next && node->next[i]) { 446 next = &graph[node->next[i]]; 447 if (next->type == item->type) 448 break; 449 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 450 node = next; 451 i = 0; 452 } else { 453 ++i; 454 } 455 } 456 if (next) 457 node = next; 458 user_pattern_size += sizeof(*item); 459 } 460 user_pattern_size += sizeof(*item); /* Handle END item. */ 461 lsize += user_pattern_size; 462 if (lsize > size) 463 return -EINVAL; 464 /* Copy the user pattern in the first entry of the buffer. */ 465 rte_memcpy(addr, pattern, user_pattern_size); 466 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 467 buf->entries = 1; 468 /* Start expanding. */ 469 memset(flow_items, 0, sizeof(flow_items)); 470 user_pattern_size -= sizeof(*item); 471 /* 472 * Check if the last valid item has spec set, need complete pattern, 473 * and the pattern can be used for expansion. 474 */ 475 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 476 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 477 /* Item type END indicates expansion is not required. */ 478 return lsize; 479 } 480 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 481 next = NULL; 482 missed = 1; 483 i = 0; 484 while (node->next && node->next[i]) { 485 next = &graph[node->next[i]]; 486 if (next->type == missed_item.type) { 487 flow_items[0].type = missed_item.type; 488 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 489 break; 490 } 491 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 492 node = next; 493 i = 0; 494 } else { 495 ++i; 496 } 497 next = NULL; 498 } 499 } 500 if (next && missed) { 501 elt = 2; /* missed item + item end. */ 502 node = next; 503 lsize += elt * sizeof(*item) + user_pattern_size; 504 if (lsize > size) 505 return -EINVAL; 506 if (node->rss_types & types) { 507 buf->entry[buf->entries].priority = 1; 508 buf->entry[buf->entries].pattern = addr; 509 buf->entries++; 510 rte_memcpy(addr, buf->entry[0].pattern, 511 user_pattern_size); 512 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 513 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 514 addr = (void *)(((uintptr_t)addr) + 515 elt * sizeof(*item)); 516 } 517 } else if (last_expand_item != NULL) { 518 node = mlx5_flow_expand_rss_adjust_node(pattern, 519 last_expand_item_idx, graph, node); 520 } 521 memset(flow_items, 0, sizeof(flow_items)); 522 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 523 node->next); 524 stack[stack_pos] = next_node; 525 node = next_node ? &graph[*next_node] : NULL; 526 while (node) { 527 flow_items[stack_pos].type = node->type; 528 if (node->rss_types & types) { 529 size_t n; 530 /* 531 * compute the number of items to copy from the 532 * expansion and copy it. 533 * When the stack_pos is 0, there are 1 element in it, 534 * plus the addition END item. 535 */ 536 elt = stack_pos + 2; 537 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 538 lsize += elt * sizeof(*item) + user_pattern_size; 539 if (lsize > size) 540 return -EINVAL; 541 n = elt * sizeof(*item); 542 buf->entry[buf->entries].priority = 543 stack_pos + 1 + missed; 544 buf->entry[buf->entries].pattern = addr; 545 buf->entries++; 546 rte_memcpy(addr, buf->entry[0].pattern, 547 user_pattern_size); 548 addr = (void *)(((uintptr_t)addr) + 549 user_pattern_size); 550 rte_memcpy(addr, &missed_item, 551 missed * sizeof(*item)); 552 addr = (void *)(((uintptr_t)addr) + 553 missed * sizeof(*item)); 554 rte_memcpy(addr, flow_items, n); 555 addr = (void *)(((uintptr_t)addr) + n); 556 } 557 /* Go deeper. */ 558 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 559 node->next) { 560 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 561 node->next); 562 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 563 rte_errno = E2BIG; 564 return -rte_errno; 565 } 566 stack[stack_pos] = next_node; 567 } else if (*(next_node + 1)) { 568 /* Follow up with the next possibility. */ 569 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 570 ++next_node); 571 } else if (!stack_pos) { 572 /* 573 * Completing the traverse over the different paths. 574 * The next_node is advanced to the terminator. 575 */ 576 ++next_node; 577 } else { 578 /* Move to the next path. */ 579 while (stack_pos) { 580 next_node = stack[--stack_pos]; 581 next_node++; 582 if (*next_node) 583 break; 584 } 585 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 586 next_node); 587 stack[stack_pos] = next_node; 588 } 589 node = next_node && *next_node ? &graph[*next_node] : NULL; 590 }; 591 return lsize; 592 } 593 594 /** 595 * Expand SQN flows into several possible flows according to the Tx queue 596 * number 597 * 598 * @param[in] buf 599 * Buffer to store the result expansion. 600 * @param[in] size 601 * Buffer size in bytes. If 0, @p buf can be NULL. 602 * @param[in] pattern 603 * User flow pattern. 604 * @param[in] sq_specs 605 * Buffer to store sq spec. 606 * 607 * @return 608 * 0 for success and negative value for failure 609 * 610 */ 611 static int 612 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 613 const struct rte_flow_item *pattern, 614 struct mlx5_rte_flow_item_sq *sq_specs) 615 { 616 const struct rte_flow_item *item; 617 bool port_representor = false; 618 size_t user_pattern_size = 0; 619 struct rte_eth_dev *dev; 620 struct mlx5_priv *priv; 621 void *addr = NULL; 622 uint16_t port_id; 623 size_t lsize; 624 int elt = 2; 625 uint16_t i; 626 627 buf->entries = 0; 628 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 629 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 630 const struct rte_flow_item_ethdev *pid_v = item->spec; 631 632 if (!pid_v) 633 return 0; 634 port_id = pid_v->port_id; 635 port_representor = true; 636 } 637 user_pattern_size += sizeof(*item); 638 } 639 if (!port_representor) 640 return 0; 641 dev = &rte_eth_devices[port_id]; 642 priv = dev->data->dev_private; 643 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 644 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 645 sizeof(buf->entry[0]) * priv->txqs_n; 646 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 647 return -EINVAL; 648 addr = buf->entry[0].pattern; 649 for (i = 0; i != priv->txqs_n; ++i) { 650 struct rte_flow_item pattern_add[] = { 651 { 652 .type = (enum rte_flow_item_type) 653 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 654 .spec = &sq_specs[i], 655 }, 656 { 657 .type = RTE_FLOW_ITEM_TYPE_END, 658 }, 659 }; 660 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 661 662 if (txq == NULL) 663 return -EINVAL; 664 buf->entry[i].pattern = addr; 665 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 666 mlx5_txq_release(dev, i); 667 rte_memcpy(addr, pattern, user_pattern_size); 668 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 669 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 670 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 671 buf->entries++; 672 } 673 return 0; 674 } 675 676 enum mlx5_expansion { 677 MLX5_EXPANSION_ROOT, 678 MLX5_EXPANSION_ROOT_OUTER, 679 MLX5_EXPANSION_OUTER_ETH, 680 MLX5_EXPANSION_OUTER_VLAN, 681 MLX5_EXPANSION_OUTER_IPV4, 682 MLX5_EXPANSION_OUTER_IPV4_UDP, 683 MLX5_EXPANSION_OUTER_IPV4_TCP, 684 MLX5_EXPANSION_OUTER_IPV4_ESP, 685 MLX5_EXPANSION_OUTER_IPV4_ICMP, 686 MLX5_EXPANSION_OUTER_IPV6, 687 MLX5_EXPANSION_OUTER_IPV6_UDP, 688 MLX5_EXPANSION_OUTER_IPV6_TCP, 689 MLX5_EXPANSION_OUTER_IPV6_ESP, 690 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 691 MLX5_EXPANSION_VXLAN, 692 MLX5_EXPANSION_STD_VXLAN, 693 MLX5_EXPANSION_L3_VXLAN, 694 MLX5_EXPANSION_VXLAN_GPE, 695 MLX5_EXPANSION_GRE, 696 MLX5_EXPANSION_NVGRE, 697 MLX5_EXPANSION_GRE_KEY, 698 MLX5_EXPANSION_MPLS, 699 MLX5_EXPANSION_ETH, 700 MLX5_EXPANSION_VLAN, 701 MLX5_EXPANSION_IPV4, 702 MLX5_EXPANSION_IPV4_UDP, 703 MLX5_EXPANSION_IPV4_TCP, 704 MLX5_EXPANSION_IPV4_ESP, 705 MLX5_EXPANSION_IPV4_ICMP, 706 MLX5_EXPANSION_IPV6, 707 MLX5_EXPANSION_IPV6_UDP, 708 MLX5_EXPANSION_IPV6_TCP, 709 MLX5_EXPANSION_IPV6_ESP, 710 MLX5_EXPANSION_IPV6_ICMP6, 711 MLX5_EXPANSION_IPV6_FRAG_EXT, 712 MLX5_EXPANSION_GTP, 713 MLX5_EXPANSION_GENEVE, 714 }; 715 716 /** Supported expansion of items. */ 717 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 718 [MLX5_EXPANSION_ROOT] = { 719 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 720 MLX5_EXPANSION_IPV4, 721 MLX5_EXPANSION_IPV6), 722 .type = RTE_FLOW_ITEM_TYPE_END, 723 }, 724 [MLX5_EXPANSION_ROOT_OUTER] = { 725 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 726 MLX5_EXPANSION_OUTER_IPV4, 727 MLX5_EXPANSION_OUTER_IPV6), 728 .type = RTE_FLOW_ITEM_TYPE_END, 729 }, 730 [MLX5_EXPANSION_OUTER_ETH] = { 731 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 732 .type = RTE_FLOW_ITEM_TYPE_ETH, 733 .rss_types = 0, 734 }, 735 [MLX5_EXPANSION_OUTER_VLAN] = { 736 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 737 MLX5_EXPANSION_OUTER_IPV6), 738 .type = RTE_FLOW_ITEM_TYPE_VLAN, 739 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 740 }, 741 [MLX5_EXPANSION_OUTER_IPV4] = { 742 .next = MLX5_FLOW_EXPAND_RSS_NEXT 743 (MLX5_EXPANSION_OUTER_IPV4_UDP, 744 MLX5_EXPANSION_OUTER_IPV4_TCP, 745 MLX5_EXPANSION_OUTER_IPV4_ESP, 746 MLX5_EXPANSION_OUTER_IPV4_ICMP, 747 MLX5_EXPANSION_GRE, 748 MLX5_EXPANSION_NVGRE, 749 MLX5_EXPANSION_IPV4, 750 MLX5_EXPANSION_IPV6), 751 .type = RTE_FLOW_ITEM_TYPE_IPV4, 752 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 753 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 754 }, 755 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 756 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 757 MLX5_EXPANSION_VXLAN_GPE, 758 MLX5_EXPANSION_MPLS, 759 MLX5_EXPANSION_GENEVE, 760 MLX5_EXPANSION_GTP), 761 .type = RTE_FLOW_ITEM_TYPE_UDP, 762 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 763 }, 764 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 765 .type = RTE_FLOW_ITEM_TYPE_TCP, 766 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 767 }, 768 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 769 .type = RTE_FLOW_ITEM_TYPE_ESP, 770 .rss_types = RTE_ETH_RSS_ESP, 771 }, 772 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 773 .type = RTE_FLOW_ITEM_TYPE_ICMP, 774 }, 775 [MLX5_EXPANSION_OUTER_IPV6] = { 776 .next = MLX5_FLOW_EXPAND_RSS_NEXT 777 (MLX5_EXPANSION_OUTER_IPV6_UDP, 778 MLX5_EXPANSION_OUTER_IPV6_TCP, 779 MLX5_EXPANSION_OUTER_IPV6_ESP, 780 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 781 MLX5_EXPANSION_IPV4, 782 MLX5_EXPANSION_IPV6, 783 MLX5_EXPANSION_GRE, 784 MLX5_EXPANSION_NVGRE), 785 .type = RTE_FLOW_ITEM_TYPE_IPV6, 786 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 787 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 788 }, 789 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 790 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 791 MLX5_EXPANSION_VXLAN_GPE, 792 MLX5_EXPANSION_MPLS, 793 MLX5_EXPANSION_GENEVE, 794 MLX5_EXPANSION_GTP), 795 .type = RTE_FLOW_ITEM_TYPE_UDP, 796 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 797 }, 798 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 799 .type = RTE_FLOW_ITEM_TYPE_TCP, 800 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 801 }, 802 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 803 .type = RTE_FLOW_ITEM_TYPE_ESP, 804 .rss_types = RTE_ETH_RSS_ESP, 805 }, 806 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 807 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 808 }, 809 [MLX5_EXPANSION_VXLAN] = { 810 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 811 MLX5_EXPANSION_IPV4, 812 MLX5_EXPANSION_IPV6), 813 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 814 }, 815 [MLX5_EXPANSION_STD_VXLAN] = { 816 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 817 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 818 }, 819 [MLX5_EXPANSION_L3_VXLAN] = { 820 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 821 MLX5_EXPANSION_IPV6), 822 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 823 }, 824 [MLX5_EXPANSION_VXLAN_GPE] = { 825 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 826 MLX5_EXPANSION_IPV4, 827 MLX5_EXPANSION_IPV6), 828 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 829 }, 830 [MLX5_EXPANSION_GRE] = { 831 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 832 MLX5_EXPANSION_IPV4, 833 MLX5_EXPANSION_IPV6, 834 MLX5_EXPANSION_GRE_KEY, 835 MLX5_EXPANSION_MPLS), 836 .type = RTE_FLOW_ITEM_TYPE_GRE, 837 }, 838 [MLX5_EXPANSION_GRE_KEY] = { 839 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 840 MLX5_EXPANSION_IPV6, 841 MLX5_EXPANSION_MPLS), 842 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 843 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 844 }, 845 [MLX5_EXPANSION_NVGRE] = { 846 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 847 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 848 }, 849 [MLX5_EXPANSION_MPLS] = { 850 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 851 MLX5_EXPANSION_IPV6, 852 MLX5_EXPANSION_ETH), 853 .type = RTE_FLOW_ITEM_TYPE_MPLS, 854 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 855 }, 856 [MLX5_EXPANSION_ETH] = { 857 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 858 .type = RTE_FLOW_ITEM_TYPE_ETH, 859 }, 860 [MLX5_EXPANSION_VLAN] = { 861 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 862 MLX5_EXPANSION_IPV6), 863 .type = RTE_FLOW_ITEM_TYPE_VLAN, 864 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 865 }, 866 [MLX5_EXPANSION_IPV4] = { 867 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 868 MLX5_EXPANSION_IPV4_TCP, 869 MLX5_EXPANSION_IPV4_ESP, 870 MLX5_EXPANSION_IPV4_ICMP), 871 .type = RTE_FLOW_ITEM_TYPE_IPV4, 872 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 873 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 874 }, 875 [MLX5_EXPANSION_IPV4_UDP] = { 876 .type = RTE_FLOW_ITEM_TYPE_UDP, 877 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 878 }, 879 [MLX5_EXPANSION_IPV4_TCP] = { 880 .type = RTE_FLOW_ITEM_TYPE_TCP, 881 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 882 }, 883 [MLX5_EXPANSION_IPV4_ESP] = { 884 .type = RTE_FLOW_ITEM_TYPE_ESP, 885 .rss_types = RTE_ETH_RSS_ESP, 886 }, 887 [MLX5_EXPANSION_IPV4_ICMP] = { 888 .type = RTE_FLOW_ITEM_TYPE_ICMP, 889 }, 890 [MLX5_EXPANSION_IPV6] = { 891 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 892 MLX5_EXPANSION_IPV6_TCP, 893 MLX5_EXPANSION_IPV6_ESP, 894 MLX5_EXPANSION_IPV6_ICMP6, 895 MLX5_EXPANSION_IPV6_FRAG_EXT), 896 .type = RTE_FLOW_ITEM_TYPE_IPV6, 897 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 898 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 899 }, 900 [MLX5_EXPANSION_IPV6_UDP] = { 901 .type = RTE_FLOW_ITEM_TYPE_UDP, 902 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 903 }, 904 [MLX5_EXPANSION_IPV6_TCP] = { 905 .type = RTE_FLOW_ITEM_TYPE_TCP, 906 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 907 }, 908 [MLX5_EXPANSION_IPV6_ESP] = { 909 .type = RTE_FLOW_ITEM_TYPE_ESP, 910 .rss_types = RTE_ETH_RSS_ESP, 911 }, 912 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 913 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 914 }, 915 [MLX5_EXPANSION_IPV6_ICMP6] = { 916 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 917 }, 918 [MLX5_EXPANSION_GTP] = { 919 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 920 MLX5_EXPANSION_IPV6), 921 .type = RTE_FLOW_ITEM_TYPE_GTP, 922 }, 923 [MLX5_EXPANSION_GENEVE] = { 924 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 925 MLX5_EXPANSION_IPV4, 926 MLX5_EXPANSION_IPV6), 927 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 928 }, 929 }; 930 931 static struct rte_flow_action_handle * 932 mlx5_action_handle_create(struct rte_eth_dev *dev, 933 const struct rte_flow_indir_action_conf *conf, 934 const struct rte_flow_action *action, 935 struct rte_flow_error *error); 936 static int mlx5_action_handle_destroy 937 (struct rte_eth_dev *dev, 938 struct rte_flow_action_handle *handle, 939 struct rte_flow_error *error); 940 static int mlx5_action_handle_update 941 (struct rte_eth_dev *dev, 942 struct rte_flow_action_handle *handle, 943 const void *update, 944 struct rte_flow_error *error); 945 static int mlx5_action_handle_query 946 (struct rte_eth_dev *dev, 947 const struct rte_flow_action_handle *handle, 948 void *data, 949 struct rte_flow_error *error); 950 static int 951 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 952 struct rte_flow_tunnel *app_tunnel, 953 struct rte_flow_action **actions, 954 uint32_t *num_of_actions, 955 struct rte_flow_error *error); 956 static int 957 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 958 struct rte_flow_tunnel *app_tunnel, 959 struct rte_flow_item **items, 960 uint32_t *num_of_items, 961 struct rte_flow_error *error); 962 static int 963 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 964 struct rte_flow_item *pmd_items, 965 uint32_t num_items, struct rte_flow_error *err); 966 static int 967 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 968 struct rte_flow_action *pmd_actions, 969 uint32_t num_actions, 970 struct rte_flow_error *err); 971 static int 972 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 973 struct rte_mbuf *m, 974 struct rte_flow_restore_info *info, 975 struct rte_flow_error *err); 976 static struct rte_flow_item_flex_handle * 977 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 978 const struct rte_flow_item_flex_conf *conf, 979 struct rte_flow_error *error); 980 static int 981 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 982 const struct rte_flow_item_flex_handle *handle, 983 struct rte_flow_error *error); 984 static int 985 mlx5_flow_info_get(struct rte_eth_dev *dev, 986 struct rte_flow_port_info *port_info, 987 struct rte_flow_queue_info *queue_info, 988 struct rte_flow_error *error); 989 static int 990 mlx5_flow_port_configure(struct rte_eth_dev *dev, 991 const struct rte_flow_port_attr *port_attr, 992 uint16_t nb_queue, 993 const struct rte_flow_queue_attr *queue_attr[], 994 struct rte_flow_error *err); 995 996 static struct rte_flow_pattern_template * 997 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 998 const struct rte_flow_pattern_template_attr *attr, 999 const struct rte_flow_item items[], 1000 struct rte_flow_error *error); 1001 1002 static int 1003 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1004 struct rte_flow_pattern_template *template, 1005 struct rte_flow_error *error); 1006 static struct rte_flow_actions_template * 1007 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1008 const struct rte_flow_actions_template_attr *attr, 1009 const struct rte_flow_action actions[], 1010 const struct rte_flow_action masks[], 1011 struct rte_flow_error *error); 1012 static int 1013 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1014 struct rte_flow_actions_template *template, 1015 struct rte_flow_error *error); 1016 1017 static struct rte_flow_template_table * 1018 mlx5_flow_table_create(struct rte_eth_dev *dev, 1019 const struct rte_flow_template_table_attr *attr, 1020 struct rte_flow_pattern_template *item_templates[], 1021 uint8_t nb_item_templates, 1022 struct rte_flow_actions_template *action_templates[], 1023 uint8_t nb_action_templates, 1024 struct rte_flow_error *error); 1025 static int 1026 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1027 struct rte_flow_template_table *table, 1028 struct rte_flow_error *error); 1029 static struct rte_flow * 1030 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1031 uint32_t queue, 1032 const struct rte_flow_op_attr *attr, 1033 struct rte_flow_template_table *table, 1034 const struct rte_flow_item items[], 1035 uint8_t pattern_template_index, 1036 const struct rte_flow_action actions[], 1037 uint8_t action_template_index, 1038 void *user_data, 1039 struct rte_flow_error *error); 1040 static struct rte_flow * 1041 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 1042 uint32_t queue, 1043 const struct rte_flow_op_attr *attr, 1044 struct rte_flow_template_table *table, 1045 uint32_t rule_index, 1046 const struct rte_flow_action actions[], 1047 uint8_t action_template_index, 1048 void *user_data, 1049 struct rte_flow_error *error); 1050 static int 1051 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1052 uint32_t queue, 1053 const struct rte_flow_op_attr *attr, 1054 struct rte_flow *flow, 1055 void *user_data, 1056 struct rte_flow_error *error); 1057 static int 1058 mlx5_flow_pull(struct rte_eth_dev *dev, 1059 uint32_t queue, 1060 struct rte_flow_op_result res[], 1061 uint16_t n_res, 1062 struct rte_flow_error *error); 1063 static int 1064 mlx5_flow_push(struct rte_eth_dev *dev, 1065 uint32_t queue, 1066 struct rte_flow_error *error); 1067 1068 static struct rte_flow_action_handle * 1069 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1070 const struct rte_flow_op_attr *attr, 1071 const struct rte_flow_indir_action_conf *conf, 1072 const struct rte_flow_action *action, 1073 void *user_data, 1074 struct rte_flow_error *error); 1075 1076 static int 1077 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1078 const struct rte_flow_op_attr *attr, 1079 struct rte_flow_action_handle *handle, 1080 const void *update, 1081 void *user_data, 1082 struct rte_flow_error *error); 1083 1084 static int 1085 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1086 const struct rte_flow_op_attr *attr, 1087 struct rte_flow_action_handle *handle, 1088 void *user_data, 1089 struct rte_flow_error *error); 1090 1091 static int 1092 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1093 const struct rte_flow_op_attr *attr, 1094 const struct rte_flow_action_handle *handle, 1095 void *data, 1096 void *user_data, 1097 struct rte_flow_error *error); 1098 1099 static const struct rte_flow_ops mlx5_flow_ops = { 1100 .validate = mlx5_flow_validate, 1101 .create = mlx5_flow_create, 1102 .destroy = mlx5_flow_destroy, 1103 .flush = mlx5_flow_flush, 1104 .isolate = mlx5_flow_isolate, 1105 .query = mlx5_flow_query, 1106 .dev_dump = mlx5_flow_dev_dump, 1107 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1108 .get_aged_flows = mlx5_flow_get_aged_flows, 1109 .action_handle_create = mlx5_action_handle_create, 1110 .action_handle_destroy = mlx5_action_handle_destroy, 1111 .action_handle_update = mlx5_action_handle_update, 1112 .action_handle_query = mlx5_action_handle_query, 1113 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1114 .tunnel_match = mlx5_flow_tunnel_match, 1115 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1116 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1117 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1118 .flex_item_create = mlx5_flow_flex_item_create, 1119 .flex_item_release = mlx5_flow_flex_item_release, 1120 .info_get = mlx5_flow_info_get, 1121 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1122 .configure = mlx5_flow_port_configure, 1123 .pattern_template_create = mlx5_flow_pattern_template_create, 1124 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1125 .actions_template_create = mlx5_flow_actions_template_create, 1126 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1127 .template_table_create = mlx5_flow_table_create, 1128 .template_table_destroy = mlx5_flow_table_destroy, 1129 .async_create = mlx5_flow_async_flow_create, 1130 .async_create_by_index = mlx5_flow_async_flow_create_by_index, 1131 .async_destroy = mlx5_flow_async_flow_destroy, 1132 .pull = mlx5_flow_pull, 1133 .push = mlx5_flow_push, 1134 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1135 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1136 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1137 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1138 }; 1139 1140 /* Tunnel information. */ 1141 struct mlx5_flow_tunnel_info { 1142 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1143 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1144 }; 1145 1146 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1147 { 1148 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1149 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1150 }, 1151 { 1152 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1153 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1154 }, 1155 { 1156 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1157 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1158 }, 1159 { 1160 .tunnel = MLX5_FLOW_LAYER_GRE, 1161 .ptype = RTE_PTYPE_TUNNEL_GRE, 1162 }, 1163 { 1164 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1165 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1166 }, 1167 { 1168 .tunnel = MLX5_FLOW_LAYER_MPLS, 1169 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1170 }, 1171 { 1172 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1173 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1174 }, 1175 { 1176 .tunnel = MLX5_FLOW_LAYER_IPIP, 1177 .ptype = RTE_PTYPE_TUNNEL_IP, 1178 }, 1179 { 1180 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1181 .ptype = RTE_PTYPE_TUNNEL_IP, 1182 }, 1183 { 1184 .tunnel = MLX5_FLOW_LAYER_GTP, 1185 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1186 }, 1187 }; 1188 1189 1190 1191 /** 1192 * Translate tag ID to register. 1193 * 1194 * @param[in] dev 1195 * Pointer to the Ethernet device structure. 1196 * @param[in] feature 1197 * The feature that request the register. 1198 * @param[in] id 1199 * The request register ID. 1200 * @param[out] error 1201 * Error description in case of any. 1202 * 1203 * @return 1204 * The request register on success, a negative errno 1205 * value otherwise and rte_errno is set. 1206 */ 1207 int 1208 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1209 enum mlx5_feature_name feature, 1210 uint32_t id, 1211 struct rte_flow_error *error) 1212 { 1213 struct mlx5_priv *priv = dev->data->dev_private; 1214 struct mlx5_sh_config *config = &priv->sh->config; 1215 enum modify_reg start_reg; 1216 bool skip_mtr_reg = false; 1217 1218 switch (feature) { 1219 case MLX5_HAIRPIN_RX: 1220 return REG_B; 1221 case MLX5_HAIRPIN_TX: 1222 return REG_A; 1223 case MLX5_METADATA_RX: 1224 switch (config->dv_xmeta_en) { 1225 case MLX5_XMETA_MODE_LEGACY: 1226 return REG_B; 1227 case MLX5_XMETA_MODE_META16: 1228 return REG_C_0; 1229 case MLX5_XMETA_MODE_META32: 1230 return REG_C_1; 1231 case MLX5_XMETA_MODE_META32_HWS: 1232 return REG_C_1; 1233 } 1234 break; 1235 case MLX5_METADATA_TX: 1236 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1237 return REG_C_1; 1238 } else { 1239 return REG_A; 1240 } 1241 case MLX5_METADATA_FDB: 1242 switch (config->dv_xmeta_en) { 1243 case MLX5_XMETA_MODE_LEGACY: 1244 return REG_NON; 1245 case MLX5_XMETA_MODE_META16: 1246 return REG_C_0; 1247 case MLX5_XMETA_MODE_META32: 1248 return REG_C_1; 1249 case MLX5_XMETA_MODE_META32_HWS: 1250 return REG_C_1; 1251 } 1252 break; 1253 case MLX5_FLOW_MARK: 1254 switch (config->dv_xmeta_en) { 1255 case MLX5_XMETA_MODE_LEGACY: 1256 case MLX5_XMETA_MODE_META32_HWS: 1257 return REG_NON; 1258 case MLX5_XMETA_MODE_META16: 1259 return REG_C_1; 1260 case MLX5_XMETA_MODE_META32: 1261 return REG_C_0; 1262 } 1263 break; 1264 case MLX5_MTR_ID: 1265 /* 1266 * If meter color and meter id share one register, flow match 1267 * should use the meter color register for match. 1268 */ 1269 if (priv->mtr_reg_share) 1270 return priv->mtr_color_reg; 1271 else 1272 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1273 REG_C_3; 1274 case MLX5_MTR_COLOR: 1275 case MLX5_ASO_FLOW_HIT: 1276 case MLX5_ASO_CONNTRACK: 1277 case MLX5_SAMPLE_ID: 1278 /* All features use the same REG_C. */ 1279 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1280 return priv->mtr_color_reg; 1281 case MLX5_COPY_MARK: 1282 /* 1283 * Metadata COPY_MARK register using is in meter suffix sub 1284 * flow while with meter. It's safe to share the same register. 1285 */ 1286 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1287 case MLX5_APP_TAG: 1288 /* 1289 * If meter is enable, it will engage the register for color 1290 * match and flow match. If meter color match is not using the 1291 * REG_C_2, need to skip the REG_C_x be used by meter color 1292 * match. 1293 * If meter is disable, free to use all available registers. 1294 */ 1295 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1296 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1297 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1298 if (id > (uint32_t)(REG_C_7 - start_reg)) 1299 return rte_flow_error_set(error, EINVAL, 1300 RTE_FLOW_ERROR_TYPE_ITEM, 1301 NULL, "invalid tag id"); 1302 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1303 return rte_flow_error_set(error, ENOTSUP, 1304 RTE_FLOW_ERROR_TYPE_ITEM, 1305 NULL, "unsupported tag id"); 1306 /* 1307 * This case means meter is using the REG_C_x great than 2. 1308 * Take care not to conflict with meter color REG_C_x. 1309 * If the available index REG_C_y >= REG_C_x, skip the 1310 * color register. 1311 */ 1312 if (skip_mtr_reg && priv->sh->flow_mreg_c 1313 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1314 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1315 return rte_flow_error_set(error, EINVAL, 1316 RTE_FLOW_ERROR_TYPE_ITEM, 1317 NULL, "invalid tag id"); 1318 if (priv->sh->flow_mreg_c 1319 [id + 1 + start_reg - REG_C_0] != REG_NON) 1320 return priv->sh->flow_mreg_c 1321 [id + 1 + start_reg - REG_C_0]; 1322 return rte_flow_error_set(error, ENOTSUP, 1323 RTE_FLOW_ERROR_TYPE_ITEM, 1324 NULL, "unsupported tag id"); 1325 } 1326 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1327 } 1328 MLX5_ASSERT(false); 1329 return rte_flow_error_set(error, EINVAL, 1330 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1331 NULL, "invalid feature name"); 1332 } 1333 1334 /** 1335 * Check extensive flow metadata register support. 1336 * 1337 * @param dev 1338 * Pointer to rte_eth_dev structure. 1339 * 1340 * @return 1341 * True if device supports extensive flow metadata register, otherwise false. 1342 */ 1343 bool 1344 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1345 { 1346 struct mlx5_priv *priv = dev->data->dev_private; 1347 1348 /* 1349 * Having available reg_c can be regarded inclusively as supporting 1350 * extensive flow metadata register, which could mean, 1351 * - metadata register copy action by modify header. 1352 * - 16 modify header actions is supported. 1353 * - reg_c's are preserved across different domain (FDB and NIC) on 1354 * packet loopback by flow lookup miss. 1355 */ 1356 return priv->sh->flow_mreg_c[2] != REG_NON; 1357 } 1358 1359 /** 1360 * Get the lowest priority. 1361 * 1362 * @param[in] dev 1363 * Pointer to the Ethernet device structure. 1364 * @param[in] attributes 1365 * Pointer to device flow rule attributes. 1366 * 1367 * @return 1368 * The value of lowest priority of flow. 1369 */ 1370 uint32_t 1371 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1372 const struct rte_flow_attr *attr) 1373 { 1374 struct mlx5_priv *priv = dev->data->dev_private; 1375 1376 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1377 return priv->sh->flow_max_priority - 2; 1378 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1379 } 1380 1381 /** 1382 * Calculate matcher priority of the flow. 1383 * 1384 * @param[in] dev 1385 * Pointer to the Ethernet device structure. 1386 * @param[in] attr 1387 * Pointer to device flow rule attributes. 1388 * @param[in] subpriority 1389 * The priority based on the items. 1390 * @param[in] external 1391 * Flow is user flow. 1392 * @return 1393 * The matcher priority of the flow. 1394 */ 1395 uint16_t 1396 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1397 const struct rte_flow_attr *attr, 1398 uint32_t subpriority, bool external) 1399 { 1400 uint16_t priority = (uint16_t)attr->priority; 1401 struct mlx5_priv *priv = dev->data->dev_private; 1402 1403 /* NIC root rules */ 1404 if (!attr->group && !attr->transfer) { 1405 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1406 priority = priv->sh->flow_max_priority - 1; 1407 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1408 /* FDB root rules */ 1409 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1410 attr->group == 0 && 1411 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1412 return (priv->sh->flow_max_priority - 1) * 3; 1413 } 1414 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1415 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1416 return priority * 3 + subpriority; 1417 } 1418 1419 /** 1420 * Verify the @p item specifications (spec, last, mask) are compatible with the 1421 * NIC capabilities. 1422 * 1423 * @param[in] item 1424 * Item specification. 1425 * @param[in] mask 1426 * @p item->mask or flow default bit-masks. 1427 * @param[in] nic_mask 1428 * Bit-masks covering supported fields by the NIC to compare with user mask. 1429 * @param[in] size 1430 * Bit-masks size in bytes. 1431 * @param[in] range_accepted 1432 * True if range of values is accepted for specific fields, false otherwise. 1433 * @param[out] error 1434 * Pointer to error structure. 1435 * 1436 * @return 1437 * 0 on success, a negative errno value otherwise and rte_errno is set. 1438 */ 1439 int 1440 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1441 const uint8_t *mask, 1442 const uint8_t *nic_mask, 1443 unsigned int size, 1444 bool range_accepted, 1445 struct rte_flow_error *error) 1446 { 1447 unsigned int i; 1448 1449 MLX5_ASSERT(nic_mask); 1450 for (i = 0; i < size; ++i) 1451 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1452 return rte_flow_error_set(error, ENOTSUP, 1453 RTE_FLOW_ERROR_TYPE_ITEM, 1454 item, 1455 "mask enables non supported" 1456 " bits"); 1457 if (!item->spec && (item->mask || item->last)) 1458 return rte_flow_error_set(error, EINVAL, 1459 RTE_FLOW_ERROR_TYPE_ITEM, item, 1460 "mask/last without a spec is not" 1461 " supported"); 1462 if (item->spec && item->last && !range_accepted) { 1463 uint8_t spec[size]; 1464 uint8_t last[size]; 1465 unsigned int i; 1466 int ret; 1467 1468 for (i = 0; i < size; ++i) { 1469 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1470 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1471 } 1472 ret = memcmp(spec, last, size); 1473 if (ret != 0) 1474 return rte_flow_error_set(error, EINVAL, 1475 RTE_FLOW_ERROR_TYPE_ITEM, 1476 item, 1477 "range is not valid"); 1478 } 1479 return 0; 1480 } 1481 1482 /** 1483 * Adjust the hash fields according to the @p flow information. 1484 * 1485 * @param[in] dev_flow. 1486 * Pointer to the mlx5_flow. 1487 * @param[in] tunnel 1488 * 1 when the hash field is for a tunnel item. 1489 * @param[in] layer_types 1490 * RTE_ETH_RSS_* types. 1491 * @param[in] hash_fields 1492 * Item hash fields. 1493 * 1494 * @return 1495 * The hash fields that should be used. 1496 */ 1497 uint64_t 1498 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1499 int tunnel __rte_unused, uint64_t layer_types, 1500 uint64_t hash_fields) 1501 { 1502 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1503 int rss_request_inner = rss_desc->level >= 2; 1504 1505 /* Check RSS hash level for tunnel. */ 1506 if (tunnel && rss_request_inner) 1507 hash_fields |= IBV_RX_HASH_INNER; 1508 else if (tunnel || rss_request_inner) 1509 return 0; 1510 #endif 1511 /* Check if requested layer matches RSS hash fields. */ 1512 if (!(rss_desc->types & layer_types)) 1513 return 0; 1514 return hash_fields; 1515 } 1516 1517 /** 1518 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1519 * if several tunnel rules are used on this queue, the tunnel ptype will be 1520 * cleared. 1521 * 1522 * @param rxq_ctrl 1523 * Rx queue to update. 1524 */ 1525 static void 1526 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1527 { 1528 unsigned int i; 1529 uint32_t tunnel_ptype = 0; 1530 1531 /* Look up for the ptype to use. */ 1532 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1533 if (!rxq_ctrl->flow_tunnels_n[i]) 1534 continue; 1535 if (!tunnel_ptype) { 1536 tunnel_ptype = tunnels_info[i].ptype; 1537 } else { 1538 tunnel_ptype = 0; 1539 break; 1540 } 1541 } 1542 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1543 } 1544 1545 /** 1546 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1547 * flow. 1548 * 1549 * @param[in] dev 1550 * Pointer to the Ethernet device structure. 1551 * @param[in] dev_handle 1552 * Pointer to device flow handle structure. 1553 */ 1554 void 1555 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1556 struct mlx5_flow_handle *dev_handle) 1557 { 1558 struct mlx5_priv *priv = dev->data->dev_private; 1559 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1560 struct mlx5_ind_table_obj *ind_tbl = NULL; 1561 unsigned int i; 1562 1563 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1564 struct mlx5_hrxq *hrxq; 1565 1566 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1567 dev_handle->rix_hrxq); 1568 if (hrxq) 1569 ind_tbl = hrxq->ind_table; 1570 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1571 struct mlx5_shared_action_rss *shared_rss; 1572 1573 shared_rss = mlx5_ipool_get 1574 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1575 dev_handle->rix_srss); 1576 if (shared_rss) 1577 ind_tbl = shared_rss->ind_tbl; 1578 } 1579 if (!ind_tbl) 1580 return; 1581 for (i = 0; i != ind_tbl->queues_n; ++i) { 1582 int idx = ind_tbl->queues[i]; 1583 struct mlx5_rxq_ctrl *rxq_ctrl; 1584 1585 if (mlx5_is_external_rxq(dev, idx)) 1586 continue; 1587 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1588 MLX5_ASSERT(rxq_ctrl != NULL); 1589 if (rxq_ctrl == NULL) 1590 continue; 1591 /* 1592 * To support metadata register copy on Tx loopback, 1593 * this must be always enabled (metadata may arive 1594 * from other port - not from local flows only. 1595 */ 1596 if (tunnel) { 1597 unsigned int j; 1598 1599 /* Increase the counter matching the flow. */ 1600 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1601 if ((tunnels_info[j].tunnel & 1602 dev_handle->layers) == 1603 tunnels_info[j].tunnel) { 1604 rxq_ctrl->flow_tunnels_n[j]++; 1605 break; 1606 } 1607 } 1608 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1609 } 1610 } 1611 } 1612 1613 static void 1614 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1615 { 1616 struct mlx5_priv *priv = dev->data->dev_private; 1617 struct mlx5_rxq_ctrl *rxq_ctrl; 1618 uint16_t port_id; 1619 1620 if (priv->sh->shared_mark_enabled) 1621 return; 1622 if (priv->master || priv->representor) { 1623 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1624 struct mlx5_priv *opriv = 1625 rte_eth_devices[port_id].data->dev_private; 1626 1627 if (!opriv || 1628 opriv->sh != priv->sh || 1629 opriv->domain_id != priv->domain_id || 1630 opriv->mark_enabled) 1631 continue; 1632 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1633 rxq_ctrl->rxq.mark = 1; 1634 } 1635 opriv->mark_enabled = 1; 1636 } 1637 } else { 1638 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1639 rxq_ctrl->rxq.mark = 1; 1640 } 1641 priv->mark_enabled = 1; 1642 } 1643 priv->sh->shared_mark_enabled = 1; 1644 } 1645 1646 /** 1647 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1648 * 1649 * @param[in] dev 1650 * Pointer to the Ethernet device structure. 1651 * @param[in] flow 1652 * Pointer to flow structure. 1653 */ 1654 static void 1655 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1656 { 1657 struct mlx5_priv *priv = dev->data->dev_private; 1658 uint32_t handle_idx; 1659 struct mlx5_flow_handle *dev_handle; 1660 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1661 1662 MLX5_ASSERT(wks); 1663 if (wks->mark) 1664 flow_rxq_mark_flag_set(dev); 1665 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1666 handle_idx, dev_handle, next) 1667 flow_drv_rxq_flags_set(dev, dev_handle); 1668 } 1669 1670 /** 1671 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1672 * device flow if no other flow uses it with the same kind of request. 1673 * 1674 * @param dev 1675 * Pointer to Ethernet device. 1676 * @param[in] dev_handle 1677 * Pointer to the device flow handle structure. 1678 */ 1679 static void 1680 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1681 struct mlx5_flow_handle *dev_handle) 1682 { 1683 struct mlx5_priv *priv = dev->data->dev_private; 1684 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1685 struct mlx5_ind_table_obj *ind_tbl = NULL; 1686 unsigned int i; 1687 1688 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1689 struct mlx5_hrxq *hrxq; 1690 1691 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1692 dev_handle->rix_hrxq); 1693 if (hrxq) 1694 ind_tbl = hrxq->ind_table; 1695 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1696 struct mlx5_shared_action_rss *shared_rss; 1697 1698 shared_rss = mlx5_ipool_get 1699 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1700 dev_handle->rix_srss); 1701 if (shared_rss) 1702 ind_tbl = shared_rss->ind_tbl; 1703 } 1704 if (!ind_tbl) 1705 return; 1706 MLX5_ASSERT(dev->data->dev_started); 1707 for (i = 0; i != ind_tbl->queues_n; ++i) { 1708 int idx = ind_tbl->queues[i]; 1709 struct mlx5_rxq_ctrl *rxq_ctrl; 1710 1711 if (mlx5_is_external_rxq(dev, idx)) 1712 continue; 1713 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1714 MLX5_ASSERT(rxq_ctrl != NULL); 1715 if (rxq_ctrl == NULL) 1716 continue; 1717 if (tunnel) { 1718 unsigned int j; 1719 1720 /* Decrease the counter matching the flow. */ 1721 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1722 if ((tunnels_info[j].tunnel & 1723 dev_handle->layers) == 1724 tunnels_info[j].tunnel) { 1725 rxq_ctrl->flow_tunnels_n[j]--; 1726 break; 1727 } 1728 } 1729 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1730 } 1731 } 1732 } 1733 1734 /** 1735 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1736 * @p flow if no other flow uses it with the same kind of request. 1737 * 1738 * @param dev 1739 * Pointer to Ethernet device. 1740 * @param[in] flow 1741 * Pointer to the flow. 1742 */ 1743 static void 1744 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1745 { 1746 struct mlx5_priv *priv = dev->data->dev_private; 1747 uint32_t handle_idx; 1748 struct mlx5_flow_handle *dev_handle; 1749 1750 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1751 handle_idx, dev_handle, next) 1752 flow_drv_rxq_flags_trim(dev, dev_handle); 1753 } 1754 1755 /** 1756 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1757 * 1758 * @param dev 1759 * Pointer to Ethernet device. 1760 */ 1761 static void 1762 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1763 { 1764 struct mlx5_priv *priv = dev->data->dev_private; 1765 unsigned int i; 1766 1767 for (i = 0; i != priv->rxqs_n; ++i) { 1768 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1769 unsigned int j; 1770 1771 if (rxq == NULL || rxq->ctrl == NULL) 1772 continue; 1773 rxq->ctrl->rxq.mark = 0; 1774 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1775 rxq->ctrl->flow_tunnels_n[j] = 0; 1776 rxq->ctrl->rxq.tunnel = 0; 1777 } 1778 priv->mark_enabled = 0; 1779 priv->sh->shared_mark_enabled = 0; 1780 } 1781 1782 /** 1783 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1784 * 1785 * @param[in] dev 1786 * Pointer to the Ethernet device structure. 1787 */ 1788 void 1789 mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev) 1790 { 1791 struct mlx5_priv *priv = dev->data->dev_private; 1792 unsigned int i; 1793 1794 for (i = 0; i != priv->rxqs_n; ++i) { 1795 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1796 struct mlx5_rxq_data *data; 1797 1798 if (rxq == NULL || rxq->ctrl == NULL) 1799 continue; 1800 data = &rxq->ctrl->rxq; 1801 if (!rte_flow_dynf_metadata_avail()) { 1802 data->dynf_meta = 0; 1803 data->flow_meta_mask = 0; 1804 data->flow_meta_offset = -1; 1805 data->flow_meta_port_mask = 0; 1806 } else { 1807 data->dynf_meta = 1; 1808 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1809 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1810 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1811 } 1812 } 1813 } 1814 1815 /* 1816 * return a pointer to the desired action in the list of actions. 1817 * 1818 * @param[in] actions 1819 * The list of actions to search the action in. 1820 * @param[in] action 1821 * The action to find. 1822 * 1823 * @return 1824 * Pointer to the action in the list, if found. NULL otherwise. 1825 */ 1826 const struct rte_flow_action * 1827 mlx5_flow_find_action(const struct rte_flow_action *actions, 1828 enum rte_flow_action_type action) 1829 { 1830 if (actions == NULL) 1831 return NULL; 1832 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1833 if (actions->type == action) 1834 return actions; 1835 return NULL; 1836 } 1837 1838 /* 1839 * Validate the flag action. 1840 * 1841 * @param[in] action_flags 1842 * Bit-fields that holds the actions detected until now. 1843 * @param[in] attr 1844 * Attributes of flow that includes this action. 1845 * @param[out] error 1846 * Pointer to error structure. 1847 * 1848 * @return 1849 * 0 on success, a negative errno value otherwise and rte_errno is set. 1850 */ 1851 int 1852 mlx5_flow_validate_action_flag(uint64_t action_flags, 1853 const struct rte_flow_attr *attr, 1854 struct rte_flow_error *error) 1855 { 1856 if (action_flags & MLX5_FLOW_ACTION_MARK) 1857 return rte_flow_error_set(error, EINVAL, 1858 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1859 "can't mark and flag in same flow"); 1860 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1861 return rte_flow_error_set(error, EINVAL, 1862 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1863 "can't have 2 flag" 1864 " actions in same flow"); 1865 if (attr->egress) 1866 return rte_flow_error_set(error, ENOTSUP, 1867 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1868 "flag action not supported for " 1869 "egress"); 1870 return 0; 1871 } 1872 1873 /* 1874 * Validate the mark action. 1875 * 1876 * @param[in] action 1877 * Pointer to the queue action. 1878 * @param[in] action_flags 1879 * Bit-fields that holds the actions detected until now. 1880 * @param[in] attr 1881 * Attributes of flow that includes this action. 1882 * @param[out] error 1883 * Pointer to error structure. 1884 * 1885 * @return 1886 * 0 on success, a negative errno value otherwise and rte_errno is set. 1887 */ 1888 int 1889 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1890 uint64_t action_flags, 1891 const struct rte_flow_attr *attr, 1892 struct rte_flow_error *error) 1893 { 1894 const struct rte_flow_action_mark *mark = action->conf; 1895 1896 if (!mark) 1897 return rte_flow_error_set(error, EINVAL, 1898 RTE_FLOW_ERROR_TYPE_ACTION, 1899 action, 1900 "configuration cannot be null"); 1901 if (mark->id >= MLX5_FLOW_MARK_MAX) 1902 return rte_flow_error_set(error, EINVAL, 1903 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1904 &mark->id, 1905 "mark id must in 0 <= id < " 1906 RTE_STR(MLX5_FLOW_MARK_MAX)); 1907 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1908 return rte_flow_error_set(error, EINVAL, 1909 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1910 "can't flag and mark in same flow"); 1911 if (action_flags & MLX5_FLOW_ACTION_MARK) 1912 return rte_flow_error_set(error, EINVAL, 1913 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1914 "can't have 2 mark actions in same" 1915 " flow"); 1916 if (attr->egress) 1917 return rte_flow_error_set(error, ENOTSUP, 1918 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1919 "mark action not supported for " 1920 "egress"); 1921 return 0; 1922 } 1923 1924 /* 1925 * Validate the drop action. 1926 * 1927 * @param[in] action_flags 1928 * Bit-fields that holds the actions detected until now. 1929 * @param[in] attr 1930 * Attributes of flow that includes this action. 1931 * @param[out] error 1932 * Pointer to error structure. 1933 * 1934 * @return 1935 * 0 on success, a negative errno value otherwise and rte_errno is set. 1936 */ 1937 int 1938 mlx5_flow_validate_action_drop(uint64_t action_flags __rte_unused, 1939 const struct rte_flow_attr *attr, 1940 struct rte_flow_error *error) 1941 { 1942 if (attr->egress) 1943 return rte_flow_error_set(error, ENOTSUP, 1944 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1945 "drop action not supported for " 1946 "egress"); 1947 return 0; 1948 } 1949 1950 /* 1951 * Validate the queue action. 1952 * 1953 * @param[in] action 1954 * Pointer to the queue action. 1955 * @param[in] action_flags 1956 * Bit-fields that holds the actions detected until now. 1957 * @param[in] dev 1958 * Pointer to the Ethernet device structure. 1959 * @param[in] attr 1960 * Attributes of flow that includes this action. 1961 * @param[out] error 1962 * Pointer to error structure. 1963 * 1964 * @return 1965 * 0 on success, a negative errno value otherwise and rte_errno is set. 1966 */ 1967 int 1968 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 1969 uint64_t action_flags, 1970 struct rte_eth_dev *dev, 1971 const struct rte_flow_attr *attr, 1972 struct rte_flow_error *error) 1973 { 1974 struct mlx5_priv *priv = dev->data->dev_private; 1975 const struct rte_flow_action_queue *queue = action->conf; 1976 1977 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 1978 return rte_flow_error_set(error, EINVAL, 1979 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1980 "can't have 2 fate actions in" 1981 " same flow"); 1982 if (attr->egress) 1983 return rte_flow_error_set(error, ENOTSUP, 1984 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1985 "queue action not supported for egress."); 1986 if (mlx5_is_external_rxq(dev, queue->index)) 1987 return 0; 1988 if (!priv->rxqs_n) 1989 return rte_flow_error_set(error, EINVAL, 1990 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1991 NULL, "No Rx queues configured"); 1992 if (queue->index >= priv->rxqs_n) 1993 return rte_flow_error_set(error, EINVAL, 1994 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1995 &queue->index, 1996 "queue index out of range"); 1997 if (mlx5_rxq_get(dev, queue->index) == NULL) 1998 return rte_flow_error_set(error, EINVAL, 1999 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2000 &queue->index, 2001 "queue is not configured"); 2002 return 0; 2003 } 2004 2005 /** 2006 * Validate queue numbers for device RSS. 2007 * 2008 * @param[in] dev 2009 * Configured device. 2010 * @param[in] queues 2011 * Array of queue numbers. 2012 * @param[in] queues_n 2013 * Size of the @p queues array. 2014 * @param[out] error 2015 * On error, filled with a textual error description. 2016 * @param[out] queue_idx 2017 * On error, filled with an offending queue index in @p queues array. 2018 * 2019 * @return 2020 * 0 on success, a negative errno code on error. 2021 */ 2022 static int 2023 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2024 const uint16_t *queues, uint32_t queues_n, 2025 const char **error, uint32_t *queue_idx) 2026 { 2027 const struct mlx5_priv *priv = dev->data->dev_private; 2028 bool is_hairpin = false; 2029 bool is_ext_rss = false; 2030 uint32_t i; 2031 2032 for (i = 0; i != queues_n; ++i) { 2033 struct mlx5_rxq_ctrl *rxq_ctrl; 2034 2035 if (mlx5_is_external_rxq(dev, queues[0])) { 2036 is_ext_rss = true; 2037 continue; 2038 } 2039 if (is_ext_rss) { 2040 *error = "Combining external and regular RSS queues is not supported"; 2041 *queue_idx = i; 2042 return -ENOTSUP; 2043 } 2044 if (queues[i] >= priv->rxqs_n) { 2045 *error = "queue index out of range"; 2046 *queue_idx = i; 2047 return -EINVAL; 2048 } 2049 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2050 if (rxq_ctrl == NULL) { 2051 *error = "queue is not configured"; 2052 *queue_idx = i; 2053 return -EINVAL; 2054 } 2055 if (i == 0 && rxq_ctrl->is_hairpin) 2056 is_hairpin = true; 2057 if (is_hairpin != rxq_ctrl->is_hairpin) { 2058 *error = "combining hairpin and regular RSS queues is not supported"; 2059 *queue_idx = i; 2060 return -ENOTSUP; 2061 } 2062 } 2063 return 0; 2064 } 2065 2066 /* 2067 * Validate the rss action. 2068 * 2069 * @param[in] dev 2070 * Pointer to the Ethernet device structure. 2071 * @param[in] action 2072 * Pointer to the queue action. 2073 * @param[out] error 2074 * Pointer to error structure. 2075 * 2076 * @return 2077 * 0 on success, a negative errno value otherwise and rte_errno is set. 2078 */ 2079 int 2080 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2081 const struct rte_flow_action *action, 2082 struct rte_flow_error *error) 2083 { 2084 struct mlx5_priv *priv = dev->data->dev_private; 2085 const struct rte_flow_action_rss *rss = action->conf; 2086 int ret; 2087 const char *message; 2088 uint32_t queue_idx; 2089 2090 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2091 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2092 return rte_flow_error_set(error, ENOTSUP, 2093 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2094 &rss->func, 2095 "RSS hash function not supported"); 2096 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2097 if (rss->level > 2) 2098 #else 2099 if (rss->level > 1) 2100 #endif 2101 return rte_flow_error_set(error, ENOTSUP, 2102 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2103 &rss->level, 2104 "tunnel RSS is not supported"); 2105 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2106 if (rss->key_len == 0 && rss->key != NULL) 2107 return rte_flow_error_set(error, ENOTSUP, 2108 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2109 &rss->key_len, 2110 "RSS hash key length 0"); 2111 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2112 return rte_flow_error_set(error, ENOTSUP, 2113 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2114 &rss->key_len, 2115 "RSS hash key too small"); 2116 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2117 return rte_flow_error_set(error, ENOTSUP, 2118 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2119 &rss->key_len, 2120 "RSS hash key too large"); 2121 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2122 return rte_flow_error_set(error, ENOTSUP, 2123 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2124 &rss->queue_num, 2125 "number of queues too large"); 2126 if (rss->types & MLX5_RSS_HF_MASK) 2127 return rte_flow_error_set(error, ENOTSUP, 2128 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2129 &rss->types, 2130 "some RSS protocols are not" 2131 " supported"); 2132 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2133 !(rss->types & RTE_ETH_RSS_IP)) 2134 return rte_flow_error_set(error, EINVAL, 2135 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2136 "L3 partial RSS requested but L3 RSS" 2137 " type not specified"); 2138 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2139 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2140 return rte_flow_error_set(error, EINVAL, 2141 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2142 "L4 partial RSS requested but L4 RSS" 2143 " type not specified"); 2144 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2145 return rte_flow_error_set(error, EINVAL, 2146 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2147 NULL, "No Rx queues configured"); 2148 if (!rss->queue_num) 2149 return rte_flow_error_set(error, EINVAL, 2150 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2151 NULL, "No queues configured"); 2152 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2153 &message, &queue_idx); 2154 if (ret != 0) { 2155 return rte_flow_error_set(error, -ret, 2156 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2157 &rss->queue[queue_idx], message); 2158 } 2159 return 0; 2160 } 2161 2162 /* 2163 * Validate the rss action. 2164 * 2165 * @param[in] action 2166 * Pointer to the queue action. 2167 * @param[in] action_flags 2168 * Bit-fields that holds the actions detected until now. 2169 * @param[in] dev 2170 * Pointer to the Ethernet device structure. 2171 * @param[in] attr 2172 * Attributes of flow that includes this action. 2173 * @param[in] item_flags 2174 * Items that were detected. 2175 * @param[out] error 2176 * Pointer to error structure. 2177 * 2178 * @return 2179 * 0 on success, a negative errno value otherwise and rte_errno is set. 2180 */ 2181 int 2182 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2183 uint64_t action_flags, 2184 struct rte_eth_dev *dev, 2185 const struct rte_flow_attr *attr, 2186 uint64_t item_flags, 2187 struct rte_flow_error *error) 2188 { 2189 const struct rte_flow_action_rss *rss = action->conf; 2190 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2191 int ret; 2192 2193 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2194 return rte_flow_error_set(error, EINVAL, 2195 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2196 "can't have 2 fate actions" 2197 " in same flow"); 2198 ret = mlx5_validate_action_rss(dev, action, error); 2199 if (ret) 2200 return ret; 2201 if (attr->egress) 2202 return rte_flow_error_set(error, ENOTSUP, 2203 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2204 "rss action not supported for " 2205 "egress"); 2206 if (rss->level > 1 && !tunnel) 2207 return rte_flow_error_set(error, EINVAL, 2208 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2209 "inner RSS is not supported for " 2210 "non-tunnel flows"); 2211 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2212 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2213 return rte_flow_error_set(error, EINVAL, 2214 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2215 "RSS on eCPRI is not supported now"); 2216 } 2217 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2218 !(item_flags & 2219 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2220 rss->level > 1) 2221 return rte_flow_error_set(error, EINVAL, 2222 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2223 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2224 return 0; 2225 } 2226 2227 /* 2228 * Validate the default miss action. 2229 * 2230 * @param[in] action_flags 2231 * Bit-fields that holds the actions detected until now. 2232 * @param[out] error 2233 * Pointer to error structure. 2234 * 2235 * @return 2236 * 0 on success, a negative errno value otherwise and rte_errno is set. 2237 */ 2238 int 2239 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2240 const struct rte_flow_attr *attr, 2241 struct rte_flow_error *error) 2242 { 2243 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2244 return rte_flow_error_set(error, EINVAL, 2245 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2246 "can't have 2 fate actions in" 2247 " same flow"); 2248 if (attr->egress) 2249 return rte_flow_error_set(error, ENOTSUP, 2250 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2251 "default miss action not supported " 2252 "for egress"); 2253 if (attr->group) 2254 return rte_flow_error_set(error, ENOTSUP, 2255 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2256 "only group 0 is supported"); 2257 if (attr->transfer) 2258 return rte_flow_error_set(error, ENOTSUP, 2259 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2260 NULL, "transfer is not supported"); 2261 return 0; 2262 } 2263 2264 /* 2265 * Validate the count action. 2266 * 2267 * @param[in] dev 2268 * Pointer to the Ethernet device structure. 2269 * @param[in] attr 2270 * Attributes of flow that includes this action. 2271 * @param[out] error 2272 * Pointer to error structure. 2273 * 2274 * @return 2275 * 0 on success, a negative errno value otherwise and rte_errno is set. 2276 */ 2277 int 2278 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2279 const struct rte_flow_attr *attr, 2280 struct rte_flow_error *error) 2281 { 2282 if (attr->egress) 2283 return rte_flow_error_set(error, ENOTSUP, 2284 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2285 "count action not supported for " 2286 "egress"); 2287 return 0; 2288 } 2289 2290 /* 2291 * Validate the ASO CT action. 2292 * 2293 * @param[in] dev 2294 * Pointer to the Ethernet device structure. 2295 * @param[in] conntrack 2296 * Pointer to the CT action profile. 2297 * @param[out] error 2298 * Pointer to error structure. 2299 * 2300 * @return 2301 * 0 on success, a negative errno value otherwise and rte_errno is set. 2302 */ 2303 int 2304 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2305 const struct rte_flow_action_conntrack *conntrack, 2306 struct rte_flow_error *error) 2307 { 2308 RTE_SET_USED(dev); 2309 2310 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2311 return rte_flow_error_set(error, EINVAL, 2312 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2313 "Invalid CT state"); 2314 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2315 return rte_flow_error_set(error, EINVAL, 2316 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2317 "Invalid last TCP packet flag"); 2318 return 0; 2319 } 2320 2321 /** 2322 * Validate ICMP6 item. 2323 * 2324 * @param[in] item 2325 * Item specification. 2326 * @param[in] item_flags 2327 * Bit-fields that holds the items detected until now. 2328 * @param[in] ext_vlan_sup 2329 * Whether extended VLAN features are supported or not. 2330 * @param[out] error 2331 * Pointer to error structure. 2332 * 2333 * @return 2334 * 0 on success, a negative errno value otherwise and rte_errno is set. 2335 */ 2336 int 2337 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2338 uint64_t item_flags, 2339 uint8_t target_protocol, 2340 struct rte_flow_error *error) 2341 { 2342 const struct rte_flow_item_icmp6 *mask = item->mask; 2343 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2344 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2345 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2346 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2347 MLX5_FLOW_LAYER_OUTER_L4; 2348 int ret; 2349 2350 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2351 return rte_flow_error_set(error, EINVAL, 2352 RTE_FLOW_ERROR_TYPE_ITEM, item, 2353 "protocol filtering not compatible" 2354 " with ICMP6 layer"); 2355 if (!(item_flags & l3m)) 2356 return rte_flow_error_set(error, EINVAL, 2357 RTE_FLOW_ERROR_TYPE_ITEM, item, 2358 "IPv6 is mandatory to filter on" 2359 " ICMP6"); 2360 if (item_flags & l4m) 2361 return rte_flow_error_set(error, EINVAL, 2362 RTE_FLOW_ERROR_TYPE_ITEM, item, 2363 "multiple L4 layers not supported"); 2364 if (!mask) 2365 mask = &rte_flow_item_icmp6_mask; 2366 ret = mlx5_flow_item_acceptable 2367 (item, (const uint8_t *)mask, 2368 (const uint8_t *)&rte_flow_item_icmp6_mask, 2369 sizeof(struct rte_flow_item_icmp6), 2370 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2371 if (ret < 0) 2372 return ret; 2373 return 0; 2374 } 2375 2376 /** 2377 * Validate ICMP6 echo request/reply item. 2378 * 2379 * @param[in] item 2380 * Item specification. 2381 * @param[in] item_flags 2382 * Bit-fields that holds the items detected until now. 2383 * @param[in] ext_vlan_sup 2384 * Whether extended VLAN features are supported or not. 2385 * @param[out] error 2386 * Pointer to error structure. 2387 * 2388 * @return 2389 * 0 on success, a negative errno value otherwise and rte_errno is set. 2390 */ 2391 int 2392 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2393 uint64_t item_flags, 2394 uint8_t target_protocol, 2395 struct rte_flow_error *error) 2396 { 2397 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2398 const struct rte_flow_item_icmp6_echo nic_mask = { 2399 .hdr.base.type = 0xff, 2400 .hdr.base.code = 0xff, 2401 .hdr.identifier = RTE_BE16(0xffff), 2402 .hdr.sequence = RTE_BE16(0xffff), 2403 }; 2404 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2405 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2406 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2407 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2408 MLX5_FLOW_LAYER_OUTER_L4; 2409 int ret; 2410 2411 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2412 return rte_flow_error_set(error, EINVAL, 2413 RTE_FLOW_ERROR_TYPE_ITEM, item, 2414 "protocol filtering not compatible" 2415 " with ICMP6 layer"); 2416 if (!(item_flags & l3m)) 2417 return rte_flow_error_set(error, EINVAL, 2418 RTE_FLOW_ERROR_TYPE_ITEM, item, 2419 "IPv6 is mandatory to filter on" 2420 " ICMP6"); 2421 if (item_flags & l4m) 2422 return rte_flow_error_set(error, EINVAL, 2423 RTE_FLOW_ERROR_TYPE_ITEM, item, 2424 "multiple L4 layers not supported"); 2425 if (!mask) 2426 mask = &nic_mask; 2427 ret = mlx5_flow_item_acceptable 2428 (item, (const uint8_t *)mask, 2429 (const uint8_t *)&nic_mask, 2430 sizeof(struct rte_flow_item_icmp6_echo), 2431 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2432 if (ret < 0) 2433 return ret; 2434 return 0; 2435 } 2436 2437 /** 2438 * Validate ICMP item. 2439 * 2440 * @param[in] item 2441 * Item specification. 2442 * @param[in] item_flags 2443 * Bit-fields that holds the items detected until now. 2444 * @param[out] error 2445 * Pointer to error structure. 2446 * 2447 * @return 2448 * 0 on success, a negative errno value otherwise and rte_errno is set. 2449 */ 2450 int 2451 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2452 uint64_t item_flags, 2453 uint8_t target_protocol, 2454 struct rte_flow_error *error) 2455 { 2456 const struct rte_flow_item_icmp *mask = item->mask; 2457 const struct rte_flow_item_icmp nic_mask = { 2458 .hdr.icmp_type = 0xff, 2459 .hdr.icmp_code = 0xff, 2460 .hdr.icmp_ident = RTE_BE16(0xffff), 2461 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2462 }; 2463 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2464 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2465 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2466 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2467 MLX5_FLOW_LAYER_OUTER_L4; 2468 int ret; 2469 2470 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2471 return rte_flow_error_set(error, EINVAL, 2472 RTE_FLOW_ERROR_TYPE_ITEM, item, 2473 "protocol filtering not compatible" 2474 " with ICMP layer"); 2475 if (!(item_flags & l3m)) 2476 return rte_flow_error_set(error, EINVAL, 2477 RTE_FLOW_ERROR_TYPE_ITEM, item, 2478 "IPv4 is mandatory to filter" 2479 " on ICMP"); 2480 if (item_flags & l4m) 2481 return rte_flow_error_set(error, EINVAL, 2482 RTE_FLOW_ERROR_TYPE_ITEM, item, 2483 "multiple L4 layers not supported"); 2484 if (!mask) 2485 mask = &nic_mask; 2486 ret = mlx5_flow_item_acceptable 2487 (item, (const uint8_t *)mask, 2488 (const uint8_t *)&nic_mask, 2489 sizeof(struct rte_flow_item_icmp), 2490 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2491 if (ret < 0) 2492 return ret; 2493 return 0; 2494 } 2495 2496 /** 2497 * Validate Ethernet item. 2498 * 2499 * @param[in] item 2500 * Item specification. 2501 * @param[in] item_flags 2502 * Bit-fields that holds the items detected until now. 2503 * @param[out] error 2504 * Pointer to error structure. 2505 * 2506 * @return 2507 * 0 on success, a negative errno value otherwise and rte_errno is set. 2508 */ 2509 int 2510 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2511 uint64_t item_flags, bool ext_vlan_sup, 2512 struct rte_flow_error *error) 2513 { 2514 const struct rte_flow_item_eth *mask = item->mask; 2515 const struct rte_flow_item_eth nic_mask = { 2516 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2517 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2518 .hdr.ether_type = RTE_BE16(0xffff), 2519 .has_vlan = ext_vlan_sup ? 1 : 0, 2520 }; 2521 int ret; 2522 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2523 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2524 MLX5_FLOW_LAYER_OUTER_L2; 2525 2526 if (item_flags & ethm) 2527 return rte_flow_error_set(error, ENOTSUP, 2528 RTE_FLOW_ERROR_TYPE_ITEM, item, 2529 "multiple L2 layers not supported"); 2530 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2531 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2532 return rte_flow_error_set(error, EINVAL, 2533 RTE_FLOW_ERROR_TYPE_ITEM, item, 2534 "L2 layer should not follow " 2535 "L3 layers"); 2536 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2537 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2538 return rte_flow_error_set(error, EINVAL, 2539 RTE_FLOW_ERROR_TYPE_ITEM, item, 2540 "L2 layer should not follow VLAN"); 2541 if (item_flags & MLX5_FLOW_LAYER_GTP) 2542 return rte_flow_error_set(error, EINVAL, 2543 RTE_FLOW_ERROR_TYPE_ITEM, item, 2544 "L2 layer should not follow GTP"); 2545 if (!mask) 2546 mask = &rte_flow_item_eth_mask; 2547 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2548 (const uint8_t *)&nic_mask, 2549 sizeof(struct rte_flow_item_eth), 2550 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2551 return ret; 2552 } 2553 2554 /** 2555 * Validate VLAN item. 2556 * 2557 * @param[in] item 2558 * Item specification. 2559 * @param[in] item_flags 2560 * Bit-fields that holds the items detected until now. 2561 * @param[in] dev 2562 * Ethernet device flow is being created on. 2563 * @param[out] error 2564 * Pointer to error structure. 2565 * 2566 * @return 2567 * 0 on success, a negative errno value otherwise and rte_errno is set. 2568 */ 2569 int 2570 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2571 uint64_t item_flags, 2572 struct rte_eth_dev *dev, 2573 struct rte_flow_error *error) 2574 { 2575 const struct rte_flow_item_vlan *spec = item->spec; 2576 const struct rte_flow_item_vlan *mask = item->mask; 2577 const struct rte_flow_item_vlan nic_mask = { 2578 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2579 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2580 }; 2581 uint16_t vlan_tag = 0; 2582 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2583 int ret; 2584 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2585 MLX5_FLOW_LAYER_INNER_L4) : 2586 (MLX5_FLOW_LAYER_OUTER_L3 | 2587 MLX5_FLOW_LAYER_OUTER_L4); 2588 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2589 MLX5_FLOW_LAYER_OUTER_VLAN; 2590 2591 if (item_flags & vlanm) 2592 return rte_flow_error_set(error, EINVAL, 2593 RTE_FLOW_ERROR_TYPE_ITEM, item, 2594 "multiple VLAN layers not supported"); 2595 else if ((item_flags & l34m) != 0) 2596 return rte_flow_error_set(error, EINVAL, 2597 RTE_FLOW_ERROR_TYPE_ITEM, item, 2598 "VLAN cannot follow L3/L4 layer"); 2599 if (!mask) 2600 mask = &rte_flow_item_vlan_mask; 2601 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2602 (const uint8_t *)&nic_mask, 2603 sizeof(struct rte_flow_item_vlan), 2604 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2605 if (ret) 2606 return ret; 2607 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2608 struct mlx5_priv *priv = dev->data->dev_private; 2609 2610 if (priv->vmwa_context) { 2611 /* 2612 * Non-NULL context means we have a virtual machine 2613 * and SR-IOV enabled, we have to create VLAN interface 2614 * to make hypervisor to setup E-Switch vport 2615 * context correctly. We avoid creating the multiple 2616 * VLAN interfaces, so we cannot support VLAN tag mask. 2617 */ 2618 return rte_flow_error_set(error, EINVAL, 2619 RTE_FLOW_ERROR_TYPE_ITEM, 2620 item, 2621 "VLAN tag mask is not" 2622 " supported in virtual" 2623 " environment"); 2624 } 2625 } 2626 if (spec) { 2627 vlan_tag = spec->hdr.vlan_tci; 2628 vlan_tag &= mask->hdr.vlan_tci; 2629 } 2630 /* 2631 * From verbs perspective an empty VLAN is equivalent 2632 * to a packet without VLAN layer. 2633 */ 2634 if (!vlan_tag) 2635 return rte_flow_error_set(error, EINVAL, 2636 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2637 item->spec, 2638 "VLAN cannot be empty"); 2639 return 0; 2640 } 2641 2642 /** 2643 * Validate IPV4 item. 2644 * 2645 * @param[in] item 2646 * Item specification. 2647 * @param[in] item_flags 2648 * Bit-fields that holds the items detected until now. 2649 * @param[in] last_item 2650 * Previous validated item in the pattern items. 2651 * @param[in] ether_type 2652 * Type in the ethernet layer header (including dot1q). 2653 * @param[in] acc_mask 2654 * Acceptable mask, if NULL default internal default mask 2655 * will be used to check whether item fields are supported. 2656 * @param[in] range_accepted 2657 * True if range of values is accepted for specific fields, false otherwise. 2658 * @param[out] error 2659 * Pointer to error structure. 2660 * 2661 * @return 2662 * 0 on success, a negative errno value otherwise and rte_errno is set. 2663 */ 2664 int 2665 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2666 uint64_t item_flags, 2667 uint64_t last_item, 2668 uint16_t ether_type, 2669 const struct rte_flow_item_ipv4 *acc_mask, 2670 bool range_accepted, 2671 struct rte_flow_error *error) 2672 { 2673 const struct rte_flow_item_ipv4 *mask = item->mask; 2674 const struct rte_flow_item_ipv4 *spec = item->spec; 2675 const struct rte_flow_item_ipv4 nic_mask = { 2676 .hdr = { 2677 .src_addr = RTE_BE32(0xffffffff), 2678 .dst_addr = RTE_BE32(0xffffffff), 2679 .type_of_service = 0xff, 2680 .next_proto_id = 0xff, 2681 }, 2682 }; 2683 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2684 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2685 MLX5_FLOW_LAYER_OUTER_L3; 2686 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2687 MLX5_FLOW_LAYER_OUTER_L4; 2688 int ret; 2689 uint8_t next_proto = 0xFF; 2690 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2691 MLX5_FLOW_LAYER_OUTER_VLAN | 2692 MLX5_FLOW_LAYER_INNER_VLAN); 2693 2694 if ((last_item & l2_vlan) && ether_type && 2695 ether_type != RTE_ETHER_TYPE_IPV4) 2696 return rte_flow_error_set(error, EINVAL, 2697 RTE_FLOW_ERROR_TYPE_ITEM, item, 2698 "IPv4 cannot follow L2/VLAN layer " 2699 "which ether type is not IPv4"); 2700 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2701 if (mask && spec) 2702 next_proto = mask->hdr.next_proto_id & 2703 spec->hdr.next_proto_id; 2704 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2705 return rte_flow_error_set(error, EINVAL, 2706 RTE_FLOW_ERROR_TYPE_ITEM, 2707 item, 2708 "multiple tunnel " 2709 "not supported"); 2710 } 2711 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2712 return rte_flow_error_set(error, EINVAL, 2713 RTE_FLOW_ERROR_TYPE_ITEM, item, 2714 "wrong tunnel type - IPv6 specified " 2715 "but IPv4 item provided"); 2716 if (item_flags & l3m) 2717 return rte_flow_error_set(error, ENOTSUP, 2718 RTE_FLOW_ERROR_TYPE_ITEM, item, 2719 "multiple L3 layers not supported"); 2720 else if (item_flags & l4m) 2721 return rte_flow_error_set(error, EINVAL, 2722 RTE_FLOW_ERROR_TYPE_ITEM, item, 2723 "L3 cannot follow an L4 layer."); 2724 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2725 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2726 return rte_flow_error_set(error, EINVAL, 2727 RTE_FLOW_ERROR_TYPE_ITEM, item, 2728 "L3 cannot follow an NVGRE layer."); 2729 if (!mask) 2730 mask = &rte_flow_item_ipv4_mask; 2731 else if (mask->hdr.next_proto_id != 0 && 2732 mask->hdr.next_proto_id != 0xff) 2733 return rte_flow_error_set(error, EINVAL, 2734 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2735 "partial mask is not supported" 2736 " for protocol"); 2737 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2738 acc_mask ? (const uint8_t *)acc_mask 2739 : (const uint8_t *)&nic_mask, 2740 sizeof(struct rte_flow_item_ipv4), 2741 range_accepted, error); 2742 if (ret < 0) 2743 return ret; 2744 return 0; 2745 } 2746 2747 /** 2748 * Validate IPV6 item. 2749 * 2750 * @param[in] item 2751 * Item specification. 2752 * @param[in] item_flags 2753 * Bit-fields that holds the items detected until now. 2754 * @param[in] last_item 2755 * Previous validated item in the pattern items. 2756 * @param[in] ether_type 2757 * Type in the ethernet layer header (including dot1q). 2758 * @param[in] acc_mask 2759 * Acceptable mask, if NULL default internal default mask 2760 * will be used to check whether item fields are supported. 2761 * @param[out] error 2762 * Pointer to error structure. 2763 * 2764 * @return 2765 * 0 on success, a negative errno value otherwise and rte_errno is set. 2766 */ 2767 int 2768 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2769 uint64_t item_flags, 2770 uint64_t last_item, 2771 uint16_t ether_type, 2772 const struct rte_flow_item_ipv6 *acc_mask, 2773 struct rte_flow_error *error) 2774 { 2775 const struct rte_flow_item_ipv6 *mask = item->mask; 2776 const struct rte_flow_item_ipv6 *spec = item->spec; 2777 const struct rte_flow_item_ipv6 nic_mask = { 2778 .hdr = { 2779 .src_addr = 2780 "\xff\xff\xff\xff\xff\xff\xff\xff" 2781 "\xff\xff\xff\xff\xff\xff\xff\xff", 2782 .dst_addr = 2783 "\xff\xff\xff\xff\xff\xff\xff\xff" 2784 "\xff\xff\xff\xff\xff\xff\xff\xff", 2785 .vtc_flow = RTE_BE32(0xffffffff), 2786 .proto = 0xff, 2787 }, 2788 }; 2789 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2790 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2791 MLX5_FLOW_LAYER_OUTER_L3; 2792 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2793 MLX5_FLOW_LAYER_OUTER_L4; 2794 int ret; 2795 uint8_t next_proto = 0xFF; 2796 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2797 MLX5_FLOW_LAYER_OUTER_VLAN | 2798 MLX5_FLOW_LAYER_INNER_VLAN); 2799 2800 if ((last_item & l2_vlan) && ether_type && 2801 ether_type != RTE_ETHER_TYPE_IPV6) 2802 return rte_flow_error_set(error, EINVAL, 2803 RTE_FLOW_ERROR_TYPE_ITEM, item, 2804 "IPv6 cannot follow L2/VLAN layer " 2805 "which ether type is not IPv6"); 2806 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2807 next_proto = spec->hdr.proto; 2808 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2809 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2810 return rte_flow_error_set(error, EINVAL, 2811 RTE_FLOW_ERROR_TYPE_ITEM, 2812 item, 2813 "multiple tunnel " 2814 "not supported"); 2815 } 2816 if (next_proto == IPPROTO_HOPOPTS || 2817 next_proto == IPPROTO_ROUTING || 2818 next_proto == IPPROTO_FRAGMENT || 2819 next_proto == IPPROTO_ESP || 2820 next_proto == IPPROTO_AH || 2821 next_proto == IPPROTO_DSTOPTS) 2822 return rte_flow_error_set(error, EINVAL, 2823 RTE_FLOW_ERROR_TYPE_ITEM, item, 2824 "IPv6 proto (next header) should " 2825 "not be set as extension header"); 2826 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2827 return rte_flow_error_set(error, EINVAL, 2828 RTE_FLOW_ERROR_TYPE_ITEM, item, 2829 "wrong tunnel type - IPv4 specified " 2830 "but IPv6 item provided"); 2831 if (item_flags & l3m) 2832 return rte_flow_error_set(error, ENOTSUP, 2833 RTE_FLOW_ERROR_TYPE_ITEM, item, 2834 "multiple L3 layers not supported"); 2835 else if (item_flags & l4m) 2836 return rte_flow_error_set(error, EINVAL, 2837 RTE_FLOW_ERROR_TYPE_ITEM, item, 2838 "L3 cannot follow an L4 layer."); 2839 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2840 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2841 return rte_flow_error_set(error, EINVAL, 2842 RTE_FLOW_ERROR_TYPE_ITEM, item, 2843 "L3 cannot follow an NVGRE layer."); 2844 if (!mask) 2845 mask = &rte_flow_item_ipv6_mask; 2846 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2847 acc_mask ? (const uint8_t *)acc_mask 2848 : (const uint8_t *)&nic_mask, 2849 sizeof(struct rte_flow_item_ipv6), 2850 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2851 if (ret < 0) 2852 return ret; 2853 return 0; 2854 } 2855 2856 /** 2857 * Validate UDP item. 2858 * 2859 * @param[in] item 2860 * Item specification. 2861 * @param[in] item_flags 2862 * Bit-fields that holds the items detected until now. 2863 * @param[in] target_protocol 2864 * The next protocol in the previous item. 2865 * @param[in] flow_mask 2866 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2867 * @param[out] error 2868 * Pointer to error structure. 2869 * 2870 * @return 2871 * 0 on success, a negative errno value otherwise and rte_errno is set. 2872 */ 2873 int 2874 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2875 uint64_t item_flags, 2876 uint8_t target_protocol, 2877 struct rte_flow_error *error) 2878 { 2879 const struct rte_flow_item_udp *mask = item->mask; 2880 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2881 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2882 MLX5_FLOW_LAYER_OUTER_L3; 2883 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2884 MLX5_FLOW_LAYER_OUTER_L4; 2885 int ret; 2886 2887 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 2888 return rte_flow_error_set(error, EINVAL, 2889 RTE_FLOW_ERROR_TYPE_ITEM, item, 2890 "protocol filtering not compatible" 2891 " with UDP layer"); 2892 if (!(item_flags & l3m)) 2893 return rte_flow_error_set(error, EINVAL, 2894 RTE_FLOW_ERROR_TYPE_ITEM, item, 2895 "L3 is mandatory to filter on L4"); 2896 if (item_flags & l4m) 2897 return rte_flow_error_set(error, EINVAL, 2898 RTE_FLOW_ERROR_TYPE_ITEM, item, 2899 "multiple L4 layers not supported"); 2900 if (!mask) 2901 mask = &rte_flow_item_udp_mask; 2902 ret = mlx5_flow_item_acceptable 2903 (item, (const uint8_t *)mask, 2904 (const uint8_t *)&rte_flow_item_udp_mask, 2905 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2906 error); 2907 if (ret < 0) 2908 return ret; 2909 return 0; 2910 } 2911 2912 /** 2913 * Validate TCP item. 2914 * 2915 * @param[in] item 2916 * Item specification. 2917 * @param[in] item_flags 2918 * Bit-fields that holds the items detected until now. 2919 * @param[in] target_protocol 2920 * The next protocol in the previous item. 2921 * @param[out] error 2922 * Pointer to error structure. 2923 * 2924 * @return 2925 * 0 on success, a negative errno value otherwise and rte_errno is set. 2926 */ 2927 int 2928 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 2929 uint64_t item_flags, 2930 uint8_t target_protocol, 2931 const struct rte_flow_item_tcp *flow_mask, 2932 struct rte_flow_error *error) 2933 { 2934 const struct rte_flow_item_tcp *mask = item->mask; 2935 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2936 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2937 MLX5_FLOW_LAYER_OUTER_L3; 2938 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2939 MLX5_FLOW_LAYER_OUTER_L4; 2940 int ret; 2941 2942 MLX5_ASSERT(flow_mask); 2943 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 2944 return rte_flow_error_set(error, EINVAL, 2945 RTE_FLOW_ERROR_TYPE_ITEM, item, 2946 "protocol filtering not compatible" 2947 " with TCP layer"); 2948 if (!(item_flags & l3m)) 2949 return rte_flow_error_set(error, EINVAL, 2950 RTE_FLOW_ERROR_TYPE_ITEM, item, 2951 "L3 is mandatory to filter on L4"); 2952 if (item_flags & l4m) 2953 return rte_flow_error_set(error, EINVAL, 2954 RTE_FLOW_ERROR_TYPE_ITEM, item, 2955 "multiple L4 layers not supported"); 2956 if (!mask) 2957 mask = &rte_flow_item_tcp_mask; 2958 ret = mlx5_flow_item_acceptable 2959 (item, (const uint8_t *)mask, 2960 (const uint8_t *)flow_mask, 2961 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 2962 error); 2963 if (ret < 0) 2964 return ret; 2965 return 0; 2966 } 2967 2968 /** 2969 * Validate VXLAN item. 2970 * 2971 * @param[in] dev 2972 * Pointer to the Ethernet device structure. 2973 * @param[in] udp_dport 2974 * UDP destination port 2975 * @param[in] item 2976 * Item specification. 2977 * @param[in] item_flags 2978 * Bit-fields that holds the items detected until now. 2979 * @param root 2980 * Whether action is on root table. 2981 * @param[out] error 2982 * Pointer to error structure. 2983 * 2984 * @return 2985 * 0 on success, a negative errno value otherwise and rte_errno is set. 2986 */ 2987 int 2988 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 2989 uint16_t udp_dport, 2990 const struct rte_flow_item *item, 2991 uint64_t item_flags, 2992 bool root, 2993 struct rte_flow_error *error) 2994 { 2995 const struct rte_flow_item_vxlan *spec = item->spec; 2996 const struct rte_flow_item_vxlan *mask = item->mask; 2997 int ret; 2998 struct mlx5_priv *priv = dev->data->dev_private; 2999 union vni { 3000 uint32_t vlan_id; 3001 uint8_t vni[4]; 3002 } id = { .vlan_id = 0, }; 3003 const struct rte_flow_item_vxlan nic_mask = { 3004 .hdr.vni = "\xff\xff\xff", 3005 .hdr.rsvd1 = 0xff, 3006 }; 3007 const struct rte_flow_item_vxlan *valid_mask; 3008 3009 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3010 return rte_flow_error_set(error, ENOTSUP, 3011 RTE_FLOW_ERROR_TYPE_ITEM, item, 3012 "multiple tunnel layers not" 3013 " supported"); 3014 valid_mask = &rte_flow_item_vxlan_mask; 3015 /* 3016 * Verify only UDPv4 is present as defined in 3017 * https://tools.ietf.org/html/rfc7348 3018 */ 3019 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3020 return rte_flow_error_set(error, EINVAL, 3021 RTE_FLOW_ERROR_TYPE_ITEM, item, 3022 "no outer UDP layer found"); 3023 if (!mask) 3024 mask = &rte_flow_item_vxlan_mask; 3025 3026 if (priv->sh->steering_format_version != 3027 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3028 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3029 /* non-root table */ 3030 if (!root && priv->sh->misc5_cap) 3031 valid_mask = &nic_mask; 3032 /* Group zero in NIC domain */ 3033 if (!root && priv->sh->tunnel_header_0_1) 3034 valid_mask = &nic_mask; 3035 } 3036 ret = mlx5_flow_item_acceptable 3037 (item, (const uint8_t *)mask, 3038 (const uint8_t *)valid_mask, 3039 sizeof(struct rte_flow_item_vxlan), 3040 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3041 if (ret < 0) 3042 return ret; 3043 if (spec) { 3044 memcpy(&id.vni[1], spec->hdr.vni, 3); 3045 memcpy(&id.vni[1], mask->hdr.vni, 3); 3046 } 3047 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3048 return rte_flow_error_set(error, ENOTSUP, 3049 RTE_FLOW_ERROR_TYPE_ITEM, item, 3050 "VXLAN tunnel must be fully defined"); 3051 return 0; 3052 } 3053 3054 /** 3055 * Validate VXLAN_GPE item. 3056 * 3057 * @param[in] item 3058 * Item specification. 3059 * @param[in] item_flags 3060 * Bit-fields that holds the items detected until now. 3061 * @param[in] priv 3062 * Pointer to the private data structure. 3063 * @param[in] target_protocol 3064 * The next protocol in the previous item. 3065 * @param[out] error 3066 * Pointer to error structure. 3067 * 3068 * @return 3069 * 0 on success, a negative errno value otherwise and rte_errno is set. 3070 */ 3071 int 3072 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3073 uint64_t item_flags, 3074 struct rte_eth_dev *dev, 3075 struct rte_flow_error *error) 3076 { 3077 struct mlx5_priv *priv = dev->data->dev_private; 3078 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3079 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3080 int ret; 3081 union vni { 3082 uint32_t vlan_id; 3083 uint8_t vni[4]; 3084 } id = { .vlan_id = 0, }; 3085 3086 if (!priv->sh->config.l3_vxlan_en) 3087 return rte_flow_error_set(error, ENOTSUP, 3088 RTE_FLOW_ERROR_TYPE_ITEM, item, 3089 "L3 VXLAN is not enabled by device" 3090 " parameter and/or not configured in" 3091 " firmware"); 3092 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3093 return rte_flow_error_set(error, ENOTSUP, 3094 RTE_FLOW_ERROR_TYPE_ITEM, item, 3095 "multiple tunnel layers not" 3096 " supported"); 3097 /* 3098 * Verify only UDPv4 is present as defined in 3099 * https://tools.ietf.org/html/rfc7348 3100 */ 3101 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3102 return rte_flow_error_set(error, EINVAL, 3103 RTE_FLOW_ERROR_TYPE_ITEM, item, 3104 "no outer UDP layer found"); 3105 if (!mask) 3106 mask = &rte_flow_item_vxlan_gpe_mask; 3107 ret = mlx5_flow_item_acceptable 3108 (item, (const uint8_t *)mask, 3109 (const uint8_t *)&rte_flow_item_vxlan_gpe_mask, 3110 sizeof(struct rte_flow_item_vxlan_gpe), 3111 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3112 if (ret < 0) 3113 return ret; 3114 if (spec) { 3115 if (spec->hdr.proto) 3116 return rte_flow_error_set(error, ENOTSUP, 3117 RTE_FLOW_ERROR_TYPE_ITEM, 3118 item, 3119 "VxLAN-GPE protocol" 3120 " not supported"); 3121 memcpy(&id.vni[1], spec->hdr.vni, 3); 3122 memcpy(&id.vni[1], mask->hdr.vni, 3); 3123 } 3124 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3125 return rte_flow_error_set(error, ENOTSUP, 3126 RTE_FLOW_ERROR_TYPE_ITEM, item, 3127 "VXLAN-GPE tunnel must be fully" 3128 " defined"); 3129 return 0; 3130 } 3131 /** 3132 * Validate GRE Key item. 3133 * 3134 * @param[in] item 3135 * Item specification. 3136 * @param[in] item_flags 3137 * Bit flags to mark detected items. 3138 * @param[in] gre_item 3139 * Pointer to gre_item 3140 * @param[out] error 3141 * Pointer to error structure. 3142 * 3143 * @return 3144 * 0 on success, a negative errno value otherwise and rte_errno is set. 3145 */ 3146 int 3147 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3148 uint64_t item_flags, 3149 const struct rte_flow_item *gre_item, 3150 struct rte_flow_error *error) 3151 { 3152 const rte_be32_t *mask = item->mask; 3153 int ret = 0; 3154 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3155 const struct rte_flow_item_gre *gre_spec; 3156 const struct rte_flow_item_gre *gre_mask; 3157 3158 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3159 return rte_flow_error_set(error, ENOTSUP, 3160 RTE_FLOW_ERROR_TYPE_ITEM, item, 3161 "Multiple GRE key not support"); 3162 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3163 return rte_flow_error_set(error, ENOTSUP, 3164 RTE_FLOW_ERROR_TYPE_ITEM, item, 3165 "No preceding GRE header"); 3166 if (item_flags & MLX5_FLOW_LAYER_INNER) 3167 return rte_flow_error_set(error, ENOTSUP, 3168 RTE_FLOW_ERROR_TYPE_ITEM, item, 3169 "GRE key following a wrong item"); 3170 gre_mask = gre_item->mask; 3171 if (!gre_mask) 3172 gre_mask = &rte_flow_item_gre_mask; 3173 gre_spec = gre_item->spec; 3174 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3175 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3176 return rte_flow_error_set(error, EINVAL, 3177 RTE_FLOW_ERROR_TYPE_ITEM, item, 3178 "Key bit must be on"); 3179 3180 if (!mask) 3181 mask = &gre_key_default_mask; 3182 ret = mlx5_flow_item_acceptable 3183 (item, (const uint8_t *)mask, 3184 (const uint8_t *)&gre_key_default_mask, 3185 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3186 return ret; 3187 } 3188 3189 /** 3190 * Validate GRE optional item. 3191 * 3192 * @param[in] dev 3193 * Pointer to the Ethernet device structure. 3194 * @param[in] item 3195 * Item specification. 3196 * @param[in] item_flags 3197 * Bit flags to mark detected items. 3198 * @param[in] attr 3199 * Flow rule attributes. 3200 * @param[in] gre_item 3201 * Pointer to gre_item 3202 * @param[out] error 3203 * Pointer to error structure. 3204 * 3205 * @return 3206 * 0 on success, a negative errno value otherwise and rte_errno is set. 3207 */ 3208 int 3209 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3210 const struct rte_flow_item *item, 3211 uint64_t item_flags, 3212 const struct rte_flow_attr *attr, 3213 const struct rte_flow_item *gre_item, 3214 struct rte_flow_error *error) 3215 { 3216 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3217 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3218 const struct rte_flow_item_gre_opt *spec = item->spec; 3219 const struct rte_flow_item_gre_opt *mask = item->mask; 3220 struct mlx5_priv *priv = dev->data->dev_private; 3221 int ret = 0; 3222 struct rte_flow_item_gre_opt nic_mask = { 3223 .checksum_rsvd = { 3224 .checksum = RTE_BE16(UINT16_MAX), 3225 .reserved1 = 0x0, 3226 }, 3227 .key = { 3228 .key = RTE_BE32(UINT32_MAX), 3229 }, 3230 .sequence = { 3231 .sequence = RTE_BE32(UINT32_MAX), 3232 }, 3233 }; 3234 3235 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3236 return rte_flow_error_set(error, ENOTSUP, 3237 RTE_FLOW_ERROR_TYPE_ITEM, item, 3238 "No preceding GRE header"); 3239 if (item_flags & MLX5_FLOW_LAYER_INNER) 3240 return rte_flow_error_set(error, ENOTSUP, 3241 RTE_FLOW_ERROR_TYPE_ITEM, item, 3242 "GRE option following a wrong item"); 3243 if (!spec || !mask) 3244 return rte_flow_error_set(error, EINVAL, 3245 RTE_FLOW_ERROR_TYPE_ITEM, item, 3246 "At least one field gre_option(checksum/key/sequence) must be specified"); 3247 if (!gre_mask) 3248 gre_mask = &rte_flow_item_gre_mask; 3249 if (mask->checksum_rsvd.checksum) 3250 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3251 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3252 return rte_flow_error_set(error, EINVAL, 3253 RTE_FLOW_ERROR_TYPE_ITEM, 3254 item, 3255 "Checksum bit must be on"); 3256 if (mask->key.key) 3257 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3258 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3259 return rte_flow_error_set(error, EINVAL, 3260 RTE_FLOW_ERROR_TYPE_ITEM, 3261 item, "Key bit must be on"); 3262 if (mask->sequence.sequence) 3263 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3264 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3265 return rte_flow_error_set(error, EINVAL, 3266 RTE_FLOW_ERROR_TYPE_ITEM, 3267 item, 3268 "Sequence bit must be on"); 3269 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3270 if (priv->sh->steering_format_version == 3271 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3272 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3273 !priv->sh->misc5_cap) || 3274 (!(priv->sh->tunnel_header_0_1 && 3275 priv->sh->tunnel_header_2_3) && 3276 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3277 return rte_flow_error_set(error, EINVAL, 3278 RTE_FLOW_ERROR_TYPE_ITEM, 3279 item, 3280 "Checksum/Sequence not supported"); 3281 } 3282 ret = mlx5_flow_item_acceptable 3283 (item, (const uint8_t *)mask, 3284 (const uint8_t *)&nic_mask, 3285 sizeof(struct rte_flow_item_gre_opt), 3286 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3287 return ret; 3288 } 3289 3290 /** 3291 * Validate GRE item. 3292 * 3293 * @param[in] item 3294 * Item specification. 3295 * @param[in] item_flags 3296 * Bit flags to mark detected items. 3297 * @param[in] target_protocol 3298 * The next protocol in the previous item. 3299 * @param[out] error 3300 * Pointer to error structure. 3301 * 3302 * @return 3303 * 0 on success, a negative errno value otherwise and rte_errno is set. 3304 */ 3305 int 3306 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3307 uint64_t item_flags, 3308 uint8_t target_protocol, 3309 struct rte_flow_error *error) 3310 { 3311 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3312 const struct rte_flow_item_gre *mask = item->mask; 3313 int ret; 3314 const struct rte_flow_item_gre nic_mask = { 3315 .c_rsvd0_ver = RTE_BE16(0xB000), 3316 .protocol = RTE_BE16(UINT16_MAX), 3317 }; 3318 3319 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3320 return rte_flow_error_set(error, EINVAL, 3321 RTE_FLOW_ERROR_TYPE_ITEM, item, 3322 "protocol filtering not compatible" 3323 " with this GRE layer"); 3324 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3325 return rte_flow_error_set(error, ENOTSUP, 3326 RTE_FLOW_ERROR_TYPE_ITEM, item, 3327 "multiple tunnel layers not" 3328 " supported"); 3329 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3330 return rte_flow_error_set(error, ENOTSUP, 3331 RTE_FLOW_ERROR_TYPE_ITEM, item, 3332 "L3 Layer is missing"); 3333 if (!mask) 3334 mask = &rte_flow_item_gre_mask; 3335 ret = mlx5_flow_item_acceptable 3336 (item, (const uint8_t *)mask, 3337 (const uint8_t *)&nic_mask, 3338 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3339 error); 3340 if (ret < 0) 3341 return ret; 3342 #ifndef HAVE_MLX5DV_DR 3343 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3344 if (spec && (spec->protocol & mask->protocol)) 3345 return rte_flow_error_set(error, ENOTSUP, 3346 RTE_FLOW_ERROR_TYPE_ITEM, item, 3347 "without MPLS support the" 3348 " specification cannot be used for" 3349 " filtering"); 3350 #endif 3351 #endif 3352 return 0; 3353 } 3354 3355 /** 3356 * Validate Geneve item. 3357 * 3358 * @param[in] item 3359 * Item specification. 3360 * @param[in] itemFlags 3361 * Bit-fields that holds the items detected until now. 3362 * @param[in] enPriv 3363 * Pointer to the private data structure. 3364 * @param[out] error 3365 * Pointer to error structure. 3366 * 3367 * @return 3368 * 0 on success, a negative errno value otherwise and rte_errno is set. 3369 */ 3370 3371 int 3372 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3373 uint64_t item_flags, 3374 struct rte_eth_dev *dev, 3375 struct rte_flow_error *error) 3376 { 3377 struct mlx5_priv *priv = dev->data->dev_private; 3378 const struct rte_flow_item_geneve *spec = item->spec; 3379 const struct rte_flow_item_geneve *mask = item->mask; 3380 int ret; 3381 uint16_t gbhdr; 3382 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3383 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3384 const struct rte_flow_item_geneve nic_mask = { 3385 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3386 .vni = "\xff\xff\xff", 3387 .protocol = RTE_BE16(UINT16_MAX), 3388 }; 3389 3390 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3391 return rte_flow_error_set(error, ENOTSUP, 3392 RTE_FLOW_ERROR_TYPE_ITEM, item, 3393 "L3 Geneve is not enabled by device" 3394 " parameter and/or not configured in" 3395 " firmware"); 3396 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3397 return rte_flow_error_set(error, ENOTSUP, 3398 RTE_FLOW_ERROR_TYPE_ITEM, item, 3399 "multiple tunnel layers not" 3400 " supported"); 3401 /* 3402 * Verify only UDPv4 is present as defined in 3403 * https://tools.ietf.org/html/rfc7348 3404 */ 3405 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3406 return rte_flow_error_set(error, EINVAL, 3407 RTE_FLOW_ERROR_TYPE_ITEM, item, 3408 "no outer UDP layer found"); 3409 if (!mask) 3410 mask = &rte_flow_item_geneve_mask; 3411 ret = mlx5_flow_item_acceptable 3412 (item, (const uint8_t *)mask, 3413 (const uint8_t *)&nic_mask, 3414 sizeof(struct rte_flow_item_geneve), 3415 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3416 if (ret) 3417 return ret; 3418 if (spec) { 3419 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3420 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3421 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3422 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3423 return rte_flow_error_set(error, ENOTSUP, 3424 RTE_FLOW_ERROR_TYPE_ITEM, 3425 item, 3426 "Geneve protocol unsupported" 3427 " fields are being used"); 3428 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3429 return rte_flow_error_set 3430 (error, ENOTSUP, 3431 RTE_FLOW_ERROR_TYPE_ITEM, 3432 item, 3433 "Unsupported Geneve options length"); 3434 } 3435 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3436 return rte_flow_error_set 3437 (error, ENOTSUP, 3438 RTE_FLOW_ERROR_TYPE_ITEM, item, 3439 "Geneve tunnel must be fully defined"); 3440 return 0; 3441 } 3442 3443 /** 3444 * Validate Geneve TLV option item. 3445 * 3446 * @param[in] item 3447 * Item specification. 3448 * @param[in] last_item 3449 * Previous validated item in the pattern items. 3450 * @param[in] geneve_item 3451 * Previous GENEVE item specification. 3452 * @param[in] dev 3453 * Pointer to the rte_eth_dev structure. 3454 * @param[out] error 3455 * Pointer to error structure. 3456 * 3457 * @return 3458 * 0 on success, a negative errno value otherwise and rte_errno is set. 3459 */ 3460 int 3461 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3462 uint64_t last_item, 3463 const struct rte_flow_item *geneve_item, 3464 struct rte_eth_dev *dev, 3465 struct rte_flow_error *error) 3466 { 3467 struct mlx5_priv *priv = dev->data->dev_private; 3468 struct mlx5_dev_ctx_shared *sh = priv->sh; 3469 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3470 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3471 uint8_t data_max_supported = 3472 hca_attr->max_geneve_tlv_option_data_len * 4; 3473 const struct rte_flow_item_geneve *geneve_spec; 3474 const struct rte_flow_item_geneve *geneve_mask; 3475 const struct rte_flow_item_geneve_opt *spec = item->spec; 3476 const struct rte_flow_item_geneve_opt *mask = item->mask; 3477 unsigned int i; 3478 unsigned int data_len; 3479 uint8_t tlv_option_len; 3480 uint16_t optlen_m, optlen_v; 3481 const struct rte_flow_item_geneve_opt full_mask = { 3482 .option_class = RTE_BE16(0xffff), 3483 .option_type = 0xff, 3484 .option_len = 0x1f, 3485 }; 3486 3487 if (!mask) 3488 mask = &rte_flow_item_geneve_opt_mask; 3489 if (!spec) 3490 return rte_flow_error_set 3491 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3492 "Geneve TLV opt class/type/length must be specified"); 3493 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3494 return rte_flow_error_set 3495 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3496 "Geneve TLV opt length exceeds the limit (31)"); 3497 /* Check if class type and length masks are full. */ 3498 if (full_mask.option_class != mask->option_class || 3499 full_mask.option_type != mask->option_type || 3500 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3501 return rte_flow_error_set 3502 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3503 "Geneve TLV opt class/type/length masks must be full"); 3504 /* Check if length is supported */ 3505 if ((uint32_t)spec->option_len > 3506 hca_attr->max_geneve_tlv_option_data_len) 3507 return rte_flow_error_set 3508 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3509 "Geneve TLV opt length not supported"); 3510 if (hca_attr->max_geneve_tlv_options > 1) 3511 DRV_LOG(DEBUG, 3512 "max_geneve_tlv_options supports more than 1 option"); 3513 /* Check GENEVE item preceding. */ 3514 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3515 return rte_flow_error_set 3516 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3517 "Geneve opt item must be preceded with Geneve item"); 3518 geneve_spec = geneve_item->spec; 3519 geneve_mask = geneve_item->mask ? geneve_item->mask : 3520 &rte_flow_item_geneve_mask; 3521 /* Check if GENEVE TLV option size doesn't exceed option length */ 3522 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3523 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3524 tlv_option_len = spec->option_len & mask->option_len; 3525 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3526 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3527 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3528 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3529 if ((optlen_v & optlen_m) <= tlv_option_len) 3530 return rte_flow_error_set 3531 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3532 "GENEVE TLV option length exceeds optlen"); 3533 } 3534 /* Check if length is 0 or data is 0. */ 3535 if (spec->data == NULL || spec->option_len == 0) 3536 return rte_flow_error_set 3537 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3538 "Geneve TLV opt with zero data/length not supported"); 3539 /* Check not all data & mask are 0. */ 3540 data_len = spec->option_len * 4; 3541 if (mask->data == NULL) { 3542 for (i = 0; i < data_len; i++) 3543 if (spec->data[i]) 3544 break; 3545 if (i == data_len) 3546 return rte_flow_error_set(error, ENOTSUP, 3547 RTE_FLOW_ERROR_TYPE_ITEM, item, 3548 "Can't match on Geneve option data 0"); 3549 } else { 3550 for (i = 0; i < data_len; i++) 3551 if (spec->data[i] & mask->data[i]) 3552 break; 3553 if (i == data_len) 3554 return rte_flow_error_set(error, ENOTSUP, 3555 RTE_FLOW_ERROR_TYPE_ITEM, item, 3556 "Can't match on Geneve option data and mask 0"); 3557 /* Check data mask supported. */ 3558 for (i = data_max_supported; i < data_len ; i++) 3559 if (mask->data[i]) 3560 return rte_flow_error_set(error, ENOTSUP, 3561 RTE_FLOW_ERROR_TYPE_ITEM, item, 3562 "Data mask is of unsupported size"); 3563 } 3564 /* Check GENEVE option is supported in NIC. */ 3565 if (!hca_attr->geneve_tlv_opt) 3566 return rte_flow_error_set 3567 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3568 "Geneve TLV opt not supported"); 3569 /* Check if we already have geneve option with different type/class. */ 3570 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3571 geneve_opt_resource = sh->geneve_tlv_option_resource; 3572 if (geneve_opt_resource != NULL) 3573 if (geneve_opt_resource->option_class != spec->option_class || 3574 geneve_opt_resource->option_type != spec->option_type || 3575 geneve_opt_resource->length != spec->option_len) { 3576 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3577 return rte_flow_error_set(error, ENOTSUP, 3578 RTE_FLOW_ERROR_TYPE_ITEM, item, 3579 "Only one Geneve TLV option supported"); 3580 } 3581 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3582 return 0; 3583 } 3584 3585 /** 3586 * Validate MPLS item. 3587 * 3588 * @param[in] dev 3589 * Pointer to the rte_eth_dev structure. 3590 * @param[in] item 3591 * Item specification. 3592 * @param[in] item_flags 3593 * Bit-fields that holds the items detected until now. 3594 * @param[in] prev_layer 3595 * The protocol layer indicated in previous item. 3596 * @param[out] error 3597 * Pointer to error structure. 3598 * 3599 * @return 3600 * 0 on success, a negative errno value otherwise and rte_errno is set. 3601 */ 3602 int 3603 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3604 const struct rte_flow_item *item __rte_unused, 3605 uint64_t item_flags __rte_unused, 3606 uint64_t prev_layer __rte_unused, 3607 struct rte_flow_error *error) 3608 { 3609 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3610 const struct rte_flow_item_mpls *mask = item->mask; 3611 struct mlx5_priv *priv = dev->data->dev_private; 3612 int ret; 3613 3614 if (!priv->sh->dev_cap.mpls_en) 3615 return rte_flow_error_set(error, ENOTSUP, 3616 RTE_FLOW_ERROR_TYPE_ITEM, item, 3617 "MPLS not supported or" 3618 " disabled in firmware" 3619 " configuration."); 3620 /* MPLS over UDP, GRE is allowed */ 3621 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3622 MLX5_FLOW_LAYER_GRE | 3623 MLX5_FLOW_LAYER_GRE_KEY))) 3624 return rte_flow_error_set(error, EINVAL, 3625 RTE_FLOW_ERROR_TYPE_ITEM, item, 3626 "protocol filtering not compatible" 3627 " with MPLS layer"); 3628 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3629 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3630 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3631 return rte_flow_error_set(error, ENOTSUP, 3632 RTE_FLOW_ERROR_TYPE_ITEM, item, 3633 "multiple tunnel layers not" 3634 " supported"); 3635 if (!mask) 3636 mask = &rte_flow_item_mpls_mask; 3637 ret = mlx5_flow_item_acceptable 3638 (item, (const uint8_t *)mask, 3639 (const uint8_t *)&rte_flow_item_mpls_mask, 3640 sizeof(struct rte_flow_item_mpls), 3641 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3642 if (ret < 0) 3643 return ret; 3644 return 0; 3645 #else 3646 return rte_flow_error_set(error, ENOTSUP, 3647 RTE_FLOW_ERROR_TYPE_ITEM, item, 3648 "MPLS is not supported by Verbs, please" 3649 " update."); 3650 #endif 3651 } 3652 3653 /** 3654 * Validate NVGRE item. 3655 * 3656 * @param[in] item 3657 * Item specification. 3658 * @param[in] item_flags 3659 * Bit flags to mark detected items. 3660 * @param[in] target_protocol 3661 * The next protocol in the previous item. 3662 * @param[out] error 3663 * Pointer to error structure. 3664 * 3665 * @return 3666 * 0 on success, a negative errno value otherwise and rte_errno is set. 3667 */ 3668 int 3669 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3670 uint64_t item_flags, 3671 uint8_t target_protocol, 3672 struct rte_flow_error *error) 3673 { 3674 const struct rte_flow_item_nvgre *mask = item->mask; 3675 int ret; 3676 3677 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3678 return rte_flow_error_set(error, EINVAL, 3679 RTE_FLOW_ERROR_TYPE_ITEM, item, 3680 "protocol filtering not compatible" 3681 " with this GRE layer"); 3682 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3683 return rte_flow_error_set(error, ENOTSUP, 3684 RTE_FLOW_ERROR_TYPE_ITEM, item, 3685 "multiple tunnel layers not" 3686 " supported"); 3687 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3688 return rte_flow_error_set(error, ENOTSUP, 3689 RTE_FLOW_ERROR_TYPE_ITEM, item, 3690 "L3 Layer is missing"); 3691 if (!mask) 3692 mask = &rte_flow_item_nvgre_mask; 3693 ret = mlx5_flow_item_acceptable 3694 (item, (const uint8_t *)mask, 3695 (const uint8_t *)&rte_flow_item_nvgre_mask, 3696 sizeof(struct rte_flow_item_nvgre), 3697 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3698 if (ret < 0) 3699 return ret; 3700 return 0; 3701 } 3702 3703 /** 3704 * Validate eCPRI item. 3705 * 3706 * @param[in] item 3707 * Item specification. 3708 * @param[in] item_flags 3709 * Bit-fields that holds the items detected until now. 3710 * @param[in] last_item 3711 * Previous validated item in the pattern items. 3712 * @param[in] ether_type 3713 * Type in the ethernet layer header (including dot1q). 3714 * @param[in] acc_mask 3715 * Acceptable mask, if NULL default internal default mask 3716 * will be used to check whether item fields are supported. 3717 * @param[out] error 3718 * Pointer to error structure. 3719 * 3720 * @return 3721 * 0 on success, a negative errno value otherwise and rte_errno is set. 3722 */ 3723 int 3724 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3725 uint64_t item_flags, 3726 uint64_t last_item, 3727 uint16_t ether_type, 3728 const struct rte_flow_item_ecpri *acc_mask, 3729 struct rte_flow_error *error) 3730 { 3731 const struct rte_flow_item_ecpri *mask = item->mask; 3732 const struct rte_flow_item_ecpri nic_mask = { 3733 .hdr = { 3734 .common = { 3735 .u32 = 3736 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3737 .type = 0xFF, 3738 }).u32), 3739 }, 3740 .dummy[0] = 0xFFFFFFFF, 3741 }, 3742 }; 3743 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3744 MLX5_FLOW_LAYER_OUTER_VLAN); 3745 struct rte_flow_item_ecpri mask_lo; 3746 3747 if (!(last_item & outer_l2_vlan) && 3748 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3749 return rte_flow_error_set(error, EINVAL, 3750 RTE_FLOW_ERROR_TYPE_ITEM, item, 3751 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3752 if ((last_item & outer_l2_vlan) && ether_type && 3753 ether_type != RTE_ETHER_TYPE_ECPRI) 3754 return rte_flow_error_set(error, EINVAL, 3755 RTE_FLOW_ERROR_TYPE_ITEM, item, 3756 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3757 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3758 return rte_flow_error_set(error, EINVAL, 3759 RTE_FLOW_ERROR_TYPE_ITEM, item, 3760 "eCPRI with tunnel is not supported right now"); 3761 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3762 return rte_flow_error_set(error, ENOTSUP, 3763 RTE_FLOW_ERROR_TYPE_ITEM, item, 3764 "multiple L3 layers not supported"); 3765 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3766 return rte_flow_error_set(error, EINVAL, 3767 RTE_FLOW_ERROR_TYPE_ITEM, item, 3768 "eCPRI cannot coexist with a TCP layer"); 3769 /* In specification, eCPRI could be over UDP layer. */ 3770 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3771 return rte_flow_error_set(error, EINVAL, 3772 RTE_FLOW_ERROR_TYPE_ITEM, item, 3773 "eCPRI over UDP layer is not yet supported right now"); 3774 /* Mask for type field in common header could be zero. */ 3775 if (!mask) 3776 mask = &rte_flow_item_ecpri_mask; 3777 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3778 /* Input mask is in big-endian format. */ 3779 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3780 return rte_flow_error_set(error, EINVAL, 3781 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3782 "partial mask is not supported for protocol"); 3783 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3784 return rte_flow_error_set(error, EINVAL, 3785 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3786 "message header mask must be after a type mask"); 3787 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3788 acc_mask ? (const uint8_t *)acc_mask 3789 : (const uint8_t *)&nic_mask, 3790 sizeof(struct rte_flow_item_ecpri), 3791 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3792 } 3793 3794 static int 3795 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3796 const struct rte_flow_attr *attr __rte_unused, 3797 const struct rte_flow_item items[] __rte_unused, 3798 const struct rte_flow_action actions[] __rte_unused, 3799 bool external __rte_unused, 3800 int hairpin __rte_unused, 3801 struct rte_flow_error *error) 3802 { 3803 return rte_flow_error_set(error, ENOTSUP, 3804 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3805 } 3806 3807 static struct mlx5_flow * 3808 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3809 const struct rte_flow_attr *attr __rte_unused, 3810 const struct rte_flow_item items[] __rte_unused, 3811 const struct rte_flow_action actions[] __rte_unused, 3812 struct rte_flow_error *error) 3813 { 3814 rte_flow_error_set(error, ENOTSUP, 3815 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3816 return NULL; 3817 } 3818 3819 static int 3820 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3821 struct mlx5_flow *dev_flow __rte_unused, 3822 const struct rte_flow_attr *attr __rte_unused, 3823 const struct rte_flow_item items[] __rte_unused, 3824 const struct rte_flow_action actions[] __rte_unused, 3825 struct rte_flow_error *error) 3826 { 3827 return rte_flow_error_set(error, ENOTSUP, 3828 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3829 } 3830 3831 static int 3832 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3833 struct rte_flow *flow __rte_unused, 3834 struct rte_flow_error *error) 3835 { 3836 return rte_flow_error_set(error, ENOTSUP, 3837 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3838 } 3839 3840 static void 3841 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3842 struct rte_flow *flow __rte_unused) 3843 { 3844 } 3845 3846 static void 3847 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 3848 struct rte_flow *flow __rte_unused) 3849 { 3850 } 3851 3852 static int 3853 flow_null_query(struct rte_eth_dev *dev __rte_unused, 3854 struct rte_flow *flow __rte_unused, 3855 const struct rte_flow_action *actions __rte_unused, 3856 void *data __rte_unused, 3857 struct rte_flow_error *error) 3858 { 3859 return rte_flow_error_set(error, ENOTSUP, 3860 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3861 } 3862 3863 static int 3864 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 3865 uint32_t domains __rte_unused, 3866 uint32_t flags __rte_unused) 3867 { 3868 return 0; 3869 } 3870 3871 int 3872 flow_null_get_aged_flows(struct rte_eth_dev *dev, 3873 void **context __rte_unused, 3874 uint32_t nb_contexts __rte_unused, 3875 struct rte_flow_error *error __rte_unused) 3876 { 3877 DRV_LOG(ERR, "port %u get aged flows is not supported.", 3878 dev->data->port_id); 3879 return -ENOTSUP; 3880 } 3881 3882 uint32_t 3883 flow_null_counter_allocate(struct rte_eth_dev *dev) 3884 { 3885 DRV_LOG(ERR, "port %u counter allocate is not supported.", 3886 dev->data->port_id); 3887 return 0; 3888 } 3889 3890 void 3891 flow_null_counter_free(struct rte_eth_dev *dev, 3892 uint32_t counter __rte_unused) 3893 { 3894 DRV_LOG(ERR, "port %u counter free is not supported.", 3895 dev->data->port_id); 3896 } 3897 3898 int 3899 flow_null_counter_query(struct rte_eth_dev *dev, 3900 uint32_t counter __rte_unused, 3901 bool clear __rte_unused, 3902 uint64_t *pkts __rte_unused, 3903 uint64_t *bytes __rte_unused, 3904 void **action __rte_unused) 3905 { 3906 DRV_LOG(ERR, "port %u counter query is not supported.", 3907 dev->data->port_id); 3908 return -ENOTSUP; 3909 } 3910 3911 /* Void driver to protect from null pointer reference. */ 3912 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 3913 .validate = flow_null_validate, 3914 .prepare = flow_null_prepare, 3915 .translate = flow_null_translate, 3916 .apply = flow_null_apply, 3917 .remove = flow_null_remove, 3918 .destroy = flow_null_destroy, 3919 .query = flow_null_query, 3920 .sync_domain = flow_null_sync_domain, 3921 .get_aged_flows = flow_null_get_aged_flows, 3922 .counter_alloc = flow_null_counter_allocate, 3923 .counter_free = flow_null_counter_free, 3924 .counter_query = flow_null_counter_query 3925 }; 3926 3927 /** 3928 * Select flow driver type according to flow attributes and device 3929 * configuration. 3930 * 3931 * @param[in] dev 3932 * Pointer to the dev structure. 3933 * @param[in] attr 3934 * Pointer to the flow attributes. 3935 * 3936 * @return 3937 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 3938 */ 3939 static enum mlx5_flow_drv_type 3940 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 3941 { 3942 struct mlx5_priv *priv = dev->data->dev_private; 3943 /* The OS can determine first a specific flow type (DV, VERBS) */ 3944 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 3945 3946 if (type != MLX5_FLOW_TYPE_MAX) 3947 return type; 3948 /* 3949 * Currently when dv_flow_en == 2, only HW steering engine is 3950 * supported. New engines can also be chosen here if ready. 3951 */ 3952 if (priv->sh->config.dv_flow_en == 2) 3953 return MLX5_FLOW_TYPE_HW; 3954 if (!attr) 3955 return MLX5_FLOW_TYPE_MIN; 3956 /* If no OS specific type - continue with DV/VERBS selection */ 3957 if (attr->transfer && priv->sh->config.dv_esw_en) 3958 type = MLX5_FLOW_TYPE_DV; 3959 if (!attr->transfer) 3960 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 3961 MLX5_FLOW_TYPE_VERBS; 3962 return type; 3963 } 3964 3965 #define flow_get_drv_ops(type) flow_drv_ops[type] 3966 3967 /** 3968 * Flow driver validation API. This abstracts calling driver specific functions. 3969 * The type of flow driver is determined according to flow attributes. 3970 * 3971 * @param[in] dev 3972 * Pointer to the dev structure. 3973 * @param[in] attr 3974 * Pointer to the flow attributes. 3975 * @param[in] items 3976 * Pointer to the list of items. 3977 * @param[in] actions 3978 * Pointer to the list of actions. 3979 * @param[in] external 3980 * This flow rule is created by request external to PMD. 3981 * @param[in] hairpin 3982 * Number of hairpin TX actions, 0 means classic flow. 3983 * @param[out] error 3984 * Pointer to the error structure. 3985 * 3986 * @return 3987 * 0 on success, a negative errno value otherwise and rte_errno is set. 3988 */ 3989 static inline int 3990 flow_drv_validate(struct rte_eth_dev *dev, 3991 const struct rte_flow_attr *attr, 3992 const struct rte_flow_item items[], 3993 const struct rte_flow_action actions[], 3994 bool external, int hairpin, struct rte_flow_error *error) 3995 { 3996 const struct mlx5_flow_driver_ops *fops; 3997 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 3998 3999 fops = flow_get_drv_ops(type); 4000 return fops->validate(dev, attr, items, actions, external, 4001 hairpin, error); 4002 } 4003 4004 /** 4005 * Flow driver preparation API. This abstracts calling driver specific 4006 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4007 * calculates the size of memory required for device flow, allocates the memory, 4008 * initializes the device flow and returns the pointer. 4009 * 4010 * @note 4011 * This function initializes device flow structure such as dv or verbs in 4012 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4013 * rest. For example, adding returning device flow to flow->dev_flow list and 4014 * setting backward reference to the flow should be done out of this function. 4015 * layers field is not filled either. 4016 * 4017 * @param[in] dev 4018 * Pointer to the dev structure. 4019 * @param[in] attr 4020 * Pointer to the flow attributes. 4021 * @param[in] items 4022 * Pointer to the list of items. 4023 * @param[in] actions 4024 * Pointer to the list of actions. 4025 * @param[in] flow_idx 4026 * This memory pool index to the flow. 4027 * @param[out] error 4028 * Pointer to the error structure. 4029 * 4030 * @return 4031 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4032 */ 4033 static inline struct mlx5_flow * 4034 flow_drv_prepare(struct rte_eth_dev *dev, 4035 const struct rte_flow *flow, 4036 const struct rte_flow_attr *attr, 4037 const struct rte_flow_item items[], 4038 const struct rte_flow_action actions[], 4039 uint32_t flow_idx, 4040 struct rte_flow_error *error) 4041 { 4042 const struct mlx5_flow_driver_ops *fops; 4043 enum mlx5_flow_drv_type type = flow->drv_type; 4044 struct mlx5_flow *mlx5_flow = NULL; 4045 4046 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4047 fops = flow_get_drv_ops(type); 4048 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4049 if (mlx5_flow) 4050 mlx5_flow->flow_idx = flow_idx; 4051 return mlx5_flow; 4052 } 4053 4054 /** 4055 * Flow driver translation API. This abstracts calling driver specific 4056 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4057 * translates a generic flow into a driver flow. flow_drv_prepare() must 4058 * precede. 4059 * 4060 * @note 4061 * dev_flow->layers could be filled as a result of parsing during translation 4062 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4063 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4064 * flow->actions could be overwritten even though all the expanded dev_flows 4065 * have the same actions. 4066 * 4067 * @param[in] dev 4068 * Pointer to the rte dev structure. 4069 * @param[in, out] dev_flow 4070 * Pointer to the mlx5 flow. 4071 * @param[in] attr 4072 * Pointer to the flow attributes. 4073 * @param[in] items 4074 * Pointer to the list of items. 4075 * @param[in] actions 4076 * Pointer to the list of actions. 4077 * @param[out] error 4078 * Pointer to the error structure. 4079 * 4080 * @return 4081 * 0 on success, a negative errno value otherwise and rte_errno is set. 4082 */ 4083 static inline int 4084 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4085 const struct rte_flow_attr *attr, 4086 const struct rte_flow_item items[], 4087 const struct rte_flow_action actions[], 4088 struct rte_flow_error *error) 4089 { 4090 const struct mlx5_flow_driver_ops *fops; 4091 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4092 4093 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4094 fops = flow_get_drv_ops(type); 4095 return fops->translate(dev, dev_flow, attr, items, actions, error); 4096 } 4097 4098 /** 4099 * Flow driver apply API. This abstracts calling driver specific functions. 4100 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4101 * translated driver flows on to device. flow_drv_translate() must precede. 4102 * 4103 * @param[in] dev 4104 * Pointer to Ethernet device structure. 4105 * @param[in, out] flow 4106 * Pointer to flow structure. 4107 * @param[out] error 4108 * Pointer to error structure. 4109 * 4110 * @return 4111 * 0 on success, a negative errno value otherwise and rte_errno is set. 4112 */ 4113 static inline int 4114 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4115 struct rte_flow_error *error) 4116 { 4117 const struct mlx5_flow_driver_ops *fops; 4118 enum mlx5_flow_drv_type type = flow->drv_type; 4119 4120 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4121 fops = flow_get_drv_ops(type); 4122 return fops->apply(dev, flow, error); 4123 } 4124 4125 /** 4126 * Flow driver destroy API. This abstracts calling driver specific functions. 4127 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4128 * on device and releases resources of the flow. 4129 * 4130 * @param[in] dev 4131 * Pointer to Ethernet device. 4132 * @param[in, out] flow 4133 * Pointer to flow structure. 4134 */ 4135 static inline void 4136 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4137 { 4138 const struct mlx5_flow_driver_ops *fops; 4139 enum mlx5_flow_drv_type type = flow->drv_type; 4140 4141 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4142 fops = flow_get_drv_ops(type); 4143 fops->destroy(dev, flow); 4144 } 4145 4146 /** 4147 * Flow driver find RSS policy tbl API. This abstracts calling driver 4148 * specific functions. Parent flow (rte_flow) should have driver 4149 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4150 * 4151 * @param[in] dev 4152 * Pointer to Ethernet device. 4153 * @param[in, out] flow 4154 * Pointer to flow structure. 4155 * @param[in] policy 4156 * Pointer to meter policy table. 4157 * @param[in] rss_desc 4158 * Pointer to rss_desc 4159 */ 4160 static struct mlx5_flow_meter_sub_policy * 4161 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4162 struct rte_flow *flow, 4163 struct mlx5_flow_meter_policy *policy, 4164 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4165 { 4166 const struct mlx5_flow_driver_ops *fops; 4167 enum mlx5_flow_drv_type type = flow->drv_type; 4168 4169 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4170 fops = flow_get_drv_ops(type); 4171 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4172 } 4173 4174 /** 4175 * Flow driver color tag rule API. This abstracts calling driver 4176 * specific functions. Parent flow (rte_flow) should have driver 4177 * type (drv_type). It will create the color tag rules in hierarchy meter. 4178 * 4179 * @param[in] dev 4180 * Pointer to Ethernet device. 4181 * @param[in, out] flow 4182 * Pointer to flow structure. 4183 * @param[in] fm 4184 * Pointer to flow meter structure. 4185 * @param[in] src_port 4186 * The src port this extra rule should use. 4187 * @param[in] item 4188 * The src port id match item. 4189 * @param[out] error 4190 * Pointer to error structure. 4191 */ 4192 static int 4193 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4194 struct rte_flow *flow, 4195 struct mlx5_flow_meter_info *fm, 4196 int32_t src_port, 4197 const struct rte_flow_item *item, 4198 struct rte_flow_error *error) 4199 { 4200 const struct mlx5_flow_driver_ops *fops; 4201 enum mlx5_flow_drv_type type = flow->drv_type; 4202 4203 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4204 fops = flow_get_drv_ops(type); 4205 return fops->meter_hierarchy_rule_create(dev, fm, 4206 src_port, item, error); 4207 } 4208 4209 /** 4210 * Get RSS action from the action list. 4211 * 4212 * @param[in] dev 4213 * Pointer to Ethernet device. 4214 * @param[in] actions 4215 * Pointer to the list of actions. 4216 * @param[in] flow 4217 * Parent flow structure pointer. 4218 * 4219 * @return 4220 * Pointer to the RSS action if exist, else return NULL. 4221 */ 4222 static const struct rte_flow_action_rss* 4223 flow_get_rss_action(struct rte_eth_dev *dev, 4224 const struct rte_flow_action actions[]) 4225 { 4226 struct mlx5_priv *priv = dev->data->dev_private; 4227 const struct rte_flow_action_rss *rss = NULL; 4228 struct mlx5_meter_policy_action_container *acg; 4229 struct mlx5_meter_policy_action_container *acy; 4230 4231 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4232 switch (actions->type) { 4233 case RTE_FLOW_ACTION_TYPE_RSS: 4234 rss = actions->conf; 4235 break; 4236 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4237 { 4238 const struct rte_flow_action_sample *sample = 4239 actions->conf; 4240 const struct rte_flow_action *act = sample->actions; 4241 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4242 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4243 rss = act->conf; 4244 break; 4245 } 4246 case RTE_FLOW_ACTION_TYPE_METER: 4247 { 4248 uint32_t mtr_idx; 4249 struct mlx5_flow_meter_info *fm; 4250 struct mlx5_flow_meter_policy *policy; 4251 const struct rte_flow_action_meter *mtr = actions->conf; 4252 4253 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4254 if (fm && !fm->def_policy) { 4255 policy = mlx5_flow_meter_policy_find(dev, 4256 fm->policy_id, NULL); 4257 MLX5_ASSERT(policy); 4258 if (policy->is_hierarchy) { 4259 policy = 4260 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4261 policy); 4262 if (!policy) 4263 return NULL; 4264 } 4265 if (policy->is_rss) { 4266 acg = 4267 &policy->act_cnt[RTE_COLOR_GREEN]; 4268 acy = 4269 &policy->act_cnt[RTE_COLOR_YELLOW]; 4270 if (acg->fate_action == 4271 MLX5_FLOW_FATE_SHARED_RSS) 4272 rss = acg->rss->conf; 4273 else if (acy->fate_action == 4274 MLX5_FLOW_FATE_SHARED_RSS) 4275 rss = acy->rss->conf; 4276 } 4277 } 4278 break; 4279 } 4280 default: 4281 break; 4282 } 4283 } 4284 return rss; 4285 } 4286 4287 /** 4288 * Get ASO age action by index. 4289 * 4290 * @param[in] dev 4291 * Pointer to the Ethernet device structure. 4292 * @param[in] age_idx 4293 * Index to the ASO age action. 4294 * 4295 * @return 4296 * The specified ASO age action. 4297 */ 4298 struct mlx5_aso_age_action* 4299 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4300 { 4301 uint16_t pool_idx = age_idx & UINT16_MAX; 4302 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4303 struct mlx5_priv *priv = dev->data->dev_private; 4304 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4305 struct mlx5_aso_age_pool *pool; 4306 4307 rte_rwlock_read_lock(&mng->resize_rwl); 4308 pool = mng->pools[pool_idx]; 4309 rte_rwlock_read_unlock(&mng->resize_rwl); 4310 return &pool->actions[offset - 1]; 4311 } 4312 4313 /* maps indirect action to translated direct in some actions array */ 4314 struct mlx5_translated_action_handle { 4315 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4316 int index; /**< Index in related array of rte_flow_action. */ 4317 }; 4318 4319 /** 4320 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4321 * direct action if translation possible. 4322 * This functionality used to run same execution path for both direct and 4323 * indirect actions on flow create. All necessary preparations for indirect 4324 * action handling should be performed on *handle* actions list returned 4325 * from this call. 4326 * 4327 * @param[in] dev 4328 * Pointer to Ethernet device. 4329 * @param[in] actions 4330 * List of actions to translate. 4331 * @param[out] handle 4332 * List to store translated indirect action object handles. 4333 * @param[in, out] indir_n 4334 * Size of *handle* array. On return should be updated with number of 4335 * indirect actions retrieved from the *actions* list. 4336 * @param[out] translated_actions 4337 * List of actions where all indirect actions were translated to direct 4338 * if possible. NULL if no translation took place. 4339 * @param[out] error 4340 * Pointer to the error structure. 4341 * 4342 * @return 4343 * 0 on success, a negative errno value otherwise and rte_errno is set. 4344 */ 4345 static int 4346 flow_action_handles_translate(struct rte_eth_dev *dev, 4347 const struct rte_flow_action actions[], 4348 struct mlx5_translated_action_handle *handle, 4349 int *indir_n, 4350 struct rte_flow_action **translated_actions, 4351 struct rte_flow_error *error) 4352 { 4353 struct mlx5_priv *priv = dev->data->dev_private; 4354 struct rte_flow_action *translated = NULL; 4355 size_t actions_size; 4356 int n; 4357 int copied_n = 0; 4358 struct mlx5_translated_action_handle *handle_end = NULL; 4359 4360 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4361 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4362 continue; 4363 if (copied_n == *indir_n) { 4364 return rte_flow_error_set 4365 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4366 NULL, "too many shared actions"); 4367 } 4368 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4369 sizeof(actions[n].conf)); 4370 handle[copied_n].index = n; 4371 copied_n++; 4372 } 4373 n++; 4374 *indir_n = copied_n; 4375 if (!copied_n) 4376 return 0; 4377 actions_size = sizeof(struct rte_flow_action) * n; 4378 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4379 if (!translated) { 4380 rte_errno = ENOMEM; 4381 return -ENOMEM; 4382 } 4383 memcpy(translated, actions, actions_size); 4384 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4385 struct mlx5_shared_action_rss *shared_rss; 4386 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4387 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4388 uint32_t idx = act_idx & 4389 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4390 4391 switch (type) { 4392 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4393 shared_rss = mlx5_ipool_get 4394 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4395 translated[handle->index].type = 4396 RTE_FLOW_ACTION_TYPE_RSS; 4397 translated[handle->index].conf = 4398 &shared_rss->origin; 4399 break; 4400 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4401 translated[handle->index].type = 4402 (enum rte_flow_action_type) 4403 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4404 translated[handle->index].conf = (void *)(uintptr_t)idx; 4405 break; 4406 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4407 translated[handle->index].type = 4408 (enum rte_flow_action_type) 4409 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4410 translated[handle->index].conf = (void *)(uintptr_t)idx; 4411 break; 4412 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4413 if (priv->sh->flow_hit_aso_en) { 4414 translated[handle->index].type = 4415 (enum rte_flow_action_type) 4416 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4417 translated[handle->index].conf = 4418 (void *)(uintptr_t)idx; 4419 break; 4420 } 4421 /* Fall-through */ 4422 case MLX5_INDIRECT_ACTION_TYPE_CT: 4423 if (priv->sh->ct_aso_en) { 4424 translated[handle->index].type = 4425 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4426 translated[handle->index].conf = 4427 (void *)(uintptr_t)idx; 4428 break; 4429 } 4430 /* Fall-through */ 4431 default: 4432 mlx5_free(translated); 4433 return rte_flow_error_set 4434 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4435 NULL, "invalid indirect action type"); 4436 } 4437 } 4438 *translated_actions = translated; 4439 return 0; 4440 } 4441 4442 /** 4443 * Get Shared RSS action from the action list. 4444 * 4445 * @param[in] dev 4446 * Pointer to Ethernet device. 4447 * @param[in] shared 4448 * Pointer to the list of actions. 4449 * @param[in] shared_n 4450 * Actions list length. 4451 * 4452 * @return 4453 * The MLX5 RSS action ID if exists, otherwise return 0. 4454 */ 4455 static uint32_t 4456 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4457 struct mlx5_translated_action_handle *handle, 4458 int shared_n) 4459 { 4460 struct mlx5_translated_action_handle *handle_end; 4461 struct mlx5_priv *priv = dev->data->dev_private; 4462 struct mlx5_shared_action_rss *shared_rss; 4463 4464 4465 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4466 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4467 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4468 uint32_t idx = act_idx & 4469 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4470 switch (type) { 4471 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4472 shared_rss = mlx5_ipool_get 4473 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4474 idx); 4475 __atomic_add_fetch(&shared_rss->refcnt, 1, 4476 __ATOMIC_RELAXED); 4477 return idx; 4478 default: 4479 break; 4480 } 4481 } 4482 return 0; 4483 } 4484 4485 static unsigned int 4486 find_graph_root(uint32_t rss_level) 4487 { 4488 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4489 MLX5_EXPANSION_ROOT_OUTER; 4490 } 4491 4492 /** 4493 * Get layer flags from the prefix flow. 4494 * 4495 * Some flows may be split to several subflows, the prefix subflow gets the 4496 * match items and the suffix sub flow gets the actions. 4497 * Some actions need the user defined match item flags to get the detail for 4498 * the action. 4499 * This function helps the suffix flow to get the item layer flags from prefix 4500 * subflow. 4501 * 4502 * @param[in] dev_flow 4503 * Pointer the created prefix subflow. 4504 * 4505 * @return 4506 * The layers get from prefix subflow. 4507 */ 4508 static inline uint64_t 4509 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4510 { 4511 uint64_t layers = 0; 4512 4513 /* 4514 * Layers bits could be localization, but usually the compiler will 4515 * help to do the optimization work for source code. 4516 * If no decap actions, use the layers directly. 4517 */ 4518 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4519 return dev_flow->handle->layers; 4520 /* Convert L3 layers with decap action. */ 4521 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4522 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4523 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4524 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4525 /* Convert L4 layers with decap action. */ 4526 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4527 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4528 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4529 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4530 return layers; 4531 } 4532 4533 /** 4534 * Get metadata split action information. 4535 * 4536 * @param[in] actions 4537 * Pointer to the list of actions. 4538 * @param[out] qrss 4539 * Pointer to the return pointer. 4540 * @param[out] qrss_type 4541 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4542 * if no QUEUE/RSS is found. 4543 * @param[out] encap_idx 4544 * Pointer to the index of the encap action if exists, otherwise the last 4545 * action index. 4546 * 4547 * @return 4548 * Total number of actions. 4549 */ 4550 static int 4551 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4552 const struct rte_flow_action **qrss, 4553 int *encap_idx) 4554 { 4555 const struct rte_flow_action_raw_encap *raw_encap; 4556 int actions_n = 0; 4557 int raw_decap_idx = -1; 4558 4559 *encap_idx = -1; 4560 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4561 switch (actions->type) { 4562 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4563 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4564 *encap_idx = actions_n; 4565 break; 4566 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4567 raw_decap_idx = actions_n; 4568 break; 4569 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4570 raw_encap = actions->conf; 4571 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4572 *encap_idx = raw_decap_idx != -1 ? 4573 raw_decap_idx : actions_n; 4574 break; 4575 case RTE_FLOW_ACTION_TYPE_QUEUE: 4576 case RTE_FLOW_ACTION_TYPE_RSS: 4577 *qrss = actions; 4578 break; 4579 default: 4580 break; 4581 } 4582 actions_n++; 4583 } 4584 if (*encap_idx == -1) 4585 *encap_idx = actions_n; 4586 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4587 return actions_n + 1; 4588 } 4589 4590 /** 4591 * Check if the action will change packet. 4592 * 4593 * @param dev 4594 * Pointer to Ethernet device. 4595 * @param[in] type 4596 * action type. 4597 * 4598 * @return 4599 * true if action will change packet, false otherwise. 4600 */ 4601 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4602 enum rte_flow_action_type type) 4603 { 4604 struct mlx5_priv *priv = dev->data->dev_private; 4605 4606 switch (type) { 4607 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4608 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4609 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4610 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4611 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4612 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4613 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4614 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4615 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4616 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4617 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4618 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4619 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4620 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4621 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4622 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4623 case RTE_FLOW_ACTION_TYPE_SET_META: 4624 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4625 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4626 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4627 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4628 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4629 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4630 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4631 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4632 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4633 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4634 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4635 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4636 return true; 4637 case RTE_FLOW_ACTION_TYPE_FLAG: 4638 case RTE_FLOW_ACTION_TYPE_MARK: 4639 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4640 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4641 return true; 4642 else 4643 return false; 4644 default: 4645 return false; 4646 } 4647 } 4648 4649 /** 4650 * Check meter action from the action list. 4651 * 4652 * @param dev 4653 * Pointer to Ethernet device. 4654 * @param[in] actions 4655 * Pointer to the list of actions. 4656 * @param[out] has_mtr 4657 * Pointer to the meter exist flag. 4658 * @param[out] has_modify 4659 * Pointer to the flag showing there's packet change action. 4660 * @param[out] meter_id 4661 * Pointer to the meter id. 4662 * 4663 * @return 4664 * Total number of actions. 4665 */ 4666 static int 4667 flow_check_meter_action(struct rte_eth_dev *dev, 4668 const struct rte_flow_action actions[], 4669 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4670 { 4671 const struct rte_flow_action_meter *mtr = NULL; 4672 int actions_n = 0; 4673 4674 MLX5_ASSERT(has_mtr); 4675 *has_mtr = false; 4676 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4677 switch (actions->type) { 4678 case RTE_FLOW_ACTION_TYPE_METER: 4679 mtr = actions->conf; 4680 *meter_id = mtr->mtr_id; 4681 *has_mtr = true; 4682 break; 4683 default: 4684 break; 4685 } 4686 if (!*has_mtr) 4687 *has_modify |= flow_check_modify_action_type(dev, 4688 actions->type); 4689 actions_n++; 4690 } 4691 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4692 return actions_n + 1; 4693 } 4694 4695 /** 4696 * Check if the flow should be split due to hairpin. 4697 * The reason for the split is that in current HW we can't 4698 * support encap and push-vlan on Rx, so if a flow contains 4699 * these actions we move it to Tx. 4700 * 4701 * @param dev 4702 * Pointer to Ethernet device. 4703 * @param[in] attr 4704 * Flow rule attributes. 4705 * @param[in] actions 4706 * Associated actions (list terminated by the END action). 4707 * 4708 * @return 4709 * > 0 the number of actions and the flow should be split, 4710 * 0 when no split required. 4711 */ 4712 static int 4713 flow_check_hairpin_split(struct rte_eth_dev *dev, 4714 const struct rte_flow_attr *attr, 4715 const struct rte_flow_action actions[]) 4716 { 4717 int queue_action = 0; 4718 int action_n = 0; 4719 int split = 0; 4720 int push_vlan = 0; 4721 const struct rte_flow_action_queue *queue; 4722 const struct rte_flow_action_rss *rss; 4723 const struct rte_flow_action_raw_encap *raw_encap; 4724 const struct rte_eth_hairpin_conf *conf; 4725 4726 if (!attr->ingress) 4727 return 0; 4728 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4729 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4730 push_vlan = 1; 4731 switch (actions->type) { 4732 case RTE_FLOW_ACTION_TYPE_QUEUE: 4733 queue = actions->conf; 4734 if (queue == NULL) 4735 return 0; 4736 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4737 if (conf == NULL || conf->tx_explicit != 0) 4738 return 0; 4739 queue_action = 1; 4740 action_n++; 4741 break; 4742 case RTE_FLOW_ACTION_TYPE_RSS: 4743 rss = actions->conf; 4744 if (rss == NULL || rss->queue_num == 0) 4745 return 0; 4746 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4747 if (conf == NULL || conf->tx_explicit != 0) 4748 return 0; 4749 queue_action = 1; 4750 action_n++; 4751 break; 4752 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4753 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4754 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4755 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4756 split++; 4757 action_n++; 4758 break; 4759 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4760 if (push_vlan) 4761 split++; 4762 action_n++; 4763 break; 4764 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4765 raw_encap = actions->conf; 4766 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4767 split++; 4768 action_n++; 4769 break; 4770 default: 4771 action_n++; 4772 break; 4773 } 4774 } 4775 if (split && queue_action) 4776 return action_n; 4777 return 0; 4778 } 4779 4780 /* Declare flow create/destroy prototype in advance. */ 4781 static uint32_t 4782 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4783 const struct rte_flow_attr *attr, 4784 const struct rte_flow_item items[], 4785 const struct rte_flow_action actions[], 4786 bool external, struct rte_flow_error *error); 4787 4788 static void 4789 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4790 uint32_t flow_idx); 4791 4792 int 4793 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4794 struct mlx5_list_entry *entry, void *cb_ctx) 4795 { 4796 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4797 struct mlx5_flow_mreg_copy_resource *mcp_res = 4798 container_of(entry, typeof(*mcp_res), hlist_ent); 4799 4800 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4801 } 4802 4803 struct mlx5_list_entry * 4804 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4805 { 4806 struct rte_eth_dev *dev = tool_ctx; 4807 struct mlx5_priv *priv = dev->data->dev_private; 4808 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4809 struct mlx5_flow_mreg_copy_resource *mcp_res; 4810 struct rte_flow_error *error = ctx->error; 4811 uint32_t idx = 0; 4812 int ret; 4813 uint32_t mark_id = *(uint32_t *)(ctx->data); 4814 struct rte_flow_attr attr = { 4815 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4816 .ingress = 1, 4817 }; 4818 struct mlx5_rte_flow_item_tag tag_spec = { 4819 .data = mark_id, 4820 }; 4821 struct rte_flow_item items[] = { 4822 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4823 }; 4824 struct rte_flow_action_mark ftag = { 4825 .id = mark_id, 4826 }; 4827 struct mlx5_flow_action_copy_mreg cp_mreg = { 4828 .dst = REG_B, 4829 .src = REG_NON, 4830 }; 4831 struct rte_flow_action_jump jump = { 4832 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4833 }; 4834 struct rte_flow_action actions[] = { 4835 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4836 }; 4837 4838 /* Fill the register fields in the flow. */ 4839 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4840 if (ret < 0) 4841 return NULL; 4842 tag_spec.id = ret; 4843 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4844 if (ret < 0) 4845 return NULL; 4846 cp_mreg.src = ret; 4847 /* Provide the full width of FLAG specific value. */ 4848 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 4849 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 4850 /* Build a new flow. */ 4851 if (mark_id != MLX5_DEFAULT_COPY_ID) { 4852 items[0] = (struct rte_flow_item){ 4853 .type = (enum rte_flow_item_type) 4854 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 4855 .spec = &tag_spec, 4856 }; 4857 items[1] = (struct rte_flow_item){ 4858 .type = RTE_FLOW_ITEM_TYPE_END, 4859 }; 4860 actions[0] = (struct rte_flow_action){ 4861 .type = (enum rte_flow_action_type) 4862 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 4863 .conf = &ftag, 4864 }; 4865 actions[1] = (struct rte_flow_action){ 4866 .type = (enum rte_flow_action_type) 4867 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4868 .conf = &cp_mreg, 4869 }; 4870 actions[2] = (struct rte_flow_action){ 4871 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4872 .conf = &jump, 4873 }; 4874 actions[3] = (struct rte_flow_action){ 4875 .type = RTE_FLOW_ACTION_TYPE_END, 4876 }; 4877 } else { 4878 /* Default rule, wildcard match. */ 4879 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 4880 items[0] = (struct rte_flow_item){ 4881 .type = RTE_FLOW_ITEM_TYPE_END, 4882 }; 4883 actions[0] = (struct rte_flow_action){ 4884 .type = (enum rte_flow_action_type) 4885 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 4886 .conf = &cp_mreg, 4887 }; 4888 actions[1] = (struct rte_flow_action){ 4889 .type = RTE_FLOW_ACTION_TYPE_JUMP, 4890 .conf = &jump, 4891 }; 4892 actions[2] = (struct rte_flow_action){ 4893 .type = RTE_FLOW_ACTION_TYPE_END, 4894 }; 4895 } 4896 /* Build a new entry. */ 4897 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4898 if (!mcp_res) { 4899 rte_errno = ENOMEM; 4900 return NULL; 4901 } 4902 mcp_res->idx = idx; 4903 mcp_res->mark_id = mark_id; 4904 /* 4905 * The copy Flows are not included in any list. There 4906 * ones are referenced from other Flows and can not 4907 * be applied, removed, deleted in arbitrary order 4908 * by list traversing. 4909 */ 4910 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 4911 &attr, items, actions, false, error); 4912 if (!mcp_res->rix_flow) { 4913 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 4914 return NULL; 4915 } 4916 return &mcp_res->hlist_ent; 4917 } 4918 4919 struct mlx5_list_entry * 4920 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 4921 void *cb_ctx __rte_unused) 4922 { 4923 struct rte_eth_dev *dev = tool_ctx; 4924 struct mlx5_priv *priv = dev->data->dev_private; 4925 struct mlx5_flow_mreg_copy_resource *mcp_res; 4926 uint32_t idx = 0; 4927 4928 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 4929 if (!mcp_res) { 4930 rte_errno = ENOMEM; 4931 return NULL; 4932 } 4933 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 4934 mcp_res->idx = idx; 4935 return &mcp_res->hlist_ent; 4936 } 4937 4938 void 4939 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4940 { 4941 struct mlx5_flow_mreg_copy_resource *mcp_res = 4942 container_of(entry, typeof(*mcp_res), hlist_ent); 4943 struct rte_eth_dev *dev = tool_ctx; 4944 struct mlx5_priv *priv = dev->data->dev_private; 4945 4946 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 4947 } 4948 4949 /** 4950 * Add a flow of copying flow metadata registers in RX_CP_TBL. 4951 * 4952 * As mark_id is unique, if there's already a registered flow for the mark_id, 4953 * return by increasing the reference counter of the resource. Otherwise, create 4954 * the resource (mcp_res) and flow. 4955 * 4956 * Flow looks like, 4957 * - If ingress port is ANY and reg_c[1] is mark_id, 4958 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 4959 * 4960 * For default flow (zero mark_id), flow is like, 4961 * - If ingress port is ANY, 4962 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 4963 * 4964 * @param dev 4965 * Pointer to Ethernet device. 4966 * @param mark_id 4967 * ID of MARK action, zero means default flow for META. 4968 * @param[out] error 4969 * Perform verbose error reporting if not NULL. 4970 * 4971 * @return 4972 * Associated resource on success, NULL otherwise and rte_errno is set. 4973 */ 4974 static struct mlx5_flow_mreg_copy_resource * 4975 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 4976 struct rte_flow_error *error) 4977 { 4978 struct mlx5_priv *priv = dev->data->dev_private; 4979 struct mlx5_list_entry *entry; 4980 struct mlx5_flow_cb_ctx ctx = { 4981 .dev = dev, 4982 .error = error, 4983 .data = &mark_id, 4984 }; 4985 4986 /* Check if already registered. */ 4987 MLX5_ASSERT(priv->mreg_cp_tbl); 4988 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 4989 if (!entry) 4990 return NULL; 4991 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 4992 hlist_ent); 4993 } 4994 4995 void 4996 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 4997 { 4998 struct mlx5_flow_mreg_copy_resource *mcp_res = 4999 container_of(entry, typeof(*mcp_res), hlist_ent); 5000 struct rte_eth_dev *dev = tool_ctx; 5001 struct mlx5_priv *priv = dev->data->dev_private; 5002 5003 MLX5_ASSERT(mcp_res->rix_flow); 5004 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5005 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5006 } 5007 5008 /** 5009 * Release flow in RX_CP_TBL. 5010 * 5011 * @param dev 5012 * Pointer to Ethernet device. 5013 * @flow 5014 * Parent flow for wich copying is provided. 5015 */ 5016 static void 5017 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5018 struct rte_flow *flow) 5019 { 5020 struct mlx5_flow_mreg_copy_resource *mcp_res; 5021 struct mlx5_priv *priv = dev->data->dev_private; 5022 5023 if (!flow->rix_mreg_copy) 5024 return; 5025 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5026 flow->rix_mreg_copy); 5027 if (!mcp_res || !priv->mreg_cp_tbl) 5028 return; 5029 MLX5_ASSERT(mcp_res->rix_flow); 5030 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5031 flow->rix_mreg_copy = 0; 5032 } 5033 5034 /** 5035 * Remove the default copy action from RX_CP_TBL. 5036 * 5037 * This functions is called in the mlx5_dev_start(). No thread safe 5038 * is guaranteed. 5039 * 5040 * @param dev 5041 * Pointer to Ethernet device. 5042 */ 5043 static void 5044 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5045 { 5046 struct mlx5_list_entry *entry; 5047 struct mlx5_priv *priv = dev->data->dev_private; 5048 struct mlx5_flow_cb_ctx ctx; 5049 uint32_t mark_id; 5050 5051 /* Check if default flow is registered. */ 5052 if (!priv->mreg_cp_tbl) 5053 return; 5054 mark_id = MLX5_DEFAULT_COPY_ID; 5055 ctx.data = &mark_id; 5056 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5057 if (!entry) 5058 return; 5059 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5060 } 5061 5062 /** 5063 * Add the default copy action in RX_CP_TBL. 5064 * 5065 * This functions is called in the mlx5_dev_start(). No thread safe 5066 * is guaranteed. 5067 * 5068 * @param dev 5069 * Pointer to Ethernet device. 5070 * @param[out] error 5071 * Perform verbose error reporting if not NULL. 5072 * 5073 * @return 5074 * 0 for success, negative value otherwise and rte_errno is set. 5075 */ 5076 static int 5077 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5078 struct rte_flow_error *error) 5079 { 5080 struct mlx5_priv *priv = dev->data->dev_private; 5081 struct mlx5_flow_mreg_copy_resource *mcp_res; 5082 struct mlx5_flow_cb_ctx ctx; 5083 uint32_t mark_id; 5084 5085 /* Check whether extensive metadata feature is engaged. */ 5086 if (!priv->sh->config.dv_flow_en || 5087 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5088 !mlx5_flow_ext_mreg_supported(dev) || 5089 !priv->sh->dv_regc0_mask) 5090 return 0; 5091 /* 5092 * Add default mreg copy flow may be called multiple time, but 5093 * only be called once in stop. Avoid register it twice. 5094 */ 5095 mark_id = MLX5_DEFAULT_COPY_ID; 5096 ctx.data = &mark_id; 5097 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5098 return 0; 5099 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5100 if (!mcp_res) 5101 return -rte_errno; 5102 return 0; 5103 } 5104 5105 /** 5106 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5107 * 5108 * All the flow having Q/RSS action should be split by 5109 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5110 * performs the following, 5111 * - CQE->flow_tag := reg_c[1] (MARK) 5112 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5113 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5114 * but there should be a flow per each MARK ID set by MARK action. 5115 * 5116 * For the aforementioned reason, if there's a MARK action in flow's action 5117 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5118 * the MARK ID to CQE's flow_tag like, 5119 * - If reg_c[1] is mark_id, 5120 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5121 * 5122 * For SET_META action which stores value in reg_c[0], as the destination is 5123 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5124 * MARK ID means the default flow. The default flow looks like, 5125 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5126 * 5127 * @param dev 5128 * Pointer to Ethernet device. 5129 * @param flow 5130 * Pointer to flow structure. 5131 * @param[in] actions 5132 * Pointer to the list of actions. 5133 * @param[out] error 5134 * Perform verbose error reporting if not NULL. 5135 * 5136 * @return 5137 * 0 on success, negative value otherwise and rte_errno is set. 5138 */ 5139 static int 5140 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5141 struct rte_flow *flow, 5142 const struct rte_flow_action *actions, 5143 struct rte_flow_error *error) 5144 { 5145 struct mlx5_priv *priv = dev->data->dev_private; 5146 struct mlx5_sh_config *config = &priv->sh->config; 5147 struct mlx5_flow_mreg_copy_resource *mcp_res; 5148 const struct rte_flow_action_mark *mark; 5149 5150 /* Check whether extensive metadata feature is engaged. */ 5151 if (!config->dv_flow_en || 5152 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5153 !mlx5_flow_ext_mreg_supported(dev) || 5154 !priv->sh->dv_regc0_mask) 5155 return 0; 5156 /* Find MARK action. */ 5157 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5158 switch (actions->type) { 5159 case RTE_FLOW_ACTION_TYPE_FLAG: 5160 mcp_res = flow_mreg_add_copy_action 5161 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5162 if (!mcp_res) 5163 return -rte_errno; 5164 flow->rix_mreg_copy = mcp_res->idx; 5165 return 0; 5166 case RTE_FLOW_ACTION_TYPE_MARK: 5167 mark = (const struct rte_flow_action_mark *) 5168 actions->conf; 5169 mcp_res = 5170 flow_mreg_add_copy_action(dev, mark->id, error); 5171 if (!mcp_res) 5172 return -rte_errno; 5173 flow->rix_mreg_copy = mcp_res->idx; 5174 return 0; 5175 default: 5176 break; 5177 } 5178 } 5179 return 0; 5180 } 5181 5182 #define MLX5_MAX_SPLIT_ACTIONS 24 5183 #define MLX5_MAX_SPLIT_ITEMS 24 5184 5185 /** 5186 * Split the hairpin flow. 5187 * Since HW can't support encap and push-vlan on Rx, we move these 5188 * actions to Tx. 5189 * If the count action is after the encap then we also 5190 * move the count action. in this case the count will also measure 5191 * the outer bytes. 5192 * 5193 * @param dev 5194 * Pointer to Ethernet device. 5195 * @param[in] actions 5196 * Associated actions (list terminated by the END action). 5197 * @param[out] actions_rx 5198 * Rx flow actions. 5199 * @param[out] actions_tx 5200 * Tx flow actions.. 5201 * @param[out] pattern_tx 5202 * The pattern items for the Tx flow. 5203 * @param[out] flow_id 5204 * The flow ID connected to this flow. 5205 * 5206 * @return 5207 * 0 on success. 5208 */ 5209 static int 5210 flow_hairpin_split(struct rte_eth_dev *dev, 5211 const struct rte_flow_action actions[], 5212 struct rte_flow_action actions_rx[], 5213 struct rte_flow_action actions_tx[], 5214 struct rte_flow_item pattern_tx[], 5215 uint32_t flow_id) 5216 { 5217 const struct rte_flow_action_raw_encap *raw_encap; 5218 const struct rte_flow_action_raw_decap *raw_decap; 5219 struct mlx5_rte_flow_action_set_tag *set_tag; 5220 struct rte_flow_action *tag_action; 5221 struct mlx5_rte_flow_item_tag *tag_item; 5222 struct rte_flow_item *item; 5223 char *addr; 5224 int push_vlan = 0; 5225 int encap = 0; 5226 5227 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5228 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5229 push_vlan = 1; 5230 switch (actions->type) { 5231 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5232 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5233 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5234 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5235 rte_memcpy(actions_tx, actions, 5236 sizeof(struct rte_flow_action)); 5237 actions_tx++; 5238 break; 5239 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5240 if (push_vlan) { 5241 rte_memcpy(actions_tx, actions, 5242 sizeof(struct rte_flow_action)); 5243 actions_tx++; 5244 } else { 5245 rte_memcpy(actions_rx, actions, 5246 sizeof(struct rte_flow_action)); 5247 actions_rx++; 5248 } 5249 break; 5250 case RTE_FLOW_ACTION_TYPE_COUNT: 5251 if (encap) { 5252 rte_memcpy(actions_tx, actions, 5253 sizeof(struct rte_flow_action)); 5254 actions_tx++; 5255 } else { 5256 rte_memcpy(actions_rx, actions, 5257 sizeof(struct rte_flow_action)); 5258 actions_rx++; 5259 } 5260 break; 5261 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5262 raw_encap = actions->conf; 5263 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5264 memcpy(actions_tx, actions, 5265 sizeof(struct rte_flow_action)); 5266 actions_tx++; 5267 encap = 1; 5268 } else { 5269 rte_memcpy(actions_rx, actions, 5270 sizeof(struct rte_flow_action)); 5271 actions_rx++; 5272 } 5273 break; 5274 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5275 raw_decap = actions->conf; 5276 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5277 memcpy(actions_tx, actions, 5278 sizeof(struct rte_flow_action)); 5279 actions_tx++; 5280 } else { 5281 rte_memcpy(actions_rx, actions, 5282 sizeof(struct rte_flow_action)); 5283 actions_rx++; 5284 } 5285 break; 5286 default: 5287 rte_memcpy(actions_rx, actions, 5288 sizeof(struct rte_flow_action)); 5289 actions_rx++; 5290 break; 5291 } 5292 } 5293 /* Add set meta action and end action for the Rx flow. */ 5294 tag_action = actions_rx; 5295 tag_action->type = (enum rte_flow_action_type) 5296 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5297 actions_rx++; 5298 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5299 actions_rx++; 5300 set_tag = (void *)actions_rx; 5301 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5302 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5303 .data = flow_id, 5304 }; 5305 MLX5_ASSERT(set_tag->id > REG_NON); 5306 tag_action->conf = set_tag; 5307 /* Create Tx item list. */ 5308 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5309 addr = (void *)&pattern_tx[2]; 5310 item = pattern_tx; 5311 item->type = (enum rte_flow_item_type) 5312 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5313 tag_item = (void *)addr; 5314 tag_item->data = flow_id; 5315 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5316 MLX5_ASSERT(set_tag->id > REG_NON); 5317 item->spec = tag_item; 5318 addr += sizeof(struct mlx5_rte_flow_item_tag); 5319 tag_item = (void *)addr; 5320 tag_item->data = UINT32_MAX; 5321 tag_item->id = UINT16_MAX; 5322 item->mask = tag_item; 5323 item->last = NULL; 5324 item++; 5325 item->type = RTE_FLOW_ITEM_TYPE_END; 5326 return 0; 5327 } 5328 5329 /** 5330 * The last stage of splitting chain, just creates the subflow 5331 * without any modification. 5332 * 5333 * @param[in] dev 5334 * Pointer to Ethernet device. 5335 * @param[in] flow 5336 * Parent flow structure pointer. 5337 * @param[in, out] sub_flow 5338 * Pointer to return the created subflow, may be NULL. 5339 * @param[in] attr 5340 * Flow rule attributes. 5341 * @param[in] items 5342 * Pattern specification (list terminated by the END pattern item). 5343 * @param[in] actions 5344 * Associated actions (list terminated by the END action). 5345 * @param[in] flow_split_info 5346 * Pointer to flow split info structure. 5347 * @param[out] error 5348 * Perform verbose error reporting if not NULL. 5349 * @return 5350 * 0 on success, negative value otherwise 5351 */ 5352 static int 5353 flow_create_split_inner(struct rte_eth_dev *dev, 5354 struct rte_flow *flow, 5355 struct mlx5_flow **sub_flow, 5356 const struct rte_flow_attr *attr, 5357 const struct rte_flow_item items[], 5358 const struct rte_flow_action actions[], 5359 struct mlx5_flow_split_info *flow_split_info, 5360 struct rte_flow_error *error) 5361 { 5362 struct mlx5_flow *dev_flow; 5363 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5364 5365 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5366 flow_split_info->flow_idx, error); 5367 if (!dev_flow) 5368 return -rte_errno; 5369 dev_flow->flow = flow; 5370 dev_flow->external = flow_split_info->external; 5371 dev_flow->skip_scale = flow_split_info->skip_scale; 5372 /* Subflow object was created, we must include one in the list. */ 5373 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5374 dev_flow->handle, next); 5375 /* 5376 * If dev_flow is as one of the suffix flow, some actions in suffix 5377 * flow may need some user defined item layer flags, and pass the 5378 * Metadata rxq mark flag to suffix flow as well. 5379 */ 5380 if (flow_split_info->prefix_layers) 5381 dev_flow->handle->layers = flow_split_info->prefix_layers; 5382 if (flow_split_info->prefix_mark) { 5383 MLX5_ASSERT(wks); 5384 wks->mark = 1; 5385 } 5386 if (sub_flow) 5387 *sub_flow = dev_flow; 5388 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5389 dev_flow->dv.table_id = flow_split_info->table_id; 5390 #endif 5391 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5392 } 5393 5394 /** 5395 * Get the sub policy of a meter. 5396 * 5397 * @param[in] dev 5398 * Pointer to Ethernet device. 5399 * @param[in] flow 5400 * Parent flow structure pointer. 5401 * @param wks 5402 * Pointer to thread flow work space. 5403 * @param[in] attr 5404 * Flow rule attributes. 5405 * @param[in] items 5406 * Pattern specification (list terminated by the END pattern item). 5407 * @param[out] error 5408 * Perform verbose error reporting if not NULL. 5409 * 5410 * @return 5411 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5412 */ 5413 static struct mlx5_flow_meter_sub_policy * 5414 get_meter_sub_policy(struct rte_eth_dev *dev, 5415 struct rte_flow *flow, 5416 struct mlx5_flow_workspace *wks, 5417 const struct rte_flow_attr *attr, 5418 const struct rte_flow_item items[], 5419 struct rte_flow_error *error) 5420 { 5421 struct mlx5_flow_meter_policy *policy; 5422 struct mlx5_flow_meter_policy *final_policy; 5423 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5424 5425 policy = wks->policy; 5426 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5427 if (final_policy->is_rss || final_policy->is_queue) { 5428 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5429 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5430 uint32_t i; 5431 5432 /* 5433 * This is a tmp dev_flow, 5434 * no need to register any matcher for it in translate. 5435 */ 5436 wks->skip_matcher_reg = 1; 5437 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5438 struct mlx5_flow dev_flow = {0}; 5439 struct mlx5_flow_handle dev_handle = { {0} }; 5440 uint8_t fate = final_policy->act_cnt[i].fate_action; 5441 5442 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5443 const struct rte_flow_action_rss *rss_act = 5444 final_policy->act_cnt[i].rss->conf; 5445 struct rte_flow_action rss_actions[2] = { 5446 [0] = { 5447 .type = RTE_FLOW_ACTION_TYPE_RSS, 5448 .conf = rss_act, 5449 }, 5450 [1] = { 5451 .type = RTE_FLOW_ACTION_TYPE_END, 5452 .conf = NULL, 5453 } 5454 }; 5455 5456 dev_flow.handle = &dev_handle; 5457 dev_flow.ingress = attr->ingress; 5458 dev_flow.flow = flow; 5459 dev_flow.external = 0; 5460 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5461 dev_flow.dv.transfer = attr->transfer; 5462 #endif 5463 /** 5464 * Translate RSS action to get rss hash fields. 5465 */ 5466 if (flow_drv_translate(dev, &dev_flow, attr, 5467 items, rss_actions, error)) 5468 goto exit; 5469 rss_desc_v[i] = wks->rss_desc; 5470 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5471 rss_desc_v[i].hash_fields = 5472 dev_flow.hash_fields; 5473 rss_desc_v[i].queue_num = 5474 rss_desc_v[i].hash_fields ? 5475 rss_desc_v[i].queue_num : 1; 5476 rss_desc_v[i].tunnel = 5477 !!(dev_flow.handle->layers & 5478 MLX5_FLOW_LAYER_TUNNEL); 5479 /* Use the RSS queues in the containers. */ 5480 rss_desc_v[i].queue = 5481 (uint16_t *)(uintptr_t)rss_act->queue; 5482 rss_desc[i] = &rss_desc_v[i]; 5483 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5484 /* This is queue action. */ 5485 rss_desc_v[i] = wks->rss_desc; 5486 rss_desc_v[i].key_len = 0; 5487 rss_desc_v[i].hash_fields = 0; 5488 rss_desc_v[i].queue = 5489 &final_policy->act_cnt[i].queue; 5490 rss_desc_v[i].queue_num = 1; 5491 rss_desc[i] = &rss_desc_v[i]; 5492 } else { 5493 rss_desc[i] = NULL; 5494 } 5495 } 5496 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5497 flow, policy, rss_desc); 5498 } else { 5499 enum mlx5_meter_domain mtr_domain = 5500 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5501 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5502 MLX5_MTR_DOMAIN_INGRESS); 5503 sub_policy = policy->sub_policys[mtr_domain][0]; 5504 } 5505 if (!sub_policy) 5506 rte_flow_error_set(error, EINVAL, 5507 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5508 "Failed to get meter sub-policy."); 5509 exit: 5510 return sub_policy; 5511 } 5512 5513 /** 5514 * Split the meter flow. 5515 * 5516 * As meter flow will split to three sub flow, other than meter 5517 * action, the other actions make sense to only meter accepts 5518 * the packet. If it need to be dropped, no other additional 5519 * actions should be take. 5520 * 5521 * One kind of special action which decapsulates the L3 tunnel 5522 * header will be in the prefix sub flow, as not to take the 5523 * L3 tunnel header into account. 5524 * 5525 * @param[in] dev 5526 * Pointer to Ethernet device. 5527 * @param[in] flow 5528 * Parent flow structure pointer. 5529 * @param wks 5530 * Pointer to thread flow work space. 5531 * @param[in] attr 5532 * Flow rule attributes. 5533 * @param[in] items 5534 * Pattern specification (list terminated by the END pattern item). 5535 * @param[out] sfx_items 5536 * Suffix flow match items (list terminated by the END pattern item). 5537 * @param[in] actions 5538 * Associated actions (list terminated by the END action). 5539 * @param[out] actions_sfx 5540 * Suffix flow actions. 5541 * @param[out] actions_pre 5542 * Prefix flow actions. 5543 * @param[out] mtr_flow_id 5544 * Pointer to meter flow id. 5545 * @param[out] error 5546 * Perform verbose error reporting if not NULL. 5547 * 5548 * @return 5549 * 0 on success, a negative errno value otherwise and rte_errno is set. 5550 */ 5551 static int 5552 flow_meter_split_prep(struct rte_eth_dev *dev, 5553 struct rte_flow *flow, 5554 struct mlx5_flow_workspace *wks, 5555 const struct rte_flow_attr *attr, 5556 const struct rte_flow_item items[], 5557 struct rte_flow_item sfx_items[], 5558 const struct rte_flow_action actions[], 5559 struct rte_flow_action actions_sfx[], 5560 struct rte_flow_action actions_pre[], 5561 uint32_t *mtr_flow_id, 5562 struct rte_flow_error *error) 5563 { 5564 struct mlx5_priv *priv = dev->data->dev_private; 5565 struct mlx5_flow_meter_info *fm = wks->fm; 5566 struct rte_flow_action *tag_action = NULL; 5567 struct rte_flow_item *tag_item; 5568 struct mlx5_rte_flow_action_set_tag *set_tag; 5569 const struct rte_flow_action_raw_encap *raw_encap; 5570 const struct rte_flow_action_raw_decap *raw_decap; 5571 struct mlx5_rte_flow_item_tag *tag_item_spec; 5572 struct mlx5_rte_flow_item_tag *tag_item_mask; 5573 uint32_t tag_id = 0; 5574 struct rte_flow_item *vlan_item_dst = NULL; 5575 const struct rte_flow_item *vlan_item_src = NULL; 5576 const struct rte_flow_item *orig_items = items; 5577 struct rte_flow_action *hw_mtr_action; 5578 struct rte_flow_action *action_pre_head = NULL; 5579 uint16_t flow_src_port = priv->representor_id; 5580 bool mtr_first; 5581 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5582 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5583 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5584 uint32_t flow_id = 0; 5585 uint32_t flow_id_reversed = 0; 5586 uint8_t flow_id_bits = 0; 5587 bool after_meter = false; 5588 int shift; 5589 5590 /* Prepare the suffix subflow items. */ 5591 tag_item = sfx_items++; 5592 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5593 int item_type = items->type; 5594 5595 switch (item_type) { 5596 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5597 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5598 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5599 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5600 return -rte_errno; 5601 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5602 flow_src_port != priv->representor_id) { 5603 if (flow_drv_mtr_hierarchy_rule_create(dev, 5604 flow, fm, 5605 flow_src_port, 5606 items, 5607 error)) 5608 return -rte_errno; 5609 } 5610 memcpy(sfx_items, items, sizeof(*sfx_items)); 5611 sfx_items++; 5612 break; 5613 case RTE_FLOW_ITEM_TYPE_VLAN: 5614 /* Determine if copy vlan item below. */ 5615 vlan_item_src = items; 5616 vlan_item_dst = sfx_items++; 5617 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5618 break; 5619 default: 5620 break; 5621 } 5622 } 5623 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5624 sfx_items++; 5625 mtr_first = priv->sh->meter_aso_en && 5626 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5627 /* For ASO meter, meter must be before tag in TX direction. */ 5628 if (mtr_first) { 5629 action_pre_head = actions_pre++; 5630 /* Leave space for tag action. */ 5631 tag_action = actions_pre++; 5632 } 5633 /* Prepare the actions for prefix and suffix flow. */ 5634 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5635 struct rte_flow_action *action_cur = NULL; 5636 5637 switch (actions->type) { 5638 case RTE_FLOW_ACTION_TYPE_METER: 5639 if (mtr_first) { 5640 action_cur = action_pre_head; 5641 } else { 5642 /* Leave space for tag action. */ 5643 tag_action = actions_pre++; 5644 action_cur = actions_pre++; 5645 } 5646 after_meter = true; 5647 break; 5648 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5649 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5650 action_cur = actions_pre++; 5651 break; 5652 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5653 raw_encap = actions->conf; 5654 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5655 action_cur = actions_pre++; 5656 break; 5657 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5658 raw_decap = actions->conf; 5659 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5660 action_cur = actions_pre++; 5661 break; 5662 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5663 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5664 if (vlan_item_dst && vlan_item_src) { 5665 memcpy(vlan_item_dst, vlan_item_src, 5666 sizeof(*vlan_item_dst)); 5667 /* 5668 * Convert to internal match item, it is used 5669 * for vlan push and set vid. 5670 */ 5671 vlan_item_dst->type = (enum rte_flow_item_type) 5672 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5673 } 5674 break; 5675 case RTE_FLOW_ACTION_TYPE_COUNT: 5676 if (fm->def_policy) 5677 action_cur = after_meter ? 5678 actions_sfx++ : actions_pre++; 5679 break; 5680 default: 5681 break; 5682 } 5683 if (!action_cur) 5684 action_cur = (fm->def_policy) ? 5685 actions_sfx++ : actions_pre++; 5686 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5687 } 5688 /* Add end action to the actions. */ 5689 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5690 if (priv->sh->meter_aso_en) { 5691 /** 5692 * For ASO meter, need to add an extra jump action explicitly, 5693 * to jump from meter to policer table. 5694 */ 5695 struct mlx5_flow_meter_sub_policy *sub_policy; 5696 struct mlx5_flow_tbl_data_entry *tbl_data; 5697 5698 if (!fm->def_policy) { 5699 sub_policy = get_meter_sub_policy(dev, flow, wks, 5700 attr, orig_items, 5701 error); 5702 if (!sub_policy) 5703 return -rte_errno; 5704 } else { 5705 enum mlx5_meter_domain mtr_domain = 5706 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5707 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5708 MLX5_MTR_DOMAIN_INGRESS); 5709 5710 sub_policy = 5711 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5712 } 5713 tbl_data = container_of(sub_policy->tbl_rsc, 5714 struct mlx5_flow_tbl_data_entry, tbl); 5715 hw_mtr_action = actions_pre++; 5716 hw_mtr_action->type = (enum rte_flow_action_type) 5717 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5718 hw_mtr_action->conf = tbl_data->jump.action; 5719 } 5720 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5721 actions_pre++; 5722 if (!tag_action) 5723 return rte_flow_error_set(error, ENOMEM, 5724 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5725 NULL, "No tag action space."); 5726 if (!mtr_flow_id) { 5727 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5728 goto exit; 5729 } 5730 /* Only default-policy Meter creates mtr flow id. */ 5731 if (fm->def_policy) { 5732 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5733 if (!tag_id) 5734 return rte_flow_error_set(error, ENOMEM, 5735 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5736 "Failed to allocate meter flow id."); 5737 flow_id = tag_id - 1; 5738 flow_id_bits = (!flow_id) ? 1 : 5739 (MLX5_REG_BITS - __builtin_clz(flow_id)); 5740 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5741 mtr_reg_bits) { 5742 mlx5_ipool_free(fm->flow_ipool, tag_id); 5743 return rte_flow_error_set(error, EINVAL, 5744 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5745 "Meter flow id exceeds max limit."); 5746 } 5747 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5748 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5749 } 5750 /* Build tag actions and items for meter_id/meter flow_id. */ 5751 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5752 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5753 tag_item_mask = tag_item_spec + 1; 5754 /* Both flow_id and meter_id share the same register. */ 5755 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5756 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5757 0, error), 5758 .offset = mtr_id_offset, 5759 .length = mtr_reg_bits, 5760 .data = flow->meter, 5761 }; 5762 /* 5763 * The color Reg bits used by flow_id are growing from 5764 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5765 */ 5766 for (shift = 0; shift < flow_id_bits; shift++) 5767 flow_id_reversed = (flow_id_reversed << 1) | 5768 ((flow_id >> shift) & 0x1); 5769 set_tag->data |= 5770 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5771 tag_item_spec->id = set_tag->id; 5772 tag_item_spec->data = set_tag->data << mtr_id_offset; 5773 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5774 tag_action->type = (enum rte_flow_action_type) 5775 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5776 tag_action->conf = set_tag; 5777 tag_item->type = (enum rte_flow_item_type) 5778 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5779 tag_item->spec = tag_item_spec; 5780 tag_item->last = NULL; 5781 tag_item->mask = tag_item_mask; 5782 exit: 5783 if (mtr_flow_id) 5784 *mtr_flow_id = tag_id; 5785 return 0; 5786 } 5787 5788 /** 5789 * Split action list having QUEUE/RSS for metadata register copy. 5790 * 5791 * Once Q/RSS action is detected in user's action list, the flow action 5792 * should be split in order to copy metadata registers, which will happen in 5793 * RX_CP_TBL like, 5794 * - CQE->flow_tag := reg_c[1] (MARK) 5795 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5796 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5797 * This is because the last action of each flow must be a terminal action 5798 * (QUEUE, RSS or DROP). 5799 * 5800 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5801 * stored and kept in the mlx5_flow structure per each sub_flow. 5802 * 5803 * The Q/RSS action is replaced with, 5804 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5805 * And the following JUMP action is added at the end, 5806 * - JUMP, to RX_CP_TBL. 5807 * 5808 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5809 * flow_create_split_metadata() routine. The flow will look like, 5810 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5811 * 5812 * @param dev 5813 * Pointer to Ethernet device. 5814 * @param[out] split_actions 5815 * Pointer to store split actions to jump to CP_TBL. 5816 * @param[in] actions 5817 * Pointer to the list of original flow actions. 5818 * @param[in] qrss 5819 * Pointer to the Q/RSS action. 5820 * @param[in] actions_n 5821 * Number of original actions. 5822 * @param[in] mtr_sfx 5823 * Check if it is in meter suffix table. 5824 * @param[out] error 5825 * Perform verbose error reporting if not NULL. 5826 * 5827 * @return 5828 * non-zero unique flow_id on success, otherwise 0 and 5829 * error/rte_error are set. 5830 */ 5831 static uint32_t 5832 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5833 struct rte_flow_action *split_actions, 5834 const struct rte_flow_action *actions, 5835 const struct rte_flow_action *qrss, 5836 int actions_n, int mtr_sfx, 5837 struct rte_flow_error *error) 5838 { 5839 struct mlx5_priv *priv = dev->data->dev_private; 5840 struct mlx5_rte_flow_action_set_tag *set_tag; 5841 struct rte_flow_action_jump *jump; 5842 const int qrss_idx = qrss - actions; 5843 uint32_t flow_id = 0; 5844 int ret = 0; 5845 5846 /* 5847 * Given actions will be split 5848 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 5849 * - Add jump to mreg CP_TBL. 5850 * As a result, there will be one more action. 5851 */ 5852 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 5853 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 5854 ++actions_n; 5855 set_tag = (void *)(split_actions + actions_n); 5856 /* 5857 * If we are not the meter suffix flow, add the tag action. 5858 * Since meter suffix flow already has the tag added. 5859 */ 5860 if (!mtr_sfx) { 5861 /* 5862 * Allocate the new subflow ID. This one is unique within 5863 * device and not shared with representors. Otherwise, 5864 * we would have to resolve multi-thread access synch 5865 * issue. Each flow on the shared device is appended 5866 * with source vport identifier, so the resulting 5867 * flows will be unique in the shared (by master and 5868 * representors) domain even if they have coinciding 5869 * IDs. 5870 */ 5871 mlx5_ipool_malloc(priv->sh->ipool 5872 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 5873 if (!flow_id) 5874 return rte_flow_error_set(error, ENOMEM, 5875 RTE_FLOW_ERROR_TYPE_ACTION, 5876 NULL, "can't allocate id " 5877 "for split Q/RSS subflow"); 5878 /* Internal SET_TAG action to set flow ID. */ 5879 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 5880 .data = flow_id, 5881 }; 5882 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 5883 if (ret < 0) 5884 return ret; 5885 set_tag->id = ret; 5886 /* Construct new actions array. */ 5887 /* Replace QUEUE/RSS action. */ 5888 split_actions[qrss_idx] = (struct rte_flow_action){ 5889 .type = (enum rte_flow_action_type) 5890 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 5891 .conf = set_tag, 5892 }; 5893 } else { 5894 /* 5895 * If we are the suffix flow of meter, tag already exist. 5896 * Set the QUEUE/RSS action to void. 5897 */ 5898 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 5899 } 5900 /* JUMP action to jump to mreg copy table (CP_TBL). */ 5901 jump = (void *)(set_tag + 1); 5902 *jump = (struct rte_flow_action_jump){ 5903 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 5904 }; 5905 split_actions[actions_n - 2] = (struct rte_flow_action){ 5906 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5907 .conf = jump, 5908 }; 5909 split_actions[actions_n - 1] = (struct rte_flow_action){ 5910 .type = RTE_FLOW_ACTION_TYPE_END, 5911 }; 5912 return flow_id; 5913 } 5914 5915 /** 5916 * Extend the given action list for Tx metadata copy. 5917 * 5918 * Copy the given action list to the ext_actions and add flow metadata register 5919 * copy action in order to copy reg_a set by WQE to reg_c[0]. 5920 * 5921 * @param[out] ext_actions 5922 * Pointer to the extended action list. 5923 * @param[in] actions 5924 * Pointer to the list of actions. 5925 * @param[in] actions_n 5926 * Number of actions in the list. 5927 * @param[out] error 5928 * Perform verbose error reporting if not NULL. 5929 * @param[in] encap_idx 5930 * The encap action index. 5931 * 5932 * @return 5933 * 0 on success, negative value otherwise 5934 */ 5935 static int 5936 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 5937 struct rte_flow_action *ext_actions, 5938 const struct rte_flow_action *actions, 5939 int actions_n, struct rte_flow_error *error, 5940 int encap_idx) 5941 { 5942 struct mlx5_flow_action_copy_mreg *cp_mreg = 5943 (struct mlx5_flow_action_copy_mreg *) 5944 (ext_actions + actions_n + 1); 5945 int ret; 5946 5947 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 5948 if (ret < 0) 5949 return ret; 5950 cp_mreg->dst = ret; 5951 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 5952 if (ret < 0) 5953 return ret; 5954 cp_mreg->src = ret; 5955 if (encap_idx != 0) 5956 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 5957 if (encap_idx == actions_n - 1) { 5958 ext_actions[actions_n - 1] = (struct rte_flow_action){ 5959 .type = (enum rte_flow_action_type) 5960 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5961 .conf = cp_mreg, 5962 }; 5963 ext_actions[actions_n] = (struct rte_flow_action){ 5964 .type = RTE_FLOW_ACTION_TYPE_END, 5965 }; 5966 } else { 5967 ext_actions[encap_idx] = (struct rte_flow_action){ 5968 .type = (enum rte_flow_action_type) 5969 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5970 .conf = cp_mreg, 5971 }; 5972 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 5973 sizeof(*ext_actions) * (actions_n - encap_idx)); 5974 } 5975 return 0; 5976 } 5977 5978 /** 5979 * Check the match action from the action list. 5980 * 5981 * @param[in] actions 5982 * Pointer to the list of actions. 5983 * @param[in] attr 5984 * Flow rule attributes. 5985 * @param[in] action 5986 * The action to be check if exist. 5987 * @param[out] match_action_pos 5988 * Pointer to the position of the matched action if exists, otherwise is -1. 5989 * @param[out] qrss_action_pos 5990 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 5991 * @param[out] modify_after_mirror 5992 * Pointer to the flag of modify action after FDB mirroring. 5993 * 5994 * @return 5995 * > 0 the total number of actions. 5996 * 0 if not found match action in action list. 5997 */ 5998 static int 5999 flow_check_match_action(const struct rte_flow_action actions[], 6000 const struct rte_flow_attr *attr, 6001 enum rte_flow_action_type action, 6002 int *match_action_pos, int *qrss_action_pos, 6003 int *modify_after_mirror) 6004 { 6005 const struct rte_flow_action_sample *sample; 6006 const struct rte_flow_action_raw_decap *decap; 6007 int actions_n = 0; 6008 uint32_t ratio = 0; 6009 int sub_type = 0; 6010 int flag = 0; 6011 int fdb_mirror = 0; 6012 6013 *match_action_pos = -1; 6014 *qrss_action_pos = -1; 6015 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6016 if (actions->type == action) { 6017 flag = 1; 6018 *match_action_pos = actions_n; 6019 } 6020 switch (actions->type) { 6021 case RTE_FLOW_ACTION_TYPE_QUEUE: 6022 case RTE_FLOW_ACTION_TYPE_RSS: 6023 *qrss_action_pos = actions_n; 6024 break; 6025 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6026 sample = actions->conf; 6027 ratio = sample->ratio; 6028 sub_type = ((const struct rte_flow_action *) 6029 (sample->actions))->type; 6030 if (ratio == 1 && attr->transfer && 6031 sub_type != RTE_FLOW_ACTION_TYPE_END) 6032 fdb_mirror = 1; 6033 break; 6034 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6035 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6036 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6037 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6038 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6039 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6040 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6041 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6042 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6043 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6044 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6045 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6046 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6047 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6048 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6049 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6050 case RTE_FLOW_ACTION_TYPE_FLAG: 6051 case RTE_FLOW_ACTION_TYPE_MARK: 6052 case RTE_FLOW_ACTION_TYPE_SET_META: 6053 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6054 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6055 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6056 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6057 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6058 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6059 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6060 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6061 case RTE_FLOW_ACTION_TYPE_METER: 6062 if (fdb_mirror) 6063 *modify_after_mirror = 1; 6064 break; 6065 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6066 decap = actions->conf; 6067 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 6068 ; 6069 actions_n++; 6070 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6071 const struct rte_flow_action_raw_encap *encap = 6072 actions->conf; 6073 if (decap->size <= 6074 MLX5_ENCAPSULATION_DECISION_SIZE && 6075 encap->size > 6076 MLX5_ENCAPSULATION_DECISION_SIZE) 6077 /* L3 encap. */ 6078 break; 6079 } 6080 if (fdb_mirror) 6081 *modify_after_mirror = 1; 6082 break; 6083 default: 6084 break; 6085 } 6086 actions_n++; 6087 } 6088 if (flag && fdb_mirror && !*modify_after_mirror) { 6089 /* FDB mirroring uses the destination array to implement 6090 * instead of FLOW_SAMPLER object. 6091 */ 6092 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6093 flag = 0; 6094 } 6095 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6096 return flag ? actions_n + 1 : 0; 6097 } 6098 6099 #define SAMPLE_SUFFIX_ITEM 3 6100 6101 /** 6102 * Split the sample flow. 6103 * 6104 * As sample flow will split to two sub flow, sample flow with 6105 * sample action, the other actions will move to new suffix flow. 6106 * 6107 * Also add unique tag id with tag action in the sample flow, 6108 * the same tag id will be as match in the suffix flow. 6109 * 6110 * @param dev 6111 * Pointer to Ethernet device. 6112 * @param[in] add_tag 6113 * Add extra tag action flag. 6114 * @param[out] sfx_items 6115 * Suffix flow match items (list terminated by the END pattern item). 6116 * @param[in] actions 6117 * Associated actions (list terminated by the END action). 6118 * @param[out] actions_sfx 6119 * Suffix flow actions. 6120 * @param[out] actions_pre 6121 * Prefix flow actions. 6122 * @param[in] actions_n 6123 * The total number of actions. 6124 * @param[in] sample_action_pos 6125 * The sample action position. 6126 * @param[in] qrss_action_pos 6127 * The Queue/RSS action position. 6128 * @param[in] jump_table 6129 * Add extra jump action flag. 6130 * @param[out] error 6131 * Perform verbose error reporting if not NULL. 6132 * 6133 * @return 6134 * 0 on success, or unique flow_id, a negative errno value 6135 * otherwise and rte_errno is set. 6136 */ 6137 static int 6138 flow_sample_split_prep(struct rte_eth_dev *dev, 6139 int add_tag, 6140 const struct rte_flow_item items[], 6141 struct rte_flow_item sfx_items[], 6142 const struct rte_flow_action actions[], 6143 struct rte_flow_action actions_sfx[], 6144 struct rte_flow_action actions_pre[], 6145 int actions_n, 6146 int sample_action_pos, 6147 int qrss_action_pos, 6148 int jump_table, 6149 struct rte_flow_error *error) 6150 { 6151 struct mlx5_priv *priv = dev->data->dev_private; 6152 struct mlx5_rte_flow_action_set_tag *set_tag; 6153 struct mlx5_rte_flow_item_tag *tag_spec; 6154 struct mlx5_rte_flow_item_tag *tag_mask; 6155 struct rte_flow_action_jump *jump_action; 6156 uint32_t tag_id = 0; 6157 int append_index = 0; 6158 int set_tag_idx = -1; 6159 int index; 6160 int ret; 6161 6162 if (sample_action_pos < 0) 6163 return rte_flow_error_set(error, EINVAL, 6164 RTE_FLOW_ERROR_TYPE_ACTION, 6165 NULL, "invalid position of sample " 6166 "action in list"); 6167 /* Prepare the actions for prefix and suffix flow. */ 6168 if (add_tag) { 6169 /* Update the new added tag action index preceding 6170 * the PUSH_VLAN or ENCAP action. 6171 */ 6172 const struct rte_flow_action_raw_encap *raw_encap; 6173 const struct rte_flow_action *action = actions; 6174 int encap_idx; 6175 int action_idx = 0; 6176 int raw_decap_idx = -1; 6177 int push_vlan_idx = -1; 6178 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6179 switch (action->type) { 6180 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6181 raw_decap_idx = action_idx; 6182 break; 6183 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6184 raw_encap = action->conf; 6185 if (raw_encap->size > 6186 MLX5_ENCAPSULATION_DECISION_SIZE) { 6187 encap_idx = raw_decap_idx != -1 ? 6188 raw_decap_idx : action_idx; 6189 if (encap_idx < sample_action_pos && 6190 push_vlan_idx == -1) 6191 set_tag_idx = encap_idx; 6192 } 6193 break; 6194 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6195 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6196 encap_idx = action_idx; 6197 if (encap_idx < sample_action_pos && 6198 push_vlan_idx == -1) 6199 set_tag_idx = encap_idx; 6200 break; 6201 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6202 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6203 if (action_idx < sample_action_pos && 6204 push_vlan_idx == -1) { 6205 set_tag_idx = action_idx; 6206 push_vlan_idx = action_idx; 6207 } 6208 break; 6209 default: 6210 break; 6211 } 6212 action_idx++; 6213 } 6214 } 6215 /* Prepare the actions for prefix and suffix flow. */ 6216 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6217 index = qrss_action_pos; 6218 /* Put the preceding the Queue/RSS action into prefix flow. */ 6219 if (index != 0) 6220 memcpy(actions_pre, actions, 6221 sizeof(struct rte_flow_action) * index); 6222 /* Put others preceding the sample action into prefix flow. */ 6223 if (sample_action_pos > index + 1) 6224 memcpy(actions_pre + index, actions + index + 1, 6225 sizeof(struct rte_flow_action) * 6226 (sample_action_pos - index - 1)); 6227 index = sample_action_pos - 1; 6228 /* Put Queue/RSS action into Suffix flow. */ 6229 memcpy(actions_sfx, actions + qrss_action_pos, 6230 sizeof(struct rte_flow_action)); 6231 actions_sfx++; 6232 } else if (add_tag && set_tag_idx >= 0) { 6233 if (set_tag_idx > 0) 6234 memcpy(actions_pre, actions, 6235 sizeof(struct rte_flow_action) * set_tag_idx); 6236 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6237 sizeof(struct rte_flow_action) * 6238 (sample_action_pos - set_tag_idx)); 6239 index = sample_action_pos; 6240 } else { 6241 index = sample_action_pos; 6242 if (index != 0) 6243 memcpy(actions_pre, actions, 6244 sizeof(struct rte_flow_action) * index); 6245 } 6246 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6247 * For CX6DX and above, metadata registers Cx preserve their value, 6248 * add an extra tag action for NIC-RX and E-Switch Domain. 6249 */ 6250 if (add_tag) { 6251 /* Prepare the prefix tag action. */ 6252 append_index++; 6253 set_tag = (void *)(actions_pre + actions_n + append_index); 6254 /* Trust VF/SF on CX5 not supported meter so that the reserved 6255 * metadata regC is REG_NON, back to use application tag 6256 * index 0. 6257 */ 6258 if (unlikely(priv->mtr_color_reg == REG_NON)) 6259 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6260 else 6261 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6262 if (ret < 0) 6263 return ret; 6264 mlx5_ipool_malloc(priv->sh->ipool 6265 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6266 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6267 .id = ret, 6268 .data = tag_id, 6269 }; 6270 /* Prepare the suffix subflow items. */ 6271 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6272 tag_spec->data = tag_id; 6273 tag_spec->id = set_tag->id; 6274 tag_mask = tag_spec + 1; 6275 tag_mask->data = UINT32_MAX; 6276 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6277 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6278 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6279 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6280 memcpy(sfx_items, items, sizeof(*sfx_items)); 6281 sfx_items++; 6282 break; 6283 } 6284 } 6285 sfx_items[0] = (struct rte_flow_item){ 6286 .type = (enum rte_flow_item_type) 6287 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6288 .spec = tag_spec, 6289 .last = NULL, 6290 .mask = tag_mask, 6291 }; 6292 sfx_items[1] = (struct rte_flow_item){ 6293 .type = (enum rte_flow_item_type) 6294 RTE_FLOW_ITEM_TYPE_END, 6295 }; 6296 /* Prepare the tag action in prefix subflow. */ 6297 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6298 actions_pre[set_tag_idx] = 6299 (struct rte_flow_action){ 6300 .type = (enum rte_flow_action_type) 6301 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6302 .conf = set_tag, 6303 }; 6304 /* Update next sample position due to add one tag action */ 6305 index += 1; 6306 } 6307 /* Copy the sample action into prefix flow. */ 6308 memcpy(actions_pre + index, actions + sample_action_pos, 6309 sizeof(struct rte_flow_action)); 6310 index += 1; 6311 /* For the modify action after the sample action in E-Switch mirroring, 6312 * Add the extra jump action in prefix subflow and jump into the next 6313 * table, then do the modify action in the new table. 6314 */ 6315 if (jump_table) { 6316 /* Prepare the prefix jump action. */ 6317 append_index++; 6318 jump_action = (void *)(actions_pre + actions_n + append_index); 6319 jump_action->group = jump_table; 6320 actions_pre[index++] = 6321 (struct rte_flow_action){ 6322 .type = (enum rte_flow_action_type) 6323 RTE_FLOW_ACTION_TYPE_JUMP, 6324 .conf = jump_action, 6325 }; 6326 } 6327 actions_pre[index] = (struct rte_flow_action){ 6328 .type = (enum rte_flow_action_type) 6329 RTE_FLOW_ACTION_TYPE_END, 6330 }; 6331 /* Put the actions after sample into Suffix flow. */ 6332 memcpy(actions_sfx, actions + sample_action_pos + 1, 6333 sizeof(struct rte_flow_action) * 6334 (actions_n - sample_action_pos - 1)); 6335 return tag_id; 6336 } 6337 6338 /** 6339 * The splitting for metadata feature. 6340 * 6341 * - Q/RSS action on NIC Rx should be split in order to pass by 6342 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6343 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6344 * 6345 * - All the actions on NIC Tx should have a mreg copy action to 6346 * copy reg_a from WQE to reg_c[0]. 6347 * 6348 * @param dev 6349 * Pointer to Ethernet device. 6350 * @param[in] flow 6351 * Parent flow structure pointer. 6352 * @param[in] attr 6353 * Flow rule attributes. 6354 * @param[in] items 6355 * Pattern specification (list terminated by the END pattern item). 6356 * @param[in] actions 6357 * Associated actions (list terminated by the END action). 6358 * @param[in] flow_split_info 6359 * Pointer to flow split info structure. 6360 * @param[out] error 6361 * Perform verbose error reporting if not NULL. 6362 * @return 6363 * 0 on success, negative value otherwise 6364 */ 6365 static int 6366 flow_create_split_metadata(struct rte_eth_dev *dev, 6367 struct rte_flow *flow, 6368 const struct rte_flow_attr *attr, 6369 const struct rte_flow_item items[], 6370 const struct rte_flow_action actions[], 6371 struct mlx5_flow_split_info *flow_split_info, 6372 struct rte_flow_error *error) 6373 { 6374 struct mlx5_priv *priv = dev->data->dev_private; 6375 struct mlx5_sh_config *config = &priv->sh->config; 6376 const struct rte_flow_action *qrss = NULL; 6377 struct rte_flow_action *ext_actions = NULL; 6378 struct mlx5_flow *dev_flow = NULL; 6379 uint32_t qrss_id = 0; 6380 int mtr_sfx = 0; 6381 size_t act_size; 6382 int actions_n; 6383 int encap_idx; 6384 int ret; 6385 6386 /* Check whether extensive metadata feature is engaged. */ 6387 if (!config->dv_flow_en || 6388 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6389 !mlx5_flow_ext_mreg_supported(dev)) 6390 return flow_create_split_inner(dev, flow, NULL, attr, items, 6391 actions, flow_split_info, error); 6392 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6393 &encap_idx); 6394 if (qrss) { 6395 /* Exclude hairpin flows from splitting. */ 6396 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6397 const struct rte_flow_action_queue *queue; 6398 6399 queue = qrss->conf; 6400 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6401 qrss = NULL; 6402 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6403 const struct rte_flow_action_rss *rss; 6404 6405 rss = qrss->conf; 6406 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6407 qrss = NULL; 6408 } 6409 } 6410 if (qrss) { 6411 /* Check if it is in meter suffix table. */ 6412 mtr_sfx = attr->group == 6413 ((attr->transfer && priv->fdb_def_rule) ? 6414 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6415 MLX5_FLOW_TABLE_LEVEL_METER); 6416 /* 6417 * Q/RSS action on NIC Rx should be split in order to pass by 6418 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6419 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6420 */ 6421 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6422 sizeof(struct rte_flow_action_set_tag) + 6423 sizeof(struct rte_flow_action_jump); 6424 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6425 SOCKET_ID_ANY); 6426 if (!ext_actions) 6427 return rte_flow_error_set(error, ENOMEM, 6428 RTE_FLOW_ERROR_TYPE_ACTION, 6429 NULL, "no memory to split " 6430 "metadata flow"); 6431 /* 6432 * Create the new actions list with removed Q/RSS action 6433 * and appended set tag and jump to register copy table 6434 * (RX_CP_TBL). We should preallocate unique tag ID here 6435 * in advance, because it is needed for set tag action. 6436 */ 6437 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6438 qrss, actions_n, 6439 mtr_sfx, error); 6440 if (!mtr_sfx && !qrss_id) { 6441 ret = -rte_errno; 6442 goto exit; 6443 } 6444 } else if (attr->egress) { 6445 /* 6446 * All the actions on NIC Tx should have a metadata register 6447 * copy action to copy reg_a from WQE to reg_c[meta] 6448 */ 6449 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6450 sizeof(struct mlx5_flow_action_copy_mreg); 6451 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6452 SOCKET_ID_ANY); 6453 if (!ext_actions) 6454 return rte_flow_error_set(error, ENOMEM, 6455 RTE_FLOW_ERROR_TYPE_ACTION, 6456 NULL, "no memory to split " 6457 "metadata flow"); 6458 /* Create the action list appended with copy register. */ 6459 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6460 actions_n, error, encap_idx); 6461 if (ret < 0) 6462 goto exit; 6463 } 6464 /* Add the unmodified original or prefix subflow. */ 6465 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6466 items, ext_actions ? ext_actions : 6467 actions, flow_split_info, error); 6468 if (ret < 0) 6469 goto exit; 6470 MLX5_ASSERT(dev_flow); 6471 if (qrss) { 6472 const struct rte_flow_attr q_attr = { 6473 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6474 .ingress = 1, 6475 }; 6476 /* Internal PMD action to set register. */ 6477 struct mlx5_rte_flow_item_tag q_tag_spec = { 6478 .data = qrss_id, 6479 .id = REG_NON, 6480 }; 6481 struct rte_flow_item q_items[] = { 6482 { 6483 .type = (enum rte_flow_item_type) 6484 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6485 .spec = &q_tag_spec, 6486 .last = NULL, 6487 .mask = NULL, 6488 }, 6489 { 6490 .type = RTE_FLOW_ITEM_TYPE_END, 6491 }, 6492 }; 6493 struct rte_flow_action q_actions[] = { 6494 { 6495 .type = qrss->type, 6496 .conf = qrss->conf, 6497 }, 6498 { 6499 .type = RTE_FLOW_ACTION_TYPE_END, 6500 }, 6501 }; 6502 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6503 6504 /* 6505 * Configure the tag item only if there is no meter subflow. 6506 * Since tag is already marked in the meter suffix subflow 6507 * we can just use the meter suffix items as is. 6508 */ 6509 if (qrss_id) { 6510 /* Not meter subflow. */ 6511 MLX5_ASSERT(!mtr_sfx); 6512 /* 6513 * Put unique id in prefix flow due to it is destroyed 6514 * after suffix flow and id will be freed after there 6515 * is no actual flows with this id and identifier 6516 * reallocation becomes possible (for example, for 6517 * other flows in other threads). 6518 */ 6519 dev_flow->handle->split_flow_id = qrss_id; 6520 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6521 error); 6522 if (ret < 0) 6523 goto exit; 6524 q_tag_spec.id = ret; 6525 } 6526 dev_flow = NULL; 6527 /* Add suffix subflow to execute Q/RSS. */ 6528 flow_split_info->prefix_layers = layers; 6529 flow_split_info->prefix_mark = 0; 6530 flow_split_info->table_id = 0; 6531 ret = flow_create_split_inner(dev, flow, &dev_flow, 6532 &q_attr, mtr_sfx ? items : 6533 q_items, q_actions, 6534 flow_split_info, error); 6535 if (ret < 0) 6536 goto exit; 6537 /* qrss ID should be freed if failed. */ 6538 qrss_id = 0; 6539 MLX5_ASSERT(dev_flow); 6540 } 6541 6542 exit: 6543 /* 6544 * We do not destroy the partially created sub_flows in case of error. 6545 * These ones are included into parent flow list and will be destroyed 6546 * by flow_drv_destroy. 6547 */ 6548 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6549 qrss_id); 6550 mlx5_free(ext_actions); 6551 return ret; 6552 } 6553 6554 /** 6555 * Create meter internal drop flow with the original pattern. 6556 * 6557 * @param dev 6558 * Pointer to Ethernet device. 6559 * @param[in] flow 6560 * Parent flow structure pointer. 6561 * @param[in] attr 6562 * Flow rule attributes. 6563 * @param[in] items 6564 * Pattern specification (list terminated by the END pattern item). 6565 * @param[in] flow_split_info 6566 * Pointer to flow split info structure. 6567 * @param[in] fm 6568 * Pointer to flow meter structure. 6569 * @param[out] error 6570 * Perform verbose error reporting if not NULL. 6571 * @return 6572 * 0 on success, negative value otherwise 6573 */ 6574 static uint32_t 6575 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6576 struct rte_flow *flow, 6577 const struct rte_flow_attr *attr, 6578 const struct rte_flow_item items[], 6579 struct mlx5_flow_split_info *flow_split_info, 6580 struct mlx5_flow_meter_info *fm, 6581 struct rte_flow_error *error) 6582 { 6583 struct mlx5_flow *dev_flow = NULL; 6584 struct rte_flow_attr drop_attr = *attr; 6585 struct rte_flow_action drop_actions[3]; 6586 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6587 6588 MLX5_ASSERT(fm->drop_cnt); 6589 drop_actions[0].type = 6590 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6591 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6592 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6593 drop_actions[1].conf = NULL; 6594 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6595 drop_actions[2].conf = NULL; 6596 drop_split_info.external = false; 6597 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6598 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6599 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6600 return flow_create_split_inner(dev, flow, &dev_flow, 6601 &drop_attr, items, drop_actions, 6602 &drop_split_info, error); 6603 } 6604 6605 /** 6606 * The splitting for meter feature. 6607 * 6608 * - The meter flow will be split to two flows as prefix and 6609 * suffix flow. The packets make sense only it pass the prefix 6610 * meter action. 6611 * 6612 * - Reg_C_5 is used for the packet to match betweend prefix and 6613 * suffix flow. 6614 * 6615 * @param dev 6616 * Pointer to Ethernet device. 6617 * @param[in] flow 6618 * Parent flow structure pointer. 6619 * @param[in] attr 6620 * Flow rule attributes. 6621 * @param[in] items 6622 * Pattern specification (list terminated by the END pattern item). 6623 * @param[in] actions 6624 * Associated actions (list terminated by the END action). 6625 * @param[in] flow_split_info 6626 * Pointer to flow split info structure. 6627 * @param[out] error 6628 * Perform verbose error reporting if not NULL. 6629 * @return 6630 * 0 on success, negative value otherwise 6631 */ 6632 static int 6633 flow_create_split_meter(struct rte_eth_dev *dev, 6634 struct rte_flow *flow, 6635 const struct rte_flow_attr *attr, 6636 const struct rte_flow_item items[], 6637 const struct rte_flow_action actions[], 6638 struct mlx5_flow_split_info *flow_split_info, 6639 struct rte_flow_error *error) 6640 { 6641 struct mlx5_priv *priv = dev->data->dev_private; 6642 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6643 struct rte_flow_action *sfx_actions = NULL; 6644 struct rte_flow_action *pre_actions = NULL; 6645 struct rte_flow_item *sfx_items = NULL; 6646 struct mlx5_flow *dev_flow = NULL; 6647 struct rte_flow_attr sfx_attr = *attr; 6648 struct mlx5_flow_meter_info *fm = NULL; 6649 uint8_t skip_scale_restore; 6650 bool has_mtr = false; 6651 bool has_modify = false; 6652 bool set_mtr_reg = true; 6653 bool is_mtr_hierarchy = false; 6654 uint32_t meter_id = 0; 6655 uint32_t mtr_idx = 0; 6656 uint32_t mtr_flow_id = 0; 6657 size_t act_size; 6658 size_t item_size; 6659 int actions_n = 0; 6660 int ret = 0; 6661 6662 if (priv->mtr_en) 6663 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6664 &has_modify, &meter_id); 6665 if (has_mtr) { 6666 if (flow->meter) { 6667 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6668 if (!fm) 6669 return rte_flow_error_set(error, EINVAL, 6670 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6671 NULL, "Meter not found."); 6672 } else { 6673 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6674 if (!fm) 6675 return rte_flow_error_set(error, EINVAL, 6676 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6677 NULL, "Meter not found."); 6678 ret = mlx5_flow_meter_attach(priv, fm, 6679 &sfx_attr, error); 6680 if (ret) 6681 return -rte_errno; 6682 flow->meter = mtr_idx; 6683 } 6684 MLX5_ASSERT(wks); 6685 wks->fm = fm; 6686 if (!fm->def_policy) { 6687 wks->policy = mlx5_flow_meter_policy_find(dev, 6688 fm->policy_id, 6689 NULL); 6690 MLX5_ASSERT(wks->policy); 6691 if (wks->policy->mark) 6692 wks->mark = 1; 6693 if (wks->policy->is_hierarchy) { 6694 wks->final_policy = 6695 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6696 wks->policy); 6697 if (!wks->final_policy) 6698 return rte_flow_error_set(error, 6699 EINVAL, 6700 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6701 "Failed to find terminal policy of hierarchy."); 6702 is_mtr_hierarchy = true; 6703 } 6704 } 6705 /* 6706 * If it isn't default-policy Meter, and 6707 * 1. Not meter hierarchy and there's no action in flow to change 6708 * packet (modify/encap/decap etc.), OR 6709 * 2. No drop count needed for this meter. 6710 * Then no need to use regC to save meter id anymore. 6711 */ 6712 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6713 set_mtr_reg = false; 6714 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6715 #define METER_PREFIX_ACTION 7 6716 act_size = (sizeof(struct rte_flow_action) * 6717 (actions_n + METER_PREFIX_ACTION)) + 6718 sizeof(struct mlx5_rte_flow_action_set_tag); 6719 /* Suffix items: tag, vlan, port id, end. */ 6720 #define METER_SUFFIX_ITEM 4 6721 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6722 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6723 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6724 0, SOCKET_ID_ANY); 6725 if (!sfx_actions) 6726 return rte_flow_error_set(error, ENOMEM, 6727 RTE_FLOW_ERROR_TYPE_ACTION, 6728 NULL, "no memory to split " 6729 "meter flow"); 6730 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6731 act_size); 6732 /* There's no suffix flow for meter of non-default policy. */ 6733 if (!fm->def_policy) 6734 pre_actions = sfx_actions + 1; 6735 else 6736 pre_actions = sfx_actions + actions_n; 6737 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6738 items, sfx_items, actions, 6739 sfx_actions, pre_actions, 6740 (set_mtr_reg ? &mtr_flow_id : NULL), 6741 error); 6742 if (ret) { 6743 ret = -rte_errno; 6744 goto exit; 6745 } 6746 /* Add the prefix subflow. */ 6747 skip_scale_restore = flow_split_info->skip_scale; 6748 flow_split_info->skip_scale |= 6749 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6750 ret = flow_create_split_inner(dev, flow, &dev_flow, 6751 attr, items, pre_actions, 6752 flow_split_info, error); 6753 flow_split_info->skip_scale = skip_scale_restore; 6754 if (ret) { 6755 if (mtr_flow_id) 6756 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6757 ret = -rte_errno; 6758 goto exit; 6759 } 6760 if (mtr_flow_id) { 6761 dev_flow->handle->split_flow_id = mtr_flow_id; 6762 dev_flow->handle->is_meter_flow_id = 1; 6763 } 6764 if (!fm->def_policy) { 6765 if (!set_mtr_reg && fm->drop_cnt) 6766 ret = 6767 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6768 &sfx_attr, items, 6769 flow_split_info, 6770 fm, error); 6771 goto exit; 6772 } 6773 /* Setting the sfx group atrr. */ 6774 sfx_attr.group = sfx_attr.transfer ? 6775 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6776 MLX5_FLOW_TABLE_LEVEL_METER; 6777 flow_split_info->prefix_layers = 6778 flow_get_prefix_layer_flags(dev_flow); 6779 flow_split_info->prefix_mark |= wks->mark; 6780 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6781 } 6782 /* Add the prefix subflow. */ 6783 ret = flow_create_split_metadata(dev, flow, 6784 &sfx_attr, sfx_items ? 6785 sfx_items : items, 6786 sfx_actions ? sfx_actions : actions, 6787 flow_split_info, error); 6788 exit: 6789 if (sfx_actions) 6790 mlx5_free(sfx_actions); 6791 return ret; 6792 } 6793 6794 /** 6795 * The splitting for sample feature. 6796 * 6797 * Once Sample action is detected in the action list, the flow actions should 6798 * be split into prefix sub flow and suffix sub flow. 6799 * 6800 * The original items remain in the prefix sub flow, all actions preceding the 6801 * sample action and the sample action itself will be copied to the prefix 6802 * sub flow, the actions following the sample action will be copied to the 6803 * suffix sub flow, Queue action always be located in the suffix sub flow. 6804 * 6805 * In order to make the packet from prefix sub flow matches with suffix sub 6806 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6807 * flow uses tag item with the unique flow id. 6808 * 6809 * @param dev 6810 * Pointer to Ethernet device. 6811 * @param[in] flow 6812 * Parent flow structure pointer. 6813 * @param[in] attr 6814 * Flow rule attributes. 6815 * @param[in] items 6816 * Pattern specification (list terminated by the END pattern item). 6817 * @param[in] actions 6818 * Associated actions (list terminated by the END action). 6819 * @param[in] flow_split_info 6820 * Pointer to flow split info structure. 6821 * @param[out] error 6822 * Perform verbose error reporting if not NULL. 6823 * @return 6824 * 0 on success, negative value otherwise 6825 */ 6826 static int 6827 flow_create_split_sample(struct rte_eth_dev *dev, 6828 struct rte_flow *flow, 6829 const struct rte_flow_attr *attr, 6830 const struct rte_flow_item items[], 6831 const struct rte_flow_action actions[], 6832 struct mlx5_flow_split_info *flow_split_info, 6833 struct rte_flow_error *error) 6834 { 6835 struct mlx5_priv *priv = dev->data->dev_private; 6836 struct rte_flow_action *sfx_actions = NULL; 6837 struct rte_flow_action *pre_actions = NULL; 6838 struct rte_flow_item *sfx_items = NULL; 6839 struct mlx5_flow *dev_flow = NULL; 6840 struct rte_flow_attr sfx_attr = *attr; 6841 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6842 struct mlx5_flow_dv_sample_resource *sample_res; 6843 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6844 struct mlx5_flow_tbl_resource *sfx_tbl; 6845 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6846 #endif 6847 size_t act_size; 6848 size_t item_size; 6849 uint32_t fdb_tx = 0; 6850 int32_t tag_id = 0; 6851 int actions_n = 0; 6852 int sample_action_pos; 6853 int qrss_action_pos; 6854 int add_tag = 0; 6855 int modify_after_mirror = 0; 6856 uint16_t jump_table = 0; 6857 const uint32_t next_ft_step = 1; 6858 int ret = 0; 6859 struct mlx5_priv *item_port_priv = NULL; 6860 const struct rte_flow_item *item; 6861 6862 if (priv->sampler_en) 6863 actions_n = flow_check_match_action(actions, attr, 6864 RTE_FLOW_ACTION_TYPE_SAMPLE, 6865 &sample_action_pos, &qrss_action_pos, 6866 &modify_after_mirror); 6867 if (actions_n) { 6868 /* The prefix actions must includes sample, tag, end. */ 6869 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 6870 + sizeof(struct mlx5_rte_flow_action_set_tag); 6871 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 6872 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6873 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 6874 item_size), 0, SOCKET_ID_ANY); 6875 if (!sfx_actions) 6876 return rte_flow_error_set(error, ENOMEM, 6877 RTE_FLOW_ERROR_TYPE_ACTION, 6878 NULL, "no memory to split " 6879 "sample flow"); 6880 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 6881 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 6882 const struct rte_flow_item_port_id *spec; 6883 6884 spec = (const struct rte_flow_item_port_id *)item->spec; 6885 if (spec) 6886 item_port_priv = 6887 mlx5_port_to_eswitch_info(spec->id, true); 6888 break; 6889 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6890 const struct rte_flow_item_ethdev *spec; 6891 6892 spec = (const struct rte_flow_item_ethdev *)item->spec; 6893 if (spec) 6894 item_port_priv = 6895 mlx5_port_to_eswitch_info(spec->port_id, true); 6896 break; 6897 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 6898 const struct rte_flow_item_ethdev *spec; 6899 6900 spec = (const struct rte_flow_item_ethdev *)item->spec; 6901 if (spec) 6902 item_port_priv = 6903 mlx5_port_to_eswitch_info(spec->port_id, true); 6904 break; 6905 } 6906 } 6907 /* The representor_id is UINT16_MAX for uplink. */ 6908 fdb_tx = (attr->transfer && 6909 flow_source_vport_representor(priv, item_port_priv)); 6910 /* 6911 * When reg_c_preserve is set, metadata registers Cx preserve 6912 * their value even through packet duplication. 6913 */ 6914 add_tag = (!fdb_tx || 6915 priv->sh->cdev->config.hca_attr.reg_c_preserve); 6916 if (add_tag) 6917 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 6918 + act_size); 6919 if (modify_after_mirror) 6920 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 6921 next_ft_step; 6922 pre_actions = sfx_actions + actions_n; 6923 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 6924 actions, sfx_actions, 6925 pre_actions, actions_n, 6926 sample_action_pos, 6927 qrss_action_pos, jump_table, 6928 error); 6929 if (tag_id < 0 || (add_tag && !tag_id)) { 6930 ret = -rte_errno; 6931 goto exit; 6932 } 6933 if (modify_after_mirror) 6934 flow_split_info->skip_scale = 6935 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6936 /* Add the prefix subflow. */ 6937 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6938 items, pre_actions, 6939 flow_split_info, error); 6940 if (ret) { 6941 ret = -rte_errno; 6942 goto exit; 6943 } 6944 dev_flow->handle->split_flow_id = tag_id; 6945 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6946 if (!modify_after_mirror) { 6947 /* Set the sfx group attr. */ 6948 sample_res = (struct mlx5_flow_dv_sample_resource *) 6949 dev_flow->dv.sample_res; 6950 sfx_tbl = (struct mlx5_flow_tbl_resource *) 6951 sample_res->normal_path_tbl; 6952 sfx_tbl_data = container_of(sfx_tbl, 6953 struct mlx5_flow_tbl_data_entry, 6954 tbl); 6955 sfx_attr.group = sfx_attr.transfer ? 6956 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 6957 } else { 6958 MLX5_ASSERT(attr->transfer); 6959 sfx_attr.group = jump_table; 6960 } 6961 flow_split_info->prefix_layers = 6962 flow_get_prefix_layer_flags(dev_flow); 6963 MLX5_ASSERT(wks); 6964 flow_split_info->prefix_mark |= wks->mark; 6965 /* Suffix group level already be scaled with factor, set 6966 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 6967 * again in translation. 6968 */ 6969 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6970 #endif 6971 } 6972 /* Add the suffix subflow. */ 6973 ret = flow_create_split_meter(dev, flow, &sfx_attr, 6974 sfx_items ? sfx_items : items, 6975 sfx_actions ? sfx_actions : actions, 6976 flow_split_info, error); 6977 exit: 6978 if (sfx_actions) 6979 mlx5_free(sfx_actions); 6980 return ret; 6981 } 6982 6983 /** 6984 * Split the flow to subflow set. The splitters might be linked 6985 * in the chain, like this: 6986 * flow_create_split_outer() calls: 6987 * flow_create_split_meter() calls: 6988 * flow_create_split_metadata(meter_subflow_0) calls: 6989 * flow_create_split_inner(metadata_subflow_0) 6990 * flow_create_split_inner(metadata_subflow_1) 6991 * flow_create_split_inner(metadata_subflow_2) 6992 * flow_create_split_metadata(meter_subflow_1) calls: 6993 * flow_create_split_inner(metadata_subflow_0) 6994 * flow_create_split_inner(metadata_subflow_1) 6995 * flow_create_split_inner(metadata_subflow_2) 6996 * 6997 * This provide flexible way to add new levels of flow splitting. 6998 * The all of successfully created subflows are included to the 6999 * parent flow dev_flow list. 7000 * 7001 * @param dev 7002 * Pointer to Ethernet device. 7003 * @param[in] flow 7004 * Parent flow structure pointer. 7005 * @param[in] attr 7006 * Flow rule attributes. 7007 * @param[in] items 7008 * Pattern specification (list terminated by the END pattern item). 7009 * @param[in] actions 7010 * Associated actions (list terminated by the END action). 7011 * @param[in] flow_split_info 7012 * Pointer to flow split info structure. 7013 * @param[out] error 7014 * Perform verbose error reporting if not NULL. 7015 * @return 7016 * 0 on success, negative value otherwise 7017 */ 7018 static int 7019 flow_create_split_outer(struct rte_eth_dev *dev, 7020 struct rte_flow *flow, 7021 const struct rte_flow_attr *attr, 7022 const struct rte_flow_item items[], 7023 const struct rte_flow_action actions[], 7024 struct mlx5_flow_split_info *flow_split_info, 7025 struct rte_flow_error *error) 7026 { 7027 int ret; 7028 7029 ret = flow_create_split_sample(dev, flow, attr, items, 7030 actions, flow_split_info, error); 7031 MLX5_ASSERT(ret <= 0); 7032 return ret; 7033 } 7034 7035 static inline struct mlx5_flow_tunnel * 7036 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7037 { 7038 struct mlx5_flow_tunnel *tunnel; 7039 7040 #pragma GCC diagnostic push 7041 #pragma GCC diagnostic ignored "-Wcast-qual" 7042 tunnel = (typeof(tunnel))flow->tunnel; 7043 #pragma GCC diagnostic pop 7044 7045 return tunnel; 7046 } 7047 7048 /** 7049 * Adjust flow RSS workspace if needed. 7050 * 7051 * @param wks 7052 * Pointer to thread flow work space. 7053 * @param rss_desc 7054 * Pointer to RSS descriptor. 7055 * @param[in] nrssq_num 7056 * New RSS queue number. 7057 * 7058 * @return 7059 * 0 on success, -1 otherwise and rte_errno is set. 7060 */ 7061 static int 7062 flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks, 7063 struct mlx5_flow_rss_desc *rss_desc, 7064 uint32_t nrssq_num) 7065 { 7066 if (likely(nrssq_num <= wks->rssq_num)) 7067 return 0; 7068 rss_desc->queue = realloc(rss_desc->queue, 7069 sizeof(*rss_desc->queue) * RTE_ALIGN(nrssq_num, 2)); 7070 if (!rss_desc->queue) { 7071 rte_errno = ENOMEM; 7072 return -1; 7073 } 7074 wks->rssq_num = RTE_ALIGN(nrssq_num, 2); 7075 return 0; 7076 } 7077 7078 /** 7079 * Create a flow and add it to @p list. 7080 * 7081 * @param dev 7082 * Pointer to Ethernet device. 7083 * @param list 7084 * Pointer to a TAILQ flow list. If this parameter NULL, 7085 * no list insertion occurred, flow is just created, 7086 * this is caller's responsibility to track the 7087 * created flow. 7088 * @param[in] attr 7089 * Flow rule attributes. 7090 * @param[in] items 7091 * Pattern specification (list terminated by the END pattern item). 7092 * @param[in] actions 7093 * Associated actions (list terminated by the END action). 7094 * @param[in] external 7095 * This flow rule is created by request external to PMD. 7096 * @param[out] error 7097 * Perform verbose error reporting if not NULL. 7098 * 7099 * @return 7100 * A flow index on success, 0 otherwise and rte_errno is set. 7101 */ 7102 static uint32_t 7103 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7104 const struct rte_flow_attr *attr, 7105 const struct rte_flow_item items[], 7106 const struct rte_flow_action original_actions[], 7107 bool external, struct rte_flow_error *error) 7108 { 7109 struct mlx5_priv *priv = dev->data->dev_private; 7110 struct rte_flow *flow = NULL; 7111 struct mlx5_flow *dev_flow; 7112 const struct rte_flow_action_rss *rss = NULL; 7113 struct mlx5_translated_action_handle 7114 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7115 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7116 union { 7117 struct mlx5_flow_expand_rss buf; 7118 uint8_t buffer[8192]; 7119 } expand_buffer; 7120 union { 7121 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7122 uint8_t buffer[2048]; 7123 } actions_rx; 7124 union { 7125 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7126 uint8_t buffer[2048]; 7127 } actions_hairpin_tx; 7128 union { 7129 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7130 uint8_t buffer[2048]; 7131 } items_tx; 7132 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7133 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7134 struct mlx5_flow_rss_desc *rss_desc; 7135 const struct rte_flow_action *p_actions_rx; 7136 uint32_t i; 7137 uint32_t idx = 0; 7138 int hairpin_flow; 7139 struct rte_flow_attr attr_tx = { .priority = 0 }; 7140 const struct rte_flow_action *actions; 7141 struct rte_flow_action *translated_actions = NULL; 7142 struct mlx5_flow_tunnel *tunnel; 7143 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7144 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7145 struct mlx5_flow_split_info flow_split_info = { 7146 .external = !!external, 7147 .skip_scale = 0, 7148 .flow_idx = 0, 7149 .prefix_mark = 0, 7150 .prefix_layers = 0, 7151 .table_id = 0 7152 }; 7153 int ret; 7154 7155 MLX5_ASSERT(wks); 7156 rss_desc = &wks->rss_desc; 7157 ret = flow_action_handles_translate(dev, original_actions, 7158 indir_actions, 7159 &indir_actions_n, 7160 &translated_actions, error); 7161 if (ret < 0) { 7162 MLX5_ASSERT(translated_actions == NULL); 7163 return 0; 7164 } 7165 actions = translated_actions ? translated_actions : original_actions; 7166 p_actions_rx = actions; 7167 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7168 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7169 external, hairpin_flow, error); 7170 if (ret < 0) 7171 goto error_before_hairpin_split; 7172 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7173 if (!flow) { 7174 rte_errno = ENOMEM; 7175 goto error_before_hairpin_split; 7176 } 7177 if (hairpin_flow > 0) { 7178 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7179 rte_errno = EINVAL; 7180 goto error_before_hairpin_split; 7181 } 7182 flow_hairpin_split(dev, actions, actions_rx.actions, 7183 actions_hairpin_tx.actions, items_tx.items, 7184 idx); 7185 p_actions_rx = actions_rx.actions; 7186 } 7187 flow_split_info.flow_idx = idx; 7188 flow->drv_type = flow_get_drv_type(dev, attr); 7189 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7190 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7191 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7192 /* RSS Action only works on NIC RX domain */ 7193 if (attr->ingress) 7194 rss = flow_get_rss_action(dev, p_actions_rx); 7195 if (rss) { 7196 if (flow_rss_workspace_adjust(wks, rss_desc, rss->queue_num)) 7197 return 0; 7198 /* 7199 * The following information is required by 7200 * mlx5_flow_hashfields_adjust() in advance. 7201 */ 7202 rss_desc->level = rss->level; 7203 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7204 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7205 } 7206 flow->dev_handles = 0; 7207 if (rss && rss->types) { 7208 unsigned int graph_root; 7209 7210 graph_root = find_graph_root(rss->level); 7211 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7212 items, rss->types, 7213 mlx5_support_expansion, graph_root); 7214 MLX5_ASSERT(ret > 0 && 7215 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7216 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7217 for (i = 0; i < buf->entries; ++i) 7218 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7219 } 7220 } else { 7221 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7222 sizeof(expand_buffer.buffer), 7223 items, sq_specs); 7224 if (ret) { 7225 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7226 NULL, "not enough memory for rte_flow"); 7227 goto error; 7228 } 7229 if (buf->entries == 0) { 7230 buf->entries = 1; 7231 buf->entry[0].pattern = (void *)(uintptr_t)items; 7232 } 7233 } 7234 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7235 indir_actions_n); 7236 for (i = 0; i < buf->entries; ++i) { 7237 /* Initialize flow split data. */ 7238 flow_split_info.prefix_layers = 0; 7239 flow_split_info.prefix_mark = 0; 7240 flow_split_info.skip_scale = 0; 7241 /* 7242 * The splitter may create multiple dev_flows, 7243 * depending on configuration. In the simplest 7244 * case it just creates unmodified original flow. 7245 */ 7246 ret = flow_create_split_outer(dev, flow, attr, 7247 buf->entry[i].pattern, 7248 p_actions_rx, &flow_split_info, 7249 error); 7250 if (ret < 0) 7251 goto error; 7252 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7253 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7254 p_actions_rx, 7255 idx, 7256 wks->flows[0].tunnel, 7257 &default_miss_ctx, 7258 error); 7259 if (ret < 0) { 7260 mlx5_free(default_miss_ctx.queue); 7261 goto error; 7262 } 7263 } 7264 } 7265 /* Create the tx flow. */ 7266 if (hairpin_flow) { 7267 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7268 attr_tx.ingress = 0; 7269 attr_tx.egress = 1; 7270 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7271 actions_hairpin_tx.actions, 7272 idx, error); 7273 if (!dev_flow) 7274 goto error; 7275 dev_flow->flow = flow; 7276 dev_flow->external = 0; 7277 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7278 dev_flow->handle, next); 7279 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7280 items_tx.items, 7281 actions_hairpin_tx.actions, error); 7282 if (ret < 0) 7283 goto error; 7284 } 7285 /* 7286 * Update the metadata register copy table. If extensive 7287 * metadata feature is enabled and registers are supported 7288 * we might create the extra rte_flow for each unique 7289 * MARK/FLAG action ID. 7290 * 7291 * The table is updated for ingress and transfer flows only, because 7292 * the egress Flows belong to the different device and 7293 * copy table should be updated in peer NIC Rx domain. 7294 */ 7295 if ((attr->ingress || attr->transfer) && 7296 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7297 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7298 if (ret) 7299 goto error; 7300 } 7301 /* 7302 * If the flow is external (from application) OR device is started, 7303 * OR mreg discover, then apply immediately. 7304 */ 7305 if (external || dev->data->dev_started || 7306 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7307 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7308 ret = flow_drv_apply(dev, flow, error); 7309 if (ret < 0) 7310 goto error; 7311 } 7312 flow->type = type; 7313 flow_rxq_flags_set(dev, flow); 7314 rte_free(translated_actions); 7315 tunnel = flow_tunnel_from_rule(wks->flows); 7316 if (tunnel) { 7317 flow->tunnel = 1; 7318 flow->tunnel_id = tunnel->tunnel_id; 7319 __atomic_add_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7320 mlx5_free(default_miss_ctx.queue); 7321 } 7322 mlx5_flow_pop_thread_workspace(); 7323 return idx; 7324 error: 7325 MLX5_ASSERT(flow); 7326 ret = rte_errno; /* Save rte_errno before cleanup. */ 7327 flow_mreg_del_copy_action(dev, flow); 7328 flow_drv_destroy(dev, flow); 7329 if (rss_desc->shared_rss) 7330 __atomic_sub_fetch(&((struct mlx5_shared_action_rss *) 7331 mlx5_ipool_get 7332 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7333 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7334 mlx5_ipool_free(priv->flows[type], idx); 7335 rte_errno = ret; /* Restore rte_errno. */ 7336 ret = rte_errno; 7337 rte_errno = ret; 7338 error_before_hairpin_split: 7339 mlx5_flow_pop_thread_workspace(); 7340 rte_free(translated_actions); 7341 return 0; 7342 } 7343 7344 /** 7345 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7346 * incoming packets to table 1. 7347 * 7348 * Other flow rules, requested for group n, will be created in 7349 * e-switch table n+1. 7350 * Jump action to e-switch group n will be created to group n+1. 7351 * 7352 * Used when working in switchdev mode, to utilise advantages of table 1 7353 * and above. 7354 * 7355 * @param dev 7356 * Pointer to Ethernet device. 7357 * 7358 * @return 7359 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7360 */ 7361 struct rte_flow * 7362 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7363 { 7364 const struct rte_flow_attr attr = { 7365 .group = 0, 7366 .priority = 0, 7367 .ingress = 0, 7368 .egress = 0, 7369 .transfer = 1, 7370 }; 7371 const struct rte_flow_item pattern = { 7372 .type = RTE_FLOW_ITEM_TYPE_END, 7373 }; 7374 struct rte_flow_action_jump jump = { 7375 .group = 1, 7376 }; 7377 const struct rte_flow_action actions[] = { 7378 { 7379 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7380 .conf = &jump, 7381 }, 7382 { 7383 .type = RTE_FLOW_ACTION_TYPE_END, 7384 }, 7385 }; 7386 struct rte_flow_error error; 7387 7388 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7389 &attr, &pattern, 7390 actions, false, &error); 7391 } 7392 7393 /** 7394 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7395 * and sq number, directs all packets to peer vport. 7396 * 7397 * @param dev 7398 * Pointer to Ethernet device. 7399 * @param sq_num 7400 * SQ number. 7401 * 7402 * @return 7403 * Flow ID on success, 0 otherwise and rte_errno is set. 7404 */ 7405 uint32_t 7406 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7407 { 7408 struct rte_flow_attr attr = { 7409 .group = 0, 7410 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7411 .ingress = 0, 7412 .egress = 0, 7413 .transfer = 1, 7414 }; 7415 struct rte_flow_item_port_id port_spec = { 7416 .id = MLX5_PORT_ESW_MGR, 7417 }; 7418 struct mlx5_rte_flow_item_sq sq_spec = { 7419 .queue = sq_num, 7420 }; 7421 struct rte_flow_item pattern[] = { 7422 { 7423 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7424 .spec = &port_spec, 7425 }, 7426 { 7427 .type = (enum rte_flow_item_type) 7428 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7429 .spec = &sq_spec, 7430 }, 7431 { 7432 .type = RTE_FLOW_ITEM_TYPE_END, 7433 }, 7434 }; 7435 struct rte_flow_action_jump jump = { 7436 .group = 1, 7437 }; 7438 struct rte_flow_action_port_id port = { 7439 .id = dev->data->port_id, 7440 }; 7441 struct rte_flow_action actions[] = { 7442 { 7443 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7444 .conf = &jump, 7445 }, 7446 { 7447 .type = RTE_FLOW_ACTION_TYPE_END, 7448 }, 7449 }; 7450 struct rte_flow_error error; 7451 7452 /* 7453 * Creates group 0, highest priority jump flow. 7454 * Matches txq to bypass kernel packets. 7455 */ 7456 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7457 false, &error) == 0) 7458 return 0; 7459 /* Create group 1, lowest priority redirect flow for txq. */ 7460 attr.group = 1; 7461 actions[0].conf = &port; 7462 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7463 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7464 actions, false, &error); 7465 } 7466 7467 /** 7468 * Validate a flow supported by the NIC. 7469 * 7470 * @see rte_flow_validate() 7471 * @see rte_flow_ops 7472 */ 7473 int 7474 mlx5_flow_validate(struct rte_eth_dev *dev, 7475 const struct rte_flow_attr *attr, 7476 const struct rte_flow_item items[], 7477 const struct rte_flow_action original_actions[], 7478 struct rte_flow_error *error) 7479 { 7480 int hairpin_flow; 7481 struct mlx5_translated_action_handle 7482 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7483 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7484 const struct rte_flow_action *actions; 7485 struct rte_flow_action *translated_actions = NULL; 7486 int ret = flow_action_handles_translate(dev, original_actions, 7487 indir_actions, 7488 &indir_actions_n, 7489 &translated_actions, error); 7490 7491 if (ret) 7492 return ret; 7493 actions = translated_actions ? translated_actions : original_actions; 7494 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7495 ret = flow_drv_validate(dev, attr, items, actions, 7496 true, hairpin_flow, error); 7497 rte_free(translated_actions); 7498 return ret; 7499 } 7500 7501 static int 7502 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7503 const struct rte_flow_attr *attr, 7504 const uint32_t orig_prio, 7505 const struct rte_flow_item *items, 7506 const struct rte_flow_action *actions, 7507 uint32_t flow_idx) 7508 { 7509 struct mlx5_priv *priv = dev->data->dev_private; 7510 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7511 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7512 struct rte_flow_error error; 7513 int len, ret; 7514 7515 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7516 if (!flow_info) { 7517 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7518 return -1; 7519 } 7520 flow_info->orig_prio = orig_prio; 7521 flow_info->attr = *attr; 7522 /* Standby mode rule awlays saves it in low priority entry. */ 7523 flow_info->flow_idx_low_prio = flow_idx; 7524 7525 /* Store matching items. */ 7526 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7527 if (ret <= 0) { 7528 DRV_LOG(ERR, "Can't get items length."); 7529 goto end; 7530 } 7531 len = RTE_ALIGN(ret, 16); 7532 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7533 if (!flow_info->items) { 7534 DRV_LOG(ERR, "No enough memory for items caching."); 7535 goto end; 7536 } 7537 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7538 if (ret <= 0) { 7539 DRV_LOG(ERR, "Can't duplicate items."); 7540 goto end; 7541 } 7542 7543 /* Store flow actions. */ 7544 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7545 if (ret <= 0) { 7546 DRV_LOG(ERR, "Can't get actions length."); 7547 goto end; 7548 } 7549 len = RTE_ALIGN(ret, 16); 7550 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7551 if (!flow_info->actions) { 7552 DRV_LOG(ERR, "No enough memory for actions caching."); 7553 goto end; 7554 } 7555 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7556 if (ret <= 0) { 7557 DRV_LOG(ERR, "Can't duplicate actions."); 7558 goto end; 7559 } 7560 7561 /* Insert to the list end. */ 7562 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7563 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7564 } else { 7565 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7566 while (LIST_NEXT(tmp_info, next)) 7567 tmp_info = LIST_NEXT(tmp_info, next); 7568 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7569 } 7570 return 0; 7571 end: 7572 if (flow_info->items) 7573 mlx5_free(flow_info->items); 7574 if (flow_info->actions) 7575 mlx5_free(flow_info->actions); 7576 mlx5_free(flow_info); 7577 return -1; 7578 } 7579 7580 static int 7581 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7582 { 7583 struct mlx5_priv *priv = dev->data->dev_private; 7584 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7585 struct mlx5_dv_flow_info *flow_info; 7586 struct rte_flow_attr attr; 7587 struct rte_flow_error error; 7588 struct rte_flow *high, *low; 7589 7590 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7591 while (flow_info) { 7592 /* DUP flow may have the same priority. */ 7593 if (flow_info->orig_prio != flow_info->attr.priority) { 7594 attr = flow_info->attr; 7595 if (orig_prio) 7596 attr.priority = flow_info->orig_prio; 7597 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7598 &attr, flow_info->items, flow_info->actions, 7599 true, &error); 7600 if (!flow_info->flow_idx_high_prio) { 7601 DRV_LOG(ERR, "Priority toggle failed internally."); 7602 goto err; 7603 } 7604 } 7605 flow_info = LIST_NEXT(flow_info, next); 7606 } 7607 /* Delete the low priority rules and swap the flow handle. */ 7608 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7609 while (flow_info) { 7610 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7611 if (flow_info->orig_prio != flow_info->attr.priority) { 7612 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7613 flow_info->flow_idx_high_prio); 7614 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7615 flow_info->flow_idx_low_prio); 7616 if (high && low) { 7617 RTE_SWAP(*low, *high); 7618 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7619 flow_info->flow_idx_low_prio); 7620 flow_info->flow_idx_high_prio = 0; 7621 } 7622 } 7623 flow_info = LIST_NEXT(flow_info, next); 7624 } 7625 return 0; 7626 err: 7627 /* Destroy preceding successful high priority rules. */ 7628 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7629 while (flow_info) { 7630 if (flow_info->orig_prio != flow_info->attr.priority) { 7631 if (flow_info->flow_idx_high_prio) 7632 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7633 flow_info->flow_idx_high_prio); 7634 else 7635 break; 7636 flow_info->flow_idx_high_prio = 0; 7637 } 7638 flow_info = LIST_NEXT(flow_info, next); 7639 } 7640 return -1; 7641 } 7642 7643 /** 7644 * Set the mode of the flow engine of a process to active or standby during live migration. 7645 * 7646 * @param[in] mode 7647 * MLX5 flow engine mode, @see `enum mlx5_flow_engine_mode`. 7648 * @param[in] flags 7649 * Flow engine mode specific flags. 7650 * 7651 * @return 7652 * Negative value on error, positive on success. 7653 */ 7654 int 7655 rte_pmd_mlx5_flow_engine_set_mode(enum mlx5_flow_engine_mode mode, uint32_t flags) 7656 { 7657 struct mlx5_priv *priv; 7658 struct mlx5_flow_engine_mode_info *mode_info; 7659 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7660 uint16_t port, port_id; 7661 uint16_t toggle_num = 0; 7662 struct rte_eth_dev *dev; 7663 enum mlx5_flow_engine_mode orig_mode; 7664 uint32_t orig_flags; 7665 bool need_toggle = false; 7666 7667 /* Check if flags combinations are supported. */ 7668 if (flags && flags != MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7669 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7670 return -1; 7671 } 7672 MLX5_ETH_FOREACH_DEV(port, NULL) { 7673 dev = &rte_eth_devices[port]; 7674 priv = dev->data->dev_private; 7675 mode_info = &priv->mode_info; 7676 /* No mode change. Assume all devices hold the same mode. */ 7677 if (mode_info->mode == mode) { 7678 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7679 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7680 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7681 port, mode_info->mode_flag); 7682 orig_mode = mode_info->mode; 7683 orig_flags = mode_info->mode_flag; 7684 goto err; 7685 } 7686 mode_info->mode_flag = flags; 7687 toggle_num++; 7688 continue; 7689 } 7690 /* Active -> standby. */ 7691 if (mode == MLX5_FLOW_ENGINE_MODE_STANDBY) { 7692 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7693 DRV_LOG(ERR, "Cached rule existed"); 7694 orig_mode = mode_info->mode; 7695 orig_flags = mode_info->mode_flag; 7696 goto err; 7697 } 7698 mode_info->mode_flag = flags; 7699 mode_info->mode = mode; 7700 toggle_num++; 7701 /* Standby -> active. */ 7702 } else if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7703 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7704 DRV_LOG(INFO, "No cached rule existed"); 7705 } else { 7706 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7707 orig_mode = mode_info->mode; 7708 orig_flags = mode_info->mode_flag; 7709 need_toggle = true; 7710 goto err; 7711 } 7712 } 7713 toggle_num++; 7714 } 7715 } 7716 if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7717 /* Clear cache flow rules. */ 7718 MLX5_ETH_FOREACH_DEV(port, NULL) { 7719 priv = rte_eth_devices[port].data->dev_private; 7720 mode_info = &priv->mode_info; 7721 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7722 while (flow_info) { 7723 tmp_info = LIST_NEXT(flow_info, next); 7724 LIST_REMOVE(flow_info, next); 7725 mlx5_free(flow_info->actions); 7726 mlx5_free(flow_info->items); 7727 mlx5_free(flow_info); 7728 flow_info = tmp_info; 7729 } 7730 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7731 } 7732 } 7733 return toggle_num; 7734 err: 7735 /* Rollback all preceding successful ports. */ 7736 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7737 if (port_id == port) 7738 break; 7739 priv = rte_eth_devices[port_id].data->dev_private; 7740 mode_info = &priv->mode_info; 7741 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7742 mlx5_flow_cache_flow_toggle(dev, false)) 7743 return -EPERM; 7744 mode_info->mode = orig_mode; 7745 mode_info->mode_flag = orig_flags; 7746 } 7747 return -EINVAL; 7748 } 7749 /** 7750 * Create a flow. 7751 * 7752 * @see rte_flow_create() 7753 * @see rte_flow_ops 7754 */ 7755 struct rte_flow * 7756 mlx5_flow_create(struct rte_eth_dev *dev, 7757 const struct rte_flow_attr *attr, 7758 const struct rte_flow_item items[], 7759 const struct rte_flow_action actions[], 7760 struct rte_flow_error *error) 7761 { 7762 struct mlx5_priv *priv = dev->data->dev_private; 7763 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7764 uint32_t prio = attr->priority; 7765 uint32_t flow_idx; 7766 7767 if (priv->sh->config.dv_flow_en == 2) { 7768 rte_flow_error_set(error, ENOTSUP, 7769 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7770 NULL, 7771 "Flow non-Q creation not supported"); 7772 return NULL; 7773 } 7774 /* 7775 * If the device is not started yet, it is not allowed to created a 7776 * flow from application. PMD default flows and traffic control flows 7777 * are not affected. 7778 */ 7779 if (unlikely(!dev->data->dev_started)) { 7780 DRV_LOG(DEBUG, "port %u is not started when " 7781 "inserting a flow", dev->data->port_id); 7782 rte_flow_error_set(error, ENODEV, 7783 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7784 NULL, 7785 "port not started"); 7786 return NULL; 7787 } 7788 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7789 if (attr->transfer || 7790 (attr->ingress && 7791 !(priv->mode_info.mode_flag & MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 7792 new_attr->priority += 1; 7793 } 7794 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 7795 if (!flow_idx) 7796 return NULL; 7797 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7798 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 7799 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7800 flow_idx = 0; 7801 } 7802 } 7803 return (void *)(uintptr_t)flow_idx; 7804 } 7805 7806 /** 7807 * Destroy a flow in a list. 7808 * 7809 * @param dev 7810 * Pointer to Ethernet device. 7811 * @param[in] flow_idx 7812 * Index of flow to destroy. 7813 */ 7814 static void 7815 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7816 uint32_t flow_idx) 7817 { 7818 struct mlx5_priv *priv = dev->data->dev_private; 7819 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7820 7821 if (!flow) 7822 return; 7823 MLX5_ASSERT(flow->type == type); 7824 /* 7825 * Update RX queue flags only if port is started, otherwise it is 7826 * already clean. 7827 */ 7828 if (dev->data->dev_started) 7829 flow_rxq_flags_trim(dev, flow); 7830 flow_drv_destroy(dev, flow); 7831 if (flow->tunnel) { 7832 struct mlx5_flow_tunnel *tunnel; 7833 7834 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7835 RTE_VERIFY(tunnel); 7836 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 7837 mlx5_flow_tunnel_free(dev, tunnel); 7838 } 7839 flow_mreg_del_copy_action(dev, flow); 7840 mlx5_ipool_free(priv->flows[type], flow_idx); 7841 } 7842 7843 /** 7844 * Destroy all flows. 7845 * 7846 * @param dev 7847 * Pointer to Ethernet device. 7848 * @param type 7849 * Flow type to be flushed. 7850 * @param active 7851 * If flushing is called actively. 7852 */ 7853 void 7854 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7855 bool active) 7856 { 7857 struct mlx5_priv *priv = dev->data->dev_private; 7858 uint32_t num_flushed = 0, fidx = 1; 7859 struct rte_flow *flow; 7860 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7861 struct mlx5_dv_flow_info *flow_info; 7862 7863 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7864 if (priv->sh->config.dv_flow_en == 2 && 7865 type == MLX5_FLOW_TYPE_GEN) { 7866 flow_hw_q_flow_flush(dev, NULL); 7867 return; 7868 } 7869 #endif 7870 7871 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7872 flow_list_destroy(dev, type, fidx); 7873 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 7874 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7875 while (flow_info) { 7876 /* Romove the cache flow info. */ 7877 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 7878 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 7879 LIST_REMOVE(flow_info, next); 7880 mlx5_free(flow_info->items); 7881 mlx5_free(flow_info->actions); 7882 mlx5_free(flow_info); 7883 break; 7884 } 7885 flow_info = LIST_NEXT(flow_info, next); 7886 } 7887 } 7888 num_flushed++; 7889 } 7890 if (active) { 7891 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 7892 dev->data->port_id, num_flushed); 7893 } 7894 } 7895 7896 /** 7897 * Stop all default actions for flows. 7898 * 7899 * @param dev 7900 * Pointer to Ethernet device. 7901 */ 7902 void 7903 mlx5_flow_stop_default(struct rte_eth_dev *dev) 7904 { 7905 flow_mreg_del_default_copy_action(dev); 7906 flow_rxq_flags_clear(dev); 7907 } 7908 7909 /** 7910 * Set rxq flag. 7911 * 7912 * @param[in] dev 7913 * Pointer to the rte_eth_dev structure. 7914 * @param[in] enable 7915 * Flag to enable or not. 7916 */ 7917 void 7918 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 7919 { 7920 struct mlx5_priv *priv = dev->data->dev_private; 7921 unsigned int i; 7922 7923 if ((!priv->mark_enabled && !enable) || 7924 (priv->mark_enabled && enable)) 7925 return; 7926 for (i = 0; i < priv->rxqs_n; ++i) { 7927 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 7928 7929 /* With RXQ start/stop feature, RXQ might be stopped. */ 7930 if (!rxq_ctrl) 7931 continue; 7932 rxq_ctrl->rxq.mark = enable; 7933 } 7934 priv->mark_enabled = enable; 7935 } 7936 7937 /** 7938 * Start all default actions for flows. 7939 * 7940 * @param dev 7941 * Pointer to Ethernet device. 7942 * @return 7943 * 0 on success, a negative errno value otherwise and rte_errno is set. 7944 */ 7945 int 7946 mlx5_flow_start_default(struct rte_eth_dev *dev) 7947 { 7948 struct rte_flow_error error; 7949 7950 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 7951 return flow_mreg_add_default_copy_action(dev, &error); 7952 } 7953 7954 /** 7955 * Release key of thread specific flow workspace data. 7956 */ 7957 void 7958 flow_release_workspace(void *data) 7959 { 7960 struct mlx5_flow_workspace *wks = data; 7961 struct mlx5_flow_workspace *next; 7962 7963 while (wks) { 7964 next = wks->next; 7965 free(wks->rss_desc.queue); 7966 free(wks); 7967 wks = next; 7968 } 7969 } 7970 7971 /** 7972 * Get thread specific current flow workspace. 7973 * 7974 * @return pointer to thread specific flow workspace data, NULL on error. 7975 */ 7976 struct mlx5_flow_workspace* 7977 mlx5_flow_get_thread_workspace(void) 7978 { 7979 struct mlx5_flow_workspace *data; 7980 7981 data = mlx5_flow_os_get_specific_workspace(); 7982 MLX5_ASSERT(data && data->inuse); 7983 if (!data || !data->inuse) 7984 DRV_LOG(ERR, "flow workspace not initialized."); 7985 return data; 7986 } 7987 7988 /** 7989 * Allocate and init new flow workspace. 7990 * 7991 * @return pointer to flow workspace data, NULL on error. 7992 */ 7993 static struct mlx5_flow_workspace* 7994 flow_alloc_thread_workspace(void) 7995 { 7996 struct mlx5_flow_workspace *data = calloc(1, sizeof(*data)); 7997 7998 if (!data) { 7999 DRV_LOG(ERR, "Failed to allocate flow workspace " 8000 "memory."); 8001 return NULL; 8002 } 8003 data->rss_desc.queue = calloc(1, 8004 sizeof(uint16_t) * MLX5_RSSQ_DEFAULT_NUM); 8005 if (!data->rss_desc.queue) 8006 goto err; 8007 data->rssq_num = MLX5_RSSQ_DEFAULT_NUM; 8008 return data; 8009 err: 8010 free(data->rss_desc.queue); 8011 free(data); 8012 return NULL; 8013 } 8014 8015 /** 8016 * Get new thread specific flow workspace. 8017 * 8018 * If current workspace inuse, create new one and set as current. 8019 * 8020 * @return pointer to thread specific flow workspace data, NULL on error. 8021 */ 8022 struct mlx5_flow_workspace* 8023 mlx5_flow_push_thread_workspace(void) 8024 { 8025 struct mlx5_flow_workspace *curr; 8026 struct mlx5_flow_workspace *data; 8027 8028 curr = mlx5_flow_os_get_specific_workspace(); 8029 if (!curr) { 8030 data = flow_alloc_thread_workspace(); 8031 if (!data) 8032 return NULL; 8033 } else if (!curr->inuse) { 8034 data = curr; 8035 } else if (curr->next) { 8036 data = curr->next; 8037 } else { 8038 data = flow_alloc_thread_workspace(); 8039 if (!data) 8040 return NULL; 8041 curr->next = data; 8042 data->prev = curr; 8043 } 8044 data->inuse = 1; 8045 data->flow_idx = 0; 8046 /* Set as current workspace */ 8047 if (mlx5_flow_os_set_specific_workspace(data)) 8048 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8049 return data; 8050 } 8051 8052 /** 8053 * Close current thread specific flow workspace. 8054 * 8055 * If previous workspace available, set it as current. 8056 * 8057 * @return pointer to thread specific flow workspace data, NULL on error. 8058 */ 8059 void 8060 mlx5_flow_pop_thread_workspace(void) 8061 { 8062 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8063 8064 if (!data) 8065 return; 8066 if (!data->inuse) { 8067 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8068 return; 8069 } 8070 data->inuse = 0; 8071 if (!data->prev) 8072 return; 8073 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8074 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8075 } 8076 8077 /** 8078 * Verify the flow list is empty 8079 * 8080 * @param dev 8081 * Pointer to Ethernet device. 8082 * 8083 * @return the number of flows not released. 8084 */ 8085 int 8086 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8087 { 8088 struct mlx5_priv *priv = dev->data->dev_private; 8089 struct rte_flow *flow; 8090 uint32_t idx = 0; 8091 int ret = 0, i; 8092 8093 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8094 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8095 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8096 dev->data->port_id, (void *)flow); 8097 ret++; 8098 } 8099 } 8100 return ret; 8101 } 8102 8103 /** 8104 * Enable default hairpin egress flow. 8105 * 8106 * @param dev 8107 * Pointer to Ethernet device. 8108 * @param sq_num 8109 * The SQ hw number. 8110 * 8111 * @return 8112 * 0 on success, a negative errno value otherwise and rte_errno is set. 8113 */ 8114 int 8115 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8116 uint32_t sq_num) 8117 { 8118 const struct rte_flow_attr attr = { 8119 .egress = 1, 8120 .priority = 0, 8121 }; 8122 struct mlx5_rte_flow_item_sq queue_spec = { 8123 .queue = sq_num, 8124 }; 8125 struct mlx5_rte_flow_item_sq queue_mask = { 8126 .queue = UINT32_MAX, 8127 }; 8128 struct rte_flow_item items[] = { 8129 { 8130 .type = (enum rte_flow_item_type) 8131 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8132 .spec = &queue_spec, 8133 .last = NULL, 8134 .mask = &queue_mask, 8135 }, 8136 { 8137 .type = RTE_FLOW_ITEM_TYPE_END, 8138 }, 8139 }; 8140 struct rte_flow_action_jump jump = { 8141 .group = MLX5_HAIRPIN_TX_TABLE, 8142 }; 8143 struct rte_flow_action actions[2]; 8144 uint32_t flow_idx; 8145 struct rte_flow_error error; 8146 8147 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8148 actions[0].conf = &jump; 8149 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8150 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8151 &attr, items, actions, false, &error); 8152 if (!flow_idx) { 8153 DRV_LOG(DEBUG, 8154 "Failed to create ctrl flow: rte_errno(%d)," 8155 " type(%d), message(%s)", 8156 rte_errno, error.type, 8157 error.message ? error.message : " (no stated reason)"); 8158 return -rte_errno; 8159 } 8160 return 0; 8161 } 8162 8163 /** 8164 * Enable a control flow configured from the control plane. 8165 * 8166 * @param dev 8167 * Pointer to Ethernet device. 8168 * @param eth_spec 8169 * An Ethernet flow spec to apply. 8170 * @param eth_mask 8171 * An Ethernet flow mask to apply. 8172 * @param vlan_spec 8173 * A VLAN flow spec to apply. 8174 * @param vlan_mask 8175 * A VLAN flow mask to apply. 8176 * 8177 * @return 8178 * 0 on success, a negative errno value otherwise and rte_errno is set. 8179 */ 8180 int 8181 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8182 struct rte_flow_item_eth *eth_spec, 8183 struct rte_flow_item_eth *eth_mask, 8184 struct rte_flow_item_vlan *vlan_spec, 8185 struct rte_flow_item_vlan *vlan_mask) 8186 { 8187 struct mlx5_priv *priv = dev->data->dev_private; 8188 const struct rte_flow_attr attr = { 8189 .ingress = 1, 8190 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8191 }; 8192 struct rte_flow_item items[] = { 8193 { 8194 .type = RTE_FLOW_ITEM_TYPE_ETH, 8195 .spec = eth_spec, 8196 .last = NULL, 8197 .mask = eth_mask, 8198 }, 8199 { 8200 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8201 RTE_FLOW_ITEM_TYPE_END, 8202 .spec = vlan_spec, 8203 .last = NULL, 8204 .mask = vlan_mask, 8205 }, 8206 { 8207 .type = RTE_FLOW_ITEM_TYPE_END, 8208 }, 8209 }; 8210 uint16_t queue[priv->reta_idx_n]; 8211 struct rte_flow_action_rss action_rss = { 8212 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8213 .level = 0, 8214 .types = priv->rss_conf.rss_hf, 8215 .key_len = priv->rss_conf.rss_key_len, 8216 .queue_num = priv->reta_idx_n, 8217 .key = priv->rss_conf.rss_key, 8218 .queue = queue, 8219 }; 8220 struct rte_flow_action actions[] = { 8221 { 8222 .type = RTE_FLOW_ACTION_TYPE_RSS, 8223 .conf = &action_rss, 8224 }, 8225 { 8226 .type = RTE_FLOW_ACTION_TYPE_END, 8227 }, 8228 }; 8229 uint32_t flow_idx; 8230 struct rte_flow_error error; 8231 unsigned int i; 8232 8233 if (!priv->reta_idx_n || !priv->rxqs_n) { 8234 return 0; 8235 } 8236 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8237 action_rss.types = 0; 8238 for (i = 0; i != priv->reta_idx_n; ++i) 8239 queue[i] = (*priv->reta_idx)[i]; 8240 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8241 &attr, items, actions, false, &error); 8242 if (!flow_idx) 8243 return -rte_errno; 8244 return 0; 8245 } 8246 8247 /** 8248 * Enable a flow control configured from the control plane. 8249 * 8250 * @param dev 8251 * Pointer to Ethernet device. 8252 * @param eth_spec 8253 * An Ethernet flow spec to apply. 8254 * @param eth_mask 8255 * An Ethernet flow mask to apply. 8256 * 8257 * @return 8258 * 0 on success, a negative errno value otherwise and rte_errno is set. 8259 */ 8260 int 8261 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8262 struct rte_flow_item_eth *eth_spec, 8263 struct rte_flow_item_eth *eth_mask) 8264 { 8265 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8266 } 8267 8268 /** 8269 * Create default miss flow rule matching lacp traffic 8270 * 8271 * @param dev 8272 * Pointer to Ethernet device. 8273 * @param eth_spec 8274 * An Ethernet flow spec to apply. 8275 * 8276 * @return 8277 * 0 on success, a negative errno value otherwise and rte_errno is set. 8278 */ 8279 int 8280 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8281 { 8282 /* 8283 * The LACP matching is done by only using ether type since using 8284 * a multicast dst mac causes kernel to give low priority to this flow. 8285 */ 8286 static const struct rte_flow_item_eth lacp_spec = { 8287 .hdr.ether_type = RTE_BE16(0x8809), 8288 }; 8289 static const struct rte_flow_item_eth lacp_mask = { 8290 .hdr.ether_type = 0xffff, 8291 }; 8292 const struct rte_flow_attr attr = { 8293 .ingress = 1, 8294 }; 8295 struct rte_flow_item items[] = { 8296 { 8297 .type = RTE_FLOW_ITEM_TYPE_ETH, 8298 .spec = &lacp_spec, 8299 .mask = &lacp_mask, 8300 }, 8301 { 8302 .type = RTE_FLOW_ITEM_TYPE_END, 8303 }, 8304 }; 8305 struct rte_flow_action actions[] = { 8306 { 8307 .type = (enum rte_flow_action_type) 8308 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8309 }, 8310 { 8311 .type = RTE_FLOW_ACTION_TYPE_END, 8312 }, 8313 }; 8314 struct rte_flow_error error; 8315 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8316 &attr, items, actions, 8317 false, &error); 8318 8319 if (!flow_idx) 8320 return -rte_errno; 8321 return 0; 8322 } 8323 8324 /** 8325 * Destroy a flow. 8326 * 8327 * @see rte_flow_destroy() 8328 * @see rte_flow_ops 8329 */ 8330 int 8331 mlx5_flow_destroy(struct rte_eth_dev *dev, 8332 struct rte_flow *flow, 8333 struct rte_flow_error *error __rte_unused) 8334 { 8335 struct mlx5_priv *priv = dev->data->dev_private; 8336 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8337 struct mlx5_dv_flow_info *flow_info; 8338 8339 if (priv->sh->config.dv_flow_en == 2) 8340 return rte_flow_error_set(error, ENOTSUP, 8341 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8342 NULL, 8343 "Flow non-Q destruction not supported"); 8344 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8345 (uintptr_t)(void *)flow); 8346 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8347 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8348 while (flow_info) { 8349 /* Romove the cache flow info. */ 8350 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8351 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8352 LIST_REMOVE(flow_info, next); 8353 mlx5_free(flow_info->items); 8354 mlx5_free(flow_info->actions); 8355 mlx5_free(flow_info); 8356 break; 8357 } 8358 flow_info = LIST_NEXT(flow_info, next); 8359 } 8360 } 8361 return 0; 8362 } 8363 8364 /** 8365 * Destroy all flows. 8366 * 8367 * @see rte_flow_flush() 8368 * @see rte_flow_ops 8369 */ 8370 int 8371 mlx5_flow_flush(struct rte_eth_dev *dev, 8372 struct rte_flow_error *error __rte_unused) 8373 { 8374 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8375 return 0; 8376 } 8377 8378 /** 8379 * Isolated mode. 8380 * 8381 * @see rte_flow_isolate() 8382 * @see rte_flow_ops 8383 */ 8384 int 8385 mlx5_flow_isolate(struct rte_eth_dev *dev, 8386 int enable, 8387 struct rte_flow_error *error) 8388 { 8389 struct mlx5_priv *priv = dev->data->dev_private; 8390 8391 if (dev->data->dev_started) { 8392 rte_flow_error_set(error, EBUSY, 8393 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8394 NULL, 8395 "port must be stopped first"); 8396 return -rte_errno; 8397 } 8398 priv->isolated = !!enable; 8399 if (enable) 8400 dev->dev_ops = &mlx5_dev_ops_isolate; 8401 else 8402 dev->dev_ops = &mlx5_dev_ops; 8403 8404 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8405 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8406 8407 return 0; 8408 } 8409 8410 /** 8411 * Query a flow. 8412 * 8413 * @see rte_flow_query() 8414 * @see rte_flow_ops 8415 */ 8416 static int 8417 flow_drv_query(struct rte_eth_dev *dev, 8418 struct rte_flow *eflow, 8419 const struct rte_flow_action *actions, 8420 void *data, 8421 struct rte_flow_error *error) 8422 { 8423 struct mlx5_priv *priv = dev->data->dev_private; 8424 const struct mlx5_flow_driver_ops *fops; 8425 struct rte_flow *flow = NULL; 8426 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8427 8428 if (priv->sh->config.dv_flow_en == 2) { 8429 #ifdef HAVE_MLX5_HWS_SUPPORT 8430 flow = eflow; 8431 ftype = MLX5_FLOW_TYPE_HW; 8432 #endif 8433 } else { 8434 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8435 (uintptr_t)(void *)eflow); 8436 } 8437 if (!flow) { 8438 return rte_flow_error_set(error, ENOENT, 8439 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8440 NULL, 8441 "invalid flow handle"); 8442 } 8443 if (ftype == MLX5_FLOW_TYPE_MIN) 8444 ftype = flow->drv_type; 8445 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8446 fops = flow_get_drv_ops(ftype); 8447 8448 return fops->query(dev, flow, actions, data, error); 8449 } 8450 8451 /** 8452 * Query a flow. 8453 * 8454 * @see rte_flow_query() 8455 * @see rte_flow_ops 8456 */ 8457 int 8458 mlx5_flow_query(struct rte_eth_dev *dev, 8459 struct rte_flow *flow, 8460 const struct rte_flow_action *actions, 8461 void *data, 8462 struct rte_flow_error *error) 8463 { 8464 int ret; 8465 8466 ret = flow_drv_query(dev, flow, actions, data, 8467 error); 8468 if (ret < 0) 8469 return ret; 8470 return 0; 8471 } 8472 8473 /** 8474 * Get rte_flow callbacks. 8475 * 8476 * @param dev 8477 * Pointer to Ethernet device structure. 8478 * @param ops 8479 * Pointer to operation-specific structure. 8480 * 8481 * @return 0 8482 */ 8483 int 8484 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8485 const struct rte_flow_ops **ops) 8486 { 8487 *ops = &mlx5_flow_ops; 8488 return 0; 8489 } 8490 8491 /** 8492 * Validate meter policy actions. 8493 * Dispatcher for action type specific validation. 8494 * 8495 * @param[in] dev 8496 * Pointer to the Ethernet device structure. 8497 * @param[in] action 8498 * The meter policy action object to validate. 8499 * @param[in] attr 8500 * Attributes of flow to determine steering domain. 8501 * @param[out] is_rss 8502 * Is RSS or not. 8503 * @param[out] domain_bitmap 8504 * Domain bitmap. 8505 * @param[out] is_def_policy 8506 * Is default policy or not. 8507 * @param[out] error 8508 * Perform verbose error reporting if not NULL. Initialized in case of 8509 * error only. 8510 * 8511 * @return 8512 * 0 on success, otherwise negative errno value. 8513 */ 8514 int 8515 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8516 const struct rte_flow_action *actions[RTE_COLORS], 8517 struct rte_flow_attr *attr, 8518 bool *is_rss, 8519 uint8_t *domain_bitmap, 8520 uint8_t *policy_mode, 8521 struct rte_mtr_error *error) 8522 { 8523 const struct mlx5_flow_driver_ops *fops; 8524 8525 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8526 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8527 domain_bitmap, policy_mode, error); 8528 } 8529 8530 /** 8531 * Destroy the meter table set. 8532 * 8533 * @param[in] dev 8534 * Pointer to Ethernet device. 8535 * @param[in] mtr_policy 8536 * Meter policy struct. 8537 */ 8538 void 8539 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8540 struct mlx5_flow_meter_policy *mtr_policy) 8541 { 8542 const struct mlx5_flow_driver_ops *fops; 8543 8544 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8545 fops->destroy_mtr_acts(dev, mtr_policy); 8546 } 8547 8548 /** 8549 * Create policy action, lock free, 8550 * (mutex should be acquired by caller). 8551 * Dispatcher for action type specific call. 8552 * 8553 * @param[in] dev 8554 * Pointer to the Ethernet device structure. 8555 * @param[in] mtr_policy 8556 * Meter policy struct. 8557 * @param[in] action 8558 * Action specification used to create meter actions. 8559 * @param[in] attr 8560 * Flow rule attributes. 8561 * @param[out] error 8562 * Perform verbose error reporting if not NULL. Initialized in case of 8563 * error only. 8564 * 8565 * @return 8566 * 0 on success, otherwise negative errno value. 8567 */ 8568 int 8569 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8570 struct mlx5_flow_meter_policy *mtr_policy, 8571 const struct rte_flow_action *actions[RTE_COLORS], 8572 struct rte_flow_attr *attr, 8573 struct rte_mtr_error *error) 8574 { 8575 const struct mlx5_flow_driver_ops *fops; 8576 8577 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8578 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8579 } 8580 8581 /** 8582 * Create policy rules, lock free, 8583 * (mutex should be acquired by caller). 8584 * Dispatcher for action type specific call. 8585 * 8586 * @param[in] dev 8587 * Pointer to the Ethernet device structure. 8588 * @param[in] mtr_policy 8589 * Meter policy struct. 8590 * 8591 * @return 8592 * 0 on success, -1 otherwise. 8593 */ 8594 int 8595 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8596 struct mlx5_flow_meter_policy *mtr_policy) 8597 { 8598 const struct mlx5_flow_driver_ops *fops; 8599 8600 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8601 return fops->create_policy_rules(dev, mtr_policy); 8602 } 8603 8604 /** 8605 * Destroy policy rules, lock free, 8606 * (mutex should be acquired by caller). 8607 * Dispatcher for action type specific call. 8608 * 8609 * @param[in] dev 8610 * Pointer to the Ethernet device structure. 8611 * @param[in] mtr_policy 8612 * Meter policy struct. 8613 */ 8614 void 8615 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8616 struct mlx5_flow_meter_policy *mtr_policy) 8617 { 8618 const struct mlx5_flow_driver_ops *fops; 8619 8620 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8621 fops->destroy_policy_rules(dev, mtr_policy); 8622 } 8623 8624 /** 8625 * Destroy the default policy table set. 8626 * 8627 * @param[in] dev 8628 * Pointer to Ethernet device. 8629 */ 8630 void 8631 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8632 { 8633 const struct mlx5_flow_driver_ops *fops; 8634 8635 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8636 fops->destroy_def_policy(dev); 8637 } 8638 8639 /** 8640 * Destroy the default policy table set. 8641 * 8642 * @param[in] dev 8643 * Pointer to Ethernet device. 8644 * 8645 * @return 8646 * 0 on success, -1 otherwise. 8647 */ 8648 int 8649 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8650 { 8651 const struct mlx5_flow_driver_ops *fops; 8652 8653 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8654 return fops->create_def_policy(dev); 8655 } 8656 8657 /** 8658 * Create the needed meter and suffix tables. 8659 * 8660 * @param[in] dev 8661 * Pointer to Ethernet device. 8662 * 8663 * @return 8664 * 0 on success, -1 otherwise. 8665 */ 8666 int 8667 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8668 struct mlx5_flow_meter_info *fm, 8669 uint32_t mtr_idx, 8670 uint8_t domain_bitmap) 8671 { 8672 const struct mlx5_flow_driver_ops *fops; 8673 8674 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8675 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8676 } 8677 8678 /** 8679 * Destroy the meter table set. 8680 * 8681 * @param[in] dev 8682 * Pointer to Ethernet device. 8683 * @param[in] tbl 8684 * Pointer to the meter table set. 8685 */ 8686 void 8687 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8688 struct mlx5_flow_meter_info *fm) 8689 { 8690 const struct mlx5_flow_driver_ops *fops; 8691 8692 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8693 fops->destroy_mtr_tbls(dev, fm); 8694 } 8695 8696 /** 8697 * Destroy the global meter drop table. 8698 * 8699 * @param[in] dev 8700 * Pointer to Ethernet device. 8701 */ 8702 void 8703 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8704 { 8705 const struct mlx5_flow_driver_ops *fops; 8706 8707 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8708 fops->destroy_mtr_drop_tbls(dev); 8709 } 8710 8711 /** 8712 * Destroy the sub policy table with RX queue. 8713 * 8714 * @param[in] dev 8715 * Pointer to Ethernet device. 8716 * @param[in] mtr_policy 8717 * Pointer to meter policy table. 8718 */ 8719 void 8720 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8721 struct mlx5_flow_meter_policy *mtr_policy) 8722 { 8723 const struct mlx5_flow_driver_ops *fops; 8724 8725 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8726 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8727 } 8728 8729 /** 8730 * Allocate the needed aso flow meter id. 8731 * 8732 * @param[in] dev 8733 * Pointer to Ethernet device. 8734 * 8735 * @return 8736 * Index to aso flow meter on success, NULL otherwise. 8737 */ 8738 uint32_t 8739 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8740 { 8741 const struct mlx5_flow_driver_ops *fops; 8742 8743 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8744 return fops->create_meter(dev); 8745 } 8746 8747 /** 8748 * Free the aso flow meter id. 8749 * 8750 * @param[in] dev 8751 * Pointer to Ethernet device. 8752 * @param[in] mtr_idx 8753 * Index to aso flow meter to be free. 8754 * 8755 * @return 8756 * 0 on success. 8757 */ 8758 void 8759 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8760 { 8761 const struct mlx5_flow_driver_ops *fops; 8762 8763 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8764 fops->free_meter(dev, mtr_idx); 8765 } 8766 8767 /** 8768 * Allocate a counter. 8769 * 8770 * @param[in] dev 8771 * Pointer to Ethernet device structure. 8772 * 8773 * @return 8774 * Index to allocated counter on success, 0 otherwise. 8775 */ 8776 uint32_t 8777 mlx5_counter_alloc(struct rte_eth_dev *dev) 8778 { 8779 struct rte_flow_attr attr = { .transfer = 0 }; 8780 8781 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8782 (dev); 8783 } 8784 8785 /** 8786 * Free a counter. 8787 * 8788 * @param[in] dev 8789 * Pointer to Ethernet device structure. 8790 * @param[in] cnt 8791 * Index to counter to be free. 8792 */ 8793 void 8794 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8795 { 8796 struct rte_flow_attr attr = { .transfer = 0 }; 8797 8798 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8799 } 8800 8801 /** 8802 * Query counter statistics. 8803 * 8804 * @param[in] dev 8805 * Pointer to Ethernet device structure. 8806 * @param[in] cnt 8807 * Index to counter to query. 8808 * @param[in] clear 8809 * Set to clear counter statistics. 8810 * @param[out] pkts 8811 * The counter hits packets number to save. 8812 * @param[out] bytes 8813 * The counter hits bytes number to save. 8814 * 8815 * @return 8816 * 0 on success, a negative errno value otherwise. 8817 */ 8818 int 8819 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8820 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8821 { 8822 struct rte_flow_attr attr = { .transfer = 0 }; 8823 8824 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8825 (dev, cnt, clear, pkts, bytes, action); 8826 } 8827 8828 /** 8829 * Get information about HWS pre-configurable resources. 8830 * 8831 * @param[in] dev 8832 * Pointer to the rte_eth_dev structure. 8833 * @param[out] port_info 8834 * Pointer to port information. 8835 * @param[out] queue_info 8836 * Pointer to queue information. 8837 * @param[out] error 8838 * Pointer to error structure. 8839 * 8840 * @return 8841 * 0 on success, a negative errno value otherwise and rte_errno is set. 8842 */ 8843 static int 8844 mlx5_flow_info_get(struct rte_eth_dev *dev, 8845 struct rte_flow_port_info *port_info, 8846 struct rte_flow_queue_info *queue_info, 8847 struct rte_flow_error *error) 8848 { 8849 const struct mlx5_flow_driver_ops *fops; 8850 struct rte_flow_attr attr = {0}; 8851 8852 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8853 return rte_flow_error_set(error, ENOTSUP, 8854 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8855 NULL, 8856 "info get with incorrect steering mode"); 8857 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8858 return fops->info_get(dev, port_info, queue_info, error); 8859 } 8860 8861 /** 8862 * Configure port HWS resources. 8863 * 8864 * @param[in] dev 8865 * Pointer to the rte_eth_dev structure. 8866 * @param[in] port_attr 8867 * Port configuration attributes. 8868 * @param[in] nb_queue 8869 * Number of queue. 8870 * @param[in] queue_attr 8871 * Array that holds attributes for each flow queue. 8872 * @param[out] error 8873 * Pointer to error structure. 8874 * 8875 * @return 8876 * 0 on success, a negative errno value otherwise and rte_errno is set. 8877 */ 8878 static int 8879 mlx5_flow_port_configure(struct rte_eth_dev *dev, 8880 const struct rte_flow_port_attr *port_attr, 8881 uint16_t nb_queue, 8882 const struct rte_flow_queue_attr *queue_attr[], 8883 struct rte_flow_error *error) 8884 { 8885 const struct mlx5_flow_driver_ops *fops; 8886 struct rte_flow_attr attr = {0}; 8887 8888 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8889 return rte_flow_error_set(error, ENOTSUP, 8890 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8891 NULL, 8892 "port configure with incorrect steering mode"); 8893 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8894 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 8895 } 8896 8897 /** 8898 * Validate item template. 8899 * 8900 * @param[in] dev 8901 * Pointer to the rte_eth_dev structure. 8902 * @param[in] attr 8903 * Pointer to the item template attributes. 8904 * @param[in] items 8905 * The template item pattern. 8906 * @param[out] error 8907 * Pointer to error structure. 8908 * 8909 * @return 8910 * 0 on success, a negative errno value otherwise and rte_errno is set. 8911 */ 8912 int 8913 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 8914 const struct rte_flow_pattern_template_attr *attr, 8915 const struct rte_flow_item items[], 8916 struct rte_flow_error *error) 8917 { 8918 const struct mlx5_flow_driver_ops *fops; 8919 struct rte_flow_attr fattr = {0}; 8920 8921 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8922 rte_flow_error_set(error, ENOTSUP, 8923 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8924 "pattern validate with incorrect steering mode"); 8925 return -ENOTSUP; 8926 } 8927 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8928 return fops->pattern_validate(dev, attr, items, error); 8929 } 8930 8931 /** 8932 * Create flow item template. 8933 * 8934 * @param[in] dev 8935 * Pointer to the rte_eth_dev structure. 8936 * @param[in] attr 8937 * Pointer to the item template attributes. 8938 * @param[in] items 8939 * The template item pattern. 8940 * @param[out] error 8941 * Pointer to error structure. 8942 * 8943 * @return 8944 * 0 on success, a negative errno value otherwise and rte_errno is set. 8945 */ 8946 static struct rte_flow_pattern_template * 8947 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 8948 const struct rte_flow_pattern_template_attr *attr, 8949 const struct rte_flow_item items[], 8950 struct rte_flow_error *error) 8951 { 8952 const struct mlx5_flow_driver_ops *fops; 8953 struct rte_flow_attr fattr = {0}; 8954 8955 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 8956 rte_flow_error_set(error, ENOTSUP, 8957 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8958 NULL, 8959 "pattern create with incorrect steering mode"); 8960 return NULL; 8961 } 8962 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8963 return fops->pattern_template_create(dev, attr, items, error); 8964 } 8965 8966 /** 8967 * Destroy flow item template. 8968 * 8969 * @param[in] dev 8970 * Pointer to the rte_eth_dev structure. 8971 * @param[in] template 8972 * Pointer to the item template to be destroyed. 8973 * @param[out] error 8974 * Pointer to error structure. 8975 * 8976 * @return 8977 * 0 on success, a negative errno value otherwise and rte_errno is set. 8978 */ 8979 static int 8980 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 8981 struct rte_flow_pattern_template *template, 8982 struct rte_flow_error *error) 8983 { 8984 const struct mlx5_flow_driver_ops *fops; 8985 struct rte_flow_attr attr = {0}; 8986 8987 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8988 return rte_flow_error_set(error, ENOTSUP, 8989 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8990 NULL, 8991 "pattern destroy with incorrect steering mode"); 8992 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 8993 return fops->pattern_template_destroy(dev, template, error); 8994 } 8995 8996 /** 8997 * Validate flow actions template. 8998 * 8999 * @param[in] dev 9000 * Pointer to the rte_eth_dev structure. 9001 * @param[in] attr 9002 * Pointer to the action template attributes. 9003 * @param[in] actions 9004 * Associated actions (list terminated by the END action). 9005 * @param[in] masks 9006 * List of actions that marks which of the action's member is constant. 9007 * @param[out] error 9008 * Pointer to error structure. 9009 * 9010 * @return 9011 * 0 on success, a negative errno value otherwise and rte_errno is set. 9012 */ 9013 int 9014 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9015 const struct rte_flow_actions_template_attr *attr, 9016 const struct rte_flow_action actions[], 9017 const struct rte_flow_action masks[], 9018 struct rte_flow_error *error) 9019 { 9020 const struct mlx5_flow_driver_ops *fops; 9021 struct rte_flow_attr fattr = {0}; 9022 9023 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9024 rte_flow_error_set(error, ENOTSUP, 9025 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9026 "actions validate with incorrect steering mode"); 9027 return -ENOTSUP; 9028 } 9029 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9030 return fops->actions_validate(dev, attr, actions, masks, error); 9031 } 9032 9033 /** 9034 * Create flow item template. 9035 * 9036 * @param[in] dev 9037 * Pointer to the rte_eth_dev structure. 9038 * @param[in] attr 9039 * Pointer to the action template attributes. 9040 * @param[in] actions 9041 * Associated actions (list terminated by the END action). 9042 * @param[in] masks 9043 * List of actions that marks which of the action's member is constant. 9044 * @param[out] error 9045 * Pointer to error structure. 9046 * 9047 * @return 9048 * 0 on success, a negative errno value otherwise and rte_errno is set. 9049 */ 9050 static struct rte_flow_actions_template * 9051 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9052 const struct rte_flow_actions_template_attr *attr, 9053 const struct rte_flow_action actions[], 9054 const struct rte_flow_action masks[], 9055 struct rte_flow_error *error) 9056 { 9057 const struct mlx5_flow_driver_ops *fops; 9058 struct rte_flow_attr fattr = {0}; 9059 9060 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9061 rte_flow_error_set(error, ENOTSUP, 9062 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9063 NULL, 9064 "action create with incorrect steering mode"); 9065 return NULL; 9066 } 9067 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9068 return fops->actions_template_create(dev, attr, actions, masks, error); 9069 } 9070 9071 /** 9072 * Destroy flow action template. 9073 * 9074 * @param[in] dev 9075 * Pointer to the rte_eth_dev structure. 9076 * @param[in] template 9077 * Pointer to the action template to be destroyed. 9078 * @param[out] error 9079 * Pointer to error structure. 9080 * 9081 * @return 9082 * 0 on success, a negative errno value otherwise and rte_errno is set. 9083 */ 9084 static int 9085 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9086 struct rte_flow_actions_template *template, 9087 struct rte_flow_error *error) 9088 { 9089 const struct mlx5_flow_driver_ops *fops; 9090 struct rte_flow_attr attr = {0}; 9091 9092 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9093 return rte_flow_error_set(error, ENOTSUP, 9094 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9095 NULL, 9096 "action destroy with incorrect steering mode"); 9097 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9098 return fops->actions_template_destroy(dev, template, error); 9099 } 9100 9101 /** 9102 * Create flow table. 9103 * 9104 * @param[in] dev 9105 * Pointer to the rte_eth_dev structure. 9106 * @param[in] attr 9107 * Pointer to the table attributes. 9108 * @param[in] item_templates 9109 * Item template array to be binded to the table. 9110 * @param[in] nb_item_templates 9111 * Number of item template. 9112 * @param[in] action_templates 9113 * Action template array to be binded to the table. 9114 * @param[in] nb_action_templates 9115 * Number of action template. 9116 * @param[out] error 9117 * Pointer to error structure. 9118 * 9119 * @return 9120 * Table on success, NULL otherwise and rte_errno is set. 9121 */ 9122 static struct rte_flow_template_table * 9123 mlx5_flow_table_create(struct rte_eth_dev *dev, 9124 const struct rte_flow_template_table_attr *attr, 9125 struct rte_flow_pattern_template *item_templates[], 9126 uint8_t nb_item_templates, 9127 struct rte_flow_actions_template *action_templates[], 9128 uint8_t nb_action_templates, 9129 struct rte_flow_error *error) 9130 { 9131 const struct mlx5_flow_driver_ops *fops; 9132 struct rte_flow_attr fattr = {0}; 9133 9134 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9135 rte_flow_error_set(error, ENOTSUP, 9136 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9137 NULL, 9138 "table create with incorrect steering mode"); 9139 return NULL; 9140 } 9141 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9142 return fops->template_table_create(dev, 9143 attr, 9144 item_templates, 9145 nb_item_templates, 9146 action_templates, 9147 nb_action_templates, 9148 error); 9149 } 9150 9151 /** 9152 * PMD destroy flow table. 9153 * 9154 * @param[in] dev 9155 * Pointer to the rte_eth_dev structure. 9156 * @param[in] table 9157 * Pointer to the table to be destroyed. 9158 * @param[out] error 9159 * Pointer to error structure. 9160 * 9161 * @return 9162 * 0 on success, a negative errno value otherwise and rte_errno is set. 9163 */ 9164 static int 9165 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9166 struct rte_flow_template_table *table, 9167 struct rte_flow_error *error) 9168 { 9169 const struct mlx5_flow_driver_ops *fops; 9170 struct rte_flow_attr attr = {0}; 9171 9172 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9173 return rte_flow_error_set(error, ENOTSUP, 9174 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9175 NULL, 9176 "table destroy with incorrect steering mode"); 9177 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9178 return fops->template_table_destroy(dev, table, error); 9179 } 9180 9181 /** 9182 * Enqueue flow creation. 9183 * 9184 * @param[in] dev 9185 * Pointer to the rte_eth_dev structure. 9186 * @param[in] queue_id 9187 * The queue to create the flow. 9188 * @param[in] attr 9189 * Pointer to the flow operation attributes. 9190 * @param[in] items 9191 * Items with flow spec value. 9192 * @param[in] pattern_template_index 9193 * The item pattern flow follows from the table. 9194 * @param[in] actions 9195 * Action with flow spec value. 9196 * @param[in] action_template_index 9197 * The action pattern flow follows from the table. 9198 * @param[in] user_data 9199 * Pointer to the user_data. 9200 * @param[out] error 9201 * Pointer to error structure. 9202 * 9203 * @return 9204 * Flow pointer on success, NULL otherwise and rte_errno is set. 9205 */ 9206 static struct rte_flow * 9207 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 9208 uint32_t queue_id, 9209 const struct rte_flow_op_attr *attr, 9210 struct rte_flow_template_table *table, 9211 const struct rte_flow_item items[], 9212 uint8_t pattern_template_index, 9213 const struct rte_flow_action actions[], 9214 uint8_t action_template_index, 9215 void *user_data, 9216 struct rte_flow_error *error) 9217 { 9218 const struct mlx5_flow_driver_ops *fops; 9219 struct rte_flow_attr fattr = {0}; 9220 9221 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9222 rte_flow_error_set(error, ENOTSUP, 9223 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9224 NULL, 9225 "flow_q create with incorrect steering mode"); 9226 return NULL; 9227 } 9228 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9229 return fops->async_flow_create(dev, queue_id, attr, table, 9230 items, pattern_template_index, 9231 actions, action_template_index, 9232 user_data, error); 9233 } 9234 9235 /** 9236 * Enqueue flow creation by index. 9237 * 9238 * @param[in] dev 9239 * Pointer to the rte_eth_dev structure. 9240 * @param[in] queue_id 9241 * The queue to create the flow. 9242 * @param[in] attr 9243 * Pointer to the flow operation attributes. 9244 * @param[in] rule_index 9245 * The item pattern flow follows from the table. 9246 * @param[in] actions 9247 * Action with flow spec value. 9248 * @param[in] action_template_index 9249 * The action pattern flow follows from the table. 9250 * @param[in] user_data 9251 * Pointer to the user_data. 9252 * @param[out] error 9253 * Pointer to error structure. 9254 * 9255 * @return 9256 * Flow pointer on success, NULL otherwise and rte_errno is set. 9257 */ 9258 static struct rte_flow * 9259 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 9260 uint32_t queue_id, 9261 const struct rte_flow_op_attr *attr, 9262 struct rte_flow_template_table *table, 9263 uint32_t rule_index, 9264 const struct rte_flow_action actions[], 9265 uint8_t action_template_index, 9266 void *user_data, 9267 struct rte_flow_error *error) 9268 { 9269 const struct mlx5_flow_driver_ops *fops; 9270 struct rte_flow_attr fattr = {0}; 9271 9272 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9273 rte_flow_error_set(error, ENOTSUP, 9274 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9275 NULL, 9276 "flow_q create with incorrect steering mode"); 9277 return NULL; 9278 } 9279 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9280 return fops->async_flow_create_by_index(dev, queue_id, attr, table, 9281 rule_index, actions, action_template_index, 9282 user_data, error); 9283 } 9284 9285 /** 9286 * Enqueue flow destruction. 9287 * 9288 * @param[in] dev 9289 * Pointer to the rte_eth_dev structure. 9290 * @param[in] queue 9291 * The queue to destroy the flow. 9292 * @param[in] attr 9293 * Pointer to the flow operation attributes. 9294 * @param[in] flow 9295 * Pointer to the flow to be destroyed. 9296 * @param[in] user_data 9297 * Pointer to the user_data. 9298 * @param[out] error 9299 * Pointer to error structure. 9300 * 9301 * @return 9302 * 0 on success, negative value otherwise and rte_errno is set. 9303 */ 9304 static int 9305 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 9306 uint32_t queue, 9307 const struct rte_flow_op_attr *attr, 9308 struct rte_flow *flow, 9309 void *user_data, 9310 struct rte_flow_error *error) 9311 { 9312 const struct mlx5_flow_driver_ops *fops; 9313 struct rte_flow_attr fattr = {0}; 9314 9315 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9316 return rte_flow_error_set(error, ENOTSUP, 9317 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9318 NULL, 9319 "flow_q destroy with incorrect steering mode"); 9320 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9321 return fops->async_flow_destroy(dev, queue, attr, flow, 9322 user_data, error); 9323 } 9324 9325 /** 9326 * Pull the enqueued flows. 9327 * 9328 * @param[in] dev 9329 * Pointer to the rte_eth_dev structure. 9330 * @param[in] queue 9331 * The queue to pull the result. 9332 * @param[in/out] res 9333 * Array to save the results. 9334 * @param[in] n_res 9335 * Available result with the array. 9336 * @param[out] error 9337 * Pointer to error structure. 9338 * 9339 * @return 9340 * Result number on success, negative value otherwise and rte_errno is set. 9341 */ 9342 static int 9343 mlx5_flow_pull(struct rte_eth_dev *dev, 9344 uint32_t queue, 9345 struct rte_flow_op_result res[], 9346 uint16_t n_res, 9347 struct rte_flow_error *error) 9348 { 9349 const struct mlx5_flow_driver_ops *fops; 9350 struct rte_flow_attr attr = {0}; 9351 9352 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9353 return rte_flow_error_set(error, ENOTSUP, 9354 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9355 NULL, 9356 "flow_q pull with incorrect steering mode"); 9357 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9358 return fops->pull(dev, queue, res, n_res, error); 9359 } 9360 9361 /** 9362 * Push the enqueued flows. 9363 * 9364 * @param[in] dev 9365 * Pointer to the rte_eth_dev structure. 9366 * @param[in] queue 9367 * The queue to push the flows. 9368 * @param[out] error 9369 * Pointer to error structure. 9370 * 9371 * @return 9372 * 0 on success, negative value otherwise and rte_errno is set. 9373 */ 9374 static int 9375 mlx5_flow_push(struct rte_eth_dev *dev, 9376 uint32_t queue, 9377 struct rte_flow_error *error) 9378 { 9379 const struct mlx5_flow_driver_ops *fops; 9380 struct rte_flow_attr attr = {0}; 9381 9382 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9383 return rte_flow_error_set(error, ENOTSUP, 9384 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9385 NULL, 9386 "flow_q push with incorrect steering mode"); 9387 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9388 return fops->push(dev, queue, error); 9389 } 9390 9391 /** 9392 * Create shared action. 9393 * 9394 * @param[in] dev 9395 * Pointer to the rte_eth_dev structure. 9396 * @param[in] queue 9397 * Which queue to be used.. 9398 * @param[in] attr 9399 * Operation attribute. 9400 * @param[in] conf 9401 * Indirect action configuration. 9402 * @param[in] action 9403 * rte_flow action detail. 9404 * @param[in] user_data 9405 * Pointer to the user_data. 9406 * @param[out] error 9407 * Pointer to error structure. 9408 * 9409 * @return 9410 * Action handle on success, NULL otherwise and rte_errno is set. 9411 */ 9412 static struct rte_flow_action_handle * 9413 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 9414 const struct rte_flow_op_attr *attr, 9415 const struct rte_flow_indir_action_conf *conf, 9416 const struct rte_flow_action *action, 9417 void *user_data, 9418 struct rte_flow_error *error) 9419 { 9420 const struct mlx5_flow_driver_ops *fops = 9421 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9422 9423 return fops->async_action_create(dev, queue, attr, conf, action, 9424 user_data, error); 9425 } 9426 9427 /** 9428 * Update shared action. 9429 * 9430 * @param[in] dev 9431 * Pointer to the rte_eth_dev structure. 9432 * @param[in] queue 9433 * Which queue to be used.. 9434 * @param[in] attr 9435 * Operation attribute. 9436 * @param[in] handle 9437 * Action handle to be updated. 9438 * @param[in] update 9439 * Update value. 9440 * @param[in] user_data 9441 * Pointer to the user_data. 9442 * @param[out] error 9443 * Pointer to error structure. 9444 * 9445 * @return 9446 * 0 on success, negative value otherwise and rte_errno is set. 9447 */ 9448 static int 9449 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 9450 const struct rte_flow_op_attr *attr, 9451 struct rte_flow_action_handle *handle, 9452 const void *update, 9453 void *user_data, 9454 struct rte_flow_error *error) 9455 { 9456 const struct mlx5_flow_driver_ops *fops = 9457 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9458 9459 return fops->async_action_update(dev, queue, attr, handle, 9460 update, user_data, error); 9461 } 9462 9463 /** 9464 * Query shared action. 9465 * 9466 * @param[in] dev 9467 * Pointer to the rte_eth_dev structure. 9468 * @param[in] queue 9469 * Which queue to be used.. 9470 * @param[in] attr 9471 * Operation attribute. 9472 * @param[in] handle 9473 * Action handle to be updated. 9474 * @param[in] data 9475 * Pointer query result data. 9476 * @param[in] user_data 9477 * Pointer to the user_data. 9478 * @param[out] error 9479 * Pointer to error structure. 9480 * 9481 * @return 9482 * 0 on success, negative value otherwise and rte_errno is set. 9483 */ 9484 static int 9485 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9486 const struct rte_flow_op_attr *attr, 9487 const struct rte_flow_action_handle *handle, 9488 void *data, 9489 void *user_data, 9490 struct rte_flow_error *error) 9491 { 9492 const struct mlx5_flow_driver_ops *fops = 9493 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9494 9495 return fops->async_action_query(dev, queue, attr, handle, 9496 data, user_data, error); 9497 } 9498 9499 /** 9500 * Destroy shared action. 9501 * 9502 * @param[in] dev 9503 * Pointer to the rte_eth_dev structure. 9504 * @param[in] queue 9505 * Which queue to be used.. 9506 * @param[in] attr 9507 * Operation attribute. 9508 * @param[in] handle 9509 * Action handle to be destroyed. 9510 * @param[in] user_data 9511 * Pointer to the user_data. 9512 * @param[out] error 9513 * Pointer to error structure. 9514 * 9515 * @return 9516 * 0 on success, negative value otherwise and rte_errno is set. 9517 */ 9518 static int 9519 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9520 const struct rte_flow_op_attr *attr, 9521 struct rte_flow_action_handle *handle, 9522 void *user_data, 9523 struct rte_flow_error *error) 9524 { 9525 const struct mlx5_flow_driver_ops *fops = 9526 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9527 9528 return fops->async_action_destroy(dev, queue, attr, handle, 9529 user_data, error); 9530 } 9531 9532 /** 9533 * Allocate a new memory for the counter values wrapped by all the needed 9534 * management. 9535 * 9536 * @param[in] sh 9537 * Pointer to mlx5_dev_ctx_shared object. 9538 * 9539 * @return 9540 * 0 on success, a negative errno value otherwise. 9541 */ 9542 static int 9543 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9544 { 9545 struct mlx5_counter_stats_mem_mng *mem_mng; 9546 volatile struct flow_counter_stats *raw_data; 9547 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9548 int size = (sizeof(struct flow_counter_stats) * 9549 MLX5_COUNTERS_PER_POOL + 9550 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9551 sizeof(struct mlx5_counter_stats_mem_mng); 9552 size_t pgsize = rte_mem_page_size(); 9553 uint8_t *mem; 9554 int ret; 9555 int i; 9556 9557 if (pgsize == (size_t)-1) { 9558 DRV_LOG(ERR, "Failed to get mem page size"); 9559 rte_errno = ENOMEM; 9560 return -ENOMEM; 9561 } 9562 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9563 if (!mem) { 9564 rte_errno = ENOMEM; 9565 return -ENOMEM; 9566 } 9567 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9568 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9569 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9570 sh->cdev->pdn, mem, size, 9571 &mem_mng->wm); 9572 if (ret) { 9573 rte_errno = errno; 9574 mlx5_free(mem); 9575 return -rte_errno; 9576 } 9577 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9578 raw_data = (volatile struct flow_counter_stats *)mem; 9579 for (i = 0; i < raws_n; ++i) { 9580 mem_mng->raws[i].mem_mng = mem_mng; 9581 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9582 } 9583 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9584 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9585 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9586 next); 9587 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9588 sh->sws_cmng.mem_mng = mem_mng; 9589 return 0; 9590 } 9591 9592 /** 9593 * Set the statistic memory to the new counter pool. 9594 * 9595 * @param[in] sh 9596 * Pointer to mlx5_dev_ctx_shared object. 9597 * @param[in] pool 9598 * Pointer to the pool to set the statistic memory. 9599 * 9600 * @return 9601 * 0 on success, a negative errno value otherwise. 9602 */ 9603 static int 9604 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9605 struct mlx5_flow_counter_pool *pool) 9606 { 9607 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9608 /* Resize statistic memory once used out. */ 9609 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9610 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9611 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9612 return -1; 9613 } 9614 rte_spinlock_lock(&pool->sl); 9615 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9616 rte_spinlock_unlock(&pool->sl); 9617 pool->raw_hw = NULL; 9618 return 0; 9619 } 9620 9621 #define MLX5_POOL_QUERY_FREQ_US 1000000 9622 9623 /** 9624 * Set the periodic procedure for triggering asynchronous batch queries for all 9625 * the counter pools. 9626 * 9627 * @param[in] sh 9628 * Pointer to mlx5_dev_ctx_shared object. 9629 */ 9630 void 9631 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9632 { 9633 uint32_t pools_n, us; 9634 9635 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9636 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9637 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9638 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9639 sh->sws_cmng.query_thread_on = 0; 9640 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9641 } else { 9642 sh->sws_cmng.query_thread_on = 1; 9643 } 9644 } 9645 9646 /** 9647 * The periodic procedure for triggering asynchronous batch queries for all the 9648 * counter pools. This function is probably called by the host thread. 9649 * 9650 * @param[in] arg 9651 * The parameter for the alarm process. 9652 */ 9653 void 9654 mlx5_flow_query_alarm(void *arg) 9655 { 9656 struct mlx5_dev_ctx_shared *sh = arg; 9657 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9658 uint16_t pool_index = cmng->pool_index; 9659 struct mlx5_flow_counter_pool *pool; 9660 uint16_t n_valid; 9661 int ret; 9662 9663 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9664 goto set_alarm; 9665 rte_spinlock_lock(&cmng->pool_update_sl); 9666 pool = cmng->pools[pool_index]; 9667 n_valid = cmng->n_valid; 9668 rte_spinlock_unlock(&cmng->pool_update_sl); 9669 /* Set the statistic memory to the new created pool. */ 9670 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9671 goto set_alarm; 9672 if (pool->raw_hw) 9673 /* There is a pool query in progress. */ 9674 goto set_alarm; 9675 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9676 if (!pool->raw_hw) 9677 /* No free counter statistics raw memory. */ 9678 goto set_alarm; 9679 /* 9680 * Identify the counters released between query trigger and query 9681 * handle more efficiently. The counter released in this gap period 9682 * should wait for a new round of query as the new arrived packets 9683 * will not be taken into account. 9684 */ 9685 pool->query_gen++; 9686 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9687 MLX5_COUNTERS_PER_POOL, 9688 NULL, NULL, 9689 pool->raw_hw->mem_mng->wm.lkey, 9690 (void *)(uintptr_t) 9691 pool->raw_hw->data, 9692 sh->devx_comp, 9693 (uint64_t)(uintptr_t)pool); 9694 if (ret) { 9695 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9696 " %d", pool->min_dcs->id); 9697 pool->raw_hw = NULL; 9698 goto set_alarm; 9699 } 9700 LIST_REMOVE(pool->raw_hw, next); 9701 cmng->pending_queries++; 9702 pool_index++; 9703 if (pool_index >= n_valid) 9704 pool_index = 0; 9705 set_alarm: 9706 cmng->pool_index = pool_index; 9707 mlx5_set_query_alarm(sh); 9708 } 9709 9710 /** 9711 * Check and callback event for new aged flow in the counter pool 9712 * 9713 * @param[in] sh 9714 * Pointer to mlx5_dev_ctx_shared object. 9715 * @param[in] pool 9716 * Pointer to Current counter pool. 9717 */ 9718 static void 9719 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9720 struct mlx5_flow_counter_pool *pool) 9721 { 9722 struct mlx5_priv *priv; 9723 struct mlx5_flow_counter *cnt; 9724 struct mlx5_age_info *age_info; 9725 struct mlx5_age_param *age_param; 9726 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9727 struct mlx5_counter_stats_raw *prev = pool->raw; 9728 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9729 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9730 uint16_t expected = AGE_CANDIDATE; 9731 uint32_t i; 9732 9733 pool->time_of_last_age_check = curr_time; 9734 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9735 cnt = MLX5_POOL_GET_CNT(pool, i); 9736 age_param = MLX5_CNT_TO_AGE(cnt); 9737 if (__atomic_load_n(&age_param->state, 9738 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9739 continue; 9740 if (cur->data[i].hits != prev->data[i].hits) { 9741 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9742 __ATOMIC_RELAXED); 9743 continue; 9744 } 9745 if (__atomic_add_fetch(&age_param->sec_since_last_hit, 9746 time_delta, 9747 __ATOMIC_RELAXED) <= age_param->timeout) 9748 continue; 9749 /** 9750 * Hold the lock first, or if between the 9751 * state AGE_TMOUT and tailq operation the 9752 * release happened, the release procedure 9753 * may delete a non-existent tailq node. 9754 */ 9755 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9756 age_info = GET_PORT_AGE_INFO(priv); 9757 rte_spinlock_lock(&age_info->aged_sl); 9758 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9759 AGE_TMOUT, false, 9760 __ATOMIC_RELAXED, 9761 __ATOMIC_RELAXED)) { 9762 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9763 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9764 } 9765 rte_spinlock_unlock(&age_info->aged_sl); 9766 } 9767 mlx5_age_event_prepare(sh); 9768 } 9769 9770 /** 9771 * Handler for the HW respond about ready values from an asynchronous batch 9772 * query. This function is probably called by the host thread. 9773 * 9774 * @param[in] sh 9775 * The pointer to the shared device context. 9776 * @param[in] async_id 9777 * The Devx async ID. 9778 * @param[in] status 9779 * The status of the completion. 9780 */ 9781 void 9782 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9783 uint64_t async_id, int status) 9784 { 9785 struct mlx5_flow_counter_pool *pool = 9786 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 9787 struct mlx5_counter_stats_raw *raw_to_free; 9788 uint8_t query_gen = pool->query_gen ^ 1; 9789 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9790 enum mlx5_counter_type cnt_type = 9791 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 9792 MLX5_COUNTER_TYPE_ORIGIN; 9793 9794 if (unlikely(status)) { 9795 raw_to_free = pool->raw_hw; 9796 } else { 9797 raw_to_free = pool->raw; 9798 if (pool->is_aged) 9799 mlx5_flow_aging_check(sh, pool); 9800 rte_spinlock_lock(&pool->sl); 9801 pool->raw = pool->raw_hw; 9802 rte_spinlock_unlock(&pool->sl); 9803 /* Be sure the new raw counters data is updated in memory. */ 9804 rte_io_wmb(); 9805 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 9806 rte_spinlock_lock(&cmng->csl[cnt_type]); 9807 TAILQ_CONCAT(&cmng->counters[cnt_type], 9808 &pool->counters[query_gen], next); 9809 rte_spinlock_unlock(&cmng->csl[cnt_type]); 9810 } 9811 } 9812 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 9813 pool->raw_hw = NULL; 9814 sh->sws_cmng.pending_queries--; 9815 } 9816 9817 static int 9818 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 9819 const struct flow_grp_info *grp_info, 9820 struct rte_flow_error *error) 9821 { 9822 if (grp_info->transfer && grp_info->external && 9823 grp_info->fdb_def_rule) { 9824 if (group == UINT32_MAX) 9825 return rte_flow_error_set 9826 (error, EINVAL, 9827 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 9828 NULL, 9829 "group index not supported"); 9830 *table = group + 1; 9831 } else { 9832 *table = group; 9833 } 9834 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 9835 return 0; 9836 } 9837 9838 /** 9839 * Translate the rte_flow group index to HW table value. 9840 * 9841 * If tunnel offload is disabled, all group ids converted to flow table 9842 * id using the standard method. 9843 * If tunnel offload is enabled, group id can be converted using the 9844 * standard or tunnel conversion method. Group conversion method 9845 * selection depends on flags in `grp_info` parameter: 9846 * - Internal (grp_info.external == 0) groups conversion uses the 9847 * standard method. 9848 * - Group ids in JUMP action converted with the tunnel conversion. 9849 * - Group id in rule attribute conversion depends on a rule type and 9850 * group id value: 9851 * ** non zero group attributes converted with the tunnel method 9852 * ** zero group attribute in non-tunnel rule is converted using the 9853 * standard method - there's only one root table 9854 * ** zero group attribute in steer tunnel rule is converted with the 9855 * standard method - single root table 9856 * ** zero group attribute in match tunnel rule is a special OvS 9857 * case: that value is used for portability reasons. That group 9858 * id is converted with the tunnel conversion method. 9859 * 9860 * @param[in] dev 9861 * Port device 9862 * @param[in] tunnel 9863 * PMD tunnel offload object 9864 * @param[in] group 9865 * rte_flow group index value. 9866 * @param[out] table 9867 * HW table value. 9868 * @param[in] grp_info 9869 * flags used for conversion 9870 * @param[out] error 9871 * Pointer to error structure. 9872 * 9873 * @return 9874 * 0 on success, a negative errno value otherwise and rte_errno is set. 9875 */ 9876 int 9877 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 9878 const struct mlx5_flow_tunnel *tunnel, 9879 uint32_t group, uint32_t *table, 9880 const struct flow_grp_info *grp_info, 9881 struct rte_flow_error *error) 9882 { 9883 int ret; 9884 bool standard_translation; 9885 9886 if (!grp_info->skip_scale && grp_info->external && 9887 group < MLX5_MAX_TABLES_EXTERNAL) 9888 group *= MLX5_FLOW_TABLE_FACTOR; 9889 if (is_tunnel_offload_active(dev)) { 9890 standard_translation = !grp_info->external || 9891 grp_info->std_tbl_fix; 9892 } else { 9893 standard_translation = true; 9894 } 9895 DRV_LOG(DEBUG, 9896 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 9897 dev->data->port_id, group, grp_info->transfer, 9898 grp_info->external, grp_info->fdb_def_rule, 9899 standard_translation ? "STANDARD" : "TUNNEL"); 9900 if (standard_translation) 9901 ret = flow_group_to_table(dev->data->port_id, group, table, 9902 grp_info, error); 9903 else 9904 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 9905 table, error); 9906 9907 return ret; 9908 } 9909 9910 /** 9911 * Discover availability of metadata reg_c's. 9912 * 9913 * Iteratively use test flows to check availability. 9914 * 9915 * @param[in] dev 9916 * Pointer to the Ethernet device structure. 9917 * 9918 * @return 9919 * 0 on success, a negative errno value otherwise and rte_errno is set. 9920 */ 9921 int 9922 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 9923 { 9924 struct mlx5_priv *priv = dev->data->dev_private; 9925 enum modify_reg idx; 9926 int n = 0; 9927 9928 /* reg_c[0] and reg_c[1] are reserved. */ 9929 priv->sh->flow_mreg_c[n++] = REG_C_0; 9930 priv->sh->flow_mreg_c[n++] = REG_C_1; 9931 /* Discover availability of other reg_c's. */ 9932 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 9933 struct rte_flow_attr attr = { 9934 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 9935 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 9936 .ingress = 1, 9937 }; 9938 struct rte_flow_item items[] = { 9939 [0] = { 9940 .type = RTE_FLOW_ITEM_TYPE_END, 9941 }, 9942 }; 9943 struct rte_flow_action actions[] = { 9944 [0] = { 9945 .type = (enum rte_flow_action_type) 9946 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 9947 .conf = &(struct mlx5_flow_action_copy_mreg){ 9948 .src = REG_C_1, 9949 .dst = idx, 9950 }, 9951 }, 9952 [1] = { 9953 .type = RTE_FLOW_ACTION_TYPE_JUMP, 9954 .conf = &(struct rte_flow_action_jump){ 9955 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 9956 }, 9957 }, 9958 [2] = { 9959 .type = RTE_FLOW_ACTION_TYPE_END, 9960 }, 9961 }; 9962 uint32_t flow_idx; 9963 struct rte_flow *flow; 9964 struct rte_flow_error error; 9965 9966 if (!priv->sh->config.dv_flow_en) 9967 break; 9968 /* Create internal flow, validation skips copy action. */ 9969 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 9970 items, actions, false, &error); 9971 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 9972 flow_idx); 9973 if (!flow) 9974 continue; 9975 priv->sh->flow_mreg_c[n++] = idx; 9976 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 9977 } 9978 for (; n < MLX5_MREG_C_NUM; ++n) 9979 priv->sh->flow_mreg_c[n] = REG_NON; 9980 priv->sh->metadata_regc_check_flag = 1; 9981 return 0; 9982 } 9983 9984 int 9985 save_dump_file(const uint8_t *data, uint32_t size, 9986 uint32_t type, uint64_t id, void *arg, FILE *file) 9987 { 9988 char line[BUF_SIZE]; 9989 uint32_t out = 0; 9990 uint32_t k; 9991 uint32_t actions_num; 9992 struct rte_flow_query_count *count; 9993 9994 memset(line, 0, BUF_SIZE); 9995 switch (type) { 9996 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 9997 actions_num = *(uint32_t *)(arg); 9998 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 9999 type, id, actions_num); 10000 break; 10001 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 10002 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 10003 type, id); 10004 break; 10005 case DR_DUMP_REC_TYPE_PMD_COUNTER: 10006 count = (struct rte_flow_query_count *)arg; 10007 fprintf(file, 10008 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 10009 type, id, count->hits, count->bytes); 10010 return 0; 10011 default: 10012 return -1; 10013 } 10014 10015 for (k = 0; k < size; k++) { 10016 /* Make sure we do not overrun the line buffer length. */ 10017 if (out >= BUF_SIZE - 4) { 10018 line[out] = '\0'; 10019 break; 10020 } 10021 out += snprintf(line + out, BUF_SIZE - out, "%02x", 10022 (data[k]) & 0xff); 10023 } 10024 fprintf(file, "%s\n", line); 10025 return 0; 10026 } 10027 10028 int 10029 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 10030 struct rte_flow_query_count *count, struct rte_flow_error *error) 10031 { 10032 struct rte_flow_action action[2]; 10033 enum mlx5_flow_drv_type ftype; 10034 const struct mlx5_flow_driver_ops *fops; 10035 10036 if (!flow) { 10037 return rte_flow_error_set(error, ENOENT, 10038 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10039 NULL, 10040 "invalid flow handle"); 10041 } 10042 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 10043 action[1].type = RTE_FLOW_ACTION_TYPE_END; 10044 if (flow->counter) { 10045 memset(count, 0, sizeof(struct rte_flow_query_count)); 10046 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 10047 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 10048 ftype < MLX5_FLOW_TYPE_MAX); 10049 fops = flow_get_drv_ops(ftype); 10050 return fops->query(dev, flow, action, count, error); 10051 } 10052 return -1; 10053 } 10054 10055 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10056 /** 10057 * Dump flow ipool data to file 10058 * 10059 * @param[in] dev 10060 * The pointer to Ethernet device. 10061 * @param[in] file 10062 * A pointer to a file for output. 10063 * @param[out] error 10064 * Perform verbose error reporting if not NULL. PMDs initialize this 10065 * structure in case of error only. 10066 * @return 10067 * 0 on success, a negative value otherwise. 10068 */ 10069 int 10070 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 10071 struct rte_flow *flow, FILE *file, 10072 struct rte_flow_error *error) 10073 { 10074 struct mlx5_priv *priv = dev->data->dev_private; 10075 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10076 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10077 uint32_t handle_idx; 10078 struct mlx5_flow_handle *dh; 10079 struct rte_flow_query_count count; 10080 uint32_t actions_num; 10081 const uint8_t *data; 10082 size_t size; 10083 uint64_t id; 10084 uint32_t type; 10085 void *action = NULL; 10086 10087 if (!flow) { 10088 return rte_flow_error_set(error, ENOENT, 10089 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10090 NULL, 10091 "invalid flow handle"); 10092 } 10093 handle_idx = flow->dev_handles; 10094 /* query counter */ 10095 if (flow->counter && 10096 (!mlx5_counter_query(dev, flow->counter, false, 10097 &count.hits, &count.bytes, &action)) && action) { 10098 id = (uint64_t)(uintptr_t)action; 10099 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10100 save_dump_file(NULL, 0, type, 10101 id, (void *)&count, file); 10102 } 10103 10104 while (handle_idx) { 10105 dh = mlx5_ipool_get(priv->sh->ipool 10106 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 10107 if (!dh) 10108 continue; 10109 handle_idx = dh->next.next; 10110 10111 /* Get modify_hdr and encap_decap buf from ipools. */ 10112 encap_decap = NULL; 10113 modify_hdr = dh->dvh.modify_hdr; 10114 10115 if (dh->dvh.rix_encap_decap) { 10116 encap_decap = mlx5_ipool_get(priv->sh->ipool 10117 [MLX5_IPOOL_DECAP_ENCAP], 10118 dh->dvh.rix_encap_decap); 10119 } 10120 if (modify_hdr) { 10121 data = (const uint8_t *)modify_hdr->actions; 10122 size = (size_t)(modify_hdr->actions_num) * 8; 10123 id = (uint64_t)(uintptr_t)modify_hdr->action; 10124 actions_num = modify_hdr->actions_num; 10125 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10126 save_dump_file(data, size, type, id, 10127 (void *)(&actions_num), file); 10128 } 10129 if (encap_decap) { 10130 data = encap_decap->buf; 10131 size = encap_decap->size; 10132 id = (uint64_t)(uintptr_t)encap_decap->action; 10133 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10134 save_dump_file(data, size, type, 10135 id, NULL, file); 10136 } 10137 } 10138 return 0; 10139 } 10140 10141 /** 10142 * Dump all flow's encap_decap/modify_hdr/counter data to file 10143 * 10144 * @param[in] dev 10145 * The pointer to Ethernet device. 10146 * @param[in] file 10147 * A pointer to a file for output. 10148 * @param[out] error 10149 * Perform verbose error reporting if not NULL. PMDs initialize this 10150 * structure in case of error only. 10151 * @return 10152 * 0 on success, a negative value otherwise. 10153 */ 10154 static int 10155 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 10156 FILE *file, struct rte_flow_error *error __rte_unused) 10157 { 10158 struct mlx5_priv *priv = dev->data->dev_private; 10159 struct mlx5_dev_ctx_shared *sh = priv->sh; 10160 struct mlx5_hlist *h; 10161 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10162 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10163 struct rte_flow_query_count count; 10164 uint32_t actions_num; 10165 const uint8_t *data; 10166 size_t size; 10167 uint64_t id; 10168 uint32_t type; 10169 uint32_t i; 10170 uint32_t j; 10171 struct mlx5_list_inconst *l_inconst; 10172 struct mlx5_list_entry *e; 10173 int lcore_index; 10174 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 10175 uint32_t max; 10176 void *action; 10177 10178 /* encap_decap hlist is lcore_share, get global core cache. */ 10179 i = MLX5_LIST_GLOBAL; 10180 h = sh->encaps_decaps; 10181 if (h) { 10182 for (j = 0; j <= h->mask; j++) { 10183 l_inconst = &h->buckets[j].l; 10184 if (!l_inconst || !l_inconst->cache[i]) 10185 continue; 10186 10187 e = LIST_FIRST(&l_inconst->cache[i]->h); 10188 while (e) { 10189 encap_decap = 10190 (struct mlx5_flow_dv_encap_decap_resource *)e; 10191 data = encap_decap->buf; 10192 size = encap_decap->size; 10193 id = (uint64_t)(uintptr_t)encap_decap->action; 10194 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10195 save_dump_file(data, size, type, 10196 id, NULL, file); 10197 e = LIST_NEXT(e, next); 10198 } 10199 } 10200 } 10201 10202 /* get modify_hdr */ 10203 h = sh->modify_cmds; 10204 if (h) { 10205 lcore_index = rte_lcore_index(rte_lcore_id()); 10206 if (unlikely(lcore_index == -1)) { 10207 lcore_index = MLX5_LIST_NLCORE; 10208 rte_spinlock_lock(&h->l_const.lcore_lock); 10209 } 10210 i = lcore_index; 10211 10212 for (j = 0; j <= h->mask; j++) { 10213 l_inconst = &h->buckets[j].l; 10214 if (!l_inconst || !l_inconst->cache[i]) 10215 continue; 10216 10217 e = LIST_FIRST(&l_inconst->cache[i]->h); 10218 while (e) { 10219 modify_hdr = 10220 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10221 data = (const uint8_t *)modify_hdr->actions; 10222 size = (size_t)(modify_hdr->actions_num) * 8; 10223 actions_num = modify_hdr->actions_num; 10224 id = (uint64_t)(uintptr_t)modify_hdr->action; 10225 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10226 save_dump_file(data, size, type, id, 10227 (void *)(&actions_num), file); 10228 e = LIST_NEXT(e, next); 10229 } 10230 } 10231 10232 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10233 rte_spinlock_unlock(&h->l_const.lcore_lock); 10234 } 10235 10236 /* get counter */ 10237 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10238 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10239 for (j = 1; j <= max; j++) { 10240 action = NULL; 10241 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10242 &count.bytes, &action)) && action) { 10243 id = (uint64_t)(uintptr_t)action; 10244 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10245 save_dump_file(NULL, 0, type, 10246 id, (void *)&count, file); 10247 } 10248 } 10249 return 0; 10250 } 10251 #endif 10252 10253 /** 10254 * Dump flow raw hw data to file 10255 * 10256 * @param[in] dev 10257 * The pointer to Ethernet device. 10258 * @param[in] file 10259 * A pointer to a file for output. 10260 * @param[out] error 10261 * Perform verbose error reporting if not NULL. PMDs initialize this 10262 * structure in case of error only. 10263 * @return 10264 * 0 on success, a negative value otherwise. 10265 */ 10266 int 10267 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10268 FILE *file, 10269 struct rte_flow_error *error __rte_unused) 10270 { 10271 struct mlx5_priv *priv = dev->data->dev_private; 10272 struct mlx5_dev_ctx_shared *sh = priv->sh; 10273 uint32_t handle_idx; 10274 int ret; 10275 struct mlx5_flow_handle *dh; 10276 struct rte_flow *flow; 10277 10278 if (!sh->config.dv_flow_en) { 10279 if (fputs("device dv flow disabled\n", file) <= 0) 10280 return -errno; 10281 return -ENOTSUP; 10282 } 10283 10284 /* dump all */ 10285 if (!flow_idx) { 10286 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10287 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10288 return -EINVAL; 10289 10290 if (sh->config.dv_flow_en == 2) 10291 return mlx5dr_debug_dump(priv->dr_ctx, file); 10292 #endif 10293 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10294 sh->rx_domain, 10295 sh->tx_domain, file); 10296 } 10297 /* dump one */ 10298 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10299 (uintptr_t)(void *)flow_idx); 10300 if (!flow) 10301 return -EINVAL; 10302 10303 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10304 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10305 #endif 10306 handle_idx = flow->dev_handles; 10307 while (handle_idx) { 10308 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10309 handle_idx); 10310 if (!dh) 10311 return -ENOENT; 10312 if (dh->drv_flow) { 10313 if (sh->config.dv_flow_en == 2) 10314 return -ENOTSUP; 10315 10316 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10317 file); 10318 if (ret) 10319 return -ENOENT; 10320 } 10321 handle_idx = dh->next.next; 10322 } 10323 return 0; 10324 } 10325 10326 /** 10327 * Get aged-out flows. 10328 * 10329 * @param[in] dev 10330 * Pointer to the Ethernet device structure. 10331 * @param[in] context 10332 * The address of an array of pointers to the aged-out flows contexts. 10333 * @param[in] nb_countexts 10334 * The length of context array pointers. 10335 * @param[out] error 10336 * Perform verbose error reporting if not NULL. Initialized in case of 10337 * error only. 10338 * 10339 * @return 10340 * how many contexts get in success, otherwise negative errno value. 10341 * if nb_contexts is 0, return the amount of all aged contexts. 10342 * if nb_contexts is not 0 , return the amount of aged flows reported 10343 * in the context array. 10344 */ 10345 int 10346 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10347 uint32_t nb_contexts, struct rte_flow_error *error) 10348 { 10349 struct rte_flow_attr attr = { .transfer = 0 }; 10350 10351 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10352 (dev, contexts, nb_contexts, error); 10353 } 10354 10355 /** 10356 * Get aged-out flows per HWS queue. 10357 * 10358 * @param[in] dev 10359 * Pointer to the Ethernet device structure. 10360 * @param[in] queue_id 10361 * Flow queue to query. 10362 * @param[in] context 10363 * The address of an array of pointers to the aged-out flows contexts. 10364 * @param[in] nb_countexts 10365 * The length of context array pointers. 10366 * @param[out] error 10367 * Perform verbose error reporting if not NULL. Initialized in case of 10368 * error only. 10369 * 10370 * @return 10371 * how many contexts get in success, otherwise negative errno value. 10372 * if nb_contexts is 0, return the amount of all aged contexts. 10373 * if nb_contexts is not 0 , return the amount of aged flows reported 10374 * in the context array. 10375 */ 10376 int 10377 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10378 void **contexts, uint32_t nb_contexts, 10379 struct rte_flow_error *error) 10380 { 10381 const struct mlx5_flow_driver_ops *fops; 10382 struct rte_flow_attr attr = { 0 }; 10383 10384 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10385 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10386 return fops->get_q_aged_flows(dev, queue_id, contexts, 10387 nb_contexts, error); 10388 } 10389 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10390 dev->data->port_id, queue_id); 10391 return rte_flow_error_set(error, ENOTSUP, 10392 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10393 "get Q aged flows with incorrect steering mode"); 10394 } 10395 10396 /* Wrapper for driver action_validate op callback */ 10397 static int 10398 flow_drv_action_validate(struct rte_eth_dev *dev, 10399 const struct rte_flow_indir_action_conf *conf, 10400 const struct rte_flow_action *action, 10401 const struct mlx5_flow_driver_ops *fops, 10402 struct rte_flow_error *error) 10403 { 10404 static const char err_msg[] = "indirect action validation unsupported"; 10405 10406 if (!fops->action_validate) { 10407 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10408 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10409 NULL, err_msg); 10410 return -rte_errno; 10411 } 10412 return fops->action_validate(dev, conf, action, error); 10413 } 10414 10415 /** 10416 * Destroys the shared action by handle. 10417 * 10418 * @param dev 10419 * Pointer to Ethernet device structure. 10420 * @param[in] handle 10421 * Handle for the indirect action object to be destroyed. 10422 * @param[out] error 10423 * Perform verbose error reporting if not NULL. PMDs initialize this 10424 * structure in case of error only. 10425 * 10426 * @return 10427 * 0 on success, a negative errno value otherwise and rte_errno is set. 10428 * 10429 * @note: wrapper for driver action_create op callback. 10430 */ 10431 static int 10432 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10433 struct rte_flow_action_handle *handle, 10434 struct rte_flow_error *error) 10435 { 10436 static const char err_msg[] = "indirect action destruction unsupported"; 10437 struct rte_flow_attr attr = { .transfer = 0 }; 10438 const struct mlx5_flow_driver_ops *fops = 10439 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10440 10441 if (!fops->action_destroy) { 10442 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10443 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10444 NULL, err_msg); 10445 return -rte_errno; 10446 } 10447 return fops->action_destroy(dev, handle, error); 10448 } 10449 10450 /* Wrapper for driver action_destroy op callback */ 10451 static int 10452 flow_drv_action_update(struct rte_eth_dev *dev, 10453 struct rte_flow_action_handle *handle, 10454 const void *update, 10455 const struct mlx5_flow_driver_ops *fops, 10456 struct rte_flow_error *error) 10457 { 10458 static const char err_msg[] = "indirect action update unsupported"; 10459 10460 if (!fops->action_update) { 10461 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10462 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10463 NULL, err_msg); 10464 return -rte_errno; 10465 } 10466 return fops->action_update(dev, handle, update, error); 10467 } 10468 10469 /* Wrapper for driver action_destroy op callback */ 10470 static int 10471 flow_drv_action_query(struct rte_eth_dev *dev, 10472 const struct rte_flow_action_handle *handle, 10473 void *data, 10474 const struct mlx5_flow_driver_ops *fops, 10475 struct rte_flow_error *error) 10476 { 10477 static const char err_msg[] = "indirect action query unsupported"; 10478 10479 if (!fops->action_query) { 10480 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10481 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10482 NULL, err_msg); 10483 return -rte_errno; 10484 } 10485 return fops->action_query(dev, handle, data, error); 10486 } 10487 10488 /** 10489 * Create indirect action for reuse in multiple flow rules. 10490 * 10491 * @param dev 10492 * Pointer to Ethernet device structure. 10493 * @param conf 10494 * Pointer to indirect action object configuration. 10495 * @param[in] action 10496 * Action configuration for indirect action object creation. 10497 * @param[out] error 10498 * Perform verbose error reporting if not NULL. PMDs initialize this 10499 * structure in case of error only. 10500 * @return 10501 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10502 */ 10503 static struct rte_flow_action_handle * 10504 mlx5_action_handle_create(struct rte_eth_dev *dev, 10505 const struct rte_flow_indir_action_conf *conf, 10506 const struct rte_flow_action *action, 10507 struct rte_flow_error *error) 10508 { 10509 static const char err_msg[] = "indirect action creation unsupported"; 10510 struct rte_flow_attr attr = { .transfer = 0 }; 10511 const struct mlx5_flow_driver_ops *fops = 10512 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10513 10514 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10515 return NULL; 10516 if (!fops->action_create) { 10517 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10518 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10519 NULL, err_msg); 10520 return NULL; 10521 } 10522 return fops->action_create(dev, conf, action, error); 10523 } 10524 10525 /** 10526 * Updates inplace the indirect action configuration pointed by *handle* 10527 * with the configuration provided as *update* argument. 10528 * The update of the indirect action configuration effects all flow rules 10529 * reusing the action via handle. 10530 * 10531 * @param dev 10532 * Pointer to Ethernet device structure. 10533 * @param[in] handle 10534 * Handle for the indirect action to be updated. 10535 * @param[in] update 10536 * Action specification used to modify the action pointed by handle. 10537 * *update* could be of same type with the action pointed by the *handle* 10538 * handle argument, or some other structures like a wrapper, depending on 10539 * the indirect action type. 10540 * @param[out] error 10541 * Perform verbose error reporting if not NULL. PMDs initialize this 10542 * structure in case of error only. 10543 * 10544 * @return 10545 * 0 on success, a negative errno value otherwise and rte_errno is set. 10546 */ 10547 static int 10548 mlx5_action_handle_update(struct rte_eth_dev *dev, 10549 struct rte_flow_action_handle *handle, 10550 const void *update, 10551 struct rte_flow_error *error) 10552 { 10553 struct rte_flow_attr attr = { .transfer = 0 }; 10554 const struct mlx5_flow_driver_ops *fops = 10555 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10556 int ret; 10557 10558 ret = flow_drv_action_validate(dev, NULL, 10559 (const struct rte_flow_action *)update, fops, error); 10560 if (ret) 10561 return ret; 10562 return flow_drv_action_update(dev, handle, update, fops, 10563 error); 10564 } 10565 10566 /** 10567 * Query the indirect action by handle. 10568 * 10569 * This function allows retrieving action-specific data such as counters. 10570 * Data is gathered by special action which may be present/referenced in 10571 * more than one flow rule definition. 10572 * 10573 * see @RTE_FLOW_ACTION_TYPE_COUNT 10574 * 10575 * @param dev 10576 * Pointer to Ethernet device structure. 10577 * @param[in] handle 10578 * Handle for the indirect action to query. 10579 * @param[in, out] data 10580 * Pointer to storage for the associated query data type. 10581 * @param[out] error 10582 * Perform verbose error reporting if not NULL. PMDs initialize this 10583 * structure in case of error only. 10584 * 10585 * @return 10586 * 0 on success, a negative errno value otherwise and rte_errno is set. 10587 */ 10588 static int 10589 mlx5_action_handle_query(struct rte_eth_dev *dev, 10590 const struct rte_flow_action_handle *handle, 10591 void *data, 10592 struct rte_flow_error *error) 10593 { 10594 struct rte_flow_attr attr = { .transfer = 0 }; 10595 const struct mlx5_flow_driver_ops *fops = 10596 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10597 10598 return flow_drv_action_query(dev, handle, data, fops, error); 10599 } 10600 10601 /** 10602 * Destroy all indirect actions (shared RSS). 10603 * 10604 * @param dev 10605 * Pointer to Ethernet device. 10606 * 10607 * @return 10608 * 0 on success, a negative errno value otherwise and rte_errno is set. 10609 */ 10610 int 10611 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10612 { 10613 struct rte_flow_error error; 10614 struct mlx5_priv *priv = dev->data->dev_private; 10615 struct mlx5_shared_action_rss *shared_rss; 10616 int ret = 0; 10617 uint32_t idx; 10618 10619 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10620 priv->rss_shared_actions, idx, shared_rss, next) { 10621 ret |= mlx5_action_handle_destroy(dev, 10622 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10623 } 10624 return ret; 10625 } 10626 10627 /** 10628 * Validate existing indirect actions against current device configuration 10629 * and attach them to device resources. 10630 * 10631 * @param dev 10632 * Pointer to Ethernet device. 10633 * 10634 * @return 10635 * 0 on success, a negative errno value otherwise and rte_errno is set. 10636 */ 10637 int 10638 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10639 { 10640 struct mlx5_priv *priv = dev->data->dev_private; 10641 int ret = 0; 10642 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10643 10644 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10645 const char *message; 10646 uint32_t queue_idx; 10647 10648 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10649 ind_tbl->queues_n, 10650 &message, &queue_idx); 10651 if (ret != 0) { 10652 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10653 dev->data->port_id, ind_tbl->queues[queue_idx], 10654 message); 10655 break; 10656 } 10657 } 10658 if (ret != 0) 10659 return ret; 10660 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10661 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10662 if (ret != 0) { 10663 DRV_LOG(ERR, "Port %u could not attach " 10664 "indirection table obj %p", 10665 dev->data->port_id, (void *)ind_tbl); 10666 goto error; 10667 } 10668 } 10669 10670 return 0; 10671 error: 10672 ind_tbl_last = ind_tbl; 10673 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10674 if (ind_tbl == ind_tbl_last) 10675 break; 10676 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10677 DRV_LOG(CRIT, "Port %u could not detach " 10678 "indirection table obj %p on rollback", 10679 dev->data->port_id, (void *)ind_tbl); 10680 } 10681 return ret; 10682 } 10683 10684 /** 10685 * Detach indirect actions of the device from its resources. 10686 * 10687 * @param dev 10688 * Pointer to Ethernet device. 10689 * 10690 * @return 10691 * 0 on success, a negative errno value otherwise and rte_errno is set. 10692 */ 10693 int 10694 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10695 { 10696 struct mlx5_priv *priv = dev->data->dev_private; 10697 int ret = 0; 10698 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10699 10700 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10701 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10702 if (ret != 0) { 10703 DRV_LOG(ERR, "Port %u could not detach " 10704 "indirection table obj %p", 10705 dev->data->port_id, (void *)ind_tbl); 10706 goto error; 10707 } 10708 } 10709 return 0; 10710 error: 10711 ind_tbl_last = ind_tbl; 10712 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10713 if (ind_tbl == ind_tbl_last) 10714 break; 10715 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10716 DRV_LOG(CRIT, "Port %u could not attach " 10717 "indirection table obj %p on rollback", 10718 dev->data->port_id, (void *)ind_tbl); 10719 } 10720 return ret; 10721 } 10722 10723 #ifndef HAVE_MLX5DV_DR 10724 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10725 #else 10726 #define MLX5_DOMAIN_SYNC_FLOW \ 10727 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10728 #endif 10729 10730 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 10731 { 10732 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 10733 const struct mlx5_flow_driver_ops *fops; 10734 int ret; 10735 struct rte_flow_attr attr = { .transfer = 0 }; 10736 10737 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10738 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 10739 if (ret > 0) 10740 ret = -ret; 10741 return ret; 10742 } 10743 10744 const struct mlx5_flow_tunnel * 10745 mlx5_get_tof(const struct rte_flow_item *item, 10746 const struct rte_flow_action *action, 10747 enum mlx5_tof_rule_type *rule_type) 10748 { 10749 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 10750 if (item->type == (typeof(item->type)) 10751 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 10752 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 10753 return flow_items_to_tunnel(item); 10754 } 10755 } 10756 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 10757 if (action->type == (typeof(action->type)) 10758 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 10759 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 10760 return flow_actions_to_tunnel(action); 10761 } 10762 } 10763 return NULL; 10764 } 10765 10766 /** 10767 * tunnel offload functionality is defined for DV environment only 10768 */ 10769 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10770 __extension__ 10771 union tunnel_offload_mark { 10772 uint32_t val; 10773 struct { 10774 uint32_t app_reserve:8; 10775 uint32_t table_id:15; 10776 uint32_t transfer:1; 10777 uint32_t _unused_:8; 10778 }; 10779 }; 10780 10781 static bool 10782 mlx5_access_tunnel_offload_db 10783 (struct rte_eth_dev *dev, 10784 bool (*match)(struct rte_eth_dev *, 10785 struct mlx5_flow_tunnel *, const void *), 10786 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 10787 void (*miss)(struct rte_eth_dev *, void *), 10788 void *ctx, bool lock_op); 10789 10790 static int 10791 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 10792 struct rte_flow *flow, 10793 const struct rte_flow_attr *attr, 10794 const struct rte_flow_action *app_actions, 10795 uint32_t flow_idx, 10796 const struct mlx5_flow_tunnel *tunnel, 10797 struct tunnel_default_miss_ctx *ctx, 10798 struct rte_flow_error *error) 10799 { 10800 struct mlx5_priv *priv = dev->data->dev_private; 10801 struct mlx5_flow *dev_flow; 10802 struct rte_flow_attr miss_attr = *attr; 10803 const struct rte_flow_item miss_items[2] = { 10804 { 10805 .type = RTE_FLOW_ITEM_TYPE_ETH, 10806 .spec = NULL, 10807 .last = NULL, 10808 .mask = NULL 10809 }, 10810 { 10811 .type = RTE_FLOW_ITEM_TYPE_END, 10812 .spec = NULL, 10813 .last = NULL, 10814 .mask = NULL 10815 } 10816 }; 10817 union tunnel_offload_mark mark_id; 10818 struct rte_flow_action_mark miss_mark; 10819 struct rte_flow_action miss_actions[3] = { 10820 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 10821 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 10822 }; 10823 const struct rte_flow_action_jump *jump_data; 10824 uint32_t i, flow_table = 0; /* prevent compilation warning */ 10825 struct flow_grp_info grp_info = { 10826 .external = 1, 10827 .transfer = attr->transfer, 10828 .fdb_def_rule = !!priv->fdb_def_rule, 10829 .std_tbl_fix = 0, 10830 }; 10831 int ret; 10832 10833 if (!attr->transfer) { 10834 uint32_t q_size; 10835 10836 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 10837 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 10838 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 10839 0, SOCKET_ID_ANY); 10840 if (!ctx->queue) 10841 return rte_flow_error_set 10842 (error, ENOMEM, 10843 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10844 NULL, "invalid default miss RSS"); 10845 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 10846 ctx->action_rss.level = 0, 10847 ctx->action_rss.types = priv->rss_conf.rss_hf, 10848 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 10849 ctx->action_rss.queue_num = priv->reta_idx_n, 10850 ctx->action_rss.key = priv->rss_conf.rss_key, 10851 ctx->action_rss.queue = ctx->queue; 10852 if (!priv->reta_idx_n || !priv->rxqs_n) 10853 return rte_flow_error_set 10854 (error, EINVAL, 10855 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10856 NULL, "invalid port configuration"); 10857 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 10858 ctx->action_rss.types = 0; 10859 for (i = 0; i != priv->reta_idx_n; ++i) 10860 ctx->queue[i] = (*priv->reta_idx)[i]; 10861 } else { 10862 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 10863 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 10864 } 10865 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 10866 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 10867 jump_data = app_actions->conf; 10868 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 10869 miss_attr.group = jump_data->group; 10870 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 10871 &flow_table, &grp_info, error); 10872 if (ret) 10873 return rte_flow_error_set(error, EINVAL, 10874 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 10875 NULL, "invalid tunnel id"); 10876 mark_id.app_reserve = 0; 10877 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 10878 mark_id.transfer = !!attr->transfer; 10879 mark_id._unused_ = 0; 10880 miss_mark.id = mark_id.val; 10881 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 10882 miss_items, miss_actions, flow_idx, error); 10883 if (!dev_flow) 10884 return -rte_errno; 10885 dev_flow->flow = flow; 10886 dev_flow->external = true; 10887 dev_flow->tunnel = tunnel; 10888 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 10889 /* Subflow object was created, we must include one in the list. */ 10890 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 10891 dev_flow->handle, next); 10892 DRV_LOG(DEBUG, 10893 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 10894 dev->data->port_id, tunnel->app_tunnel.type, 10895 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 10896 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 10897 miss_actions, error); 10898 if (!ret) 10899 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 10900 error); 10901 10902 return ret; 10903 } 10904 10905 static const struct mlx5_flow_tbl_data_entry * 10906 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 10907 { 10908 struct mlx5_priv *priv = dev->data->dev_private; 10909 struct mlx5_dev_ctx_shared *sh = priv->sh; 10910 struct mlx5_list_entry *he; 10911 union tunnel_offload_mark mbits = { .val = mark }; 10912 union mlx5_flow_tbl_key table_key = { 10913 { 10914 .level = tunnel_id_to_flow_tbl(mbits.table_id), 10915 .id = 0, 10916 .reserved = 0, 10917 .dummy = 0, 10918 .is_fdb = !!mbits.transfer, 10919 .is_egress = 0, 10920 } 10921 }; 10922 struct mlx5_flow_cb_ctx ctx = { 10923 .data = &table_key.v64, 10924 }; 10925 10926 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 10927 return he ? 10928 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 10929 } 10930 10931 static void 10932 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 10933 struct mlx5_list_entry *entry) 10934 { 10935 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10936 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10937 10938 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10939 tunnel_flow_tbl_to_id(tte->flow_table)); 10940 mlx5_free(tte); 10941 } 10942 10943 static int 10944 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 10945 struct mlx5_list_entry *entry, void *cb_ctx) 10946 { 10947 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10948 union tunnel_tbl_key tbl = { 10949 .val = *(uint64_t *)(ctx->data), 10950 }; 10951 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 10952 10953 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 10954 } 10955 10956 static struct mlx5_list_entry * 10957 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 10958 { 10959 struct mlx5_dev_ctx_shared *sh = tool_ctx; 10960 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 10961 struct tunnel_tbl_entry *tte; 10962 union tunnel_tbl_key tbl = { 10963 .val = *(uint64_t *)(ctx->data), 10964 }; 10965 10966 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 10967 sizeof(*tte), 0, 10968 SOCKET_ID_ANY); 10969 if (!tte) 10970 goto err; 10971 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10972 &tte->flow_table); 10973 if (tte->flow_table >= MLX5_MAX_TABLES) { 10974 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 10975 tte->flow_table); 10976 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 10977 tte->flow_table); 10978 goto err; 10979 } else if (!tte->flow_table) { 10980 goto err; 10981 } 10982 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 10983 tte->tunnel_id = tbl.tunnel_id; 10984 tte->group = tbl.group; 10985 return &tte->hash; 10986 err: 10987 if (tte) 10988 mlx5_free(tte); 10989 return NULL; 10990 } 10991 10992 static struct mlx5_list_entry * 10993 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 10994 struct mlx5_list_entry *oentry, 10995 void *cb_ctx __rte_unused) 10996 { 10997 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 10998 0, SOCKET_ID_ANY); 10999 11000 if (!tte) 11001 return NULL; 11002 memcpy(tte, oentry, sizeof(*tte)); 11003 return &tte->hash; 11004 } 11005 11006 static void 11007 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 11008 struct mlx5_list_entry *entry) 11009 { 11010 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11011 11012 mlx5_free(tte); 11013 } 11014 11015 static uint32_t 11016 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 11017 const struct mlx5_flow_tunnel *tunnel, 11018 uint32_t group, uint32_t *table, 11019 struct rte_flow_error *error) 11020 { 11021 struct mlx5_list_entry *he; 11022 struct tunnel_tbl_entry *tte; 11023 union tunnel_tbl_key key = { 11024 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 11025 .group = group 11026 }; 11027 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11028 struct mlx5_hlist *group_hash; 11029 struct mlx5_flow_cb_ctx ctx = { 11030 .data = &key.val, 11031 }; 11032 11033 group_hash = tunnel ? tunnel->groups : thub->groups; 11034 he = mlx5_hlist_register(group_hash, key.val, &ctx); 11035 if (!he) 11036 return rte_flow_error_set(error, EINVAL, 11037 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 11038 NULL, 11039 "tunnel group index not supported"); 11040 tte = container_of(he, typeof(*tte), hash); 11041 *table = tte->flow_table; 11042 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 11043 dev->data->port_id, key.tunnel_id, group, *table); 11044 return 0; 11045 } 11046 11047 static void 11048 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 11049 struct mlx5_flow_tunnel *tunnel) 11050 { 11051 struct mlx5_priv *priv = dev->data->dev_private; 11052 struct mlx5_indexed_pool *ipool; 11053 11054 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11055 dev->data->port_id, tunnel->tunnel_id); 11056 LIST_REMOVE(tunnel, chain); 11057 mlx5_hlist_destroy(tunnel->groups); 11058 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11059 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11060 } 11061 11062 static bool 11063 mlx5_access_tunnel_offload_db 11064 (struct rte_eth_dev *dev, 11065 bool (*match)(struct rte_eth_dev *, 11066 struct mlx5_flow_tunnel *, const void *), 11067 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11068 void (*miss)(struct rte_eth_dev *, void *), 11069 void *ctx, bool lock_op) 11070 { 11071 bool verdict = false; 11072 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11073 struct mlx5_flow_tunnel *tunnel; 11074 11075 rte_spinlock_lock(&thub->sl); 11076 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11077 verdict = match(dev, tunnel, (const void *)ctx); 11078 if (verdict) 11079 break; 11080 } 11081 if (!lock_op) 11082 rte_spinlock_unlock(&thub->sl); 11083 if (verdict && hit) 11084 hit(dev, tunnel, ctx); 11085 if (!verdict && miss) 11086 miss(dev, ctx); 11087 if (lock_op) 11088 rte_spinlock_unlock(&thub->sl); 11089 11090 return verdict; 11091 } 11092 11093 struct tunnel_db_find_tunnel_id_ctx { 11094 uint32_t tunnel_id; 11095 struct mlx5_flow_tunnel *tunnel; 11096 }; 11097 11098 static bool 11099 find_tunnel_id_match(struct rte_eth_dev *dev, 11100 struct mlx5_flow_tunnel *tunnel, const void *x) 11101 { 11102 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11103 11104 RTE_SET_USED(dev); 11105 return tunnel->tunnel_id == ctx->tunnel_id; 11106 } 11107 11108 static void 11109 find_tunnel_id_hit(struct rte_eth_dev *dev, 11110 struct mlx5_flow_tunnel *tunnel, void *x) 11111 { 11112 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11113 RTE_SET_USED(dev); 11114 ctx->tunnel = tunnel; 11115 } 11116 11117 static struct mlx5_flow_tunnel * 11118 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11119 { 11120 struct tunnel_db_find_tunnel_id_ctx ctx = { 11121 .tunnel_id = id, 11122 }; 11123 11124 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11125 find_tunnel_id_hit, NULL, &ctx, true); 11126 11127 return ctx.tunnel; 11128 } 11129 11130 static struct mlx5_flow_tunnel * 11131 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11132 const struct rte_flow_tunnel *app_tunnel) 11133 { 11134 struct mlx5_priv *priv = dev->data->dev_private; 11135 struct mlx5_indexed_pool *ipool; 11136 struct mlx5_flow_tunnel *tunnel; 11137 uint32_t id; 11138 11139 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11140 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11141 if (!tunnel) 11142 return NULL; 11143 if (id >= MLX5_MAX_TUNNELS) { 11144 mlx5_ipool_free(ipool, id); 11145 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11146 return NULL; 11147 } 11148 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11149 priv->sh, 11150 mlx5_flow_tunnel_grp2tbl_create_cb, 11151 mlx5_flow_tunnel_grp2tbl_match_cb, 11152 mlx5_flow_tunnel_grp2tbl_remove_cb, 11153 mlx5_flow_tunnel_grp2tbl_clone_cb, 11154 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11155 if (!tunnel->groups) { 11156 mlx5_ipool_free(ipool, id); 11157 return NULL; 11158 } 11159 /* initiate new PMD tunnel */ 11160 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11161 tunnel->tunnel_id = id; 11162 tunnel->action.type = (typeof(tunnel->action.type)) 11163 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11164 tunnel->action.conf = tunnel; 11165 tunnel->item.type = (typeof(tunnel->item.type)) 11166 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11167 tunnel->item.spec = tunnel; 11168 tunnel->item.last = NULL; 11169 tunnel->item.mask = NULL; 11170 11171 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11172 dev->data->port_id, tunnel->tunnel_id); 11173 11174 return tunnel; 11175 } 11176 11177 struct tunnel_db_get_tunnel_ctx { 11178 const struct rte_flow_tunnel *app_tunnel; 11179 struct mlx5_flow_tunnel *tunnel; 11180 }; 11181 11182 static bool get_tunnel_match(struct rte_eth_dev *dev, 11183 struct mlx5_flow_tunnel *tunnel, const void *x) 11184 { 11185 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11186 11187 RTE_SET_USED(dev); 11188 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11189 sizeof(*ctx->app_tunnel)); 11190 } 11191 11192 static void get_tunnel_hit(struct rte_eth_dev *dev, 11193 struct mlx5_flow_tunnel *tunnel, void *x) 11194 { 11195 /* called under tunnel spinlock protection */ 11196 struct tunnel_db_get_tunnel_ctx *ctx = x; 11197 11198 RTE_SET_USED(dev); 11199 tunnel->refctn++; 11200 ctx->tunnel = tunnel; 11201 } 11202 11203 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11204 { 11205 /* called under tunnel spinlock protection */ 11206 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11207 struct tunnel_db_get_tunnel_ctx *ctx = x; 11208 11209 rte_spinlock_unlock(&thub->sl); 11210 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11211 rte_spinlock_lock(&thub->sl); 11212 if (ctx->tunnel) { 11213 ctx->tunnel->refctn = 1; 11214 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11215 } 11216 } 11217 11218 11219 static int 11220 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11221 const struct rte_flow_tunnel *app_tunnel, 11222 struct mlx5_flow_tunnel **tunnel) 11223 { 11224 struct tunnel_db_get_tunnel_ctx ctx = { 11225 .app_tunnel = app_tunnel, 11226 }; 11227 11228 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11229 get_tunnel_miss, &ctx, true); 11230 *tunnel = ctx.tunnel; 11231 return ctx.tunnel ? 0 : -ENOMEM; 11232 } 11233 11234 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11235 { 11236 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11237 11238 if (!thub) 11239 return; 11240 if (!LIST_EMPTY(&thub->tunnels)) 11241 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11242 mlx5_hlist_destroy(thub->groups); 11243 mlx5_free(thub); 11244 } 11245 11246 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11247 { 11248 int err; 11249 struct mlx5_flow_tunnel_hub *thub; 11250 11251 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11252 0, SOCKET_ID_ANY); 11253 if (!thub) 11254 return -ENOMEM; 11255 LIST_INIT(&thub->tunnels); 11256 rte_spinlock_init(&thub->sl); 11257 thub->groups = mlx5_hlist_create("flow groups", 64, 11258 false, true, sh, 11259 mlx5_flow_tunnel_grp2tbl_create_cb, 11260 mlx5_flow_tunnel_grp2tbl_match_cb, 11261 mlx5_flow_tunnel_grp2tbl_remove_cb, 11262 mlx5_flow_tunnel_grp2tbl_clone_cb, 11263 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11264 if (!thub->groups) { 11265 err = -rte_errno; 11266 goto err; 11267 } 11268 sh->tunnel_hub = thub; 11269 11270 return 0; 11271 11272 err: 11273 if (thub->groups) 11274 mlx5_hlist_destroy(thub->groups); 11275 if (thub) 11276 mlx5_free(thub); 11277 return err; 11278 } 11279 11280 static inline int 11281 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11282 struct rte_flow_tunnel *tunnel, 11283 struct rte_flow_error *error) 11284 { 11285 struct mlx5_priv *priv = dev->data->dev_private; 11286 11287 if (!priv->sh->config.dv_flow_en) 11288 return rte_flow_error_set(error, ENOTSUP, 11289 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11290 "flow DV interface is off"); 11291 if (!is_tunnel_offload_active(dev)) 11292 return rte_flow_error_set(error, ENOTSUP, 11293 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11294 "tunnel offload was not activated"); 11295 if (!tunnel) 11296 return rte_flow_error_set(error, EINVAL, 11297 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11298 "no application tunnel"); 11299 switch (tunnel->type) { 11300 default: 11301 return rte_flow_error_set(error, EINVAL, 11302 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11303 "unsupported tunnel type"); 11304 case RTE_FLOW_ITEM_TYPE_VXLAN: 11305 case RTE_FLOW_ITEM_TYPE_GRE: 11306 case RTE_FLOW_ITEM_TYPE_NVGRE: 11307 case RTE_FLOW_ITEM_TYPE_GENEVE: 11308 break; 11309 } 11310 return 0; 11311 } 11312 11313 static int 11314 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11315 struct rte_flow_tunnel *app_tunnel, 11316 struct rte_flow_action **actions, 11317 uint32_t *num_of_actions, 11318 struct rte_flow_error *error) 11319 { 11320 struct mlx5_flow_tunnel *tunnel; 11321 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11322 11323 if (ret) 11324 return ret; 11325 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11326 if (ret < 0) { 11327 return rte_flow_error_set(error, ret, 11328 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11329 "failed to initialize pmd tunnel"); 11330 } 11331 *actions = &tunnel->action; 11332 *num_of_actions = 1; 11333 return 0; 11334 } 11335 11336 static int 11337 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11338 struct rte_flow_tunnel *app_tunnel, 11339 struct rte_flow_item **items, 11340 uint32_t *num_of_items, 11341 struct rte_flow_error *error) 11342 { 11343 struct mlx5_flow_tunnel *tunnel; 11344 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11345 11346 if (ret) 11347 return ret; 11348 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11349 if (ret < 0) { 11350 return rte_flow_error_set(error, ret, 11351 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11352 "failed to initialize pmd tunnel"); 11353 } 11354 *items = &tunnel->item; 11355 *num_of_items = 1; 11356 return 0; 11357 } 11358 11359 struct tunnel_db_element_release_ctx { 11360 struct rte_flow_item *items; 11361 struct rte_flow_action *actions; 11362 uint32_t num_elements; 11363 struct rte_flow_error *error; 11364 int ret; 11365 }; 11366 11367 static bool 11368 tunnel_element_release_match(struct rte_eth_dev *dev, 11369 struct mlx5_flow_tunnel *tunnel, const void *x) 11370 { 11371 const struct tunnel_db_element_release_ctx *ctx = x; 11372 11373 RTE_SET_USED(dev); 11374 if (ctx->num_elements != 1) 11375 return false; 11376 else if (ctx->items) 11377 return ctx->items == &tunnel->item; 11378 else if (ctx->actions) 11379 return ctx->actions == &tunnel->action; 11380 11381 return false; 11382 } 11383 11384 static void 11385 tunnel_element_release_hit(struct rte_eth_dev *dev, 11386 struct mlx5_flow_tunnel *tunnel, void *x) 11387 { 11388 struct tunnel_db_element_release_ctx *ctx = x; 11389 ctx->ret = 0; 11390 if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED)) 11391 mlx5_flow_tunnel_free(dev, tunnel); 11392 } 11393 11394 static void 11395 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11396 { 11397 struct tunnel_db_element_release_ctx *ctx = x; 11398 RTE_SET_USED(dev); 11399 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11400 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11401 "invalid argument"); 11402 } 11403 11404 static int 11405 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11406 struct rte_flow_item *pmd_items, 11407 uint32_t num_items, struct rte_flow_error *err) 11408 { 11409 struct tunnel_db_element_release_ctx ctx = { 11410 .items = pmd_items, 11411 .actions = NULL, 11412 .num_elements = num_items, 11413 .error = err, 11414 }; 11415 11416 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11417 tunnel_element_release_hit, 11418 tunnel_element_release_miss, &ctx, false); 11419 11420 return ctx.ret; 11421 } 11422 11423 static int 11424 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11425 struct rte_flow_action *pmd_actions, 11426 uint32_t num_actions, struct rte_flow_error *err) 11427 { 11428 struct tunnel_db_element_release_ctx ctx = { 11429 .items = NULL, 11430 .actions = pmd_actions, 11431 .num_elements = num_actions, 11432 .error = err, 11433 }; 11434 11435 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11436 tunnel_element_release_hit, 11437 tunnel_element_release_miss, &ctx, false); 11438 11439 return ctx.ret; 11440 } 11441 11442 static int 11443 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11444 struct rte_mbuf *m, 11445 struct rte_flow_restore_info *info, 11446 struct rte_flow_error *err) 11447 { 11448 uint64_t ol_flags = m->ol_flags; 11449 const struct mlx5_flow_tbl_data_entry *tble; 11450 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11451 11452 if (!is_tunnel_offload_active(dev)) { 11453 info->flags = 0; 11454 return 0; 11455 } 11456 11457 if ((ol_flags & mask) != mask) 11458 goto err; 11459 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11460 if (!tble) { 11461 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11462 dev->data->port_id, m->hash.fdir.hi); 11463 goto err; 11464 } 11465 MLX5_ASSERT(tble->tunnel); 11466 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11467 info->group_id = tble->group_id; 11468 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11469 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11470 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11471 11472 return 0; 11473 11474 err: 11475 return rte_flow_error_set(err, EINVAL, 11476 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11477 "failed to get restore info"); 11478 } 11479 11480 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11481 static int 11482 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11483 __rte_unused struct rte_flow_tunnel *app_tunnel, 11484 __rte_unused struct rte_flow_action **actions, 11485 __rte_unused uint32_t *num_of_actions, 11486 __rte_unused struct rte_flow_error *error) 11487 { 11488 return -ENOTSUP; 11489 } 11490 11491 static int 11492 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11493 __rte_unused struct rte_flow_tunnel *app_tunnel, 11494 __rte_unused struct rte_flow_item **items, 11495 __rte_unused uint32_t *num_of_items, 11496 __rte_unused struct rte_flow_error *error) 11497 { 11498 return -ENOTSUP; 11499 } 11500 11501 static int 11502 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11503 __rte_unused struct rte_flow_item *pmd_items, 11504 __rte_unused uint32_t num_items, 11505 __rte_unused struct rte_flow_error *err) 11506 { 11507 return -ENOTSUP; 11508 } 11509 11510 static int 11511 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11512 __rte_unused struct rte_flow_action *pmd_action, 11513 __rte_unused uint32_t num_actions, 11514 __rte_unused struct rte_flow_error *err) 11515 { 11516 return -ENOTSUP; 11517 } 11518 11519 static int 11520 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11521 __rte_unused struct rte_mbuf *m, 11522 __rte_unused struct rte_flow_restore_info *i, 11523 __rte_unused struct rte_flow_error *err) 11524 { 11525 return -ENOTSUP; 11526 } 11527 11528 static int 11529 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11530 __rte_unused struct rte_flow *flow, 11531 __rte_unused const struct rte_flow_attr *attr, 11532 __rte_unused const struct rte_flow_action *actions, 11533 __rte_unused uint32_t flow_idx, 11534 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11535 __rte_unused struct tunnel_default_miss_ctx *ctx, 11536 __rte_unused struct rte_flow_error *error) 11537 { 11538 return -ENOTSUP; 11539 } 11540 11541 static struct mlx5_flow_tunnel * 11542 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11543 __rte_unused uint32_t id) 11544 { 11545 return NULL; 11546 } 11547 11548 static void 11549 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11550 __rte_unused struct mlx5_flow_tunnel *tunnel) 11551 { 11552 } 11553 11554 static uint32_t 11555 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11556 __rte_unused const struct mlx5_flow_tunnel *t, 11557 __rte_unused uint32_t group, 11558 __rte_unused uint32_t *table, 11559 struct rte_flow_error *error) 11560 { 11561 return rte_flow_error_set(error, ENOTSUP, 11562 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11563 "tunnel offload requires DV support"); 11564 } 11565 11566 void 11567 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11568 __rte_unused uint16_t port_id) 11569 { 11570 } 11571 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11572 11573 /* Flex flow item API */ 11574 static struct rte_flow_item_flex_handle * 11575 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11576 const struct rte_flow_item_flex_conf *conf, 11577 struct rte_flow_error *error) 11578 { 11579 static const char err_msg[] = "flex item creation unsupported"; 11580 struct mlx5_priv *priv = dev->data->dev_private; 11581 struct rte_flow_attr attr = { .transfer = 0 }; 11582 const struct mlx5_flow_driver_ops *fops = 11583 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11584 11585 if (!priv->pci_dev) { 11586 rte_flow_error_set(error, ENOTSUP, 11587 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11588 "create flex item on PF only"); 11589 return NULL; 11590 } 11591 switch (priv->pci_dev->id.device_id) { 11592 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11593 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11594 break; 11595 default: 11596 rte_flow_error_set(error, ENOTSUP, 11597 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11598 "flex item available on BlueField ports only"); 11599 return NULL; 11600 } 11601 if (!fops->item_create) { 11602 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11603 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11604 NULL, err_msg); 11605 return NULL; 11606 } 11607 return fops->item_create(dev, conf, error); 11608 } 11609 11610 static int 11611 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11612 const struct rte_flow_item_flex_handle *handle, 11613 struct rte_flow_error *error) 11614 { 11615 static const char err_msg[] = "flex item release unsupported"; 11616 struct rte_flow_attr attr = { .transfer = 0 }; 11617 const struct mlx5_flow_driver_ops *fops = 11618 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11619 11620 if (!fops->item_release) { 11621 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11622 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11623 NULL, err_msg); 11624 return -rte_errno; 11625 } 11626 return fops->item_release(dev, handle, error); 11627 } 11628 11629 static void 11630 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11631 { 11632 int ret; 11633 struct rte_flow_error error; 11634 11635 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11636 char *item_name; 11637 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11638 sizeof(item_name), 11639 (void *)(uintptr_t)item->type, &error); 11640 if (ret > 0) 11641 printf("%s ", item_name); 11642 else 11643 printf("%d\n", (int)item->type); 11644 } 11645 printf("END\n"); 11646 } 11647 11648 static int 11649 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11650 { 11651 const struct rte_flow_item_udp *spec = udp_item->spec; 11652 const struct rte_flow_item_udp *mask = udp_item->mask; 11653 uint16_t udp_dport = 0; 11654 11655 if (spec != NULL) { 11656 if (!mask) 11657 mask = &rte_flow_item_udp_mask; 11658 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11659 mask->hdr.dst_port); 11660 } 11661 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11662 } 11663 11664 static const struct mlx5_flow_expand_node * 11665 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11666 unsigned int item_idx, 11667 const struct mlx5_flow_expand_node graph[], 11668 const struct mlx5_flow_expand_node *node) 11669 { 11670 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11671 11672 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11673 node != NULL && 11674 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11675 /* 11676 * The expansion node is VXLAN and it is also the last 11677 * expandable item in the pattern, so need to continue 11678 * expansion of the inner tunnel. 11679 */ 11680 MLX5_ASSERT(item_idx > 0); 11681 prev_item = pattern + item_idx - 1; 11682 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11683 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11684 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11685 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11686 } 11687 return node; 11688 } 11689 11690 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11691 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11692 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11693 }; 11694 11695 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11696 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11697 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11698 { 9, 10, 11 }, { 12, 13, 14 }, 11699 }; 11700 11701 /** 11702 * Discover the number of available flow priorities. 11703 * 11704 * @param dev 11705 * Ethernet device. 11706 * 11707 * @return 11708 * On success, number of available flow priorities. 11709 * On failure, a negative errno-style code and rte_errno is set. 11710 */ 11711 int 11712 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11713 { 11714 static const uint16_t vprio[] = {8, 16}; 11715 const struct mlx5_priv *priv = dev->data->dev_private; 11716 const struct mlx5_flow_driver_ops *fops; 11717 enum mlx5_flow_drv_type type; 11718 int ret; 11719 11720 type = mlx5_flow_os_get_type(); 11721 if (type == MLX5_FLOW_TYPE_MAX) { 11722 type = MLX5_FLOW_TYPE_VERBS; 11723 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11724 type = MLX5_FLOW_TYPE_DV; 11725 } 11726 fops = flow_get_drv_ops(type); 11727 if (fops->discover_priorities == NULL) { 11728 DRV_LOG(ERR, "Priority discovery not supported"); 11729 rte_errno = ENOTSUP; 11730 return -rte_errno; 11731 } 11732 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 11733 if (ret < 0) 11734 return ret; 11735 switch (ret) { 11736 case 8: 11737 ret = RTE_DIM(priority_map_3); 11738 break; 11739 case 16: 11740 ret = RTE_DIM(priority_map_5); 11741 break; 11742 default: 11743 rte_errno = ENOTSUP; 11744 DRV_LOG(ERR, 11745 "port %u maximum priority: %d expected 8/16", 11746 dev->data->port_id, ret); 11747 return -rte_errno; 11748 } 11749 DRV_LOG(INFO, "port %u supported flow priorities:" 11750 " 0-%d for ingress or egress root table," 11751 " 0-%d for non-root table or transfer root table.", 11752 dev->data->port_id, ret - 2, 11753 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 11754 return ret; 11755 } 11756 11757 /** 11758 * Adjust flow priority based on the highest layer and the request priority. 11759 * 11760 * @param[in] dev 11761 * Pointer to the Ethernet device structure. 11762 * @param[in] priority 11763 * The rule base priority. 11764 * @param[in] subpriority 11765 * The priority based on the items. 11766 * 11767 * @return 11768 * The new priority. 11769 */ 11770 uint32_t 11771 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 11772 uint32_t subpriority) 11773 { 11774 uint32_t res = 0; 11775 struct mlx5_priv *priv = dev->data->dev_private; 11776 11777 switch (priv->sh->flow_max_priority) { 11778 case RTE_DIM(priority_map_3): 11779 res = priority_map_3[priority][subpriority]; 11780 break; 11781 case RTE_DIM(priority_map_5): 11782 res = priority_map_5[priority][subpriority]; 11783 break; 11784 } 11785 return res; 11786 } 11787 11788 /** 11789 * Get the priority for sending traffic to kernel table. 11790 * 11791 * @param[in] dev 11792 * Pointer to the Ethernet device structure. 11793 * 11794 * @return 11795 * On success: the value of priority for sending traffic to kernel table 11796 * On failure: -1 11797 */ 11798 uint32_t 11799 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 11800 { 11801 struct mlx5_priv *priv = dev->data->dev_private; 11802 uint32_t res; 11803 11804 switch (priv->sh->flow_max_priority) { 11805 case RTE_DIM(priority_map_5): 11806 res = 15; 11807 break; 11808 case RTE_DIM(priority_map_3): 11809 res = 7; 11810 break; 11811 default: 11812 DRV_LOG(ERR, 11813 "port %u maximum priority: %d expected 8/16", 11814 dev->data->port_id, priv->sh->flow_max_priority); 11815 res = (uint32_t)-1; 11816 } 11817 return res; 11818 } 11819 11820 /** 11821 * Get the E-Switch Manager vport id. 11822 * 11823 * @param[in] dev 11824 * Pointer to the Ethernet device structure. 11825 * 11826 * @return 11827 * The vport id. 11828 */ 11829 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 11830 { 11831 struct mlx5_priv *priv = dev->data->dev_private; 11832 struct mlx5_common_device *cdev = priv->sh->cdev; 11833 11834 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 11835 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 11836 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 11837 11838 if (priv->pci_dev == NULL) 11839 return 0; 11840 switch (priv->pci_dev->id.device_id) { 11841 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 11842 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11843 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11844 /* 11845 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 11846 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 11847 * ECPF vport ID is selected and not the host port (0) in any BF case. 11848 */ 11849 return (int16_t)MLX5_ECPF_VPORT_ID; 11850 default: 11851 return MLX5_PF_VPORT_ID; 11852 } 11853 } 11854 11855 /** 11856 * Parse item to get the vport id. 11857 * 11858 * @param[in] dev 11859 * Pointer to the Ethernet device structure. 11860 * @param[in] item 11861 * The src port id match item. 11862 * @param[out] vport_id 11863 * Pointer to put the vport id. 11864 * @param[out] all_ports 11865 * Indicate if the item matches all ports. 11866 * @param[out] error 11867 * Pointer to error structure. 11868 * 11869 * @return 11870 * 0 on success, a negative errno value otherwise and rte_errno is set. 11871 */ 11872 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 11873 const struct rte_flow_item *item, 11874 uint16_t *vport_id, 11875 bool *all_ports, 11876 struct rte_flow_error *error) 11877 { 11878 struct mlx5_priv *port_priv; 11879 const struct rte_flow_item_port_id *pid_v = NULL; 11880 const struct rte_flow_item_ethdev *dev_v = NULL; 11881 uint32_t esw_mgr_port; 11882 uint32_t src_port; 11883 11884 if (all_ports) 11885 *all_ports = false; 11886 switch (item->type) { 11887 case RTE_FLOW_ITEM_TYPE_PORT_ID: 11888 pid_v = item->spec; 11889 if (!pid_v) 11890 return 0; 11891 src_port = pid_v->id; 11892 esw_mgr_port = MLX5_PORT_ESW_MGR; 11893 break; 11894 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 11895 dev_v = item->spec; 11896 if (!dev_v) { 11897 if (all_ports) 11898 *all_ports = true; 11899 return 0; 11900 } 11901 src_port = dev_v->port_id; 11902 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11903 break; 11904 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 11905 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11906 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 11907 break; 11908 default: 11909 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11910 NULL, "Incorrect item type."); 11911 } 11912 if (src_port == esw_mgr_port) { 11913 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 11914 } else { 11915 port_priv = mlx5_port_to_eswitch_info(src_port, false); 11916 if (!port_priv) 11917 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 11918 NULL, "Failed to get port info."); 11919 *vport_id = port_priv->representor_id; 11920 } 11921 11922 return 0; 11923 } 11924 11925 int 11926 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 11927 uint16_t *proxy_port_id, 11928 struct rte_flow_error *error) 11929 { 11930 const struct mlx5_priv *priv = dev->data->dev_private; 11931 uint16_t port_id; 11932 11933 if (!priv->sh->config.dv_esw_en) 11934 return rte_flow_error_set(error, EINVAL, 11935 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11936 NULL, 11937 "unable to provide a proxy port" 11938 " without E-Switch configured"); 11939 if (!priv->master && !priv->representor) 11940 return rte_flow_error_set(error, EINVAL, 11941 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11942 NULL, 11943 "unable to provide a proxy port" 11944 " for port which is not a master" 11945 " or a representor port"); 11946 if (priv->master) { 11947 *proxy_port_id = dev->data->port_id; 11948 return 0; 11949 } 11950 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 11951 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 11952 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 11953 11954 if (port_priv->master && 11955 port_priv->domain_id == priv->domain_id) { 11956 *proxy_port_id = port_id; 11957 return 0; 11958 } 11959 } 11960 return rte_flow_error_set(error, ENODEV, 11961 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 11962 NULL, "unable to find a proxy port"); 11963 } 11964