1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2016 6WIND S.A. 3 * Copyright 2016 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdalign.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdbool.h> 10 #include <sys/queue.h> 11 12 #include <rte_common.h> 13 #include <rte_ether.h> 14 #include <ethdev_driver.h> 15 #include <rte_eal_paging.h> 16 #include <rte_flow.h> 17 #include <rte_cycles.h> 18 #include <rte_flow_driver.h> 19 #include <rte_malloc.h> 20 #include <rte_ip.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_devx_cmds.h> 24 #include <mlx5_prm.h> 25 #include <mlx5_malloc.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_flow.h" 30 #include "mlx5_flow_os.h" 31 #include "mlx5_rx.h" 32 #include "mlx5_tx.h" 33 #include "mlx5_common_os.h" 34 #include "rte_pmd_mlx5.h" 35 36 /* 37 * Shared array for quick translation between port_id and vport mask/values 38 * used for HWS rules. 39 */ 40 struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS]; 41 42 /* 43 * A global structure to save the available REG_C_x for tags usage. 44 * The Meter color REG (ASO) and the last available one will be reserved 45 * for PMD internal usage. 46 * Since there is no "port" concept in the driver, it is assumed that the 47 * available tags set will be the minimum intersection. 48 * 3 - in FDB mode / 5 - in legacy mode 49 */ 50 uint32_t mlx5_flow_hw_avl_tags_init_cnt; 51 enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON}; 52 enum modify_reg mlx5_flow_hw_aso_tag; 53 54 struct tunnel_default_miss_ctx { 55 uint16_t *queue; 56 __extension__ 57 union { 58 struct rte_flow_action_rss action_rss; 59 struct rte_flow_action_queue miss_queue; 60 struct rte_flow_action_jump miss_jump; 61 uint8_t raw[0]; 62 }; 63 }; 64 65 static int 66 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 67 struct rte_flow *flow, 68 const struct rte_flow_attr *attr, 69 const struct rte_flow_action *app_actions, 70 uint32_t flow_idx, 71 const struct mlx5_flow_tunnel *tunnel, 72 struct tunnel_default_miss_ctx *ctx, 73 struct rte_flow_error *error); 74 static struct mlx5_flow_tunnel * 75 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id); 76 static void 77 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, struct mlx5_flow_tunnel *tunnel); 78 static uint32_t 79 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 80 const struct mlx5_flow_tunnel *tunnel, 81 uint32_t group, uint32_t *table, 82 struct rte_flow_error *error); 83 84 /** Device flow drivers. */ 85 extern const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops; 86 87 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops; 88 89 const struct mlx5_flow_driver_ops *flow_drv_ops[] = { 90 [MLX5_FLOW_TYPE_MIN] = &mlx5_flow_null_drv_ops, 91 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 92 [MLX5_FLOW_TYPE_DV] = &mlx5_flow_dv_drv_ops, 93 #endif 94 #ifdef HAVE_MLX5_HWS_SUPPORT 95 [MLX5_FLOW_TYPE_HW] = &mlx5_flow_hw_drv_ops, 96 #endif 97 [MLX5_FLOW_TYPE_VERBS] = &mlx5_flow_verbs_drv_ops, 98 [MLX5_FLOW_TYPE_MAX] = &mlx5_flow_null_drv_ops 99 }; 100 101 /** Helper macro to build input graph for mlx5_flow_expand_rss(). */ 102 #define MLX5_FLOW_EXPAND_RSS_NEXT(...) \ 103 (const int []){ \ 104 __VA_ARGS__, 0, \ 105 } 106 107 /** Node object of input graph for mlx5_flow_expand_rss(). */ 108 struct mlx5_flow_expand_node { 109 const int *const next; 110 /**< 111 * List of next node indexes. Index 0 is interpreted as a terminator. 112 */ 113 const enum rte_flow_item_type type; 114 /**< Pattern item type of current node. */ 115 uint64_t rss_types; 116 /**< 117 * RSS types bit-field associated with this node 118 * (see RTE_ETH_RSS_* definitions). 119 */ 120 uint64_t node_flags; 121 /**< 122 * Bit-fields that define how the node is used in the expansion. 123 * (see MLX5_EXPANSION_NODE_* definitions). 124 */ 125 }; 126 127 /** Keep same format with mlx5_flow_expand_rss to share the buffer for expansion. */ 128 struct mlx5_flow_expand_sqn { 129 uint32_t entries; /** Number of entries */ 130 struct { 131 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 132 uint32_t priority; /**< Priority offset for each expansion. */ 133 } entry[]; 134 }; 135 136 /* Optional expand field. The expansion alg will not go deeper. */ 137 #define MLX5_EXPANSION_NODE_OPTIONAL (UINT64_C(1) << 0) 138 139 /* The node is not added implicitly as expansion to the flow pattern. 140 * If the node type does not match the flow pattern item type, the 141 * expansion alg will go deeper to its next items. 142 * In the current implementation, the list of next nodes indexes can 143 * have up to one node with this flag set and it has to be the last 144 * node index (before the list terminator). 145 */ 146 #define MLX5_EXPANSION_NODE_EXPLICIT (UINT64_C(1) << 1) 147 148 /** Object returned by mlx5_flow_expand_rss(). */ 149 struct mlx5_flow_expand_rss { 150 uint32_t entries; 151 /**< Number of entries @p patterns and @p priorities. */ 152 struct { 153 struct rte_flow_item *pattern; /**< Expanded pattern array. */ 154 uint32_t priority; /**< Priority offset for each expansion. */ 155 } entry[]; 156 }; 157 158 static void 159 mlx5_dbg__print_pattern(const struct rte_flow_item *item); 160 161 static const struct mlx5_flow_expand_node * 162 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 163 unsigned int item_idx, 164 const struct mlx5_flow_expand_node graph[], 165 const struct mlx5_flow_expand_node *node); 166 167 static __rte_always_inline int 168 mlx5_need_cache_flow(const struct mlx5_priv *priv, 169 const struct rte_flow_attr *attr) 170 { 171 return priv->isolated && priv->sh->config.dv_flow_en == 1 && 172 (attr ? !attr->group : true) && 173 priv->mode_info.mode == MLX5_FLOW_ENGINE_MODE_STANDBY && 174 (!priv->sh->config.dv_esw_en || !priv->sh->config.fdb_def_rule); 175 } 176 177 static bool 178 mlx5_flow_is_rss_expandable_item(const struct rte_flow_item *item) 179 { 180 switch (item->type) { 181 case RTE_FLOW_ITEM_TYPE_ETH: 182 case RTE_FLOW_ITEM_TYPE_VLAN: 183 case RTE_FLOW_ITEM_TYPE_IPV4: 184 case RTE_FLOW_ITEM_TYPE_IPV6: 185 case RTE_FLOW_ITEM_TYPE_UDP: 186 case RTE_FLOW_ITEM_TYPE_TCP: 187 case RTE_FLOW_ITEM_TYPE_ESP: 188 case RTE_FLOW_ITEM_TYPE_ICMP: 189 case RTE_FLOW_ITEM_TYPE_ICMP6: 190 case RTE_FLOW_ITEM_TYPE_VXLAN: 191 case RTE_FLOW_ITEM_TYPE_NVGRE: 192 case RTE_FLOW_ITEM_TYPE_GRE: 193 case RTE_FLOW_ITEM_TYPE_GENEVE: 194 case RTE_FLOW_ITEM_TYPE_MPLS: 195 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 196 case RTE_FLOW_ITEM_TYPE_GRE_KEY: 197 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT: 198 case RTE_FLOW_ITEM_TYPE_GTP: 199 return true; 200 default: 201 break; 202 } 203 return false; 204 } 205 206 /** 207 * Network Service Header (NSH) and its next protocol values 208 * are described in RFC-8393. 209 */ 210 static enum rte_flow_item_type 211 mlx5_nsh_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 212 { 213 enum rte_flow_item_type type; 214 215 switch (proto_mask & proto_spec) { 216 case 0: 217 type = RTE_FLOW_ITEM_TYPE_VOID; 218 break; 219 case RTE_VXLAN_GPE_TYPE_IPV4: 220 type = RTE_FLOW_ITEM_TYPE_IPV4; 221 break; 222 case RTE_VXLAN_GPE_TYPE_IPV6: 223 type = RTE_VXLAN_GPE_TYPE_IPV6; 224 break; 225 case RTE_VXLAN_GPE_TYPE_ETH: 226 type = RTE_FLOW_ITEM_TYPE_ETH; 227 break; 228 default: 229 type = RTE_FLOW_ITEM_TYPE_END; 230 } 231 return type; 232 } 233 234 static enum rte_flow_item_type 235 mlx5_inet_proto_to_item_type(uint8_t proto_spec, uint8_t proto_mask) 236 { 237 enum rte_flow_item_type type; 238 239 switch (proto_mask & proto_spec) { 240 case 0: 241 type = RTE_FLOW_ITEM_TYPE_VOID; 242 break; 243 case IPPROTO_UDP: 244 type = RTE_FLOW_ITEM_TYPE_UDP; 245 break; 246 case IPPROTO_TCP: 247 type = RTE_FLOW_ITEM_TYPE_TCP; 248 break; 249 case IPPROTO_IPIP: 250 type = RTE_FLOW_ITEM_TYPE_IPV4; 251 break; 252 case IPPROTO_IPV6: 253 type = RTE_FLOW_ITEM_TYPE_IPV6; 254 break; 255 case IPPROTO_ESP: 256 type = RTE_FLOW_ITEM_TYPE_ESP; 257 break; 258 default: 259 type = RTE_FLOW_ITEM_TYPE_END; 260 } 261 return type; 262 } 263 264 static enum rte_flow_item_type 265 mlx5_ethertype_to_item_type(rte_be16_t type_spec, 266 rte_be16_t type_mask, bool is_tunnel) 267 { 268 enum rte_flow_item_type type; 269 270 switch (rte_be_to_cpu_16(type_spec & type_mask)) { 271 case 0: 272 type = RTE_FLOW_ITEM_TYPE_VOID; 273 break; 274 case RTE_ETHER_TYPE_TEB: 275 type = is_tunnel ? 276 RTE_FLOW_ITEM_TYPE_ETH : RTE_FLOW_ITEM_TYPE_END; 277 break; 278 case RTE_ETHER_TYPE_VLAN: 279 type = !is_tunnel ? 280 RTE_FLOW_ITEM_TYPE_VLAN : RTE_FLOW_ITEM_TYPE_END; 281 break; 282 case RTE_ETHER_TYPE_IPV4: 283 type = RTE_FLOW_ITEM_TYPE_IPV4; 284 break; 285 case RTE_ETHER_TYPE_IPV6: 286 type = RTE_FLOW_ITEM_TYPE_IPV6; 287 break; 288 default: 289 type = RTE_FLOW_ITEM_TYPE_END; 290 } 291 return type; 292 } 293 294 static enum rte_flow_item_type 295 mlx5_flow_expand_rss_item_complete(const struct rte_flow_item *item) 296 { 297 #define MLX5_XSET_ITEM_MASK_SPEC(type, fld) \ 298 do { \ 299 const void *m = item->mask; \ 300 const void *s = item->spec; \ 301 mask = m ? \ 302 ((const struct rte_flow_item_##type *)m)->fld : \ 303 rte_flow_item_##type##_mask.fld; \ 304 spec = ((const struct rte_flow_item_##type *)s)->fld; \ 305 } while (0) 306 307 enum rte_flow_item_type ret; 308 uint16_t spec, mask; 309 310 if (item == NULL || item->spec == NULL) 311 return RTE_FLOW_ITEM_TYPE_VOID; 312 switch (item->type) { 313 case RTE_FLOW_ITEM_TYPE_ETH: 314 MLX5_XSET_ITEM_MASK_SPEC(eth, hdr.ether_type); 315 if (!mask) 316 return RTE_FLOW_ITEM_TYPE_VOID; 317 ret = mlx5_ethertype_to_item_type(spec, mask, false); 318 break; 319 case RTE_FLOW_ITEM_TYPE_VLAN: 320 MLX5_XSET_ITEM_MASK_SPEC(vlan, hdr.eth_proto); 321 if (!mask) 322 return RTE_FLOW_ITEM_TYPE_VOID; 323 ret = mlx5_ethertype_to_item_type(spec, mask, false); 324 break; 325 case RTE_FLOW_ITEM_TYPE_IPV4: 326 MLX5_XSET_ITEM_MASK_SPEC(ipv4, hdr.next_proto_id); 327 if (!mask) 328 return RTE_FLOW_ITEM_TYPE_VOID; 329 ret = mlx5_inet_proto_to_item_type(spec, mask); 330 break; 331 case RTE_FLOW_ITEM_TYPE_IPV6: 332 MLX5_XSET_ITEM_MASK_SPEC(ipv6, hdr.proto); 333 if (!mask) 334 return RTE_FLOW_ITEM_TYPE_VOID; 335 ret = mlx5_inet_proto_to_item_type(spec, mask); 336 break; 337 case RTE_FLOW_ITEM_TYPE_GENEVE: 338 MLX5_XSET_ITEM_MASK_SPEC(geneve, protocol); 339 ret = mlx5_ethertype_to_item_type(spec, mask, true); 340 break; 341 case RTE_FLOW_ITEM_TYPE_GRE: 342 MLX5_XSET_ITEM_MASK_SPEC(gre, protocol); 343 ret = mlx5_ethertype_to_item_type(spec, mask, true); 344 break; 345 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: 346 MLX5_XSET_ITEM_MASK_SPEC(vxlan_gpe, hdr.proto); 347 ret = mlx5_nsh_proto_to_item_type(spec, mask); 348 break; 349 default: 350 ret = RTE_FLOW_ITEM_TYPE_VOID; 351 break; 352 } 353 return ret; 354 #undef MLX5_XSET_ITEM_MASK_SPEC 355 } 356 357 static const int * 358 mlx5_flow_expand_rss_skip_explicit(const struct mlx5_flow_expand_node graph[], 359 const int *next_node) 360 { 361 const struct mlx5_flow_expand_node *node = NULL; 362 const int *next = next_node; 363 364 while (next && *next) { 365 /* 366 * Skip the nodes with the MLX5_EXPANSION_NODE_EXPLICIT 367 * flag set, because they were not found in the flow pattern. 368 */ 369 node = &graph[*next]; 370 if (!(node->node_flags & MLX5_EXPANSION_NODE_EXPLICIT)) 371 break; 372 next = node->next; 373 } 374 return next; 375 } 376 377 #define MLX5_RSS_EXP_ELT_N 32 378 379 /** 380 * Expand RSS flows into several possible flows according to the RSS hash 381 * fields requested and the driver capabilities. 382 * 383 * @param[out] buf 384 * Buffer to store the result expansion. 385 * @param[in] size 386 * Buffer size in bytes. If 0, @p buf can be NULL. 387 * @param[in] pattern 388 * User flow pattern. 389 * @param[in] types 390 * RSS types to expand (see RTE_ETH_RSS_* definitions). 391 * @param[in] graph 392 * Input graph to expand @p pattern according to @p types. 393 * @param[in] graph_root_index 394 * Index of root node in @p graph, typically 0. 395 * 396 * @return 397 * A positive value representing the size of @p buf in bytes regardless of 398 * @p size on success, a negative errno value otherwise and rte_errno is 399 * set, the following errors are defined: 400 * 401 * -E2BIG: graph-depth @p graph is too deep. 402 * -EINVAL: @p size has not enough space for expanded pattern. 403 */ 404 static int 405 mlx5_flow_expand_rss(struct mlx5_flow_expand_rss *buf, size_t size, 406 const struct rte_flow_item *pattern, uint64_t types, 407 const struct mlx5_flow_expand_node graph[], 408 int graph_root_index) 409 { 410 const struct rte_flow_item *item; 411 const struct mlx5_flow_expand_node *node = &graph[graph_root_index]; 412 const int *next_node; 413 const int *stack[MLX5_RSS_EXP_ELT_N]; 414 int stack_pos = 0; 415 struct rte_flow_item flow_items[MLX5_RSS_EXP_ELT_N]; 416 unsigned int i, item_idx, last_expand_item_idx = 0; 417 size_t lsize; 418 size_t user_pattern_size = 0; 419 void *addr = NULL; 420 const struct mlx5_flow_expand_node *next = NULL; 421 struct rte_flow_item missed_item; 422 int missed = 0; 423 int elt = 0; 424 const struct rte_flow_item *last_expand_item = NULL; 425 426 memset(&missed_item, 0, sizeof(missed_item)); 427 lsize = offsetof(struct mlx5_flow_expand_rss, entry) + 428 MLX5_RSS_EXP_ELT_N * sizeof(buf->entry[0]); 429 if (lsize > size) 430 return -EINVAL; 431 buf->entry[0].priority = 0; 432 buf->entry[0].pattern = (void *)&buf->entry[MLX5_RSS_EXP_ELT_N]; 433 buf->entries = 0; 434 addr = buf->entry[0].pattern; 435 for (item = pattern, item_idx = 0; 436 item->type != RTE_FLOW_ITEM_TYPE_END; 437 item++, item_idx++) { 438 if (!mlx5_flow_is_rss_expandable_item(item)) { 439 user_pattern_size += sizeof(*item); 440 continue; 441 } 442 last_expand_item = item; 443 last_expand_item_idx = item_idx; 444 i = 0; 445 while (node->next && node->next[i]) { 446 next = &graph[node->next[i]]; 447 if (next->type == item->type) 448 break; 449 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 450 node = next; 451 i = 0; 452 } else { 453 ++i; 454 } 455 } 456 if (next) 457 node = next; 458 user_pattern_size += sizeof(*item); 459 } 460 user_pattern_size += sizeof(*item); /* Handle END item. */ 461 lsize += user_pattern_size; 462 if (lsize > size) 463 return -EINVAL; 464 /* Copy the user pattern in the first entry of the buffer. */ 465 rte_memcpy(addr, pattern, user_pattern_size); 466 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 467 buf->entries = 1; 468 /* Start expanding. */ 469 memset(flow_items, 0, sizeof(flow_items)); 470 user_pattern_size -= sizeof(*item); 471 /* 472 * Check if the last valid item has spec set, need complete pattern, 473 * and the pattern can be used for expansion. 474 */ 475 missed_item.type = mlx5_flow_expand_rss_item_complete(last_expand_item); 476 if (missed_item.type == RTE_FLOW_ITEM_TYPE_END) { 477 /* Item type END indicates expansion is not required. */ 478 return lsize; 479 } 480 if (missed_item.type != RTE_FLOW_ITEM_TYPE_VOID) { 481 next = NULL; 482 missed = 1; 483 i = 0; 484 while (node->next && node->next[i]) { 485 next = &graph[node->next[i]]; 486 if (next->type == missed_item.type) { 487 flow_items[0].type = missed_item.type; 488 flow_items[1].type = RTE_FLOW_ITEM_TYPE_END; 489 break; 490 } 491 if (next->node_flags & MLX5_EXPANSION_NODE_EXPLICIT) { 492 node = next; 493 i = 0; 494 } else { 495 ++i; 496 } 497 next = NULL; 498 } 499 } 500 if (next && missed) { 501 elt = 2; /* missed item + item end. */ 502 node = next; 503 lsize += elt * sizeof(*item) + user_pattern_size; 504 if (lsize > size) 505 return -EINVAL; 506 if (node->rss_types & types) { 507 buf->entry[buf->entries].priority = 1; 508 buf->entry[buf->entries].pattern = addr; 509 buf->entries++; 510 rte_memcpy(addr, buf->entry[0].pattern, 511 user_pattern_size); 512 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 513 rte_memcpy(addr, flow_items, elt * sizeof(*item)); 514 addr = (void *)(((uintptr_t)addr) + 515 elt * sizeof(*item)); 516 } 517 } else if (last_expand_item != NULL) { 518 node = mlx5_flow_expand_rss_adjust_node(pattern, 519 last_expand_item_idx, graph, node); 520 } 521 memset(flow_items, 0, sizeof(flow_items)); 522 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 523 node->next); 524 stack[stack_pos] = next_node; 525 node = next_node ? &graph[*next_node] : NULL; 526 while (node) { 527 flow_items[stack_pos].type = node->type; 528 if (node->rss_types & types) { 529 size_t n; 530 /* 531 * compute the number of items to copy from the 532 * expansion and copy it. 533 * When the stack_pos is 0, there are 1 element in it, 534 * plus the addition END item. 535 */ 536 elt = stack_pos + 2; 537 flow_items[stack_pos + 1].type = RTE_FLOW_ITEM_TYPE_END; 538 lsize += elt * sizeof(*item) + user_pattern_size; 539 if (lsize > size) 540 return -EINVAL; 541 n = elt * sizeof(*item); 542 MLX5_ASSERT((buf->entries) < MLX5_RSS_EXP_ELT_N); 543 buf->entry[buf->entries].priority = 544 stack_pos + 1 + missed; 545 buf->entry[buf->entries].pattern = addr; 546 buf->entries++; 547 rte_memcpy(addr, buf->entry[0].pattern, 548 user_pattern_size); 549 addr = (void *)(((uintptr_t)addr) + 550 user_pattern_size); 551 rte_memcpy(addr, &missed_item, 552 missed * sizeof(*item)); 553 addr = (void *)(((uintptr_t)addr) + 554 missed * sizeof(*item)); 555 rte_memcpy(addr, flow_items, n); 556 addr = (void *)(((uintptr_t)addr) + n); 557 } 558 /* Go deeper. */ 559 if (!(node->node_flags & MLX5_EXPANSION_NODE_OPTIONAL) && 560 node->next) { 561 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 562 node->next); 563 if (stack_pos++ == MLX5_RSS_EXP_ELT_N) { 564 rte_errno = E2BIG; 565 return -rte_errno; 566 } 567 stack[stack_pos] = next_node; 568 } else if (*(next_node + 1)) { 569 /* Follow up with the next possibility. */ 570 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 571 ++next_node); 572 } else if (!stack_pos) { 573 /* 574 * Completing the traverse over the different paths. 575 * The next_node is advanced to the terminator. 576 */ 577 ++next_node; 578 } else { 579 /* Move to the next path. */ 580 while (stack_pos) { 581 next_node = stack[--stack_pos]; 582 next_node++; 583 if (*next_node) 584 break; 585 } 586 next_node = mlx5_flow_expand_rss_skip_explicit(graph, 587 next_node); 588 stack[stack_pos] = next_node; 589 } 590 node = next_node && *next_node ? &graph[*next_node] : NULL; 591 }; 592 return lsize; 593 } 594 595 /** 596 * Expand SQN flows into several possible flows according to the Tx queue 597 * number 598 * 599 * @param[in] buf 600 * Buffer to store the result expansion. 601 * @param[in] size 602 * Buffer size in bytes. If 0, @p buf can be NULL. 603 * @param[in] pattern 604 * User flow pattern. 605 * @param[in] sq_specs 606 * Buffer to store sq spec. 607 * 608 * @return 609 * 0 for success and negative value for failure 610 * 611 */ 612 static int 613 mlx5_flow_expand_sqn(struct mlx5_flow_expand_sqn *buf, size_t size, 614 const struct rte_flow_item *pattern, 615 struct mlx5_rte_flow_item_sq *sq_specs) 616 { 617 const struct rte_flow_item *item; 618 bool port_representor = false; 619 size_t user_pattern_size = 0; 620 struct rte_eth_dev *dev; 621 struct mlx5_priv *priv; 622 void *addr = NULL; 623 uint16_t port_id; 624 size_t lsize; 625 int elt = 2; 626 uint16_t i; 627 628 buf->entries = 0; 629 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 630 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 631 const struct rte_flow_item_ethdev *pid_v = item->spec; 632 633 if (!pid_v) 634 return 0; 635 port_id = pid_v->port_id; 636 port_representor = true; 637 } 638 user_pattern_size += sizeof(*item); 639 } 640 if (!port_representor) 641 return 0; 642 dev = &rte_eth_devices[port_id]; 643 priv = dev->data->dev_private; 644 buf->entry[0].pattern = (void *)&buf->entry[priv->txqs_n]; 645 lsize = offsetof(struct mlx5_flow_expand_sqn, entry) + 646 sizeof(buf->entry[0]) * priv->txqs_n; 647 if (lsize + (user_pattern_size + sizeof(struct rte_flow_item) * elt) * priv->txqs_n > size) 648 return -EINVAL; 649 addr = buf->entry[0].pattern; 650 for (i = 0; i != priv->txqs_n; ++i) { 651 struct rte_flow_item pattern_add[] = { 652 { 653 .type = (enum rte_flow_item_type) 654 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 655 .spec = &sq_specs[i], 656 }, 657 { 658 .type = RTE_FLOW_ITEM_TYPE_END, 659 }, 660 }; 661 struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i); 662 663 if (txq == NULL) 664 return -EINVAL; 665 buf->entry[i].pattern = addr; 666 sq_specs[i].queue = mlx5_txq_get_sqn(txq); 667 mlx5_txq_release(dev, i); 668 rte_memcpy(addr, pattern, user_pattern_size); 669 addr = (void *)(((uintptr_t)addr) + user_pattern_size); 670 rte_memcpy(addr, pattern_add, sizeof(struct rte_flow_item) * elt); 671 addr = (void *)(((uintptr_t)addr) + sizeof(struct rte_flow_item) * elt); 672 buf->entries++; 673 } 674 return 0; 675 } 676 677 enum mlx5_expansion { 678 MLX5_EXPANSION_ROOT, 679 MLX5_EXPANSION_ROOT_OUTER, 680 MLX5_EXPANSION_OUTER_ETH, 681 MLX5_EXPANSION_OUTER_VLAN, 682 MLX5_EXPANSION_OUTER_IPV4, 683 MLX5_EXPANSION_OUTER_IPV4_UDP, 684 MLX5_EXPANSION_OUTER_IPV4_TCP, 685 MLX5_EXPANSION_OUTER_IPV4_ESP, 686 MLX5_EXPANSION_OUTER_IPV4_ICMP, 687 MLX5_EXPANSION_OUTER_IPV6, 688 MLX5_EXPANSION_OUTER_IPV6_UDP, 689 MLX5_EXPANSION_OUTER_IPV6_TCP, 690 MLX5_EXPANSION_OUTER_IPV6_ESP, 691 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 692 MLX5_EXPANSION_VXLAN, 693 MLX5_EXPANSION_STD_VXLAN, 694 MLX5_EXPANSION_L3_VXLAN, 695 MLX5_EXPANSION_VXLAN_GPE, 696 MLX5_EXPANSION_GRE, 697 MLX5_EXPANSION_NVGRE, 698 MLX5_EXPANSION_GRE_KEY, 699 MLX5_EXPANSION_MPLS, 700 MLX5_EXPANSION_ETH, 701 MLX5_EXPANSION_VLAN, 702 MLX5_EXPANSION_IPV4, 703 MLX5_EXPANSION_IPV4_UDP, 704 MLX5_EXPANSION_IPV4_TCP, 705 MLX5_EXPANSION_IPV4_ESP, 706 MLX5_EXPANSION_IPV4_ICMP, 707 MLX5_EXPANSION_IPV6, 708 MLX5_EXPANSION_IPV6_UDP, 709 MLX5_EXPANSION_IPV6_TCP, 710 MLX5_EXPANSION_IPV6_ESP, 711 MLX5_EXPANSION_IPV6_ICMP6, 712 MLX5_EXPANSION_IPV6_FRAG_EXT, 713 MLX5_EXPANSION_GTP, 714 MLX5_EXPANSION_GENEVE, 715 }; 716 717 /** Supported expansion of items. */ 718 static const struct mlx5_flow_expand_node mlx5_support_expansion[] = { 719 [MLX5_EXPANSION_ROOT] = { 720 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 721 MLX5_EXPANSION_IPV4, 722 MLX5_EXPANSION_IPV6), 723 .type = RTE_FLOW_ITEM_TYPE_END, 724 }, 725 [MLX5_EXPANSION_ROOT_OUTER] = { 726 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_ETH, 727 MLX5_EXPANSION_OUTER_IPV4, 728 MLX5_EXPANSION_OUTER_IPV6), 729 .type = RTE_FLOW_ITEM_TYPE_END, 730 }, 731 [MLX5_EXPANSION_OUTER_ETH] = { 732 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_VLAN), 733 .type = RTE_FLOW_ITEM_TYPE_ETH, 734 .rss_types = 0, 735 }, 736 [MLX5_EXPANSION_OUTER_VLAN] = { 737 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_OUTER_IPV4, 738 MLX5_EXPANSION_OUTER_IPV6), 739 .type = RTE_FLOW_ITEM_TYPE_VLAN, 740 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 741 }, 742 [MLX5_EXPANSION_OUTER_IPV4] = { 743 .next = MLX5_FLOW_EXPAND_RSS_NEXT 744 (MLX5_EXPANSION_OUTER_IPV4_UDP, 745 MLX5_EXPANSION_OUTER_IPV4_TCP, 746 MLX5_EXPANSION_OUTER_IPV4_ESP, 747 MLX5_EXPANSION_OUTER_IPV4_ICMP, 748 MLX5_EXPANSION_GRE, 749 MLX5_EXPANSION_NVGRE, 750 MLX5_EXPANSION_IPV4, 751 MLX5_EXPANSION_IPV6), 752 .type = RTE_FLOW_ITEM_TYPE_IPV4, 753 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 754 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 755 }, 756 [MLX5_EXPANSION_OUTER_IPV4_UDP] = { 757 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 758 MLX5_EXPANSION_VXLAN_GPE, 759 MLX5_EXPANSION_MPLS, 760 MLX5_EXPANSION_GENEVE, 761 MLX5_EXPANSION_GTP), 762 .type = RTE_FLOW_ITEM_TYPE_UDP, 763 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 764 }, 765 [MLX5_EXPANSION_OUTER_IPV4_TCP] = { 766 .type = RTE_FLOW_ITEM_TYPE_TCP, 767 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 768 }, 769 [MLX5_EXPANSION_OUTER_IPV4_ESP] = { 770 .type = RTE_FLOW_ITEM_TYPE_ESP, 771 .rss_types = RTE_ETH_RSS_ESP, 772 }, 773 [MLX5_EXPANSION_OUTER_IPV4_ICMP] = { 774 .type = RTE_FLOW_ITEM_TYPE_ICMP, 775 }, 776 [MLX5_EXPANSION_OUTER_IPV6] = { 777 .next = MLX5_FLOW_EXPAND_RSS_NEXT 778 (MLX5_EXPANSION_OUTER_IPV6_UDP, 779 MLX5_EXPANSION_OUTER_IPV6_TCP, 780 MLX5_EXPANSION_OUTER_IPV6_ESP, 781 MLX5_EXPANSION_OUTER_IPV6_ICMP6, 782 MLX5_EXPANSION_IPV4, 783 MLX5_EXPANSION_IPV6, 784 MLX5_EXPANSION_GRE, 785 MLX5_EXPANSION_NVGRE), 786 .type = RTE_FLOW_ITEM_TYPE_IPV6, 787 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 788 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 789 }, 790 [MLX5_EXPANSION_OUTER_IPV6_UDP] = { 791 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VXLAN, 792 MLX5_EXPANSION_VXLAN_GPE, 793 MLX5_EXPANSION_MPLS, 794 MLX5_EXPANSION_GENEVE, 795 MLX5_EXPANSION_GTP), 796 .type = RTE_FLOW_ITEM_TYPE_UDP, 797 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 798 }, 799 [MLX5_EXPANSION_OUTER_IPV6_TCP] = { 800 .type = RTE_FLOW_ITEM_TYPE_TCP, 801 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 802 }, 803 [MLX5_EXPANSION_OUTER_IPV6_ESP] = { 804 .type = RTE_FLOW_ITEM_TYPE_ESP, 805 .rss_types = RTE_ETH_RSS_ESP, 806 }, 807 [MLX5_EXPANSION_OUTER_IPV6_ICMP6] = { 808 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 809 }, 810 [MLX5_EXPANSION_VXLAN] = { 811 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 812 MLX5_EXPANSION_IPV4, 813 MLX5_EXPANSION_IPV6), 814 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 815 }, 816 [MLX5_EXPANSION_STD_VXLAN] = { 817 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 818 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 819 }, 820 [MLX5_EXPANSION_L3_VXLAN] = { 821 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 822 MLX5_EXPANSION_IPV6), 823 .type = RTE_FLOW_ITEM_TYPE_VXLAN, 824 }, 825 [MLX5_EXPANSION_VXLAN_GPE] = { 826 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 827 MLX5_EXPANSION_IPV4, 828 MLX5_EXPANSION_IPV6), 829 .type = RTE_FLOW_ITEM_TYPE_VXLAN_GPE, 830 }, 831 [MLX5_EXPANSION_GRE] = { 832 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 833 MLX5_EXPANSION_IPV4, 834 MLX5_EXPANSION_IPV6, 835 MLX5_EXPANSION_GRE_KEY, 836 MLX5_EXPANSION_MPLS), 837 .type = RTE_FLOW_ITEM_TYPE_GRE, 838 }, 839 [MLX5_EXPANSION_GRE_KEY] = { 840 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 841 MLX5_EXPANSION_IPV6, 842 MLX5_EXPANSION_MPLS), 843 .type = RTE_FLOW_ITEM_TYPE_GRE_KEY, 844 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 845 }, 846 [MLX5_EXPANSION_NVGRE] = { 847 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH), 848 .type = RTE_FLOW_ITEM_TYPE_NVGRE, 849 }, 850 [MLX5_EXPANSION_MPLS] = { 851 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 852 MLX5_EXPANSION_IPV6, 853 MLX5_EXPANSION_ETH), 854 .type = RTE_FLOW_ITEM_TYPE_MPLS, 855 .node_flags = MLX5_EXPANSION_NODE_OPTIONAL, 856 }, 857 [MLX5_EXPANSION_ETH] = { 858 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_VLAN), 859 .type = RTE_FLOW_ITEM_TYPE_ETH, 860 }, 861 [MLX5_EXPANSION_VLAN] = { 862 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 863 MLX5_EXPANSION_IPV6), 864 .type = RTE_FLOW_ITEM_TYPE_VLAN, 865 .node_flags = MLX5_EXPANSION_NODE_EXPLICIT, 866 }, 867 [MLX5_EXPANSION_IPV4] = { 868 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4_UDP, 869 MLX5_EXPANSION_IPV4_TCP, 870 MLX5_EXPANSION_IPV4_ESP, 871 MLX5_EXPANSION_IPV4_ICMP), 872 .type = RTE_FLOW_ITEM_TYPE_IPV4, 873 .rss_types = RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | 874 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 875 }, 876 [MLX5_EXPANSION_IPV4_UDP] = { 877 .type = RTE_FLOW_ITEM_TYPE_UDP, 878 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_UDP, 879 }, 880 [MLX5_EXPANSION_IPV4_TCP] = { 881 .type = RTE_FLOW_ITEM_TYPE_TCP, 882 .rss_types = RTE_ETH_RSS_NONFRAG_IPV4_TCP, 883 }, 884 [MLX5_EXPANSION_IPV4_ESP] = { 885 .type = RTE_FLOW_ITEM_TYPE_ESP, 886 .rss_types = RTE_ETH_RSS_ESP, 887 }, 888 [MLX5_EXPANSION_IPV4_ICMP] = { 889 .type = RTE_FLOW_ITEM_TYPE_ICMP, 890 }, 891 [MLX5_EXPANSION_IPV6] = { 892 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV6_UDP, 893 MLX5_EXPANSION_IPV6_TCP, 894 MLX5_EXPANSION_IPV6_ESP, 895 MLX5_EXPANSION_IPV6_ICMP6, 896 MLX5_EXPANSION_IPV6_FRAG_EXT), 897 .type = RTE_FLOW_ITEM_TYPE_IPV6, 898 .rss_types = RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | 899 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 900 }, 901 [MLX5_EXPANSION_IPV6_UDP] = { 902 .type = RTE_FLOW_ITEM_TYPE_UDP, 903 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_UDP, 904 }, 905 [MLX5_EXPANSION_IPV6_TCP] = { 906 .type = RTE_FLOW_ITEM_TYPE_TCP, 907 .rss_types = RTE_ETH_RSS_NONFRAG_IPV6_TCP, 908 }, 909 [MLX5_EXPANSION_IPV6_ESP] = { 910 .type = RTE_FLOW_ITEM_TYPE_ESP, 911 .rss_types = RTE_ETH_RSS_ESP, 912 }, 913 [MLX5_EXPANSION_IPV6_FRAG_EXT] = { 914 .type = RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT, 915 }, 916 [MLX5_EXPANSION_IPV6_ICMP6] = { 917 .type = RTE_FLOW_ITEM_TYPE_ICMP6, 918 }, 919 [MLX5_EXPANSION_GTP] = { 920 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_IPV4, 921 MLX5_EXPANSION_IPV6), 922 .type = RTE_FLOW_ITEM_TYPE_GTP, 923 }, 924 [MLX5_EXPANSION_GENEVE] = { 925 .next = MLX5_FLOW_EXPAND_RSS_NEXT(MLX5_EXPANSION_ETH, 926 MLX5_EXPANSION_IPV4, 927 MLX5_EXPANSION_IPV6), 928 .type = RTE_FLOW_ITEM_TYPE_GENEVE, 929 }, 930 }; 931 932 static struct rte_flow_action_handle * 933 mlx5_action_handle_create(struct rte_eth_dev *dev, 934 const struct rte_flow_indir_action_conf *conf, 935 const struct rte_flow_action *action, 936 struct rte_flow_error *error); 937 static int mlx5_action_handle_destroy 938 (struct rte_eth_dev *dev, 939 struct rte_flow_action_handle *handle, 940 struct rte_flow_error *error); 941 static int mlx5_action_handle_update 942 (struct rte_eth_dev *dev, 943 struct rte_flow_action_handle *handle, 944 const void *update, 945 struct rte_flow_error *error); 946 static int mlx5_action_handle_query 947 (struct rte_eth_dev *dev, 948 const struct rte_flow_action_handle *handle, 949 void *data, 950 struct rte_flow_error *error); 951 static int 952 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 953 struct rte_flow_tunnel *app_tunnel, 954 struct rte_flow_action **actions, 955 uint32_t *num_of_actions, 956 struct rte_flow_error *error); 957 static int 958 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 959 struct rte_flow_tunnel *app_tunnel, 960 struct rte_flow_item **items, 961 uint32_t *num_of_items, 962 struct rte_flow_error *error); 963 static int 964 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 965 struct rte_flow_item *pmd_items, 966 uint32_t num_items, struct rte_flow_error *err); 967 static int 968 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 969 struct rte_flow_action *pmd_actions, 970 uint32_t num_actions, 971 struct rte_flow_error *err); 972 static int 973 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 974 struct rte_mbuf *m, 975 struct rte_flow_restore_info *info, 976 struct rte_flow_error *err); 977 static struct rte_flow_item_flex_handle * 978 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 979 const struct rte_flow_item_flex_conf *conf, 980 struct rte_flow_error *error); 981 static int 982 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 983 const struct rte_flow_item_flex_handle *handle, 984 struct rte_flow_error *error); 985 static int 986 mlx5_flow_info_get(struct rte_eth_dev *dev, 987 struct rte_flow_port_info *port_info, 988 struct rte_flow_queue_info *queue_info, 989 struct rte_flow_error *error); 990 static int 991 mlx5_flow_port_configure(struct rte_eth_dev *dev, 992 const struct rte_flow_port_attr *port_attr, 993 uint16_t nb_queue, 994 const struct rte_flow_queue_attr *queue_attr[], 995 struct rte_flow_error *err); 996 997 static struct rte_flow_pattern_template * 998 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 999 const struct rte_flow_pattern_template_attr *attr, 1000 const struct rte_flow_item items[], 1001 struct rte_flow_error *error); 1002 1003 static int 1004 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 1005 struct rte_flow_pattern_template *template, 1006 struct rte_flow_error *error); 1007 static struct rte_flow_actions_template * 1008 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 1009 const struct rte_flow_actions_template_attr *attr, 1010 const struct rte_flow_action actions[], 1011 const struct rte_flow_action masks[], 1012 struct rte_flow_error *error); 1013 static int 1014 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 1015 struct rte_flow_actions_template *template, 1016 struct rte_flow_error *error); 1017 1018 static struct rte_flow_template_table * 1019 mlx5_flow_table_create(struct rte_eth_dev *dev, 1020 const struct rte_flow_template_table_attr *attr, 1021 struct rte_flow_pattern_template *item_templates[], 1022 uint8_t nb_item_templates, 1023 struct rte_flow_actions_template *action_templates[], 1024 uint8_t nb_action_templates, 1025 struct rte_flow_error *error); 1026 static int 1027 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 1028 struct rte_flow_template_table *table, 1029 struct rte_flow_error *error); 1030 static struct rte_flow * 1031 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 1032 uint32_t queue, 1033 const struct rte_flow_op_attr *attr, 1034 struct rte_flow_template_table *table, 1035 const struct rte_flow_item items[], 1036 uint8_t pattern_template_index, 1037 const struct rte_flow_action actions[], 1038 uint8_t action_template_index, 1039 void *user_data, 1040 struct rte_flow_error *error); 1041 static struct rte_flow * 1042 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 1043 uint32_t queue, 1044 const struct rte_flow_op_attr *attr, 1045 struct rte_flow_template_table *table, 1046 uint32_t rule_index, 1047 const struct rte_flow_action actions[], 1048 uint8_t action_template_index, 1049 void *user_data, 1050 struct rte_flow_error *error); 1051 static int 1052 mlx5_flow_async_flow_update(struct rte_eth_dev *dev, 1053 uint32_t queue, 1054 const struct rte_flow_op_attr *attr, 1055 struct rte_flow *flow, 1056 const struct rte_flow_action actions[], 1057 uint8_t action_template_index, 1058 void *user_data, 1059 struct rte_flow_error *error); 1060 static int 1061 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 1062 uint32_t queue, 1063 const struct rte_flow_op_attr *attr, 1064 struct rte_flow *flow, 1065 void *user_data, 1066 struct rte_flow_error *error); 1067 static int 1068 mlx5_flow_pull(struct rte_eth_dev *dev, 1069 uint32_t queue, 1070 struct rte_flow_op_result res[], 1071 uint16_t n_res, 1072 struct rte_flow_error *error); 1073 static int 1074 mlx5_flow_push(struct rte_eth_dev *dev, 1075 uint32_t queue, 1076 struct rte_flow_error *error); 1077 1078 static struct rte_flow_action_handle * 1079 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 1080 const struct rte_flow_op_attr *attr, 1081 const struct rte_flow_indir_action_conf *conf, 1082 const struct rte_flow_action *action, 1083 void *user_data, 1084 struct rte_flow_error *error); 1085 1086 static int 1087 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 1088 const struct rte_flow_op_attr *attr, 1089 struct rte_flow_action_handle *handle, 1090 const void *update, 1091 void *user_data, 1092 struct rte_flow_error *error); 1093 1094 static int 1095 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 1096 const struct rte_flow_op_attr *attr, 1097 struct rte_flow_action_handle *handle, 1098 void *user_data, 1099 struct rte_flow_error *error); 1100 1101 static int 1102 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 1103 const struct rte_flow_op_attr *attr, 1104 const struct rte_flow_action_handle *handle, 1105 void *data, 1106 void *user_data, 1107 struct rte_flow_error *error); 1108 static int 1109 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 1110 struct rte_flow_action_handle *handle, 1111 const void *update, void *query, 1112 enum rte_flow_query_update_mode qu_mode, 1113 struct rte_flow_error *error); 1114 static int 1115 mlx5_flow_async_action_handle_query_update 1116 (struct rte_eth_dev *dev, uint32_t queue_id, 1117 const struct rte_flow_op_attr *op_attr, 1118 struct rte_flow_action_handle *action_handle, 1119 const void *update, void *query, 1120 enum rte_flow_query_update_mode qu_mode, 1121 void *user_data, struct rte_flow_error *error); 1122 1123 static const struct rte_flow_ops mlx5_flow_ops = { 1124 .validate = mlx5_flow_validate, 1125 .create = mlx5_flow_create, 1126 .destroy = mlx5_flow_destroy, 1127 .flush = mlx5_flow_flush, 1128 .isolate = mlx5_flow_isolate, 1129 .query = mlx5_flow_query, 1130 .dev_dump = mlx5_flow_dev_dump, 1131 .get_q_aged_flows = mlx5_flow_get_q_aged_flows, 1132 .get_aged_flows = mlx5_flow_get_aged_flows, 1133 .action_handle_create = mlx5_action_handle_create, 1134 .action_handle_destroy = mlx5_action_handle_destroy, 1135 .action_handle_update = mlx5_action_handle_update, 1136 .action_handle_query = mlx5_action_handle_query, 1137 .action_handle_query_update = mlx5_action_handle_query_update, 1138 .tunnel_decap_set = mlx5_flow_tunnel_decap_set, 1139 .tunnel_match = mlx5_flow_tunnel_match, 1140 .tunnel_action_decap_release = mlx5_flow_tunnel_action_release, 1141 .tunnel_item_release = mlx5_flow_tunnel_item_release, 1142 .get_restore_info = mlx5_flow_tunnel_get_restore_info, 1143 .flex_item_create = mlx5_flow_flex_item_create, 1144 .flex_item_release = mlx5_flow_flex_item_release, 1145 .info_get = mlx5_flow_info_get, 1146 .pick_transfer_proxy = mlx5_flow_pick_transfer_proxy, 1147 .configure = mlx5_flow_port_configure, 1148 .pattern_template_create = mlx5_flow_pattern_template_create, 1149 .pattern_template_destroy = mlx5_flow_pattern_template_destroy, 1150 .actions_template_create = mlx5_flow_actions_template_create, 1151 .actions_template_destroy = mlx5_flow_actions_template_destroy, 1152 .template_table_create = mlx5_flow_table_create, 1153 .template_table_destroy = mlx5_flow_table_destroy, 1154 .async_create = mlx5_flow_async_flow_create, 1155 .async_create_by_index = mlx5_flow_async_flow_create_by_index, 1156 .async_destroy = mlx5_flow_async_flow_destroy, 1157 .pull = mlx5_flow_pull, 1158 .push = mlx5_flow_push, 1159 .async_action_handle_create = mlx5_flow_async_action_handle_create, 1160 .async_action_handle_update = mlx5_flow_async_action_handle_update, 1161 .async_action_handle_query_update = 1162 mlx5_flow_async_action_handle_query_update, 1163 .async_action_handle_query = mlx5_flow_async_action_handle_query, 1164 .async_action_handle_destroy = mlx5_flow_async_action_handle_destroy, 1165 .async_actions_update = mlx5_flow_async_flow_update, 1166 }; 1167 1168 /* Tunnel information. */ 1169 struct mlx5_flow_tunnel_info { 1170 uint64_t tunnel; /**< Tunnel bit (see MLX5_FLOW_*). */ 1171 uint32_t ptype; /**< Tunnel Ptype (see RTE_PTYPE_*). */ 1172 }; 1173 1174 static struct mlx5_flow_tunnel_info tunnels_info[] = { 1175 { 1176 .tunnel = MLX5_FLOW_LAYER_VXLAN, 1177 .ptype = RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L4_UDP, 1178 }, 1179 { 1180 .tunnel = MLX5_FLOW_LAYER_GENEVE, 1181 .ptype = RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L4_UDP, 1182 }, 1183 { 1184 .tunnel = MLX5_FLOW_LAYER_VXLAN_GPE, 1185 .ptype = RTE_PTYPE_TUNNEL_VXLAN_GPE | RTE_PTYPE_L4_UDP, 1186 }, 1187 { 1188 .tunnel = MLX5_FLOW_LAYER_GRE, 1189 .ptype = RTE_PTYPE_TUNNEL_GRE, 1190 }, 1191 { 1192 .tunnel = MLX5_FLOW_LAYER_MPLS | MLX5_FLOW_LAYER_OUTER_L4_UDP, 1193 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_UDP | RTE_PTYPE_L4_UDP, 1194 }, 1195 { 1196 .tunnel = MLX5_FLOW_LAYER_MPLS, 1197 .ptype = RTE_PTYPE_TUNNEL_MPLS_IN_GRE, 1198 }, 1199 { 1200 .tunnel = MLX5_FLOW_LAYER_NVGRE, 1201 .ptype = RTE_PTYPE_TUNNEL_NVGRE, 1202 }, 1203 { 1204 .tunnel = MLX5_FLOW_LAYER_IPIP, 1205 .ptype = RTE_PTYPE_TUNNEL_IP, 1206 }, 1207 { 1208 .tunnel = MLX5_FLOW_LAYER_IPV6_ENCAP, 1209 .ptype = RTE_PTYPE_TUNNEL_IP, 1210 }, 1211 { 1212 .tunnel = MLX5_FLOW_LAYER_GTP, 1213 .ptype = RTE_PTYPE_TUNNEL_GTPU, 1214 }, 1215 }; 1216 1217 1218 1219 /** 1220 * Translate tag ID to register. 1221 * 1222 * @param[in] dev 1223 * Pointer to the Ethernet device structure. 1224 * @param[in] feature 1225 * The feature that request the register. 1226 * @param[in] id 1227 * The request register ID. 1228 * @param[out] error 1229 * Error description in case of any. 1230 * 1231 * @return 1232 * The request register on success, a negative errno 1233 * value otherwise and rte_errno is set. 1234 */ 1235 int 1236 mlx5_flow_get_reg_id(struct rte_eth_dev *dev, 1237 enum mlx5_feature_name feature, 1238 uint32_t id, 1239 struct rte_flow_error *error) 1240 { 1241 struct mlx5_priv *priv = dev->data->dev_private; 1242 struct mlx5_sh_config *config = &priv->sh->config; 1243 enum modify_reg start_reg; 1244 bool skip_mtr_reg = false; 1245 1246 switch (feature) { 1247 case MLX5_HAIRPIN_RX: 1248 return REG_B; 1249 case MLX5_HAIRPIN_TX: 1250 return REG_A; 1251 case MLX5_METADATA_RX: 1252 switch (config->dv_xmeta_en) { 1253 case MLX5_XMETA_MODE_LEGACY: 1254 return REG_B; 1255 case MLX5_XMETA_MODE_META16: 1256 return REG_C_0; 1257 case MLX5_XMETA_MODE_META32: 1258 return REG_C_1; 1259 case MLX5_XMETA_MODE_META32_HWS: 1260 return REG_C_1; 1261 } 1262 break; 1263 case MLX5_METADATA_TX: 1264 if (config->dv_flow_en == 2 && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) { 1265 return REG_C_1; 1266 } else { 1267 return REG_A; 1268 } 1269 case MLX5_METADATA_FDB: 1270 switch (config->dv_xmeta_en) { 1271 case MLX5_XMETA_MODE_LEGACY: 1272 return REG_NON; 1273 case MLX5_XMETA_MODE_META16: 1274 return REG_C_0; 1275 case MLX5_XMETA_MODE_META32: 1276 return REG_C_1; 1277 case MLX5_XMETA_MODE_META32_HWS: 1278 return REG_C_1; 1279 } 1280 break; 1281 case MLX5_FLOW_MARK: 1282 switch (config->dv_xmeta_en) { 1283 case MLX5_XMETA_MODE_LEGACY: 1284 case MLX5_XMETA_MODE_META32_HWS: 1285 return REG_NON; 1286 case MLX5_XMETA_MODE_META16: 1287 return REG_C_1; 1288 case MLX5_XMETA_MODE_META32: 1289 return REG_C_0; 1290 } 1291 break; 1292 case MLX5_MTR_ID: 1293 /* 1294 * If meter color and meter id share one register, flow match 1295 * should use the meter color register for match. 1296 */ 1297 if (priv->mtr_reg_share) 1298 return priv->mtr_color_reg; 1299 else 1300 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1301 REG_C_3; 1302 case MLX5_MTR_COLOR: 1303 case MLX5_ASO_FLOW_HIT: 1304 case MLX5_ASO_CONNTRACK: 1305 case MLX5_SAMPLE_ID: 1306 /* All features use the same REG_C. */ 1307 MLX5_ASSERT(priv->mtr_color_reg != REG_NON); 1308 return priv->mtr_color_reg; 1309 case MLX5_COPY_MARK: 1310 /* 1311 * Metadata COPY_MARK register using is in meter suffix sub 1312 * flow while with meter. It's safe to share the same register. 1313 */ 1314 return priv->mtr_color_reg != REG_C_2 ? REG_C_2 : REG_C_3; 1315 case MLX5_APP_TAG: 1316 /* 1317 * If meter is enable, it will engage the register for color 1318 * match and flow match. If meter color match is not using the 1319 * REG_C_2, need to skip the REG_C_x be used by meter color 1320 * match. 1321 * If meter is disable, free to use all available registers. 1322 */ 1323 start_reg = priv->mtr_color_reg != REG_C_2 ? REG_C_2 : 1324 (priv->mtr_reg_share ? REG_C_3 : REG_C_4); 1325 skip_mtr_reg = !!(priv->mtr_en && start_reg == REG_C_2); 1326 if (id > (uint32_t)(REG_C_7 - start_reg)) 1327 return rte_flow_error_set(error, EINVAL, 1328 RTE_FLOW_ERROR_TYPE_ITEM, 1329 NULL, "invalid tag id"); 1330 if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON) 1331 return rte_flow_error_set(error, ENOTSUP, 1332 RTE_FLOW_ERROR_TYPE_ITEM, 1333 NULL, "unsupported tag id"); 1334 /* 1335 * This case means meter is using the REG_C_x great than 2. 1336 * Take care not to conflict with meter color REG_C_x. 1337 * If the available index REG_C_y >= REG_C_x, skip the 1338 * color register. 1339 */ 1340 if (skip_mtr_reg && priv->sh->flow_mreg_c 1341 [id + start_reg - REG_C_0] >= priv->mtr_color_reg) { 1342 if (id >= (uint32_t)(REG_C_7 - start_reg)) 1343 return rte_flow_error_set(error, EINVAL, 1344 RTE_FLOW_ERROR_TYPE_ITEM, 1345 NULL, "invalid tag id"); 1346 if (priv->sh->flow_mreg_c 1347 [id + 1 + start_reg - REG_C_0] != REG_NON) 1348 return priv->sh->flow_mreg_c 1349 [id + 1 + start_reg - REG_C_0]; 1350 return rte_flow_error_set(error, ENOTSUP, 1351 RTE_FLOW_ERROR_TYPE_ITEM, 1352 NULL, "unsupported tag id"); 1353 } 1354 return priv->sh->flow_mreg_c[id + start_reg - REG_C_0]; 1355 } 1356 MLX5_ASSERT(false); 1357 return rte_flow_error_set(error, EINVAL, 1358 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 1359 NULL, "invalid feature name"); 1360 } 1361 1362 /** 1363 * Check extensive flow metadata register support. 1364 * 1365 * @param dev 1366 * Pointer to rte_eth_dev structure. 1367 * 1368 * @return 1369 * True if device supports extensive flow metadata register, otherwise false. 1370 */ 1371 bool 1372 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev) 1373 { 1374 struct mlx5_priv *priv = dev->data->dev_private; 1375 1376 /* 1377 * Having available reg_c can be regarded inclusively as supporting 1378 * extensive flow metadata register, which could mean, 1379 * - metadata register copy action by modify header. 1380 * - 16 modify header actions is supported. 1381 * - reg_c's are preserved across different domain (FDB and NIC) on 1382 * packet loopback by flow lookup miss. 1383 */ 1384 return priv->sh->flow_mreg_c[2] != REG_NON; 1385 } 1386 1387 /** 1388 * Get the lowest priority. 1389 * 1390 * @param[in] dev 1391 * Pointer to the Ethernet device structure. 1392 * @param[in] attributes 1393 * Pointer to device flow rule attributes. 1394 * 1395 * @return 1396 * The value of lowest priority of flow. 1397 */ 1398 uint32_t 1399 mlx5_get_lowest_priority(struct rte_eth_dev *dev, 1400 const struct rte_flow_attr *attr) 1401 { 1402 struct mlx5_priv *priv = dev->data->dev_private; 1403 1404 if (!attr->group && !(attr->transfer && priv->fdb_def_rule)) 1405 return priv->sh->flow_max_priority - 2; 1406 return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1; 1407 } 1408 1409 /** 1410 * Calculate matcher priority of the flow. 1411 * 1412 * @param[in] dev 1413 * Pointer to the Ethernet device structure. 1414 * @param[in] attr 1415 * Pointer to device flow rule attributes. 1416 * @param[in] subpriority 1417 * The priority based on the items. 1418 * @param[in] external 1419 * Flow is user flow. 1420 * @return 1421 * The matcher priority of the flow. 1422 */ 1423 uint16_t 1424 mlx5_get_matcher_priority(struct rte_eth_dev *dev, 1425 const struct rte_flow_attr *attr, 1426 uint32_t subpriority, bool external) 1427 { 1428 uint16_t priority = (uint16_t)attr->priority; 1429 struct mlx5_priv *priv = dev->data->dev_private; 1430 1431 /* NIC root rules */ 1432 if (!attr->group && !attr->transfer) { 1433 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1434 priority = priv->sh->flow_max_priority - 1; 1435 return mlx5_os_flow_adjust_priority(dev, priority, subpriority); 1436 /* FDB root rules */ 1437 } else if (attr->transfer && (!external || !priv->fdb_def_rule) && 1438 attr->group == 0 && 1439 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) { 1440 return (priv->sh->flow_max_priority - 1) * 3; 1441 } 1442 if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) 1443 priority = MLX5_NON_ROOT_FLOW_MAX_PRIO; 1444 return priority * 3 + subpriority; 1445 } 1446 1447 /** 1448 * Verify the @p item specifications (spec, last, mask) are compatible with the 1449 * NIC capabilities. 1450 * 1451 * @param[in] item 1452 * Item specification. 1453 * @param[in] mask 1454 * @p item->mask or flow default bit-masks. 1455 * @param[in] nic_mask 1456 * Bit-masks covering supported fields by the NIC to compare with user mask. 1457 * @param[in] size 1458 * Bit-masks size in bytes. 1459 * @param[in] range_accepted 1460 * True if range of values is accepted for specific fields, false otherwise. 1461 * @param[out] error 1462 * Pointer to error structure. 1463 * 1464 * @return 1465 * 0 on success, a negative errno value otherwise and rte_errno is set. 1466 */ 1467 int 1468 mlx5_flow_item_acceptable(const struct rte_flow_item *item, 1469 const uint8_t *mask, 1470 const uint8_t *nic_mask, 1471 unsigned int size, 1472 bool range_accepted, 1473 struct rte_flow_error *error) 1474 { 1475 unsigned int i; 1476 1477 MLX5_ASSERT(nic_mask); 1478 for (i = 0; i < size; ++i) 1479 if ((nic_mask[i] | mask[i]) != nic_mask[i]) 1480 return rte_flow_error_set(error, ENOTSUP, 1481 RTE_FLOW_ERROR_TYPE_ITEM, 1482 item, 1483 "mask enables non supported" 1484 " bits"); 1485 if (!item->spec && (item->mask || item->last)) 1486 return rte_flow_error_set(error, EINVAL, 1487 RTE_FLOW_ERROR_TYPE_ITEM, item, 1488 "mask/last without a spec is not" 1489 " supported"); 1490 if (item->spec && item->last && !range_accepted) { 1491 uint8_t spec[size]; 1492 uint8_t last[size]; 1493 unsigned int i; 1494 int ret; 1495 1496 for (i = 0; i < size; ++i) { 1497 spec[i] = ((const uint8_t *)item->spec)[i] & mask[i]; 1498 last[i] = ((const uint8_t *)item->last)[i] & mask[i]; 1499 } 1500 ret = memcmp(spec, last, size); 1501 if (ret != 0) 1502 return rte_flow_error_set(error, EINVAL, 1503 RTE_FLOW_ERROR_TYPE_ITEM, 1504 item, 1505 "range is not valid"); 1506 } 1507 return 0; 1508 } 1509 1510 /** 1511 * Adjust the hash fields according to the @p flow information. 1512 * 1513 * @param[in] dev_flow. 1514 * Pointer to the mlx5_flow. 1515 * @param[in] tunnel 1516 * 1 when the hash field is for a tunnel item. 1517 * @param[in] layer_types 1518 * RTE_ETH_RSS_* types. 1519 * @param[in] hash_fields 1520 * Item hash fields. 1521 * 1522 * @return 1523 * The hash fields that should be used. 1524 */ 1525 uint64_t 1526 mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_desc *rss_desc, 1527 int tunnel __rte_unused, uint64_t layer_types, 1528 uint64_t hash_fields) 1529 { 1530 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 1531 int rss_request_inner = rss_desc->level >= 2; 1532 1533 /* Check RSS hash level for tunnel. */ 1534 if (tunnel && rss_request_inner) 1535 hash_fields |= IBV_RX_HASH_INNER; 1536 else if (tunnel || rss_request_inner) 1537 return 0; 1538 #endif 1539 /* Check if requested layer matches RSS hash fields. */ 1540 if (!(rss_desc->types & layer_types)) 1541 return 0; 1542 return hash_fields; 1543 } 1544 1545 /** 1546 * Lookup and set the ptype in the data Rx part. A single Ptype can be used, 1547 * if several tunnel rules are used on this queue, the tunnel ptype will be 1548 * cleared. 1549 * 1550 * @param rxq_ctrl 1551 * Rx queue to update. 1552 */ 1553 static void 1554 flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) 1555 { 1556 unsigned int i; 1557 uint32_t tunnel_ptype = 0; 1558 1559 /* Look up for the ptype to use. */ 1560 for (i = 0; i != MLX5_FLOW_TUNNEL; ++i) { 1561 if (!rxq_ctrl->flow_tunnels_n[i]) 1562 continue; 1563 if (!tunnel_ptype) { 1564 tunnel_ptype = tunnels_info[i].ptype; 1565 } else { 1566 tunnel_ptype = 0; 1567 break; 1568 } 1569 } 1570 rxq_ctrl->rxq.tunnel = tunnel_ptype; 1571 } 1572 1573 /** 1574 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) according to the device 1575 * flow. 1576 * 1577 * @param[in] dev 1578 * Pointer to the Ethernet device structure. 1579 * @param[in] dev_handle 1580 * Pointer to device flow handle structure. 1581 */ 1582 void 1583 flow_drv_rxq_flags_set(struct rte_eth_dev *dev, 1584 struct mlx5_flow_handle *dev_handle) 1585 { 1586 struct mlx5_priv *priv = dev->data->dev_private; 1587 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1588 struct mlx5_ind_table_obj *ind_tbl = NULL; 1589 unsigned int i; 1590 1591 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1592 struct mlx5_hrxq *hrxq; 1593 1594 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1595 dev_handle->rix_hrxq); 1596 if (hrxq) 1597 ind_tbl = hrxq->ind_table; 1598 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1599 struct mlx5_shared_action_rss *shared_rss; 1600 1601 shared_rss = mlx5_ipool_get 1602 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1603 dev_handle->rix_srss); 1604 if (shared_rss) 1605 ind_tbl = shared_rss->ind_tbl; 1606 } 1607 if (!ind_tbl) 1608 return; 1609 for (i = 0; i != ind_tbl->queues_n; ++i) { 1610 int idx = ind_tbl->queues[i]; 1611 struct mlx5_rxq_ctrl *rxq_ctrl; 1612 1613 if (mlx5_is_external_rxq(dev, idx)) 1614 continue; 1615 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1616 MLX5_ASSERT(rxq_ctrl != NULL); 1617 if (rxq_ctrl == NULL) 1618 continue; 1619 /* 1620 * To support metadata register copy on Tx loopback, 1621 * this must be always enabled (metadata may arive 1622 * from other port - not from local flows only. 1623 */ 1624 if (tunnel) { 1625 unsigned int j; 1626 1627 /* Increase the counter matching the flow. */ 1628 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1629 if ((tunnels_info[j].tunnel & 1630 dev_handle->layers) == 1631 tunnels_info[j].tunnel) { 1632 rxq_ctrl->flow_tunnels_n[j]++; 1633 break; 1634 } 1635 } 1636 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1637 } 1638 } 1639 } 1640 1641 static void 1642 flow_rxq_mark_flag_set(struct rte_eth_dev *dev) 1643 { 1644 struct mlx5_priv *priv = dev->data->dev_private; 1645 struct mlx5_rxq_ctrl *rxq_ctrl; 1646 uint16_t port_id; 1647 1648 if (priv->sh->shared_mark_enabled) 1649 return; 1650 if (priv->master || priv->representor) { 1651 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 1652 struct mlx5_priv *opriv = 1653 rte_eth_devices[port_id].data->dev_private; 1654 1655 if (!opriv || 1656 opriv->sh != priv->sh || 1657 opriv->domain_id != priv->domain_id || 1658 opriv->mark_enabled) 1659 continue; 1660 LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) { 1661 rxq_ctrl->rxq.mark = 1; 1662 } 1663 opriv->mark_enabled = 1; 1664 } 1665 } else { 1666 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 1667 rxq_ctrl->rxq.mark = 1; 1668 } 1669 priv->mark_enabled = 1; 1670 } 1671 priv->sh->shared_mark_enabled = 1; 1672 } 1673 1674 /** 1675 * Set the Rx queue flags (Mark/Flag and Tunnel Ptypes) for a flow 1676 * 1677 * @param[in] dev 1678 * Pointer to the Ethernet device structure. 1679 * @param[in] flow 1680 * Pointer to flow structure. 1681 */ 1682 static void 1683 flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) 1684 { 1685 struct mlx5_priv *priv = dev->data->dev_private; 1686 uint32_t handle_idx; 1687 struct mlx5_flow_handle *dev_handle; 1688 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 1689 1690 MLX5_ASSERT(wks); 1691 if (wks->mark) 1692 flow_rxq_mark_flag_set(dev); 1693 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1694 handle_idx, dev_handle, next) 1695 flow_drv_rxq_flags_set(dev, dev_handle); 1696 } 1697 1698 /** 1699 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1700 * device flow if no other flow uses it with the same kind of request. 1701 * 1702 * @param dev 1703 * Pointer to Ethernet device. 1704 * @param[in] dev_handle 1705 * Pointer to the device flow handle structure. 1706 */ 1707 static void 1708 flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, 1709 struct mlx5_flow_handle *dev_handle) 1710 { 1711 struct mlx5_priv *priv = dev->data->dev_private; 1712 const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL); 1713 struct mlx5_ind_table_obj *ind_tbl = NULL; 1714 unsigned int i; 1715 1716 if (dev_handle->fate_action == MLX5_FLOW_FATE_QUEUE) { 1717 struct mlx5_hrxq *hrxq; 1718 1719 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], 1720 dev_handle->rix_hrxq); 1721 if (hrxq) 1722 ind_tbl = hrxq->ind_table; 1723 } else if (dev_handle->fate_action == MLX5_FLOW_FATE_SHARED_RSS) { 1724 struct mlx5_shared_action_rss *shared_rss; 1725 1726 shared_rss = mlx5_ipool_get 1727 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 1728 dev_handle->rix_srss); 1729 if (shared_rss) 1730 ind_tbl = shared_rss->ind_tbl; 1731 } 1732 if (!ind_tbl) 1733 return; 1734 MLX5_ASSERT(dev->data->dev_started); 1735 for (i = 0; i != ind_tbl->queues_n; ++i) { 1736 int idx = ind_tbl->queues[i]; 1737 struct mlx5_rxq_ctrl *rxq_ctrl; 1738 1739 if (mlx5_is_external_rxq(dev, idx)) 1740 continue; 1741 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 1742 MLX5_ASSERT(rxq_ctrl != NULL); 1743 if (rxq_ctrl == NULL) 1744 continue; 1745 if (tunnel) { 1746 unsigned int j; 1747 1748 /* Decrease the counter matching the flow. */ 1749 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) { 1750 if ((tunnels_info[j].tunnel & 1751 dev_handle->layers) == 1752 tunnels_info[j].tunnel) { 1753 rxq_ctrl->flow_tunnels_n[j]--; 1754 break; 1755 } 1756 } 1757 flow_rxq_tunnel_ptype_update(rxq_ctrl); 1758 } 1759 } 1760 } 1761 1762 /** 1763 * Clear the Rx queue flags (Mark/Flag and Tunnel Ptype) associated with the 1764 * @p flow if no other flow uses it with the same kind of request. 1765 * 1766 * @param dev 1767 * Pointer to Ethernet device. 1768 * @param[in] flow 1769 * Pointer to the flow. 1770 */ 1771 static void 1772 flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) 1773 { 1774 struct mlx5_priv *priv = dev->data->dev_private; 1775 uint32_t handle_idx; 1776 struct mlx5_flow_handle *dev_handle; 1777 1778 SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, 1779 handle_idx, dev_handle, next) 1780 flow_drv_rxq_flags_trim(dev, dev_handle); 1781 } 1782 1783 /** 1784 * Clear the Mark/Flag and Tunnel ptype information in all Rx queues. 1785 * 1786 * @param dev 1787 * Pointer to Ethernet device. 1788 */ 1789 static void 1790 flow_rxq_flags_clear(struct rte_eth_dev *dev) 1791 { 1792 struct mlx5_priv *priv = dev->data->dev_private; 1793 unsigned int i; 1794 1795 for (i = 0; i != priv->rxqs_n; ++i) { 1796 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1797 unsigned int j; 1798 1799 if (rxq == NULL || rxq->ctrl == NULL) 1800 continue; 1801 rxq->ctrl->rxq.mark = 0; 1802 for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) 1803 rxq->ctrl->flow_tunnels_n[j] = 0; 1804 rxq->ctrl->rxq.tunnel = 0; 1805 } 1806 priv->mark_enabled = 0; 1807 priv->sh->shared_mark_enabled = 0; 1808 } 1809 1810 static uint64_t mlx5_restore_info_dynflag; 1811 1812 int 1813 mlx5_flow_rx_metadata_negotiate(struct rte_eth_dev *dev, uint64_t *features) 1814 { 1815 struct mlx5_priv *priv = dev->data->dev_private; 1816 uint64_t supported = 0; 1817 1818 if (!is_tunnel_offload_active(dev)) { 1819 supported |= RTE_ETH_RX_METADATA_USER_FLAG; 1820 supported |= RTE_ETH_RX_METADATA_USER_MARK; 1821 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) { 1822 DRV_LOG(DEBUG, 1823 "tunnel offload was not activated, consider setting dv_xmeta_en=%d", 1824 MLX5_XMETA_MODE_MISS_INFO); 1825 } 1826 } else { 1827 supported |= RTE_ETH_RX_METADATA_TUNNEL_ID; 1828 if ((*features & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0 && 1829 mlx5_restore_info_dynflag == 0) 1830 mlx5_restore_info_dynflag = rte_flow_restore_info_dynflag(); 1831 } 1832 1833 if (((*features & supported) & RTE_ETH_RX_METADATA_TUNNEL_ID) != 0) 1834 priv->tunnel_enabled = 1; 1835 else 1836 priv->tunnel_enabled = 0; 1837 1838 *features &= supported; 1839 return 0; 1840 } 1841 1842 /** 1843 * Set the Rx queue dynamic metadata (mask and offset) for a flow 1844 * 1845 * @param[in] dev 1846 * Pointer to the Ethernet device structure. 1847 */ 1848 void 1849 mlx5_flow_rxq_dynf_set(struct rte_eth_dev *dev) 1850 { 1851 struct mlx5_priv *priv = dev->data->dev_private; 1852 uint64_t mark_flag = RTE_MBUF_F_RX_FDIR_ID; 1853 unsigned int i; 1854 1855 if (priv->tunnel_enabled) 1856 mark_flag |= mlx5_restore_info_dynflag; 1857 1858 for (i = 0; i != priv->rxqs_n; ++i) { 1859 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1860 struct mlx5_rxq_data *data; 1861 1862 if (rxq == NULL || rxq->ctrl == NULL) 1863 continue; 1864 data = &rxq->ctrl->rxq; 1865 if (!rte_flow_dynf_metadata_avail()) { 1866 data->dynf_meta = 0; 1867 data->flow_meta_mask = 0; 1868 data->flow_meta_offset = -1; 1869 data->flow_meta_port_mask = 0; 1870 } else { 1871 data->dynf_meta = 1; 1872 data->flow_meta_mask = rte_flow_dynf_metadata_mask; 1873 data->flow_meta_offset = rte_flow_dynf_metadata_offs; 1874 data->flow_meta_port_mask = priv->sh->dv_meta_mask; 1875 } 1876 data->mark_flag = mark_flag; 1877 } 1878 } 1879 1880 /* 1881 * return a pointer to the desired action in the list of actions. 1882 * 1883 * @param[in] actions 1884 * The list of actions to search the action in. 1885 * @param[in] action 1886 * The action to find. 1887 * 1888 * @return 1889 * Pointer to the action in the list, if found. NULL otherwise. 1890 */ 1891 const struct rte_flow_action * 1892 mlx5_flow_find_action(const struct rte_flow_action *actions, 1893 enum rte_flow_action_type action) 1894 { 1895 if (actions == NULL) 1896 return NULL; 1897 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) 1898 if (actions->type == action) 1899 return actions; 1900 return NULL; 1901 } 1902 1903 /* 1904 * Validate the flag action. 1905 * 1906 * @param[in] action_flags 1907 * Bit-fields that holds the actions detected until now. 1908 * @param[in] attr 1909 * Attributes of flow that includes this action. 1910 * @param[out] error 1911 * Pointer to error structure. 1912 * 1913 * @return 1914 * 0 on success, a negative errno value otherwise and rte_errno is set. 1915 */ 1916 int 1917 mlx5_flow_validate_action_flag(uint64_t action_flags, 1918 const struct rte_flow_attr *attr, 1919 struct rte_flow_error *error) 1920 { 1921 if (action_flags & MLX5_FLOW_ACTION_MARK) 1922 return rte_flow_error_set(error, EINVAL, 1923 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1924 "can't mark and flag in same flow"); 1925 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1926 return rte_flow_error_set(error, EINVAL, 1927 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1928 "can't have 2 flag" 1929 " actions in same flow"); 1930 if (attr->egress) 1931 return rte_flow_error_set(error, ENOTSUP, 1932 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1933 "flag action not supported for " 1934 "egress"); 1935 return 0; 1936 } 1937 1938 /* 1939 * Validate the mark action. 1940 * 1941 * @param[in] action 1942 * Pointer to the queue action. 1943 * @param[in] action_flags 1944 * Bit-fields that holds the actions detected until now. 1945 * @param[in] attr 1946 * Attributes of flow that includes this action. 1947 * @param[out] error 1948 * Pointer to error structure. 1949 * 1950 * @return 1951 * 0 on success, a negative errno value otherwise and rte_errno is set. 1952 */ 1953 int 1954 mlx5_flow_validate_action_mark(const struct rte_flow_action *action, 1955 uint64_t action_flags, 1956 const struct rte_flow_attr *attr, 1957 struct rte_flow_error *error) 1958 { 1959 const struct rte_flow_action_mark *mark = action->conf; 1960 1961 if (!mark) 1962 return rte_flow_error_set(error, EINVAL, 1963 RTE_FLOW_ERROR_TYPE_ACTION, 1964 action, 1965 "configuration cannot be null"); 1966 if (mark->id >= MLX5_FLOW_MARK_MAX) 1967 return rte_flow_error_set(error, EINVAL, 1968 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 1969 &mark->id, 1970 "mark id must in 0 <= id < " 1971 RTE_STR(MLX5_FLOW_MARK_MAX)); 1972 if (action_flags & MLX5_FLOW_ACTION_FLAG) 1973 return rte_flow_error_set(error, EINVAL, 1974 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1975 "can't flag and mark in same flow"); 1976 if (action_flags & MLX5_FLOW_ACTION_MARK) 1977 return rte_flow_error_set(error, EINVAL, 1978 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 1979 "can't have 2 mark actions in same" 1980 " flow"); 1981 if (attr->egress) 1982 return rte_flow_error_set(error, ENOTSUP, 1983 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 1984 "mark action not supported for " 1985 "egress"); 1986 return 0; 1987 } 1988 1989 /* 1990 * Validate the drop action. 1991 * 1992 * @param[in] dev 1993 * Pointer to the Ethernet device structure. 1994 * @param[in] is_root 1995 * True if flow is validated for root table. False otherwise. 1996 * @param[in] attr 1997 * Attributes of flow that includes this action. 1998 * @param[out] error 1999 * Pointer to error structure. 2000 * 2001 * @return 2002 * 0 on success, a negative errno value otherwise and rte_errno is set. 2003 */ 2004 int 2005 mlx5_flow_validate_action_drop(struct rte_eth_dev *dev, 2006 bool is_root, 2007 const struct rte_flow_attr *attr, 2008 struct rte_flow_error *error) 2009 { 2010 struct mlx5_priv *priv = dev->data->dev_private; 2011 2012 if (priv->sh->config.dv_flow_en == 0 && attr->egress) 2013 return rte_flow_error_set(error, ENOTSUP, 2014 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2015 "drop action not supported for " 2016 "egress"); 2017 if (priv->sh->config.dv_flow_en == 1 && is_root && (attr->egress || attr->transfer) && 2018 !priv->sh->dr_root_drop_action_en) { 2019 return rte_flow_error_set(error, ENOTSUP, 2020 RTE_FLOW_ERROR_TYPE_ATTR, NULL, 2021 "drop action not supported for " 2022 "egress and transfer on group 0"); 2023 } 2024 return 0; 2025 } 2026 2027 /* 2028 * Validate the queue action. 2029 * 2030 * @param[in] action 2031 * Pointer to the queue action. 2032 * @param[in] action_flags 2033 * Bit-fields that holds the actions detected until now. 2034 * @param[in] dev 2035 * Pointer to the Ethernet device structure. 2036 * @param[in] attr 2037 * Attributes of flow that includes this action. 2038 * @param[out] error 2039 * Pointer to error structure. 2040 * 2041 * @return 2042 * 0 on success, a negative errno value otherwise and rte_errno is set. 2043 */ 2044 int 2045 mlx5_flow_validate_action_queue(const struct rte_flow_action *action, 2046 uint64_t action_flags, 2047 struct rte_eth_dev *dev, 2048 const struct rte_flow_attr *attr, 2049 struct rte_flow_error *error) 2050 { 2051 struct mlx5_priv *priv = dev->data->dev_private; 2052 const struct rte_flow_action_queue *queue = action->conf; 2053 2054 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2055 return rte_flow_error_set(error, EINVAL, 2056 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2057 "can't have 2 fate actions in" 2058 " same flow"); 2059 if (attr->egress) 2060 return rte_flow_error_set(error, ENOTSUP, 2061 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2062 "queue action not supported for egress."); 2063 if (mlx5_is_external_rxq(dev, queue->index)) 2064 return 0; 2065 if (!priv->rxqs_n) 2066 return rte_flow_error_set(error, EINVAL, 2067 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2068 NULL, "No Rx queues configured"); 2069 if (queue->index >= priv->rxqs_n) 2070 return rte_flow_error_set(error, EINVAL, 2071 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2072 &queue->index, 2073 "queue index out of range"); 2074 if (mlx5_rxq_get(dev, queue->index) == NULL) 2075 return rte_flow_error_set(error, EINVAL, 2076 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2077 &queue->index, 2078 "queue is not configured"); 2079 return 0; 2080 } 2081 2082 /** 2083 * Validate queue numbers for device RSS. 2084 * 2085 * @param[in] dev 2086 * Configured device. 2087 * @param[in] queues 2088 * Array of queue numbers. 2089 * @param[in] queues_n 2090 * Size of the @p queues array. 2091 * @param[out] error 2092 * On error, filled with a textual error description. 2093 * @param[out] queue_idx 2094 * On error, filled with an offending queue index in @p queues array. 2095 * 2096 * @return 2097 * 0 on success, a negative errno code on error. 2098 */ 2099 static int 2100 mlx5_validate_rss_queues(struct rte_eth_dev *dev, 2101 const uint16_t *queues, uint32_t queues_n, 2102 const char **error, uint32_t *queue_idx) 2103 { 2104 const struct mlx5_priv *priv = dev->data->dev_private; 2105 bool is_hairpin = false; 2106 bool is_ext_rss = false; 2107 uint32_t i; 2108 2109 for (i = 0; i != queues_n; ++i) { 2110 struct mlx5_rxq_ctrl *rxq_ctrl; 2111 2112 if (mlx5_is_external_rxq(dev, queues[0])) { 2113 is_ext_rss = true; 2114 continue; 2115 } 2116 if (is_ext_rss) { 2117 *error = "Combining external and regular RSS queues is not supported"; 2118 *queue_idx = i; 2119 return -ENOTSUP; 2120 } 2121 if (queues[i] >= priv->rxqs_n) { 2122 *error = "queue index out of range"; 2123 *queue_idx = i; 2124 return -EINVAL; 2125 } 2126 rxq_ctrl = mlx5_rxq_ctrl_get(dev, queues[i]); 2127 if (rxq_ctrl == NULL) { 2128 *error = "queue is not configured"; 2129 *queue_idx = i; 2130 return -EINVAL; 2131 } 2132 if (i == 0 && rxq_ctrl->is_hairpin) 2133 is_hairpin = true; 2134 if (is_hairpin != rxq_ctrl->is_hairpin) { 2135 *error = "combining hairpin and regular RSS queues is not supported"; 2136 *queue_idx = i; 2137 return -ENOTSUP; 2138 } 2139 } 2140 return 0; 2141 } 2142 2143 /* 2144 * Validate the rss action. 2145 * 2146 * @param[in] dev 2147 * Pointer to the Ethernet device structure. 2148 * @param[in] action 2149 * Pointer to the queue action. 2150 * @param[out] error 2151 * Pointer to error structure. 2152 * 2153 * @return 2154 * 0 on success, a negative errno value otherwise and rte_errno is set. 2155 */ 2156 int 2157 mlx5_validate_action_rss(struct rte_eth_dev *dev, 2158 const struct rte_flow_action *action, 2159 struct rte_flow_error *error) 2160 { 2161 struct mlx5_priv *priv = dev->data->dev_private; 2162 const struct rte_flow_action_rss *rss = action->conf; 2163 int ret; 2164 const char *message; 2165 uint32_t queue_idx; 2166 2167 if (rss->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) { 2168 DRV_LOG(WARNING, "port %u symmetric RSS supported with SORT", 2169 dev->data->port_id); 2170 } else if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT && 2171 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) 2172 return rte_flow_error_set(error, ENOTSUP, 2173 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2174 &rss->func, 2175 "RSS hash function not supported"); 2176 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 2177 if (rss->level > 2) 2178 #else 2179 if (rss->level > 1) 2180 #endif 2181 return rte_flow_error_set(error, ENOTSUP, 2182 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2183 &rss->level, 2184 "tunnel RSS is not supported"); 2185 /* allow RSS key_len 0 in case of NULL (default) RSS key. */ 2186 if (rss->key_len == 0 && rss->key != NULL) 2187 return rte_flow_error_set(error, ENOTSUP, 2188 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2189 &rss->key_len, 2190 "RSS hash key length 0"); 2191 if (rss->key_len > 0 && rss->key_len < MLX5_RSS_HASH_KEY_LEN) 2192 return rte_flow_error_set(error, ENOTSUP, 2193 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2194 &rss->key_len, 2195 "RSS hash key too small"); 2196 if (rss->key_len > MLX5_RSS_HASH_KEY_LEN) 2197 return rte_flow_error_set(error, ENOTSUP, 2198 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2199 &rss->key_len, 2200 "RSS hash key too large"); 2201 if (rss->queue_num > priv->sh->dev_cap.ind_table_max_size) 2202 return rte_flow_error_set(error, ENOTSUP, 2203 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2204 &rss->queue_num, 2205 "number of queues too large"); 2206 if (rss->types & MLX5_RSS_HF_MASK) 2207 return rte_flow_error_set(error, ENOTSUP, 2208 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2209 &rss->types, 2210 "some RSS protocols are not" 2211 " supported"); 2212 if ((rss->types & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) && 2213 !(rss->types & RTE_ETH_RSS_IP)) 2214 return rte_flow_error_set(error, EINVAL, 2215 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2216 "L3 partial RSS requested but L3 RSS" 2217 " type not specified"); 2218 if ((rss->types & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) && 2219 !(rss->types & (RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP))) 2220 return rte_flow_error_set(error, EINVAL, 2221 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2222 "L4 partial RSS requested but L4 RSS" 2223 " type not specified"); 2224 if (!priv->rxqs_n && priv->ext_rxqs == NULL) 2225 return rte_flow_error_set(error, EINVAL, 2226 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2227 NULL, "No Rx queues configured"); 2228 if (!rss->queue_num) 2229 return rte_flow_error_set(error, EINVAL, 2230 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2231 NULL, "No queues configured"); 2232 ret = mlx5_validate_rss_queues(dev, rss->queue, rss->queue_num, 2233 &message, &queue_idx); 2234 if (ret != 0) { 2235 return rte_flow_error_set(error, -ret, 2236 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 2237 &rss->queue[queue_idx], message); 2238 } 2239 return 0; 2240 } 2241 2242 /* 2243 * Validate the rss action. 2244 * 2245 * @param[in] action 2246 * Pointer to the queue action. 2247 * @param[in] action_flags 2248 * Bit-fields that holds the actions detected until now. 2249 * @param[in] dev 2250 * Pointer to the Ethernet device structure. 2251 * @param[in] attr 2252 * Attributes of flow that includes this action. 2253 * @param[in] item_flags 2254 * Items that were detected. 2255 * @param[out] error 2256 * Pointer to error structure. 2257 * 2258 * @return 2259 * 0 on success, a negative errno value otherwise and rte_errno is set. 2260 */ 2261 int 2262 mlx5_flow_validate_action_rss(const struct rte_flow_action *action, 2263 uint64_t action_flags, 2264 struct rte_eth_dev *dev, 2265 const struct rte_flow_attr *attr, 2266 uint64_t item_flags, 2267 struct rte_flow_error *error) 2268 { 2269 const struct rte_flow_action_rss *rss = action->conf; 2270 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2271 int ret; 2272 2273 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2274 return rte_flow_error_set(error, EINVAL, 2275 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2276 "can't have 2 fate actions" 2277 " in same flow"); 2278 ret = mlx5_validate_action_rss(dev, action, error); 2279 if (ret) 2280 return ret; 2281 if (attr->egress) 2282 return rte_flow_error_set(error, ENOTSUP, 2283 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2284 "rss action not supported for " 2285 "egress"); 2286 if (rss->level > 1 && !tunnel) 2287 return rte_flow_error_set(error, EINVAL, 2288 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2289 "inner RSS is not supported for " 2290 "non-tunnel flows"); 2291 if ((item_flags & MLX5_FLOW_LAYER_ECPRI) && 2292 !(item_flags & MLX5_FLOW_LAYER_INNER_L4_UDP)) { 2293 return rte_flow_error_set(error, EINVAL, 2294 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 2295 "RSS on eCPRI is not supported now"); 2296 } 2297 if ((item_flags & MLX5_FLOW_LAYER_MPLS) && 2298 !(item_flags & 2299 (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3)) && 2300 rss->level > 1) 2301 return rte_flow_error_set(error, EINVAL, 2302 RTE_FLOW_ERROR_TYPE_ITEM, NULL, 2303 "MPLS inner RSS needs to specify inner L2/L3 items after MPLS in pattern"); 2304 return 0; 2305 } 2306 2307 /* 2308 * Validate the default miss action. 2309 * 2310 * @param[in] action_flags 2311 * Bit-fields that holds the actions detected until now. 2312 * @param[out] error 2313 * Pointer to error structure. 2314 * 2315 * @return 2316 * 0 on success, a negative errno value otherwise and rte_errno is set. 2317 */ 2318 int 2319 mlx5_flow_validate_action_default_miss(uint64_t action_flags, 2320 const struct rte_flow_attr *attr, 2321 struct rte_flow_error *error) 2322 { 2323 if (action_flags & MLX5_FLOW_FATE_ACTIONS) 2324 return rte_flow_error_set(error, EINVAL, 2325 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2326 "can't have 2 fate actions in" 2327 " same flow"); 2328 if (attr->egress) 2329 return rte_flow_error_set(error, ENOTSUP, 2330 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2331 "default miss action not supported " 2332 "for egress"); 2333 if (attr->group) 2334 return rte_flow_error_set(error, ENOTSUP, 2335 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, NULL, 2336 "only group 0 is supported"); 2337 if (attr->transfer) 2338 return rte_flow_error_set(error, ENOTSUP, 2339 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, 2340 NULL, "transfer is not supported"); 2341 return 0; 2342 } 2343 2344 /* 2345 * Validate the count action. 2346 * 2347 * @param[in] dev 2348 * Pointer to the Ethernet device structure. 2349 * @param[in] attr 2350 * Attributes of flow that includes this action. 2351 * @param[out] error 2352 * Pointer to error structure. 2353 * 2354 * @return 2355 * 0 on success, a negative errno value otherwise and rte_errno is set. 2356 */ 2357 int 2358 mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, 2359 const struct rte_flow_attr *attr, 2360 struct rte_flow_error *error) 2361 { 2362 if (attr->egress) 2363 return rte_flow_error_set(error, ENOTSUP, 2364 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, 2365 "count action not supported for " 2366 "egress"); 2367 return 0; 2368 } 2369 2370 /* 2371 * Validate the ASO CT action. 2372 * 2373 * @param[in] dev 2374 * Pointer to the Ethernet device structure. 2375 * @param[in] conntrack 2376 * Pointer to the CT action profile. 2377 * @param[out] error 2378 * Pointer to error structure. 2379 * 2380 * @return 2381 * 0 on success, a negative errno value otherwise and rte_errno is set. 2382 */ 2383 int 2384 mlx5_validate_action_ct(struct rte_eth_dev *dev, 2385 const struct rte_flow_action_conntrack *conntrack, 2386 struct rte_flow_error *error) 2387 { 2388 RTE_SET_USED(dev); 2389 2390 if (conntrack->state > RTE_FLOW_CONNTRACK_STATE_TIME_WAIT) 2391 return rte_flow_error_set(error, EINVAL, 2392 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2393 "Invalid CT state"); 2394 if (conntrack->last_index > RTE_FLOW_CONNTRACK_FLAG_RST) 2395 return rte_flow_error_set(error, EINVAL, 2396 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2397 "Invalid last TCP packet flag"); 2398 return 0; 2399 } 2400 2401 /** 2402 * Validate the level value for modify field action. 2403 * 2404 * @param[in] data 2405 * Pointer to the rte_flow_action_modify_data structure either src or dst. 2406 * @param[out] error 2407 * Pointer to error structure. 2408 * 2409 * @return 2410 * 0 on success, a negative errno value otherwise and rte_errno is set. 2411 */ 2412 int 2413 flow_validate_modify_field_level(const struct rte_flow_action_modify_data *data, 2414 struct rte_flow_error *error) 2415 { 2416 if (data->level == 0) 2417 return 0; 2418 if (data->field != RTE_FLOW_FIELD_TAG && 2419 data->field != (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) 2420 return rte_flow_error_set(error, ENOTSUP, 2421 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2422 "inner header fields modification is not supported"); 2423 if (data->tag_index != 0) 2424 return rte_flow_error_set(error, EINVAL, 2425 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 2426 "tag array can be provided using 'level' or 'tag_index' fields, not both"); 2427 /* 2428 * The tag array for RTE_FLOW_FIELD_TAG type is provided using 2429 * 'tag_index' field. In old API, it was provided using 'level' field 2430 * and it is still supported for backwards compatibility. 2431 */ 2432 DRV_LOG(DEBUG, "tag array provided in 'level' field instead of 'tag_index' field."); 2433 return 0; 2434 } 2435 2436 /** 2437 * Validate ICMP6 item. 2438 * 2439 * @param[in] item 2440 * Item specification. 2441 * @param[in] item_flags 2442 * Bit-fields that holds the items detected until now. 2443 * @param[in] ext_vlan_sup 2444 * Whether extended VLAN features are supported or not. 2445 * @param[out] error 2446 * Pointer to error structure. 2447 * 2448 * @return 2449 * 0 on success, a negative errno value otherwise and rte_errno is set. 2450 */ 2451 int 2452 mlx5_flow_validate_item_icmp6(const struct rte_flow_item *item, 2453 uint64_t item_flags, 2454 uint8_t target_protocol, 2455 struct rte_flow_error *error) 2456 { 2457 const struct rte_flow_item_icmp6 *mask = item->mask; 2458 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2459 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2460 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2461 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2462 MLX5_FLOW_LAYER_OUTER_L4; 2463 int ret; 2464 2465 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2466 return rte_flow_error_set(error, EINVAL, 2467 RTE_FLOW_ERROR_TYPE_ITEM, item, 2468 "protocol filtering not compatible" 2469 " with ICMP6 layer"); 2470 if (!(item_flags & l3m)) 2471 return rte_flow_error_set(error, EINVAL, 2472 RTE_FLOW_ERROR_TYPE_ITEM, item, 2473 "IPv6 is mandatory to filter on" 2474 " ICMP6"); 2475 if (item_flags & l4m) 2476 return rte_flow_error_set(error, EINVAL, 2477 RTE_FLOW_ERROR_TYPE_ITEM, item, 2478 "multiple L4 layers not supported"); 2479 if (!mask) 2480 mask = &rte_flow_item_icmp6_mask; 2481 ret = mlx5_flow_item_acceptable 2482 (item, (const uint8_t *)mask, 2483 (const uint8_t *)&rte_flow_item_icmp6_mask, 2484 sizeof(struct rte_flow_item_icmp6), 2485 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2486 if (ret < 0) 2487 return ret; 2488 return 0; 2489 } 2490 2491 /** 2492 * Validate ICMP6 echo request/reply item. 2493 * 2494 * @param[in] item 2495 * Item specification. 2496 * @param[in] item_flags 2497 * Bit-fields that holds the items detected until now. 2498 * @param[in] ext_vlan_sup 2499 * Whether extended VLAN features are supported or not. 2500 * @param[out] error 2501 * Pointer to error structure. 2502 * 2503 * @return 2504 * 0 on success, a negative errno value otherwise and rte_errno is set. 2505 */ 2506 int 2507 mlx5_flow_validate_item_icmp6_echo(const struct rte_flow_item *item, 2508 uint64_t item_flags, 2509 uint8_t target_protocol, 2510 struct rte_flow_error *error) 2511 { 2512 const struct rte_flow_item_icmp6_echo *mask = item->mask; 2513 const struct rte_flow_item_icmp6_echo nic_mask = { 2514 .hdr.base.type = 0xff, 2515 .hdr.base.code = 0xff, 2516 .hdr.identifier = RTE_BE16(0xffff), 2517 .hdr.sequence = RTE_BE16(0xffff), 2518 }; 2519 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2520 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : 2521 MLX5_FLOW_LAYER_OUTER_L3_IPV6; 2522 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2523 MLX5_FLOW_LAYER_OUTER_L4; 2524 int ret; 2525 2526 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMPV6) 2527 return rte_flow_error_set(error, EINVAL, 2528 RTE_FLOW_ERROR_TYPE_ITEM, item, 2529 "protocol filtering not compatible" 2530 " with ICMP6 layer"); 2531 if (!(item_flags & l3m)) 2532 return rte_flow_error_set(error, EINVAL, 2533 RTE_FLOW_ERROR_TYPE_ITEM, item, 2534 "IPv6 is mandatory to filter on" 2535 " ICMP6"); 2536 if (item_flags & l4m) 2537 return rte_flow_error_set(error, EINVAL, 2538 RTE_FLOW_ERROR_TYPE_ITEM, item, 2539 "multiple L4 layers not supported"); 2540 if (!mask) 2541 mask = &nic_mask; 2542 ret = mlx5_flow_item_acceptable 2543 (item, (const uint8_t *)mask, 2544 (const uint8_t *)&nic_mask, 2545 sizeof(struct rte_flow_item_icmp6_echo), 2546 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2547 if (ret < 0) 2548 return ret; 2549 return 0; 2550 } 2551 2552 /** 2553 * Validate ICMP item. 2554 * 2555 * @param[in] item 2556 * Item specification. 2557 * @param[in] item_flags 2558 * Bit-fields that holds the items detected until now. 2559 * @param[out] error 2560 * Pointer to error structure. 2561 * 2562 * @return 2563 * 0 on success, a negative errno value otherwise and rte_errno is set. 2564 */ 2565 int 2566 mlx5_flow_validate_item_icmp(const struct rte_flow_item *item, 2567 uint64_t item_flags, 2568 uint8_t target_protocol, 2569 struct rte_flow_error *error) 2570 { 2571 const struct rte_flow_item_icmp *mask = item->mask; 2572 const struct rte_flow_item_icmp nic_mask = { 2573 .hdr.icmp_type = 0xff, 2574 .hdr.icmp_code = 0xff, 2575 .hdr.icmp_ident = RTE_BE16(0xffff), 2576 .hdr.icmp_seq_nb = RTE_BE16(0xffff), 2577 }; 2578 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2579 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : 2580 MLX5_FLOW_LAYER_OUTER_L3_IPV4; 2581 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2582 MLX5_FLOW_LAYER_OUTER_L4; 2583 int ret; 2584 2585 if (target_protocol != 0xFF && target_protocol != IPPROTO_ICMP) 2586 return rte_flow_error_set(error, EINVAL, 2587 RTE_FLOW_ERROR_TYPE_ITEM, item, 2588 "protocol filtering not compatible" 2589 " with ICMP layer"); 2590 if (!(item_flags & l3m)) 2591 return rte_flow_error_set(error, EINVAL, 2592 RTE_FLOW_ERROR_TYPE_ITEM, item, 2593 "IPv4 is mandatory to filter" 2594 " on ICMP"); 2595 if (item_flags & l4m) 2596 return rte_flow_error_set(error, EINVAL, 2597 RTE_FLOW_ERROR_TYPE_ITEM, item, 2598 "multiple L4 layers not supported"); 2599 if (!mask) 2600 mask = &nic_mask; 2601 ret = mlx5_flow_item_acceptable 2602 (item, (const uint8_t *)mask, 2603 (const uint8_t *)&nic_mask, 2604 sizeof(struct rte_flow_item_icmp), 2605 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2606 if (ret < 0) 2607 return ret; 2608 return 0; 2609 } 2610 2611 /** 2612 * Validate Ethernet item. 2613 * 2614 * @param[in] item 2615 * Item specification. 2616 * @param[in] item_flags 2617 * Bit-fields that holds the items detected until now. 2618 * @param[out] error 2619 * Pointer to error structure. 2620 * 2621 * @return 2622 * 0 on success, a negative errno value otherwise and rte_errno is set. 2623 */ 2624 int 2625 mlx5_flow_validate_item_eth(const struct rte_flow_item *item, 2626 uint64_t item_flags, bool ext_vlan_sup, 2627 struct rte_flow_error *error) 2628 { 2629 const struct rte_flow_item_eth *mask = item->mask; 2630 const struct rte_flow_item_eth nic_mask = { 2631 .hdr.dst_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2632 .hdr.src_addr.addr_bytes = "\xff\xff\xff\xff\xff\xff", 2633 .hdr.ether_type = RTE_BE16(0xffff), 2634 .has_vlan = ext_vlan_sup ? 1 : 0, 2635 }; 2636 int ret; 2637 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2638 const uint64_t ethm = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : 2639 MLX5_FLOW_LAYER_OUTER_L2; 2640 2641 if (item_flags & ethm) 2642 return rte_flow_error_set(error, ENOTSUP, 2643 RTE_FLOW_ERROR_TYPE_ITEM, item, 2644 "multiple L2 layers not supported"); 2645 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_L3)) || 2646 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_L3))) 2647 return rte_flow_error_set(error, EINVAL, 2648 RTE_FLOW_ERROR_TYPE_ITEM, item, 2649 "L2 layer should not follow " 2650 "L3 layers"); 2651 if ((!tunnel && (item_flags & MLX5_FLOW_LAYER_OUTER_VLAN)) || 2652 (tunnel && (item_flags & MLX5_FLOW_LAYER_INNER_VLAN))) 2653 return rte_flow_error_set(error, EINVAL, 2654 RTE_FLOW_ERROR_TYPE_ITEM, item, 2655 "L2 layer should not follow VLAN"); 2656 if (item_flags & MLX5_FLOW_LAYER_GTP) 2657 return rte_flow_error_set(error, EINVAL, 2658 RTE_FLOW_ERROR_TYPE_ITEM, item, 2659 "L2 layer should not follow GTP"); 2660 if (!mask) 2661 mask = &rte_flow_item_eth_mask; 2662 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2663 (const uint8_t *)&nic_mask, 2664 sizeof(struct rte_flow_item_eth), 2665 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2666 return ret; 2667 } 2668 2669 /** 2670 * Validate VLAN item. 2671 * 2672 * @param[in] item 2673 * Item specification. 2674 * @param[in] item_flags 2675 * Bit-fields that holds the items detected until now. 2676 * @param[in] dev 2677 * Ethernet device flow is being created on. 2678 * @param[out] error 2679 * Pointer to error structure. 2680 * 2681 * @return 2682 * 0 on success, a negative errno value otherwise and rte_errno is set. 2683 */ 2684 int 2685 mlx5_flow_validate_item_vlan(const struct rte_flow_item *item, 2686 uint64_t item_flags, 2687 struct rte_eth_dev *dev, 2688 struct rte_flow_error *error) 2689 { 2690 const struct rte_flow_item_vlan *spec = item->spec; 2691 const struct rte_flow_item_vlan *mask = item->mask; 2692 const struct rte_flow_item_vlan nic_mask = { 2693 .hdr.vlan_tci = RTE_BE16(UINT16_MAX), 2694 .hdr.eth_proto = RTE_BE16(UINT16_MAX), 2695 }; 2696 uint16_t vlan_tag = 0; 2697 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2698 int ret; 2699 const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 | 2700 MLX5_FLOW_LAYER_INNER_L4) : 2701 (MLX5_FLOW_LAYER_OUTER_L3 | 2702 MLX5_FLOW_LAYER_OUTER_L4); 2703 const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN : 2704 MLX5_FLOW_LAYER_OUTER_VLAN; 2705 2706 if (item_flags & vlanm) 2707 return rte_flow_error_set(error, EINVAL, 2708 RTE_FLOW_ERROR_TYPE_ITEM, item, 2709 "multiple VLAN layers not supported"); 2710 else if ((item_flags & l34m) != 0) 2711 return rte_flow_error_set(error, EINVAL, 2712 RTE_FLOW_ERROR_TYPE_ITEM, item, 2713 "VLAN cannot follow L3/L4 layer"); 2714 if (!mask) 2715 mask = &rte_flow_item_vlan_mask; 2716 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2717 (const uint8_t *)&nic_mask, 2718 sizeof(struct rte_flow_item_vlan), 2719 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2720 if (ret) 2721 return ret; 2722 if (!tunnel && mask->hdr.vlan_tci != RTE_BE16(0x0fff)) { 2723 struct mlx5_priv *priv = dev->data->dev_private; 2724 2725 if (priv->vmwa_context) { 2726 /* 2727 * Non-NULL context means we have a virtual machine 2728 * and SR-IOV enabled, we have to create VLAN interface 2729 * to make hypervisor to setup E-Switch vport 2730 * context correctly. We avoid creating the multiple 2731 * VLAN interfaces, so we cannot support VLAN tag mask. 2732 */ 2733 return rte_flow_error_set(error, EINVAL, 2734 RTE_FLOW_ERROR_TYPE_ITEM, 2735 item, 2736 "VLAN tag mask is not" 2737 " supported in virtual" 2738 " environment"); 2739 } 2740 } 2741 if (spec) { 2742 vlan_tag = spec->hdr.vlan_tci; 2743 vlan_tag &= mask->hdr.vlan_tci; 2744 } 2745 /* 2746 * From verbs perspective an empty VLAN is equivalent 2747 * to a packet without VLAN layer. 2748 */ 2749 if (!vlan_tag) 2750 return rte_flow_error_set(error, EINVAL, 2751 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 2752 item->spec, 2753 "VLAN cannot be empty"); 2754 return 0; 2755 } 2756 2757 /** 2758 * Validate IPV4 item. 2759 * 2760 * @param[in] item 2761 * Item specification. 2762 * @param[in] item_flags 2763 * Bit-fields that holds the items detected until now. 2764 * @param[in] last_item 2765 * Previous validated item in the pattern items. 2766 * @param[in] ether_type 2767 * Type in the ethernet layer header (including dot1q). 2768 * @param[in] acc_mask 2769 * Acceptable mask, if NULL default internal default mask 2770 * will be used to check whether item fields are supported. 2771 * @param[in] range_accepted 2772 * True if range of values is accepted for specific fields, false otherwise. 2773 * @param[out] error 2774 * Pointer to error structure. 2775 * 2776 * @return 2777 * 0 on success, a negative errno value otherwise and rte_errno is set. 2778 */ 2779 int 2780 mlx5_flow_validate_item_ipv4(const struct rte_flow_item *item, 2781 uint64_t item_flags, 2782 uint64_t last_item, 2783 uint16_t ether_type, 2784 const struct rte_flow_item_ipv4 *acc_mask, 2785 bool range_accepted, 2786 struct rte_flow_error *error) 2787 { 2788 const struct rte_flow_item_ipv4 *mask = item->mask; 2789 const struct rte_flow_item_ipv4 *spec = item->spec; 2790 const struct rte_flow_item_ipv4 nic_mask = { 2791 .hdr = { 2792 .src_addr = RTE_BE32(0xffffffff), 2793 .dst_addr = RTE_BE32(0xffffffff), 2794 .type_of_service = 0xff, 2795 .next_proto_id = 0xff, 2796 }, 2797 }; 2798 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2799 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2800 MLX5_FLOW_LAYER_OUTER_L3; 2801 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2802 MLX5_FLOW_LAYER_OUTER_L4; 2803 int ret; 2804 uint8_t next_proto = 0xFF; 2805 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2806 MLX5_FLOW_LAYER_OUTER_VLAN | 2807 MLX5_FLOW_LAYER_INNER_VLAN); 2808 2809 if ((last_item & l2_vlan) && ether_type && 2810 ether_type != RTE_ETHER_TYPE_IPV4) 2811 return rte_flow_error_set(error, EINVAL, 2812 RTE_FLOW_ERROR_TYPE_ITEM, item, 2813 "IPv4 cannot follow L2/VLAN layer " 2814 "which ether type is not IPv4"); 2815 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2816 if (mask && spec) 2817 next_proto = mask->hdr.next_proto_id & 2818 spec->hdr.next_proto_id; 2819 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2820 return rte_flow_error_set(error, EINVAL, 2821 RTE_FLOW_ERROR_TYPE_ITEM, 2822 item, 2823 "multiple tunnel " 2824 "not supported"); 2825 } 2826 if (item_flags & MLX5_FLOW_LAYER_IPV6_ENCAP) 2827 return rte_flow_error_set(error, EINVAL, 2828 RTE_FLOW_ERROR_TYPE_ITEM, item, 2829 "wrong tunnel type - IPv6 specified " 2830 "but IPv4 item provided"); 2831 if (item_flags & l3m) 2832 return rte_flow_error_set(error, ENOTSUP, 2833 RTE_FLOW_ERROR_TYPE_ITEM, item, 2834 "multiple L3 layers not supported"); 2835 else if (item_flags & l4m) 2836 return rte_flow_error_set(error, EINVAL, 2837 RTE_FLOW_ERROR_TYPE_ITEM, item, 2838 "L3 cannot follow an L4 layer."); 2839 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2840 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2841 return rte_flow_error_set(error, EINVAL, 2842 RTE_FLOW_ERROR_TYPE_ITEM, item, 2843 "L3 cannot follow an NVGRE layer."); 2844 if (!mask) 2845 mask = &rte_flow_item_ipv4_mask; 2846 else if (mask->hdr.next_proto_id != 0 && 2847 mask->hdr.next_proto_id != 0xff) 2848 return rte_flow_error_set(error, EINVAL, 2849 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 2850 "partial mask is not supported" 2851 " for protocol"); 2852 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2853 acc_mask ? (const uint8_t *)acc_mask 2854 : (const uint8_t *)&nic_mask, 2855 sizeof(struct rte_flow_item_ipv4), 2856 range_accepted, error); 2857 if (ret < 0) 2858 return ret; 2859 return 0; 2860 } 2861 2862 /** 2863 * Validate IPV6 item. 2864 * 2865 * @param[in] item 2866 * Item specification. 2867 * @param[in] item_flags 2868 * Bit-fields that holds the items detected until now. 2869 * @param[in] last_item 2870 * Previous validated item in the pattern items. 2871 * @param[in] ether_type 2872 * Type in the ethernet layer header (including dot1q). 2873 * @param[in] acc_mask 2874 * Acceptable mask, if NULL default internal default mask 2875 * will be used to check whether item fields are supported. 2876 * @param[out] error 2877 * Pointer to error structure. 2878 * 2879 * @return 2880 * 0 on success, a negative errno value otherwise and rte_errno is set. 2881 */ 2882 int 2883 mlx5_flow_validate_item_ipv6(const struct rte_flow_item *item, 2884 uint64_t item_flags, 2885 uint64_t last_item, 2886 uint16_t ether_type, 2887 const struct rte_flow_item_ipv6 *acc_mask, 2888 struct rte_flow_error *error) 2889 { 2890 const struct rte_flow_item_ipv6 *mask = item->mask; 2891 const struct rte_flow_item_ipv6 *spec = item->spec; 2892 const struct rte_flow_item_ipv6 nic_mask = { 2893 .hdr = { 2894 .src_addr = 2895 "\xff\xff\xff\xff\xff\xff\xff\xff" 2896 "\xff\xff\xff\xff\xff\xff\xff\xff", 2897 .dst_addr = 2898 "\xff\xff\xff\xff\xff\xff\xff\xff" 2899 "\xff\xff\xff\xff\xff\xff\xff\xff", 2900 .vtc_flow = RTE_BE32(0xffffffff), 2901 .proto = 0xff, 2902 }, 2903 }; 2904 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2905 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2906 MLX5_FLOW_LAYER_OUTER_L3; 2907 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2908 MLX5_FLOW_LAYER_OUTER_L4; 2909 int ret; 2910 uint8_t next_proto = 0xFF; 2911 const uint64_t l2_vlan = (MLX5_FLOW_LAYER_L2 | 2912 MLX5_FLOW_LAYER_OUTER_VLAN | 2913 MLX5_FLOW_LAYER_INNER_VLAN); 2914 2915 if ((last_item & l2_vlan) && ether_type && 2916 ether_type != RTE_ETHER_TYPE_IPV6) 2917 return rte_flow_error_set(error, EINVAL, 2918 RTE_FLOW_ERROR_TYPE_ITEM, item, 2919 "IPv6 cannot follow L2/VLAN layer " 2920 "which ether type is not IPv6"); 2921 if (mask && mask->hdr.proto == UINT8_MAX && spec) 2922 next_proto = spec->hdr.proto; 2923 if (item_flags & MLX5_FLOW_LAYER_IPIP) { 2924 if (next_proto == IPPROTO_IPIP || next_proto == IPPROTO_IPV6) 2925 return rte_flow_error_set(error, EINVAL, 2926 RTE_FLOW_ERROR_TYPE_ITEM, 2927 item, 2928 "multiple tunnel " 2929 "not supported"); 2930 } 2931 if (next_proto == IPPROTO_HOPOPTS || 2932 next_proto == IPPROTO_ROUTING || 2933 next_proto == IPPROTO_FRAGMENT || 2934 next_proto == IPPROTO_ESP || 2935 next_proto == IPPROTO_AH || 2936 next_proto == IPPROTO_DSTOPTS) 2937 return rte_flow_error_set(error, EINVAL, 2938 RTE_FLOW_ERROR_TYPE_ITEM, item, 2939 "IPv6 proto (next header) should " 2940 "not be set as extension header"); 2941 if (item_flags & MLX5_FLOW_LAYER_IPIP) 2942 return rte_flow_error_set(error, EINVAL, 2943 RTE_FLOW_ERROR_TYPE_ITEM, item, 2944 "wrong tunnel type - IPv4 specified " 2945 "but IPv6 item provided"); 2946 if (item_flags & l3m) 2947 return rte_flow_error_set(error, ENOTSUP, 2948 RTE_FLOW_ERROR_TYPE_ITEM, item, 2949 "multiple L3 layers not supported"); 2950 else if (item_flags & l4m) 2951 return rte_flow_error_set(error, EINVAL, 2952 RTE_FLOW_ERROR_TYPE_ITEM, item, 2953 "L3 cannot follow an L4 layer."); 2954 else if ((item_flags & MLX5_FLOW_LAYER_NVGRE) && 2955 !(item_flags & MLX5_FLOW_LAYER_INNER_L2)) 2956 return rte_flow_error_set(error, EINVAL, 2957 RTE_FLOW_ERROR_TYPE_ITEM, item, 2958 "L3 cannot follow an NVGRE layer."); 2959 if (!mask) 2960 mask = &rte_flow_item_ipv6_mask; 2961 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 2962 acc_mask ? (const uint8_t *)acc_mask 2963 : (const uint8_t *)&nic_mask, 2964 sizeof(struct rte_flow_item_ipv6), 2965 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 2966 if (ret < 0) 2967 return ret; 2968 return 0; 2969 } 2970 2971 /** 2972 * Validate UDP item. 2973 * 2974 * @param[in] item 2975 * Item specification. 2976 * @param[in] item_flags 2977 * Bit-fields that holds the items detected until now. 2978 * @param[in] target_protocol 2979 * The next protocol in the previous item. 2980 * @param[in] flow_mask 2981 * mlx5 flow-specific (DV, verbs, etc.) supported header fields mask. 2982 * @param[out] error 2983 * Pointer to error structure. 2984 * 2985 * @return 2986 * 0 on success, a negative errno value otherwise and rte_errno is set. 2987 */ 2988 int 2989 mlx5_flow_validate_item_udp(const struct rte_flow_item *item, 2990 uint64_t item_flags, 2991 uint8_t target_protocol, 2992 struct rte_flow_error *error) 2993 { 2994 const struct rte_flow_item_udp *mask = item->mask; 2995 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 2996 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 2997 MLX5_FLOW_LAYER_OUTER_L3; 2998 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 2999 MLX5_FLOW_LAYER_OUTER_L4; 3000 int ret; 3001 3002 if (target_protocol != 0xff && target_protocol != IPPROTO_UDP) 3003 return rte_flow_error_set(error, EINVAL, 3004 RTE_FLOW_ERROR_TYPE_ITEM, item, 3005 "protocol filtering not compatible" 3006 " with UDP layer"); 3007 if (!(item_flags & l3m)) 3008 return rte_flow_error_set(error, EINVAL, 3009 RTE_FLOW_ERROR_TYPE_ITEM, item, 3010 "L3 is mandatory to filter on L4"); 3011 if (item_flags & l4m) 3012 return rte_flow_error_set(error, EINVAL, 3013 RTE_FLOW_ERROR_TYPE_ITEM, item, 3014 "multiple L4 layers not supported"); 3015 if (!mask) 3016 mask = &rte_flow_item_udp_mask; 3017 ret = mlx5_flow_item_acceptable 3018 (item, (const uint8_t *)mask, 3019 (const uint8_t *)&rte_flow_item_udp_mask, 3020 sizeof(struct rte_flow_item_udp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3021 error); 3022 if (ret < 0) 3023 return ret; 3024 return 0; 3025 } 3026 3027 /** 3028 * Validate TCP item. 3029 * 3030 * @param[in] item 3031 * Item specification. 3032 * @param[in] item_flags 3033 * Bit-fields that holds the items detected until now. 3034 * @param[in] target_protocol 3035 * The next protocol in the previous item. 3036 * @param[out] error 3037 * Pointer to error structure. 3038 * 3039 * @return 3040 * 0 on success, a negative errno value otherwise and rte_errno is set. 3041 */ 3042 int 3043 mlx5_flow_validate_item_tcp(const struct rte_flow_item *item, 3044 uint64_t item_flags, 3045 uint8_t target_protocol, 3046 const struct rte_flow_item_tcp *flow_mask, 3047 struct rte_flow_error *error) 3048 { 3049 const struct rte_flow_item_tcp *mask = item->mask; 3050 const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); 3051 const uint64_t l3m = tunnel ? MLX5_FLOW_LAYER_INNER_L3 : 3052 MLX5_FLOW_LAYER_OUTER_L3; 3053 const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 : 3054 MLX5_FLOW_LAYER_OUTER_L4; 3055 int ret; 3056 3057 MLX5_ASSERT(flow_mask); 3058 if (target_protocol != 0xff && target_protocol != IPPROTO_TCP) 3059 return rte_flow_error_set(error, EINVAL, 3060 RTE_FLOW_ERROR_TYPE_ITEM, item, 3061 "protocol filtering not compatible" 3062 " with TCP layer"); 3063 if (!(item_flags & l3m)) 3064 return rte_flow_error_set(error, EINVAL, 3065 RTE_FLOW_ERROR_TYPE_ITEM, item, 3066 "L3 is mandatory to filter on L4"); 3067 if (item_flags & l4m) 3068 return rte_flow_error_set(error, EINVAL, 3069 RTE_FLOW_ERROR_TYPE_ITEM, item, 3070 "multiple L4 layers not supported"); 3071 if (!mask) 3072 mask = &rte_flow_item_tcp_mask; 3073 ret = mlx5_flow_item_acceptable 3074 (item, (const uint8_t *)mask, 3075 (const uint8_t *)flow_mask, 3076 sizeof(struct rte_flow_item_tcp), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3077 error); 3078 if (ret < 0) 3079 return ret; 3080 return 0; 3081 } 3082 3083 /** 3084 * Validate VXLAN item. 3085 * 3086 * @param[in] dev 3087 * Pointer to the Ethernet device structure. 3088 * @param[in] udp_dport 3089 * UDP destination port 3090 * @param[in] item 3091 * Item specification. 3092 * @param[in] item_flags 3093 * Bit-fields that holds the items detected until now. 3094 * @param root 3095 * Whether action is on root table. 3096 * @param[out] error 3097 * Pointer to error structure. 3098 * 3099 * @return 3100 * 0 on success, a negative errno value otherwise and rte_errno is set. 3101 */ 3102 int 3103 mlx5_flow_validate_item_vxlan(struct rte_eth_dev *dev, 3104 uint16_t udp_dport, 3105 const struct rte_flow_item *item, 3106 uint64_t item_flags, 3107 bool root, 3108 struct rte_flow_error *error) 3109 { 3110 const struct rte_flow_item_vxlan *spec = item->spec; 3111 const struct rte_flow_item_vxlan *mask = item->mask; 3112 int ret; 3113 struct mlx5_priv *priv = dev->data->dev_private; 3114 union vni { 3115 uint32_t vlan_id; 3116 uint8_t vni[4]; 3117 } id = { .vlan_id = 0, }; 3118 const struct rte_flow_item_vxlan nic_mask = { 3119 .hdr.vni = "\xff\xff\xff", 3120 .hdr.rsvd1 = 0xff, 3121 }; 3122 const struct rte_flow_item_vxlan *valid_mask; 3123 3124 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3125 return rte_flow_error_set(error, ENOTSUP, 3126 RTE_FLOW_ERROR_TYPE_ITEM, item, 3127 "multiple tunnel layers not" 3128 " supported"); 3129 valid_mask = &rte_flow_item_vxlan_mask; 3130 /* 3131 * Verify only UDPv4 is present as defined in 3132 * https://tools.ietf.org/html/rfc7348 3133 */ 3134 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3135 return rte_flow_error_set(error, EINVAL, 3136 RTE_FLOW_ERROR_TYPE_ITEM, item, 3137 "no outer UDP layer found"); 3138 if (!mask) 3139 mask = &rte_flow_item_vxlan_mask; 3140 3141 if (priv->sh->steering_format_version != 3142 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3143 !udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN) { 3144 /* non-root table */ 3145 if (!root && priv->sh->misc5_cap) 3146 valid_mask = &nic_mask; 3147 /* Group zero in NIC domain */ 3148 if (!root && priv->sh->tunnel_header_0_1) 3149 valid_mask = &nic_mask; 3150 } 3151 ret = mlx5_flow_item_acceptable 3152 (item, (const uint8_t *)mask, 3153 (const uint8_t *)valid_mask, 3154 sizeof(struct rte_flow_item_vxlan), 3155 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3156 if (ret < 0) 3157 return ret; 3158 if (spec) { 3159 memcpy(&id.vni[1], spec->hdr.vni, 3); 3160 memcpy(&id.vni[1], mask->hdr.vni, 3); 3161 } 3162 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3163 return rte_flow_error_set(error, ENOTSUP, 3164 RTE_FLOW_ERROR_TYPE_ITEM, item, 3165 "VXLAN tunnel must be fully defined"); 3166 return 0; 3167 } 3168 3169 /** 3170 * Validate VXLAN_GPE item. 3171 * 3172 * @param[in] item 3173 * Item specification. 3174 * @param[in] item_flags 3175 * Bit-fields that holds the items detected until now. 3176 * @param[in] priv 3177 * Pointer to the private data structure. 3178 * @param[in] target_protocol 3179 * The next protocol in the previous item. 3180 * @param[out] error 3181 * Pointer to error structure. 3182 * 3183 * @return 3184 * 0 on success, a negative errno value otherwise and rte_errno is set. 3185 */ 3186 int 3187 mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, 3188 uint64_t item_flags, 3189 struct rte_eth_dev *dev, 3190 struct rte_flow_error *error) 3191 { 3192 struct mlx5_priv *priv = dev->data->dev_private; 3193 const struct rte_flow_item_vxlan_gpe *spec = item->spec; 3194 const struct rte_flow_item_vxlan_gpe *mask = item->mask; 3195 int ret; 3196 union vni { 3197 uint32_t vlan_id; 3198 uint8_t vni[4]; 3199 } id = { .vlan_id = 0, }; 3200 3201 struct rte_flow_item_vxlan_gpe nic_mask = { 3202 .vni = "\xff\xff\xff", 3203 .protocol = 0xff, 3204 }; 3205 3206 if (!priv->sh->config.l3_vxlan_en) 3207 return rte_flow_error_set(error, ENOTSUP, 3208 RTE_FLOW_ERROR_TYPE_ITEM, item, 3209 "L3 VXLAN is not enabled by device" 3210 " parameter and/or not configured in" 3211 " firmware"); 3212 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3213 return rte_flow_error_set(error, ENOTSUP, 3214 RTE_FLOW_ERROR_TYPE_ITEM, item, 3215 "multiple tunnel layers not" 3216 " supported"); 3217 /* 3218 * Verify only UDPv4 is present as defined in 3219 * https://tools.ietf.org/html/rfc7348 3220 */ 3221 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3222 return rte_flow_error_set(error, EINVAL, 3223 RTE_FLOW_ERROR_TYPE_ITEM, item, 3224 "no outer UDP layer found"); 3225 if (!mask) 3226 mask = &rte_flow_item_vxlan_gpe_mask; 3227 ret = mlx5_flow_item_acceptable 3228 (item, (const uint8_t *)mask, 3229 (const uint8_t *)&nic_mask, 3230 sizeof(struct rte_flow_item_vxlan_gpe), 3231 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3232 if (ret < 0) 3233 return ret; 3234 if (spec) { 3235 memcpy(&id.vni[1], spec->hdr.vni, 3); 3236 memcpy(&id.vni[1], mask->hdr.vni, 3); 3237 } 3238 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3239 return rte_flow_error_set(error, ENOTSUP, 3240 RTE_FLOW_ERROR_TYPE_ITEM, item, 3241 "VXLAN-GPE tunnel must be fully" 3242 " defined"); 3243 return 0; 3244 } 3245 /** 3246 * Validate GRE Key item. 3247 * 3248 * @param[in] item 3249 * Item specification. 3250 * @param[in] item_flags 3251 * Bit flags to mark detected items. 3252 * @param[in] gre_item 3253 * Pointer to gre_item 3254 * @param[out] error 3255 * Pointer to error structure. 3256 * 3257 * @return 3258 * 0 on success, a negative errno value otherwise and rte_errno is set. 3259 */ 3260 int 3261 mlx5_flow_validate_item_gre_key(const struct rte_flow_item *item, 3262 uint64_t item_flags, 3263 const struct rte_flow_item *gre_item, 3264 struct rte_flow_error *error) 3265 { 3266 const rte_be32_t *mask = item->mask; 3267 int ret = 0; 3268 rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX); 3269 const struct rte_flow_item_gre *gre_spec; 3270 const struct rte_flow_item_gre *gre_mask; 3271 3272 if (item_flags & MLX5_FLOW_LAYER_GRE_KEY) 3273 return rte_flow_error_set(error, ENOTSUP, 3274 RTE_FLOW_ERROR_TYPE_ITEM, item, 3275 "Multiple GRE key not support"); 3276 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3277 return rte_flow_error_set(error, ENOTSUP, 3278 RTE_FLOW_ERROR_TYPE_ITEM, item, 3279 "No preceding GRE header"); 3280 if (item_flags & MLX5_FLOW_LAYER_INNER) 3281 return rte_flow_error_set(error, ENOTSUP, 3282 RTE_FLOW_ERROR_TYPE_ITEM, item, 3283 "GRE key following a wrong item"); 3284 gre_mask = gre_item->mask; 3285 if (!gre_mask) 3286 gre_mask = &rte_flow_item_gre_mask; 3287 gre_spec = gre_item->spec; 3288 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3289 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3290 return rte_flow_error_set(error, EINVAL, 3291 RTE_FLOW_ERROR_TYPE_ITEM, item, 3292 "Key bit must be on"); 3293 3294 if (!mask) 3295 mask = &gre_key_default_mask; 3296 ret = mlx5_flow_item_acceptable 3297 (item, (const uint8_t *)mask, 3298 (const uint8_t *)&gre_key_default_mask, 3299 sizeof(rte_be32_t), MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3300 return ret; 3301 } 3302 3303 /** 3304 * Validate GRE optional item. 3305 * 3306 * @param[in] dev 3307 * Pointer to the Ethernet device structure. 3308 * @param[in] item 3309 * Item specification. 3310 * @param[in] item_flags 3311 * Bit flags to mark detected items. 3312 * @param[in] attr 3313 * Flow rule attributes. 3314 * @param[in] gre_item 3315 * Pointer to gre_item 3316 * @param[out] error 3317 * Pointer to error structure. 3318 * 3319 * @return 3320 * 0 on success, a negative errno value otherwise and rte_errno is set. 3321 */ 3322 int 3323 mlx5_flow_validate_item_gre_option(struct rte_eth_dev *dev, 3324 const struct rte_flow_item *item, 3325 uint64_t item_flags, 3326 const struct rte_flow_attr *attr, 3327 const struct rte_flow_item *gre_item, 3328 struct rte_flow_error *error) 3329 { 3330 const struct rte_flow_item_gre *gre_spec = gre_item->spec; 3331 const struct rte_flow_item_gre *gre_mask = gre_item->mask; 3332 const struct rte_flow_item_gre_opt *spec = item->spec; 3333 const struct rte_flow_item_gre_opt *mask = item->mask; 3334 struct mlx5_priv *priv = dev->data->dev_private; 3335 int ret = 0; 3336 struct rte_flow_item_gre_opt nic_mask = { 3337 .checksum_rsvd = { 3338 .checksum = RTE_BE16(UINT16_MAX), 3339 .reserved1 = 0x0, 3340 }, 3341 .key = { 3342 .key = RTE_BE32(UINT32_MAX), 3343 }, 3344 .sequence = { 3345 .sequence = RTE_BE32(UINT32_MAX), 3346 }, 3347 }; 3348 3349 if (!(item_flags & MLX5_FLOW_LAYER_GRE)) 3350 return rte_flow_error_set(error, ENOTSUP, 3351 RTE_FLOW_ERROR_TYPE_ITEM, item, 3352 "No preceding GRE header"); 3353 if (item_flags & MLX5_FLOW_LAYER_INNER) 3354 return rte_flow_error_set(error, ENOTSUP, 3355 RTE_FLOW_ERROR_TYPE_ITEM, item, 3356 "GRE option following a wrong item"); 3357 if (!spec || !mask) 3358 return rte_flow_error_set(error, EINVAL, 3359 RTE_FLOW_ERROR_TYPE_ITEM, item, 3360 "At least one field gre_option(checksum/key/sequence) must be specified"); 3361 if (!gre_mask) 3362 gre_mask = &rte_flow_item_gre_mask; 3363 if (mask->checksum_rsvd.checksum) 3364 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x8000)) && 3365 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x8000))) 3366 return rte_flow_error_set(error, EINVAL, 3367 RTE_FLOW_ERROR_TYPE_ITEM, 3368 item, 3369 "Checksum bit must be on"); 3370 if (mask->key.key) 3371 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x2000)) && 3372 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x2000))) 3373 return rte_flow_error_set(error, EINVAL, 3374 RTE_FLOW_ERROR_TYPE_ITEM, 3375 item, "Key bit must be on"); 3376 if (mask->sequence.sequence) 3377 if (gre_spec && (gre_mask->c_rsvd0_ver & RTE_BE16(0x1000)) && 3378 !(gre_spec->c_rsvd0_ver & RTE_BE16(0x1000))) 3379 return rte_flow_error_set(error, EINVAL, 3380 RTE_FLOW_ERROR_TYPE_ITEM, 3381 item, 3382 "Sequence bit must be on"); 3383 if (mask->checksum_rsvd.checksum || mask->sequence.sequence) { 3384 if (priv->sh->steering_format_version == 3385 MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 || 3386 ((attr->group || (attr->transfer && priv->fdb_def_rule)) && 3387 !priv->sh->misc5_cap) || 3388 (!(priv->sh->tunnel_header_0_1 && 3389 priv->sh->tunnel_header_2_3) && 3390 !attr->group && (!attr->transfer || !priv->fdb_def_rule))) 3391 return rte_flow_error_set(error, EINVAL, 3392 RTE_FLOW_ERROR_TYPE_ITEM, 3393 item, 3394 "Checksum/Sequence not supported"); 3395 } 3396 ret = mlx5_flow_item_acceptable 3397 (item, (const uint8_t *)mask, 3398 (const uint8_t *)&nic_mask, 3399 sizeof(struct rte_flow_item_gre_opt), 3400 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3401 return ret; 3402 } 3403 3404 /** 3405 * Validate GRE item. 3406 * 3407 * @param[in] item 3408 * Item specification. 3409 * @param[in] item_flags 3410 * Bit flags to mark detected items. 3411 * @param[in] target_protocol 3412 * The next protocol in the previous item. 3413 * @param[out] error 3414 * Pointer to error structure. 3415 * 3416 * @return 3417 * 0 on success, a negative errno value otherwise and rte_errno is set. 3418 */ 3419 int 3420 mlx5_flow_validate_item_gre(const struct rte_flow_item *item, 3421 uint64_t item_flags, 3422 uint8_t target_protocol, 3423 struct rte_flow_error *error) 3424 { 3425 const struct rte_flow_item_gre *spec __rte_unused = item->spec; 3426 const struct rte_flow_item_gre *mask = item->mask; 3427 int ret; 3428 const struct rte_flow_item_gre nic_mask = { 3429 .c_rsvd0_ver = RTE_BE16(0xB000), 3430 .protocol = RTE_BE16(UINT16_MAX), 3431 }; 3432 3433 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3434 return rte_flow_error_set(error, EINVAL, 3435 RTE_FLOW_ERROR_TYPE_ITEM, item, 3436 "protocol filtering not compatible" 3437 " with this GRE layer"); 3438 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3439 return rte_flow_error_set(error, ENOTSUP, 3440 RTE_FLOW_ERROR_TYPE_ITEM, item, 3441 "multiple tunnel layers not" 3442 " supported"); 3443 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3444 return rte_flow_error_set(error, ENOTSUP, 3445 RTE_FLOW_ERROR_TYPE_ITEM, item, 3446 "L3 Layer is missing"); 3447 if (!mask) 3448 mask = &rte_flow_item_gre_mask; 3449 ret = mlx5_flow_item_acceptable 3450 (item, (const uint8_t *)mask, 3451 (const uint8_t *)&nic_mask, 3452 sizeof(struct rte_flow_item_gre), MLX5_ITEM_RANGE_NOT_ACCEPTED, 3453 error); 3454 if (ret < 0) 3455 return ret; 3456 #ifndef HAVE_MLX5DV_DR 3457 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT 3458 if (spec && (spec->protocol & mask->protocol)) 3459 return rte_flow_error_set(error, ENOTSUP, 3460 RTE_FLOW_ERROR_TYPE_ITEM, item, 3461 "without MPLS support the" 3462 " specification cannot be used for" 3463 " filtering"); 3464 #endif 3465 #endif 3466 return 0; 3467 } 3468 3469 /** 3470 * Validate Geneve item. 3471 * 3472 * @param[in] item 3473 * Item specification. 3474 * @param[in] itemFlags 3475 * Bit-fields that holds the items detected until now. 3476 * @param[in] enPriv 3477 * Pointer to the private data structure. 3478 * @param[out] error 3479 * Pointer to error structure. 3480 * 3481 * @return 3482 * 0 on success, a negative errno value otherwise and rte_errno is set. 3483 */ 3484 3485 int 3486 mlx5_flow_validate_item_geneve(const struct rte_flow_item *item, 3487 uint64_t item_flags, 3488 struct rte_eth_dev *dev, 3489 struct rte_flow_error *error) 3490 { 3491 struct mlx5_priv *priv = dev->data->dev_private; 3492 const struct rte_flow_item_geneve *spec = item->spec; 3493 const struct rte_flow_item_geneve *mask = item->mask; 3494 int ret; 3495 uint16_t gbhdr; 3496 uint8_t opt_len = priv->sh->cdev->config.hca_attr.geneve_max_opt_len ? 3497 MLX5_GENEVE_OPT_LEN_1 : MLX5_GENEVE_OPT_LEN_0; 3498 const struct rte_flow_item_geneve nic_mask = { 3499 .ver_opt_len_o_c_rsvd0 = RTE_BE16(0x3f80), 3500 .vni = "\xff\xff\xff", 3501 .protocol = RTE_BE16(UINT16_MAX), 3502 }; 3503 3504 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_geneve_rx) 3505 return rte_flow_error_set(error, ENOTSUP, 3506 RTE_FLOW_ERROR_TYPE_ITEM, item, 3507 "L3 Geneve is not enabled by device" 3508 " parameter and/or not configured in" 3509 " firmware"); 3510 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3511 return rte_flow_error_set(error, ENOTSUP, 3512 RTE_FLOW_ERROR_TYPE_ITEM, item, 3513 "multiple tunnel layers not" 3514 " supported"); 3515 /* 3516 * Verify only UDPv4 is present as defined in 3517 * https://tools.ietf.org/html/rfc7348 3518 */ 3519 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP)) 3520 return rte_flow_error_set(error, EINVAL, 3521 RTE_FLOW_ERROR_TYPE_ITEM, item, 3522 "no outer UDP layer found"); 3523 if (!mask) 3524 mask = &rte_flow_item_geneve_mask; 3525 ret = mlx5_flow_item_acceptable 3526 (item, (const uint8_t *)mask, 3527 (const uint8_t *)&nic_mask, 3528 sizeof(struct rte_flow_item_geneve), 3529 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3530 if (ret) 3531 return ret; 3532 if (spec) { 3533 gbhdr = rte_be_to_cpu_16(spec->ver_opt_len_o_c_rsvd0); 3534 if (MLX5_GENEVE_VER_VAL(gbhdr) || 3535 MLX5_GENEVE_CRITO_VAL(gbhdr) || 3536 MLX5_GENEVE_RSVD_VAL(gbhdr) || spec->rsvd1) 3537 return rte_flow_error_set(error, ENOTSUP, 3538 RTE_FLOW_ERROR_TYPE_ITEM, 3539 item, 3540 "Geneve protocol unsupported" 3541 " fields are being used"); 3542 if (MLX5_GENEVE_OPTLEN_VAL(gbhdr) > opt_len) 3543 return rte_flow_error_set 3544 (error, ENOTSUP, 3545 RTE_FLOW_ERROR_TYPE_ITEM, 3546 item, 3547 "Unsupported Geneve options length"); 3548 } 3549 if (!(item_flags & MLX5_FLOW_LAYER_OUTER)) 3550 return rte_flow_error_set 3551 (error, ENOTSUP, 3552 RTE_FLOW_ERROR_TYPE_ITEM, item, 3553 "Geneve tunnel must be fully defined"); 3554 return 0; 3555 } 3556 3557 /** 3558 * Validate Geneve TLV option item. 3559 * 3560 * @param[in] item 3561 * Item specification. 3562 * @param[in] last_item 3563 * Previous validated item in the pattern items. 3564 * @param[in] geneve_item 3565 * Previous GENEVE item specification. 3566 * @param[in] dev 3567 * Pointer to the rte_eth_dev structure. 3568 * @param[out] error 3569 * Pointer to error structure. 3570 * 3571 * @return 3572 * 0 on success, a negative errno value otherwise and rte_errno is set. 3573 */ 3574 int 3575 mlx5_flow_validate_item_geneve_opt(const struct rte_flow_item *item, 3576 uint64_t last_item, 3577 const struct rte_flow_item *geneve_item, 3578 struct rte_eth_dev *dev, 3579 struct rte_flow_error *error) 3580 { 3581 struct mlx5_priv *priv = dev->data->dev_private; 3582 struct mlx5_dev_ctx_shared *sh = priv->sh; 3583 struct mlx5_geneve_tlv_option_resource *geneve_opt_resource; 3584 struct mlx5_hca_attr *hca_attr = &sh->cdev->config.hca_attr; 3585 uint8_t data_max_supported = 3586 hca_attr->max_geneve_tlv_option_data_len * 4; 3587 const struct rte_flow_item_geneve *geneve_spec; 3588 const struct rte_flow_item_geneve *geneve_mask; 3589 const struct rte_flow_item_geneve_opt *spec = item->spec; 3590 const struct rte_flow_item_geneve_opt *mask = item->mask; 3591 unsigned int i; 3592 unsigned int data_len; 3593 uint8_t tlv_option_len; 3594 uint16_t optlen_m, optlen_v; 3595 const struct rte_flow_item_geneve_opt full_mask = { 3596 .option_class = RTE_BE16(0xffff), 3597 .option_type = 0xff, 3598 .option_len = 0x1f, 3599 }; 3600 3601 if (!mask) 3602 mask = &rte_flow_item_geneve_opt_mask; 3603 if (!spec) 3604 return rte_flow_error_set 3605 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3606 "Geneve TLV opt class/type/length must be specified"); 3607 if ((uint32_t)spec->option_len > MLX5_GENEVE_OPTLEN_MASK) 3608 return rte_flow_error_set 3609 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3610 "Geneve TLV opt length exceeds the limit (31)"); 3611 /* Check if class type and length masks are full. */ 3612 if (full_mask.option_class != mask->option_class || 3613 full_mask.option_type != mask->option_type || 3614 full_mask.option_len != (mask->option_len & full_mask.option_len)) 3615 return rte_flow_error_set 3616 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3617 "Geneve TLV opt class/type/length masks must be full"); 3618 /* Check if length is supported */ 3619 if ((uint32_t)spec->option_len > 3620 hca_attr->max_geneve_tlv_option_data_len) 3621 return rte_flow_error_set 3622 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3623 "Geneve TLV opt length not supported"); 3624 if (hca_attr->max_geneve_tlv_options > 1) 3625 DRV_LOG(DEBUG, 3626 "max_geneve_tlv_options supports more than 1 option"); 3627 /* Check GENEVE item preceding. */ 3628 if (!geneve_item || !(last_item & MLX5_FLOW_LAYER_GENEVE)) 3629 return rte_flow_error_set 3630 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3631 "Geneve opt item must be preceded with Geneve item"); 3632 geneve_spec = geneve_item->spec; 3633 geneve_mask = geneve_item->mask ? geneve_item->mask : 3634 &rte_flow_item_geneve_mask; 3635 /* Check if GENEVE TLV option size doesn't exceed option length */ 3636 if (geneve_spec && (geneve_mask->ver_opt_len_o_c_rsvd0 || 3637 geneve_spec->ver_opt_len_o_c_rsvd0)) { 3638 tlv_option_len = spec->option_len & mask->option_len; 3639 optlen_v = rte_be_to_cpu_16(geneve_spec->ver_opt_len_o_c_rsvd0); 3640 optlen_v = MLX5_GENEVE_OPTLEN_VAL(optlen_v); 3641 optlen_m = rte_be_to_cpu_16(geneve_mask->ver_opt_len_o_c_rsvd0); 3642 optlen_m = MLX5_GENEVE_OPTLEN_VAL(optlen_m); 3643 if ((optlen_v & optlen_m) <= tlv_option_len) 3644 return rte_flow_error_set 3645 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3646 "GENEVE TLV option length exceeds optlen"); 3647 } 3648 /* Check if length is 0 or data is 0. */ 3649 if (spec->data == NULL || spec->option_len == 0) 3650 return rte_flow_error_set 3651 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3652 "Geneve TLV opt with zero data/length not supported"); 3653 /* Check not all data & mask are 0. */ 3654 data_len = spec->option_len * 4; 3655 if (mask->data == NULL) { 3656 for (i = 0; i < data_len; i++) 3657 if (spec->data[i]) 3658 break; 3659 if (i == data_len) 3660 return rte_flow_error_set(error, ENOTSUP, 3661 RTE_FLOW_ERROR_TYPE_ITEM, item, 3662 "Can't match on Geneve option data 0"); 3663 } else { 3664 for (i = 0; i < data_len; i++) 3665 if (spec->data[i] & mask->data[i]) 3666 break; 3667 if (i == data_len) 3668 return rte_flow_error_set(error, ENOTSUP, 3669 RTE_FLOW_ERROR_TYPE_ITEM, item, 3670 "Can't match on Geneve option data and mask 0"); 3671 /* Check data mask supported. */ 3672 for (i = data_max_supported; i < data_len ; i++) 3673 if (mask->data[i]) 3674 return rte_flow_error_set(error, ENOTSUP, 3675 RTE_FLOW_ERROR_TYPE_ITEM, item, 3676 "Data mask is of unsupported size"); 3677 } 3678 /* Check GENEVE option is supported in NIC. */ 3679 if (!hca_attr->geneve_tlv_opt) 3680 return rte_flow_error_set 3681 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, 3682 "Geneve TLV opt not supported"); 3683 /* Check if we already have geneve option with different type/class. */ 3684 rte_spinlock_lock(&sh->geneve_tlv_opt_sl); 3685 geneve_opt_resource = sh->geneve_tlv_option_resource; 3686 if (geneve_opt_resource != NULL) 3687 if (geneve_opt_resource->option_class != spec->option_class || 3688 geneve_opt_resource->option_type != spec->option_type || 3689 geneve_opt_resource->length != spec->option_len) { 3690 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3691 return rte_flow_error_set(error, ENOTSUP, 3692 RTE_FLOW_ERROR_TYPE_ITEM, item, 3693 "Only one Geneve TLV option supported"); 3694 } 3695 rte_spinlock_unlock(&sh->geneve_tlv_opt_sl); 3696 return 0; 3697 } 3698 3699 /** 3700 * Validate MPLS item. 3701 * 3702 * @param[in] dev 3703 * Pointer to the rte_eth_dev structure. 3704 * @param[in] item 3705 * Item specification. 3706 * @param[in] item_flags 3707 * Bit-fields that holds the items detected until now. 3708 * @param[in] prev_layer 3709 * The protocol layer indicated in previous item. 3710 * @param[out] error 3711 * Pointer to error structure. 3712 * 3713 * @return 3714 * 0 on success, a negative errno value otherwise and rte_errno is set. 3715 */ 3716 int 3717 mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, 3718 const struct rte_flow_item *item __rte_unused, 3719 uint64_t item_flags __rte_unused, 3720 uint64_t prev_layer __rte_unused, 3721 struct rte_flow_error *error) 3722 { 3723 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT 3724 const struct rte_flow_item_mpls *mask = item->mask; 3725 struct mlx5_priv *priv = dev->data->dev_private; 3726 int ret; 3727 3728 if (!priv->sh->dev_cap.mpls_en) 3729 return rte_flow_error_set(error, ENOTSUP, 3730 RTE_FLOW_ERROR_TYPE_ITEM, item, 3731 "MPLS not supported or" 3732 " disabled in firmware" 3733 " configuration."); 3734 /* MPLS over UDP, GRE is allowed */ 3735 if (!(prev_layer & (MLX5_FLOW_LAYER_OUTER_L4_UDP | 3736 MLX5_FLOW_LAYER_GRE | 3737 MLX5_FLOW_LAYER_GRE_KEY))) 3738 return rte_flow_error_set(error, EINVAL, 3739 RTE_FLOW_ERROR_TYPE_ITEM, item, 3740 "protocol filtering not compatible" 3741 " with MPLS layer"); 3742 /* Multi-tunnel isn't allowed but MPLS over GRE is an exception. */ 3743 if ((item_flags & MLX5_FLOW_LAYER_TUNNEL) && 3744 !(item_flags & MLX5_FLOW_LAYER_GRE)) 3745 return rte_flow_error_set(error, ENOTSUP, 3746 RTE_FLOW_ERROR_TYPE_ITEM, item, 3747 "multiple tunnel layers not" 3748 " supported"); 3749 if (!mask) 3750 mask = &rte_flow_item_mpls_mask; 3751 ret = mlx5_flow_item_acceptable 3752 (item, (const uint8_t *)mask, 3753 (const uint8_t *)&rte_flow_item_mpls_mask, 3754 sizeof(struct rte_flow_item_mpls), 3755 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3756 if (ret < 0) 3757 return ret; 3758 return 0; 3759 #else 3760 return rte_flow_error_set(error, ENOTSUP, 3761 RTE_FLOW_ERROR_TYPE_ITEM, item, 3762 "MPLS is not supported by Verbs, please" 3763 " update."); 3764 #endif 3765 } 3766 3767 /** 3768 * Validate NVGRE item. 3769 * 3770 * @param[in] item 3771 * Item specification. 3772 * @param[in] item_flags 3773 * Bit flags to mark detected items. 3774 * @param[in] target_protocol 3775 * The next protocol in the previous item. 3776 * @param[out] error 3777 * Pointer to error structure. 3778 * 3779 * @return 3780 * 0 on success, a negative errno value otherwise and rte_errno is set. 3781 */ 3782 int 3783 mlx5_flow_validate_item_nvgre(const struct rte_flow_item *item, 3784 uint64_t item_flags, 3785 uint8_t target_protocol, 3786 struct rte_flow_error *error) 3787 { 3788 const struct rte_flow_item_nvgre *mask = item->mask; 3789 int ret; 3790 3791 if (target_protocol != 0xff && target_protocol != IPPROTO_GRE) 3792 return rte_flow_error_set(error, EINVAL, 3793 RTE_FLOW_ERROR_TYPE_ITEM, item, 3794 "protocol filtering not compatible" 3795 " with this GRE layer"); 3796 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3797 return rte_flow_error_set(error, ENOTSUP, 3798 RTE_FLOW_ERROR_TYPE_ITEM, item, 3799 "multiple tunnel layers not" 3800 " supported"); 3801 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L3)) 3802 return rte_flow_error_set(error, ENOTSUP, 3803 RTE_FLOW_ERROR_TYPE_ITEM, item, 3804 "L3 Layer is missing"); 3805 if (!mask) 3806 mask = &rte_flow_item_nvgre_mask; 3807 ret = mlx5_flow_item_acceptable 3808 (item, (const uint8_t *)mask, 3809 (const uint8_t *)&rte_flow_item_nvgre_mask, 3810 sizeof(struct rte_flow_item_nvgre), 3811 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3812 if (ret < 0) 3813 return ret; 3814 return 0; 3815 } 3816 3817 /** 3818 * Validate eCPRI item. 3819 * 3820 * @param[in] item 3821 * Item specification. 3822 * @param[in] item_flags 3823 * Bit-fields that holds the items detected until now. 3824 * @param[in] last_item 3825 * Previous validated item in the pattern items. 3826 * @param[in] ether_type 3827 * Type in the ethernet layer header (including dot1q). 3828 * @param[in] acc_mask 3829 * Acceptable mask, if NULL default internal default mask 3830 * will be used to check whether item fields are supported. 3831 * @param[out] error 3832 * Pointer to error structure. 3833 * 3834 * @return 3835 * 0 on success, a negative errno value otherwise and rte_errno is set. 3836 */ 3837 int 3838 mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, 3839 uint64_t item_flags, 3840 uint64_t last_item, 3841 uint16_t ether_type, 3842 const struct rte_flow_item_ecpri *acc_mask, 3843 struct rte_flow_error *error) 3844 { 3845 const struct rte_flow_item_ecpri *mask = item->mask; 3846 const struct rte_flow_item_ecpri nic_mask = { 3847 .hdr = { 3848 .common = { 3849 .u32 = 3850 RTE_BE32(((const struct rte_ecpri_common_hdr) { 3851 .type = 0xFF, 3852 }).u32), 3853 }, 3854 .dummy[0] = 0xFFFFFFFF, 3855 }, 3856 }; 3857 const uint64_t outer_l2_vlan = (MLX5_FLOW_LAYER_OUTER_L2 | 3858 MLX5_FLOW_LAYER_OUTER_VLAN); 3859 struct rte_flow_item_ecpri mask_lo; 3860 3861 if (!(last_item & outer_l2_vlan) && 3862 last_item != MLX5_FLOW_LAYER_OUTER_L4_UDP) 3863 return rte_flow_error_set(error, EINVAL, 3864 RTE_FLOW_ERROR_TYPE_ITEM, item, 3865 "eCPRI can only follow L2/VLAN layer or UDP layer"); 3866 if ((last_item & outer_l2_vlan) && ether_type && 3867 ether_type != RTE_ETHER_TYPE_ECPRI) 3868 return rte_flow_error_set(error, EINVAL, 3869 RTE_FLOW_ERROR_TYPE_ITEM, item, 3870 "eCPRI cannot follow L2/VLAN layer which ether type is not 0xAEFE"); 3871 if (item_flags & MLX5_FLOW_LAYER_TUNNEL) 3872 return rte_flow_error_set(error, EINVAL, 3873 RTE_FLOW_ERROR_TYPE_ITEM, item, 3874 "eCPRI with tunnel is not supported right now"); 3875 if (item_flags & MLX5_FLOW_LAYER_OUTER_L3) 3876 return rte_flow_error_set(error, ENOTSUP, 3877 RTE_FLOW_ERROR_TYPE_ITEM, item, 3878 "multiple L3 layers not supported"); 3879 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP) 3880 return rte_flow_error_set(error, EINVAL, 3881 RTE_FLOW_ERROR_TYPE_ITEM, item, 3882 "eCPRI cannot coexist with a TCP layer"); 3883 /* In specification, eCPRI could be over UDP layer. */ 3884 else if (item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP) 3885 return rte_flow_error_set(error, EINVAL, 3886 RTE_FLOW_ERROR_TYPE_ITEM, item, 3887 "eCPRI over UDP layer is not yet supported right now"); 3888 /* Mask for type field in common header could be zero. */ 3889 if (!mask) 3890 mask = &rte_flow_item_ecpri_mask; 3891 mask_lo.hdr.common.u32 = rte_be_to_cpu_32(mask->hdr.common.u32); 3892 /* Input mask is in big-endian format. */ 3893 if (mask_lo.hdr.common.type != 0 && mask_lo.hdr.common.type != 0xff) 3894 return rte_flow_error_set(error, EINVAL, 3895 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3896 "partial mask is not supported for protocol"); 3897 else if (mask_lo.hdr.common.type == 0 && mask->hdr.dummy[0] != 0) 3898 return rte_flow_error_set(error, EINVAL, 3899 RTE_FLOW_ERROR_TYPE_ITEM_MASK, mask, 3900 "message header mask must be after a type mask"); 3901 return mlx5_flow_item_acceptable(item, (const uint8_t *)mask, 3902 acc_mask ? (const uint8_t *)acc_mask 3903 : (const uint8_t *)&nic_mask, 3904 sizeof(struct rte_flow_item_ecpri), 3905 MLX5_ITEM_RANGE_NOT_ACCEPTED, error); 3906 } 3907 3908 /** 3909 * Validate the NSH item. 3910 * 3911 * @param[in] dev 3912 * Pointer to Ethernet device on which flow rule is being created on. 3913 * @param[out] error 3914 * Pointer to error structure. 3915 * 3916 * @return 3917 * 0 on success, a negative errno value otherwise and rte_errno is set. 3918 */ 3919 int 3920 mlx5_flow_validate_item_nsh(struct rte_eth_dev *dev, 3921 const struct rte_flow_item *item, 3922 struct rte_flow_error *error) 3923 { 3924 struct mlx5_priv *priv = dev->data->dev_private; 3925 3926 if (item->mask) { 3927 return rte_flow_error_set(error, ENOTSUP, 3928 RTE_FLOW_ERROR_TYPE_ITEM, item, 3929 "NSH fields matching is not supported"); 3930 } 3931 3932 if (!priv->sh->config.dv_flow_en) { 3933 return rte_flow_error_set(error, ENOTSUP, 3934 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 3935 NULL, "NSH support requires DV flow interface"); 3936 } 3937 3938 if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_vxlan_gpe_nsh) { 3939 return rte_flow_error_set(error, ENOTSUP, 3940 RTE_FLOW_ERROR_TYPE_ITEM, item, 3941 "Current FW does not support matching on NSH"); 3942 } 3943 3944 return 0; 3945 } 3946 3947 static int 3948 flow_null_validate(struct rte_eth_dev *dev __rte_unused, 3949 const struct rte_flow_attr *attr __rte_unused, 3950 const struct rte_flow_item items[] __rte_unused, 3951 const struct rte_flow_action actions[] __rte_unused, 3952 bool external __rte_unused, 3953 int hairpin __rte_unused, 3954 struct rte_flow_error *error) 3955 { 3956 return rte_flow_error_set(error, ENOTSUP, 3957 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3958 } 3959 3960 static struct mlx5_flow * 3961 flow_null_prepare(struct rte_eth_dev *dev __rte_unused, 3962 const struct rte_flow_attr *attr __rte_unused, 3963 const struct rte_flow_item items[] __rte_unused, 3964 const struct rte_flow_action actions[] __rte_unused, 3965 struct rte_flow_error *error) 3966 { 3967 rte_flow_error_set(error, ENOTSUP, 3968 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3969 return NULL; 3970 } 3971 3972 static int 3973 flow_null_translate(struct rte_eth_dev *dev __rte_unused, 3974 struct mlx5_flow *dev_flow __rte_unused, 3975 const struct rte_flow_attr *attr __rte_unused, 3976 const struct rte_flow_item items[] __rte_unused, 3977 const struct rte_flow_action actions[] __rte_unused, 3978 struct rte_flow_error *error) 3979 { 3980 return rte_flow_error_set(error, ENOTSUP, 3981 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3982 } 3983 3984 static int 3985 flow_null_apply(struct rte_eth_dev *dev __rte_unused, 3986 struct rte_flow *flow __rte_unused, 3987 struct rte_flow_error *error) 3988 { 3989 return rte_flow_error_set(error, ENOTSUP, 3990 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 3991 } 3992 3993 static void 3994 flow_null_remove(struct rte_eth_dev *dev __rte_unused, 3995 struct rte_flow *flow __rte_unused) 3996 { 3997 } 3998 3999 static void 4000 flow_null_destroy(struct rte_eth_dev *dev __rte_unused, 4001 struct rte_flow *flow __rte_unused) 4002 { 4003 } 4004 4005 static int 4006 flow_null_query(struct rte_eth_dev *dev __rte_unused, 4007 struct rte_flow *flow __rte_unused, 4008 const struct rte_flow_action *actions __rte_unused, 4009 void *data __rte_unused, 4010 struct rte_flow_error *error) 4011 { 4012 return rte_flow_error_set(error, ENOTSUP, 4013 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, NULL); 4014 } 4015 4016 static int 4017 flow_null_sync_domain(struct rte_eth_dev *dev __rte_unused, 4018 uint32_t domains __rte_unused, 4019 uint32_t flags __rte_unused) 4020 { 4021 return 0; 4022 } 4023 4024 int 4025 flow_null_get_aged_flows(struct rte_eth_dev *dev, 4026 void **context __rte_unused, 4027 uint32_t nb_contexts __rte_unused, 4028 struct rte_flow_error *error __rte_unused) 4029 { 4030 DRV_LOG(ERR, "port %u get aged flows is not supported.", 4031 dev->data->port_id); 4032 return -ENOTSUP; 4033 } 4034 4035 uint32_t 4036 flow_null_counter_allocate(struct rte_eth_dev *dev) 4037 { 4038 DRV_LOG(ERR, "port %u counter allocate is not supported.", 4039 dev->data->port_id); 4040 return 0; 4041 } 4042 4043 void 4044 flow_null_counter_free(struct rte_eth_dev *dev, 4045 uint32_t counter __rte_unused) 4046 { 4047 DRV_LOG(ERR, "port %u counter free is not supported.", 4048 dev->data->port_id); 4049 } 4050 4051 int 4052 flow_null_counter_query(struct rte_eth_dev *dev, 4053 uint32_t counter __rte_unused, 4054 bool clear __rte_unused, 4055 uint64_t *pkts __rte_unused, 4056 uint64_t *bytes __rte_unused, 4057 void **action __rte_unused) 4058 { 4059 DRV_LOG(ERR, "port %u counter query is not supported.", 4060 dev->data->port_id); 4061 return -ENOTSUP; 4062 } 4063 4064 /* Void driver to protect from null pointer reference. */ 4065 const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { 4066 .validate = flow_null_validate, 4067 .prepare = flow_null_prepare, 4068 .translate = flow_null_translate, 4069 .apply = flow_null_apply, 4070 .remove = flow_null_remove, 4071 .destroy = flow_null_destroy, 4072 .query = flow_null_query, 4073 .sync_domain = flow_null_sync_domain, 4074 .get_aged_flows = flow_null_get_aged_flows, 4075 .counter_alloc = flow_null_counter_allocate, 4076 .counter_free = flow_null_counter_free, 4077 .counter_query = flow_null_counter_query 4078 }; 4079 4080 /** 4081 * Select flow driver type according to flow attributes and device 4082 * configuration. 4083 * 4084 * @param[in] dev 4085 * Pointer to the dev structure. 4086 * @param[in] attr 4087 * Pointer to the flow attributes. 4088 * 4089 * @return 4090 * flow driver type, MLX5_FLOW_TYPE_MAX otherwise. 4091 */ 4092 static enum mlx5_flow_drv_type 4093 flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) 4094 { 4095 struct mlx5_priv *priv = dev->data->dev_private; 4096 /* The OS can determine first a specific flow type (DV, VERBS) */ 4097 enum mlx5_flow_drv_type type = mlx5_flow_os_get_type(); 4098 4099 if (type != MLX5_FLOW_TYPE_MAX) 4100 return type; 4101 /* 4102 * Currently when dv_flow_en == 2, only HW steering engine is 4103 * supported. New engines can also be chosen here if ready. 4104 */ 4105 if (priv->sh->config.dv_flow_en == 2) 4106 return MLX5_FLOW_TYPE_HW; 4107 if (!attr) 4108 return MLX5_FLOW_TYPE_MIN; 4109 /* If no OS specific type - continue with DV/VERBS selection */ 4110 if (attr->transfer && priv->sh->config.dv_esw_en) 4111 type = MLX5_FLOW_TYPE_DV; 4112 if (!attr->transfer) 4113 type = priv->sh->config.dv_flow_en ? MLX5_FLOW_TYPE_DV : 4114 MLX5_FLOW_TYPE_VERBS; 4115 return type; 4116 } 4117 4118 #define flow_get_drv_ops(type) flow_drv_ops[type] 4119 4120 /** 4121 * Flow driver validation API. This abstracts calling driver specific functions. 4122 * The type of flow driver is determined according to flow attributes. 4123 * 4124 * @param[in] dev 4125 * Pointer to the dev structure. 4126 * @param[in] attr 4127 * Pointer to the flow attributes. 4128 * @param[in] items 4129 * Pointer to the list of items. 4130 * @param[in] actions 4131 * Pointer to the list of actions. 4132 * @param[in] external 4133 * This flow rule is created by request external to PMD. 4134 * @param[in] hairpin 4135 * Number of hairpin TX actions, 0 means classic flow. 4136 * @param[out] error 4137 * Pointer to the error structure. 4138 * 4139 * @return 4140 * 0 on success, a negative errno value otherwise and rte_errno is set. 4141 */ 4142 static inline int 4143 flow_drv_validate(struct rte_eth_dev *dev, 4144 const struct rte_flow_attr *attr, 4145 const struct rte_flow_item items[], 4146 const struct rte_flow_action actions[], 4147 bool external, int hairpin, struct rte_flow_error *error) 4148 { 4149 const struct mlx5_flow_driver_ops *fops; 4150 enum mlx5_flow_drv_type type = flow_get_drv_type(dev, attr); 4151 4152 fops = flow_get_drv_ops(type); 4153 return fops->validate(dev, attr, items, actions, external, 4154 hairpin, error); 4155 } 4156 4157 /** 4158 * Flow driver preparation API. This abstracts calling driver specific 4159 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4160 * calculates the size of memory required for device flow, allocates the memory, 4161 * initializes the device flow and returns the pointer. 4162 * 4163 * @note 4164 * This function initializes device flow structure such as dv or verbs in 4165 * struct mlx5_flow. However, it is caller's responsibility to initialize the 4166 * rest. For example, adding returning device flow to flow->dev_flow list and 4167 * setting backward reference to the flow should be done out of this function. 4168 * layers field is not filled either. 4169 * 4170 * @param[in] dev 4171 * Pointer to the dev structure. 4172 * @param[in] attr 4173 * Pointer to the flow attributes. 4174 * @param[in] items 4175 * Pointer to the list of items. 4176 * @param[in] actions 4177 * Pointer to the list of actions. 4178 * @param[in] flow_idx 4179 * This memory pool index to the flow. 4180 * @param[out] error 4181 * Pointer to the error structure. 4182 * 4183 * @return 4184 * Pointer to device flow on success, otherwise NULL and rte_errno is set. 4185 */ 4186 static inline struct mlx5_flow * 4187 flow_drv_prepare(struct rte_eth_dev *dev, 4188 const struct rte_flow *flow, 4189 const struct rte_flow_attr *attr, 4190 const struct rte_flow_item items[], 4191 const struct rte_flow_action actions[], 4192 uint32_t flow_idx, 4193 struct rte_flow_error *error) 4194 { 4195 const struct mlx5_flow_driver_ops *fops; 4196 enum mlx5_flow_drv_type type = flow->drv_type; 4197 struct mlx5_flow *mlx5_flow = NULL; 4198 4199 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4200 fops = flow_get_drv_ops(type); 4201 mlx5_flow = fops->prepare(dev, attr, items, actions, error); 4202 if (mlx5_flow) 4203 mlx5_flow->flow_idx = flow_idx; 4204 return mlx5_flow; 4205 } 4206 4207 /** 4208 * Flow driver translation API. This abstracts calling driver specific 4209 * functions. Parent flow (rte_flow) should have driver type (drv_type). It 4210 * translates a generic flow into a driver flow. flow_drv_prepare() must 4211 * precede. 4212 * 4213 * @note 4214 * dev_flow->layers could be filled as a result of parsing during translation 4215 * if needed by flow_drv_apply(). dev_flow->flow->actions can also be filled 4216 * if necessary. As a flow can have multiple dev_flows by RSS flow expansion, 4217 * flow->actions could be overwritten even though all the expanded dev_flows 4218 * have the same actions. 4219 * 4220 * @param[in] dev 4221 * Pointer to the rte dev structure. 4222 * @param[in, out] dev_flow 4223 * Pointer to the mlx5 flow. 4224 * @param[in] attr 4225 * Pointer to the flow attributes. 4226 * @param[in] items 4227 * Pointer to the list of items. 4228 * @param[in] actions 4229 * Pointer to the list of actions. 4230 * @param[out] error 4231 * Pointer to the error structure. 4232 * 4233 * @return 4234 * 0 on success, a negative errno value otherwise and rte_errno is set. 4235 */ 4236 static inline int 4237 flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, 4238 const struct rte_flow_attr *attr, 4239 const struct rte_flow_item items[], 4240 const struct rte_flow_action actions[], 4241 struct rte_flow_error *error) 4242 { 4243 const struct mlx5_flow_driver_ops *fops; 4244 enum mlx5_flow_drv_type type = dev_flow->flow->drv_type; 4245 4246 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4247 fops = flow_get_drv_ops(type); 4248 return fops->translate(dev, dev_flow, attr, items, actions, error); 4249 } 4250 4251 /** 4252 * Flow driver apply API. This abstracts calling driver specific functions. 4253 * Parent flow (rte_flow) should have driver type (drv_type). It applies 4254 * translated driver flows on to device. flow_drv_translate() must precede. 4255 * 4256 * @param[in] dev 4257 * Pointer to Ethernet device structure. 4258 * @param[in, out] flow 4259 * Pointer to flow structure. 4260 * @param[out] error 4261 * Pointer to error structure. 4262 * 4263 * @return 4264 * 0 on success, a negative errno value otherwise and rte_errno is set. 4265 */ 4266 static inline int 4267 flow_drv_apply(struct rte_eth_dev *dev, struct rte_flow *flow, 4268 struct rte_flow_error *error) 4269 { 4270 const struct mlx5_flow_driver_ops *fops; 4271 enum mlx5_flow_drv_type type = flow->drv_type; 4272 4273 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4274 fops = flow_get_drv_ops(type); 4275 return fops->apply(dev, flow, error); 4276 } 4277 4278 /** 4279 * Flow driver destroy API. This abstracts calling driver specific functions. 4280 * Parent flow (rte_flow) should have driver type (drv_type). It removes a flow 4281 * on device and releases resources of the flow. 4282 * 4283 * @param[in] dev 4284 * Pointer to Ethernet device. 4285 * @param[in, out] flow 4286 * Pointer to flow structure. 4287 */ 4288 static inline void 4289 flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) 4290 { 4291 const struct mlx5_flow_driver_ops *fops; 4292 enum mlx5_flow_drv_type type = flow->drv_type; 4293 4294 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4295 fops = flow_get_drv_ops(type); 4296 fops->destroy(dev, flow); 4297 } 4298 4299 /** 4300 * Flow driver find RSS policy tbl API. This abstracts calling driver 4301 * specific functions. Parent flow (rte_flow) should have driver 4302 * type (drv_type). It will find the RSS policy table that has the rss_desc. 4303 * 4304 * @param[in] dev 4305 * Pointer to Ethernet device. 4306 * @param[in, out] flow 4307 * Pointer to flow structure. 4308 * @param[in] policy 4309 * Pointer to meter policy table. 4310 * @param[in] rss_desc 4311 * Pointer to rss_desc 4312 */ 4313 static struct mlx5_flow_meter_sub_policy * 4314 flow_drv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev, 4315 struct rte_flow *flow, 4316 struct mlx5_flow_meter_policy *policy, 4317 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS]) 4318 { 4319 const struct mlx5_flow_driver_ops *fops; 4320 enum mlx5_flow_drv_type type = flow->drv_type; 4321 4322 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4323 fops = flow_get_drv_ops(type); 4324 return fops->meter_sub_policy_rss_prepare(dev, policy, rss_desc); 4325 } 4326 4327 /** 4328 * Flow driver color tag rule API. This abstracts calling driver 4329 * specific functions. Parent flow (rte_flow) should have driver 4330 * type (drv_type). It will create the color tag rules in hierarchy meter. 4331 * 4332 * @param[in] dev 4333 * Pointer to Ethernet device. 4334 * @param[in, out] flow 4335 * Pointer to flow structure. 4336 * @param[in] fm 4337 * Pointer to flow meter structure. 4338 * @param[in] src_port 4339 * The src port this extra rule should use. 4340 * @param[in] item 4341 * The src port id match item. 4342 * @param[out] error 4343 * Pointer to error structure. 4344 */ 4345 static int 4346 flow_drv_mtr_hierarchy_rule_create(struct rte_eth_dev *dev, 4347 struct rte_flow *flow, 4348 struct mlx5_flow_meter_info *fm, 4349 int32_t src_port, 4350 const struct rte_flow_item *item, 4351 struct rte_flow_error *error) 4352 { 4353 const struct mlx5_flow_driver_ops *fops; 4354 enum mlx5_flow_drv_type type = flow->drv_type; 4355 4356 MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX); 4357 fops = flow_get_drv_ops(type); 4358 return fops->meter_hierarchy_rule_create(dev, fm, 4359 src_port, item, error); 4360 } 4361 4362 /** 4363 * Get RSS action from the action list. 4364 * 4365 * @param[in] dev 4366 * Pointer to Ethernet device. 4367 * @param[in] actions 4368 * Pointer to the list of actions. 4369 * @param[in] flow 4370 * Parent flow structure pointer. 4371 * 4372 * @return 4373 * Pointer to the RSS action if exist, else return NULL. 4374 */ 4375 static const struct rte_flow_action_rss* 4376 flow_get_rss_action(struct rte_eth_dev *dev, 4377 const struct rte_flow_action actions[]) 4378 { 4379 struct mlx5_priv *priv = dev->data->dev_private; 4380 const struct rte_flow_action_rss *rss = NULL; 4381 struct mlx5_meter_policy_action_container *acg; 4382 struct mlx5_meter_policy_action_container *acy; 4383 4384 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4385 switch (actions->type) { 4386 case RTE_FLOW_ACTION_TYPE_RSS: 4387 rss = actions->conf; 4388 break; 4389 case RTE_FLOW_ACTION_TYPE_SAMPLE: 4390 { 4391 const struct rte_flow_action_sample *sample = 4392 actions->conf; 4393 const struct rte_flow_action *act = sample->actions; 4394 for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) 4395 if (act->type == RTE_FLOW_ACTION_TYPE_RSS) 4396 rss = act->conf; 4397 break; 4398 } 4399 case RTE_FLOW_ACTION_TYPE_METER: 4400 { 4401 uint32_t mtr_idx; 4402 struct mlx5_flow_meter_info *fm; 4403 struct mlx5_flow_meter_policy *policy; 4404 const struct rte_flow_action_meter *mtr = actions->conf; 4405 4406 fm = mlx5_flow_meter_find(priv, mtr->mtr_id, &mtr_idx); 4407 if (fm && !fm->def_policy) { 4408 policy = mlx5_flow_meter_policy_find(dev, 4409 fm->policy_id, NULL); 4410 MLX5_ASSERT(policy); 4411 if (policy->is_hierarchy) { 4412 policy = 4413 mlx5_flow_meter_hierarchy_get_final_policy(dev, 4414 policy); 4415 if (!policy) 4416 return NULL; 4417 } 4418 if (policy->is_rss) { 4419 acg = 4420 &policy->act_cnt[RTE_COLOR_GREEN]; 4421 acy = 4422 &policy->act_cnt[RTE_COLOR_YELLOW]; 4423 if (acg->fate_action == 4424 MLX5_FLOW_FATE_SHARED_RSS) 4425 rss = acg->rss->conf; 4426 else if (acy->fate_action == 4427 MLX5_FLOW_FATE_SHARED_RSS) 4428 rss = acy->rss->conf; 4429 } 4430 } 4431 break; 4432 } 4433 default: 4434 break; 4435 } 4436 } 4437 return rss; 4438 } 4439 4440 /** 4441 * Get ASO age action by index. 4442 * 4443 * @param[in] dev 4444 * Pointer to the Ethernet device structure. 4445 * @param[in] age_idx 4446 * Index to the ASO age action. 4447 * 4448 * @return 4449 * The specified ASO age action. 4450 */ 4451 struct mlx5_aso_age_action* 4452 flow_aso_age_get_by_idx(struct rte_eth_dev *dev, uint32_t age_idx) 4453 { 4454 uint16_t pool_idx = age_idx & UINT16_MAX; 4455 uint16_t offset = (age_idx >> 16) & UINT16_MAX; 4456 struct mlx5_priv *priv = dev->data->dev_private; 4457 struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng; 4458 struct mlx5_aso_age_pool *pool; 4459 4460 rte_rwlock_read_lock(&mng->resize_rwl); 4461 pool = mng->pools[pool_idx]; 4462 rte_rwlock_read_unlock(&mng->resize_rwl); 4463 return &pool->actions[offset - 1]; 4464 } 4465 4466 /* maps indirect action to translated direct in some actions array */ 4467 struct mlx5_translated_action_handle { 4468 struct rte_flow_action_handle *action; /**< Indirect action handle. */ 4469 int index; /**< Index in related array of rte_flow_action. */ 4470 }; 4471 4472 /** 4473 * Translates actions of type RTE_FLOW_ACTION_TYPE_INDIRECT to related 4474 * direct action if translation possible. 4475 * This functionality used to run same execution path for both direct and 4476 * indirect actions on flow create. All necessary preparations for indirect 4477 * action handling should be performed on *handle* actions list returned 4478 * from this call. 4479 * 4480 * @param[in] dev 4481 * Pointer to Ethernet device. 4482 * @param[in] actions 4483 * List of actions to translate. 4484 * @param[out] handle 4485 * List to store translated indirect action object handles. 4486 * @param[in, out] indir_n 4487 * Size of *handle* array. On return should be updated with number of 4488 * indirect actions retrieved from the *actions* list. 4489 * @param[out] translated_actions 4490 * List of actions where all indirect actions were translated to direct 4491 * if possible. NULL if no translation took place. 4492 * @param[out] error 4493 * Pointer to the error structure. 4494 * 4495 * @return 4496 * 0 on success, a negative errno value otherwise and rte_errno is set. 4497 */ 4498 static int 4499 flow_action_handles_translate(struct rte_eth_dev *dev, 4500 const struct rte_flow_action actions[], 4501 struct mlx5_translated_action_handle *handle, 4502 int *indir_n, 4503 struct rte_flow_action **translated_actions, 4504 struct rte_flow_error *error) 4505 { 4506 struct mlx5_priv *priv = dev->data->dev_private; 4507 struct rte_flow_action *translated = NULL; 4508 size_t actions_size; 4509 int n; 4510 int copied_n = 0; 4511 struct mlx5_translated_action_handle *handle_end = NULL; 4512 4513 for (n = 0; actions[n].type != RTE_FLOW_ACTION_TYPE_END; n++) { 4514 if (actions[n].type != RTE_FLOW_ACTION_TYPE_INDIRECT) 4515 continue; 4516 if (copied_n == *indir_n) { 4517 return rte_flow_error_set 4518 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION_NUM, 4519 NULL, "too many shared actions"); 4520 } 4521 rte_memcpy(&handle[copied_n].action, &actions[n].conf, 4522 sizeof(actions[n].conf)); 4523 handle[copied_n].index = n; 4524 copied_n++; 4525 } 4526 n++; 4527 *indir_n = copied_n; 4528 if (!copied_n) 4529 return 0; 4530 actions_size = sizeof(struct rte_flow_action) * n; 4531 translated = mlx5_malloc(MLX5_MEM_ZERO, actions_size, 0, SOCKET_ID_ANY); 4532 if (!translated) { 4533 rte_errno = ENOMEM; 4534 return -ENOMEM; 4535 } 4536 memcpy(translated, actions, actions_size); 4537 for (handle_end = handle + copied_n; handle < handle_end; handle++) { 4538 struct mlx5_shared_action_rss *shared_rss; 4539 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4540 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4541 uint32_t idx = act_idx & 4542 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4543 4544 switch (type) { 4545 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4546 shared_rss = mlx5_ipool_get 4547 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx); 4548 translated[handle->index].type = 4549 RTE_FLOW_ACTION_TYPE_RSS; 4550 translated[handle->index].conf = 4551 &shared_rss->origin; 4552 break; 4553 case MLX5_INDIRECT_ACTION_TYPE_COUNT: 4554 translated[handle->index].type = 4555 (enum rte_flow_action_type) 4556 MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 4557 translated[handle->index].conf = (void *)(uintptr_t)idx; 4558 break; 4559 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 4560 translated[handle->index].type = 4561 (enum rte_flow_action_type) 4562 MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK; 4563 translated[handle->index].conf = (void *)(uintptr_t)idx; 4564 break; 4565 case MLX5_INDIRECT_ACTION_TYPE_AGE: 4566 if (priv->sh->flow_hit_aso_en) { 4567 translated[handle->index].type = 4568 (enum rte_flow_action_type) 4569 MLX5_RTE_FLOW_ACTION_TYPE_AGE; 4570 translated[handle->index].conf = 4571 (void *)(uintptr_t)idx; 4572 break; 4573 } 4574 /* Fall-through */ 4575 case MLX5_INDIRECT_ACTION_TYPE_CT: 4576 if (priv->sh->ct_aso_en) { 4577 translated[handle->index].type = 4578 RTE_FLOW_ACTION_TYPE_CONNTRACK; 4579 translated[handle->index].conf = 4580 (void *)(uintptr_t)idx; 4581 break; 4582 } 4583 /* Fall-through */ 4584 default: 4585 mlx5_free(translated); 4586 return rte_flow_error_set 4587 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, 4588 NULL, "invalid indirect action type"); 4589 } 4590 } 4591 *translated_actions = translated; 4592 return 0; 4593 } 4594 4595 /** 4596 * Get Shared RSS action from the action list. 4597 * 4598 * @param[in] dev 4599 * Pointer to Ethernet device. 4600 * @param[in] shared 4601 * Pointer to the list of actions. 4602 * @param[in] shared_n 4603 * Actions list length. 4604 * 4605 * @return 4606 * The MLX5 RSS action ID if exists, otherwise return 0. 4607 */ 4608 static uint32_t 4609 flow_get_shared_rss_action(struct rte_eth_dev *dev, 4610 struct mlx5_translated_action_handle *handle, 4611 int shared_n) 4612 { 4613 struct mlx5_translated_action_handle *handle_end; 4614 struct mlx5_priv *priv = dev->data->dev_private; 4615 struct mlx5_shared_action_rss *shared_rss; 4616 4617 4618 for (handle_end = handle + shared_n; handle < handle_end; handle++) { 4619 uint32_t act_idx = (uint32_t)(uintptr_t)handle->action; 4620 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 4621 uint32_t idx = act_idx & 4622 ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1); 4623 switch (type) { 4624 case MLX5_INDIRECT_ACTION_TYPE_RSS: 4625 shared_rss = mlx5_ipool_get 4626 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 4627 idx); 4628 __atomic_fetch_add(&shared_rss->refcnt, 1, 4629 __ATOMIC_RELAXED); 4630 return idx; 4631 default: 4632 break; 4633 } 4634 } 4635 return 0; 4636 } 4637 4638 static unsigned int 4639 find_graph_root(uint32_t rss_level) 4640 { 4641 return rss_level < 2 ? MLX5_EXPANSION_ROOT : 4642 MLX5_EXPANSION_ROOT_OUTER; 4643 } 4644 4645 /** 4646 * Get layer flags from the prefix flow. 4647 * 4648 * Some flows may be split to several subflows, the prefix subflow gets the 4649 * match items and the suffix sub flow gets the actions. 4650 * Some actions need the user defined match item flags to get the detail for 4651 * the action. 4652 * This function helps the suffix flow to get the item layer flags from prefix 4653 * subflow. 4654 * 4655 * @param[in] dev_flow 4656 * Pointer the created prefix subflow. 4657 * 4658 * @return 4659 * The layers get from prefix subflow. 4660 */ 4661 static inline uint64_t 4662 flow_get_prefix_layer_flags(struct mlx5_flow *dev_flow) 4663 { 4664 uint64_t layers = 0; 4665 4666 /* 4667 * Layers bits could be localization, but usually the compiler will 4668 * help to do the optimization work for source code. 4669 * If no decap actions, use the layers directly. 4670 */ 4671 if (!(dev_flow->act_flags & MLX5_FLOW_ACTION_DECAP)) 4672 return dev_flow->handle->layers; 4673 /* Convert L3 layers with decap action. */ 4674 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4) 4675 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4; 4676 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6) 4677 layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6; 4678 /* Convert L4 layers with decap action. */ 4679 if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP) 4680 layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP; 4681 else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP) 4682 layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP; 4683 return layers; 4684 } 4685 4686 /** 4687 * Get metadata split action information. 4688 * 4689 * @param[in] actions 4690 * Pointer to the list of actions. 4691 * @param[out] qrss 4692 * Pointer to the return pointer. 4693 * @param[out] qrss_type 4694 * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned 4695 * if no QUEUE/RSS is found. 4696 * @param[out] encap_idx 4697 * Pointer to the index of the encap action if exists, otherwise the last 4698 * action index. 4699 * 4700 * @return 4701 * Total number of actions. 4702 */ 4703 static int 4704 flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], 4705 const struct rte_flow_action **qrss, 4706 int *encap_idx) 4707 { 4708 const struct rte_flow_action_raw_encap *raw_encap; 4709 int actions_n = 0; 4710 int raw_decap_idx = -1; 4711 4712 *encap_idx = -1; 4713 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4714 switch (actions->type) { 4715 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4716 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4717 *encap_idx = actions_n; 4718 break; 4719 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4720 raw_decap_idx = actions_n; 4721 break; 4722 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4723 raw_encap = actions->conf; 4724 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4725 *encap_idx = raw_decap_idx != -1 ? 4726 raw_decap_idx : actions_n; 4727 break; 4728 case RTE_FLOW_ACTION_TYPE_QUEUE: 4729 case RTE_FLOW_ACTION_TYPE_RSS: 4730 *qrss = actions; 4731 break; 4732 default: 4733 break; 4734 } 4735 actions_n++; 4736 } 4737 if (*encap_idx == -1) 4738 *encap_idx = actions_n; 4739 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4740 return actions_n + 1; 4741 } 4742 4743 /** 4744 * Check if the action will change packet. 4745 * 4746 * @param dev 4747 * Pointer to Ethernet device. 4748 * @param[in] type 4749 * action type. 4750 * 4751 * @return 4752 * true if action will change packet, false otherwise. 4753 */ 4754 static bool flow_check_modify_action_type(struct rte_eth_dev *dev, 4755 enum rte_flow_action_type type) 4756 { 4757 struct mlx5_priv *priv = dev->data->dev_private; 4758 4759 switch (type) { 4760 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 4761 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 4762 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 4763 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 4764 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 4765 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 4766 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 4767 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 4768 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 4769 case RTE_FLOW_ACTION_TYPE_SET_TTL: 4770 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 4771 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 4772 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 4773 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 4774 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 4775 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 4776 case RTE_FLOW_ACTION_TYPE_SET_META: 4777 case RTE_FLOW_ACTION_TYPE_SET_TAG: 4778 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 4779 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4780 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4781 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4782 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4783 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 4784 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4785 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 4786 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4787 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 4788 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 4789 return true; 4790 case RTE_FLOW_ACTION_TYPE_FLAG: 4791 case RTE_FLOW_ACTION_TYPE_MARK: 4792 if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY && 4793 priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) 4794 return true; 4795 else 4796 return false; 4797 default: 4798 return false; 4799 } 4800 } 4801 4802 /** 4803 * Check meter action from the action list. 4804 * 4805 * @param dev 4806 * Pointer to Ethernet device. 4807 * @param[in] actions 4808 * Pointer to the list of actions. 4809 * @param[out] has_mtr 4810 * Pointer to the meter exist flag. 4811 * @param[out] has_modify 4812 * Pointer to the flag showing there's packet change action. 4813 * @param[out] meter_id 4814 * Pointer to the meter id. 4815 * 4816 * @return 4817 * Total number of actions. 4818 */ 4819 static int 4820 flow_check_meter_action(struct rte_eth_dev *dev, 4821 const struct rte_flow_action actions[], 4822 bool *has_mtr, bool *has_modify, uint32_t *meter_id) 4823 { 4824 const struct rte_flow_action_meter *mtr = NULL; 4825 int actions_n = 0; 4826 4827 MLX5_ASSERT(has_mtr); 4828 *has_mtr = false; 4829 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4830 switch (actions->type) { 4831 case RTE_FLOW_ACTION_TYPE_METER: 4832 mtr = actions->conf; 4833 *meter_id = mtr->mtr_id; 4834 *has_mtr = true; 4835 break; 4836 default: 4837 break; 4838 } 4839 if (!*has_mtr) 4840 *has_modify |= flow_check_modify_action_type(dev, 4841 actions->type); 4842 actions_n++; 4843 } 4844 /* Count RTE_FLOW_ACTION_TYPE_END. */ 4845 return actions_n + 1; 4846 } 4847 4848 /** 4849 * Check if the flow should be split due to hairpin. 4850 * The reason for the split is that in current HW we can't 4851 * support encap and push-vlan on Rx, so if a flow contains 4852 * these actions we move it to Tx. 4853 * 4854 * @param dev 4855 * Pointer to Ethernet device. 4856 * @param[in] attr 4857 * Flow rule attributes. 4858 * @param[in] actions 4859 * Associated actions (list terminated by the END action). 4860 * 4861 * @return 4862 * > 0 the number of actions and the flow should be split, 4863 * 0 when no split required. 4864 */ 4865 static int 4866 flow_check_hairpin_split(struct rte_eth_dev *dev, 4867 const struct rte_flow_attr *attr, 4868 const struct rte_flow_action actions[]) 4869 { 4870 int queue_action = 0; 4871 int action_n = 0; 4872 int split = 0; 4873 int push_vlan = 0; 4874 const struct rte_flow_action_queue *queue; 4875 const struct rte_flow_action_rss *rss; 4876 const struct rte_flow_action_raw_encap *raw_encap; 4877 const struct rte_eth_hairpin_conf *conf; 4878 4879 if (!attr->ingress) 4880 return 0; 4881 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 4882 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 4883 push_vlan = 1; 4884 switch (actions->type) { 4885 case RTE_FLOW_ACTION_TYPE_QUEUE: 4886 queue = actions->conf; 4887 if (queue == NULL) 4888 return 0; 4889 conf = mlx5_rxq_get_hairpin_conf(dev, queue->index); 4890 if (conf == NULL || conf->tx_explicit != 0) 4891 return 0; 4892 queue_action = 1; 4893 action_n++; 4894 break; 4895 case RTE_FLOW_ACTION_TYPE_RSS: 4896 rss = actions->conf; 4897 if (rss == NULL || rss->queue_num == 0) 4898 return 0; 4899 conf = mlx5_rxq_get_hairpin_conf(dev, rss->queue[0]); 4900 if (conf == NULL || conf->tx_explicit != 0) 4901 return 0; 4902 queue_action = 1; 4903 action_n++; 4904 break; 4905 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 4906 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 4907 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 4908 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 4909 split++; 4910 action_n++; 4911 break; 4912 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 4913 if (push_vlan) 4914 split++; 4915 action_n++; 4916 break; 4917 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 4918 raw_encap = actions->conf; 4919 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 4920 split++; 4921 action_n++; 4922 break; 4923 default: 4924 action_n++; 4925 break; 4926 } 4927 } 4928 if (split && queue_action) 4929 return action_n; 4930 return 0; 4931 } 4932 4933 /* Declare flow create/destroy prototype in advance. */ 4934 static uint32_t 4935 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4936 const struct rte_flow_attr *attr, 4937 const struct rte_flow_item items[], 4938 const struct rte_flow_action actions[], 4939 bool external, struct rte_flow_error *error); 4940 4941 static void 4942 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 4943 uint32_t flow_idx); 4944 4945 int 4946 flow_dv_mreg_match_cb(void *tool_ctx __rte_unused, 4947 struct mlx5_list_entry *entry, void *cb_ctx) 4948 { 4949 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4950 struct mlx5_flow_mreg_copy_resource *mcp_res = 4951 container_of(entry, typeof(*mcp_res), hlist_ent); 4952 4953 return mcp_res->mark_id != *(uint32_t *)(ctx->data); 4954 } 4955 4956 struct mlx5_list_entry * 4957 flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx) 4958 { 4959 struct rte_eth_dev *dev = tool_ctx; 4960 struct mlx5_priv *priv = dev->data->dev_private; 4961 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 4962 struct mlx5_flow_mreg_copy_resource *mcp_res; 4963 struct rte_flow_error *error = ctx->error; 4964 uint32_t idx = 0; 4965 int ret; 4966 uint32_t mark_id = *(uint32_t *)(ctx->data); 4967 struct rte_flow_attr attr = { 4968 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 4969 .ingress = 1, 4970 }; 4971 struct mlx5_rte_flow_item_tag tag_spec = { 4972 .data = mark_id, 4973 }; 4974 struct rte_flow_item items[] = { 4975 [1] = { .type = RTE_FLOW_ITEM_TYPE_END, }, 4976 }; 4977 struct rte_flow_action_mark ftag = { 4978 .id = mark_id, 4979 }; 4980 struct mlx5_flow_action_copy_mreg cp_mreg = { 4981 .dst = REG_B, 4982 .src = REG_NON, 4983 }; 4984 struct rte_flow_action_jump jump = { 4985 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 4986 }; 4987 struct rte_flow_action actions[] = { 4988 [3] = { .type = RTE_FLOW_ACTION_TYPE_END, }, 4989 }; 4990 4991 /* Fill the register fields in the flow. */ 4992 ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error); 4993 if (ret < 0) 4994 return NULL; 4995 tag_spec.id = ret; 4996 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 4997 if (ret < 0) 4998 return NULL; 4999 cp_mreg.src = ret; 5000 /* Provide the full width of FLAG specific value. */ 5001 if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT)) 5002 tag_spec.data = MLX5_FLOW_MARK_DEFAULT; 5003 /* Build a new flow. */ 5004 if (mark_id != MLX5_DEFAULT_COPY_ID) { 5005 items[0] = (struct rte_flow_item){ 5006 .type = (enum rte_flow_item_type) 5007 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 5008 .spec = &tag_spec, 5009 }; 5010 items[1] = (struct rte_flow_item){ 5011 .type = RTE_FLOW_ITEM_TYPE_END, 5012 }; 5013 actions[0] = (struct rte_flow_action){ 5014 .type = (enum rte_flow_action_type) 5015 MLX5_RTE_FLOW_ACTION_TYPE_MARK, 5016 .conf = &ftag, 5017 }; 5018 actions[1] = (struct rte_flow_action){ 5019 .type = (enum rte_flow_action_type) 5020 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5021 .conf = &cp_mreg, 5022 }; 5023 actions[2] = (struct rte_flow_action){ 5024 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5025 .conf = &jump, 5026 }; 5027 actions[3] = (struct rte_flow_action){ 5028 .type = RTE_FLOW_ACTION_TYPE_END, 5029 }; 5030 } else { 5031 /* Default rule, wildcard match. */ 5032 attr.priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR; 5033 items[0] = (struct rte_flow_item){ 5034 .type = RTE_FLOW_ITEM_TYPE_END, 5035 }; 5036 actions[0] = (struct rte_flow_action){ 5037 .type = (enum rte_flow_action_type) 5038 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 5039 .conf = &cp_mreg, 5040 }; 5041 actions[1] = (struct rte_flow_action){ 5042 .type = RTE_FLOW_ACTION_TYPE_JUMP, 5043 .conf = &jump, 5044 }; 5045 actions[2] = (struct rte_flow_action){ 5046 .type = RTE_FLOW_ACTION_TYPE_END, 5047 }; 5048 } 5049 /* Build a new entry. */ 5050 mcp_res = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5051 if (!mcp_res) { 5052 rte_errno = ENOMEM; 5053 return NULL; 5054 } 5055 mcp_res->idx = idx; 5056 mcp_res->mark_id = mark_id; 5057 /* 5058 * The copy Flows are not included in any list. There 5059 * ones are referenced from other Flows and can not 5060 * be applied, removed, deleted in arbitrary order 5061 * by list traversing. 5062 */ 5063 mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP, 5064 &attr, items, actions, false, error); 5065 if (!mcp_res->rix_flow) { 5066 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx); 5067 return NULL; 5068 } 5069 return &mcp_res->hlist_ent; 5070 } 5071 5072 struct mlx5_list_entry * 5073 flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry, 5074 void *cb_ctx __rte_unused) 5075 { 5076 struct rte_eth_dev *dev = tool_ctx; 5077 struct mlx5_priv *priv = dev->data->dev_private; 5078 struct mlx5_flow_mreg_copy_resource *mcp_res; 5079 uint32_t idx = 0; 5080 5081 mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx); 5082 if (!mcp_res) { 5083 rte_errno = ENOMEM; 5084 return NULL; 5085 } 5086 memcpy(mcp_res, oentry, sizeof(*mcp_res)); 5087 mcp_res->idx = idx; 5088 return &mcp_res->hlist_ent; 5089 } 5090 5091 void 5092 flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5093 { 5094 struct mlx5_flow_mreg_copy_resource *mcp_res = 5095 container_of(entry, typeof(*mcp_res), hlist_ent); 5096 struct rte_eth_dev *dev = tool_ctx; 5097 struct mlx5_priv *priv = dev->data->dev_private; 5098 5099 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5100 } 5101 5102 /** 5103 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5104 * 5105 * As mark_id is unique, if there's already a registered flow for the mark_id, 5106 * return by increasing the reference counter of the resource. Otherwise, create 5107 * the resource (mcp_res) and flow. 5108 * 5109 * Flow looks like, 5110 * - If ingress port is ANY and reg_c[1] is mark_id, 5111 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5112 * 5113 * For default flow (zero mark_id), flow is like, 5114 * - If ingress port is ANY, 5115 * reg_b := reg_c[0] and jump to RX_ACT_TBL. 5116 * 5117 * @param dev 5118 * Pointer to Ethernet device. 5119 * @param mark_id 5120 * ID of MARK action, zero means default flow for META. 5121 * @param[out] error 5122 * Perform verbose error reporting if not NULL. 5123 * 5124 * @return 5125 * Associated resource on success, NULL otherwise and rte_errno is set. 5126 */ 5127 static struct mlx5_flow_mreg_copy_resource * 5128 flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id, 5129 struct rte_flow_error *error) 5130 { 5131 struct mlx5_priv *priv = dev->data->dev_private; 5132 struct mlx5_list_entry *entry; 5133 struct mlx5_flow_cb_ctx ctx = { 5134 .dev = dev, 5135 .error = error, 5136 .data = &mark_id, 5137 }; 5138 5139 /* Check if already registered. */ 5140 MLX5_ASSERT(priv->mreg_cp_tbl); 5141 entry = mlx5_hlist_register(priv->mreg_cp_tbl, mark_id, &ctx); 5142 if (!entry) 5143 return NULL; 5144 return container_of(entry, struct mlx5_flow_mreg_copy_resource, 5145 hlist_ent); 5146 } 5147 5148 void 5149 flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 5150 { 5151 struct mlx5_flow_mreg_copy_resource *mcp_res = 5152 container_of(entry, typeof(*mcp_res), hlist_ent); 5153 struct rte_eth_dev *dev = tool_ctx; 5154 struct mlx5_priv *priv = dev->data->dev_private; 5155 5156 MLX5_ASSERT(mcp_res->rix_flow); 5157 flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow); 5158 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx); 5159 } 5160 5161 /** 5162 * Release flow in RX_CP_TBL. 5163 * 5164 * @param dev 5165 * Pointer to Ethernet device. 5166 * @flow 5167 * Parent flow for wich copying is provided. 5168 */ 5169 static void 5170 flow_mreg_del_copy_action(struct rte_eth_dev *dev, 5171 struct rte_flow *flow) 5172 { 5173 struct mlx5_flow_mreg_copy_resource *mcp_res; 5174 struct mlx5_priv *priv = dev->data->dev_private; 5175 5176 if (!flow->rix_mreg_copy) 5177 return; 5178 mcp_res = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MCP], 5179 flow->rix_mreg_copy); 5180 if (!mcp_res || !priv->mreg_cp_tbl) 5181 return; 5182 MLX5_ASSERT(mcp_res->rix_flow); 5183 mlx5_hlist_unregister(priv->mreg_cp_tbl, &mcp_res->hlist_ent); 5184 flow->rix_mreg_copy = 0; 5185 } 5186 5187 /** 5188 * Remove the default copy action from RX_CP_TBL. 5189 * 5190 * This functions is called in the mlx5_dev_start(). No thread safe 5191 * is guaranteed. 5192 * 5193 * @param dev 5194 * Pointer to Ethernet device. 5195 */ 5196 static void 5197 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev) 5198 { 5199 struct mlx5_list_entry *entry; 5200 struct mlx5_priv *priv = dev->data->dev_private; 5201 struct mlx5_flow_cb_ctx ctx; 5202 uint32_t mark_id; 5203 5204 /* Check if default flow is registered. */ 5205 if (!priv->mreg_cp_tbl) 5206 return; 5207 mark_id = MLX5_DEFAULT_COPY_ID; 5208 ctx.data = &mark_id; 5209 entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx); 5210 if (!entry) 5211 return; 5212 mlx5_hlist_unregister(priv->mreg_cp_tbl, entry); 5213 } 5214 5215 /** 5216 * Add the default copy action in RX_CP_TBL. 5217 * 5218 * This functions is called in the mlx5_dev_start(). No thread safe 5219 * is guaranteed. 5220 * 5221 * @param dev 5222 * Pointer to Ethernet device. 5223 * @param[out] error 5224 * Perform verbose error reporting if not NULL. 5225 * 5226 * @return 5227 * 0 for success, negative value otherwise and rte_errno is set. 5228 */ 5229 static int 5230 flow_mreg_add_default_copy_action(struct rte_eth_dev *dev, 5231 struct rte_flow_error *error) 5232 { 5233 struct mlx5_priv *priv = dev->data->dev_private; 5234 struct mlx5_flow_mreg_copy_resource *mcp_res; 5235 struct mlx5_flow_cb_ctx ctx; 5236 uint32_t mark_id; 5237 5238 /* Check whether extensive metadata feature is engaged. */ 5239 if (!priv->sh->config.dv_flow_en || 5240 priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5241 !mlx5_flow_ext_mreg_supported(dev) || 5242 !priv->sh->dv_regc0_mask) 5243 return 0; 5244 /* 5245 * Add default mreg copy flow may be called multiple time, but 5246 * only be called once in stop. Avoid register it twice. 5247 */ 5248 mark_id = MLX5_DEFAULT_COPY_ID; 5249 ctx.data = &mark_id; 5250 if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx)) 5251 return 0; 5252 mcp_res = flow_mreg_add_copy_action(dev, mark_id, error); 5253 if (!mcp_res) 5254 return -rte_errno; 5255 return 0; 5256 } 5257 5258 /** 5259 * Add a flow of copying flow metadata registers in RX_CP_TBL. 5260 * 5261 * All the flow having Q/RSS action should be split by 5262 * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL 5263 * performs the following, 5264 * - CQE->flow_tag := reg_c[1] (MARK) 5265 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5266 * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1] 5267 * but there should be a flow per each MARK ID set by MARK action. 5268 * 5269 * For the aforementioned reason, if there's a MARK action in flow's action 5270 * list, a corresponding flow should be added to the RX_CP_TBL in order to copy 5271 * the MARK ID to CQE's flow_tag like, 5272 * - If reg_c[1] is mark_id, 5273 * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5274 * 5275 * For SET_META action which stores value in reg_c[0], as the destination is 5276 * also a flow metadata register (reg_b), adding a default flow is enough. Zero 5277 * MARK ID means the default flow. The default flow looks like, 5278 * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL. 5279 * 5280 * @param dev 5281 * Pointer to Ethernet device. 5282 * @param flow 5283 * Pointer to flow structure. 5284 * @param[in] actions 5285 * Pointer to the list of actions. 5286 * @param[out] error 5287 * Perform verbose error reporting if not NULL. 5288 * 5289 * @return 5290 * 0 on success, negative value otherwise and rte_errno is set. 5291 */ 5292 static int 5293 flow_mreg_update_copy_table(struct rte_eth_dev *dev, 5294 struct rte_flow *flow, 5295 const struct rte_flow_action *actions, 5296 struct rte_flow_error *error) 5297 { 5298 struct mlx5_priv *priv = dev->data->dev_private; 5299 struct mlx5_sh_config *config = &priv->sh->config; 5300 struct mlx5_flow_mreg_copy_resource *mcp_res; 5301 const struct rte_flow_action_mark *mark; 5302 5303 /* Check whether extensive metadata feature is engaged. */ 5304 if (!config->dv_flow_en || 5305 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 5306 !mlx5_flow_ext_mreg_supported(dev) || 5307 !priv->sh->dv_regc0_mask) 5308 return 0; 5309 /* Find MARK action. */ 5310 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5311 switch (actions->type) { 5312 case RTE_FLOW_ACTION_TYPE_FLAG: 5313 mcp_res = flow_mreg_add_copy_action 5314 (dev, MLX5_FLOW_MARK_DEFAULT, error); 5315 if (!mcp_res) 5316 return -rte_errno; 5317 flow->rix_mreg_copy = mcp_res->idx; 5318 return 0; 5319 case RTE_FLOW_ACTION_TYPE_MARK: 5320 mark = (const struct rte_flow_action_mark *) 5321 actions->conf; 5322 mcp_res = 5323 flow_mreg_add_copy_action(dev, mark->id, error); 5324 if (!mcp_res) 5325 return -rte_errno; 5326 flow->rix_mreg_copy = mcp_res->idx; 5327 return 0; 5328 default: 5329 break; 5330 } 5331 } 5332 return 0; 5333 } 5334 5335 #define MLX5_MAX_SPLIT_ACTIONS 24 5336 #define MLX5_MAX_SPLIT_ITEMS 24 5337 5338 /** 5339 * Split the hairpin flow. 5340 * Since HW can't support encap and push-vlan on Rx, we move these 5341 * actions to Tx. 5342 * If the count action is after the encap then we also 5343 * move the count action. in this case the count will also measure 5344 * the outer bytes. 5345 * 5346 * @param dev 5347 * Pointer to Ethernet device. 5348 * @param[in] actions 5349 * Associated actions (list terminated by the END action). 5350 * @param[out] actions_rx 5351 * Rx flow actions. 5352 * @param[out] actions_tx 5353 * Tx flow actions.. 5354 * @param[out] pattern_tx 5355 * The pattern items for the Tx flow. 5356 * @param[out] flow_id 5357 * The flow ID connected to this flow. 5358 * 5359 * @return 5360 * 0 on success. 5361 */ 5362 static int 5363 flow_hairpin_split(struct rte_eth_dev *dev, 5364 const struct rte_flow_action actions[], 5365 struct rte_flow_action actions_rx[], 5366 struct rte_flow_action actions_tx[], 5367 struct rte_flow_item pattern_tx[], 5368 uint32_t flow_id) 5369 { 5370 const struct rte_flow_action_raw_encap *raw_encap; 5371 const struct rte_flow_action_raw_decap *raw_decap; 5372 struct mlx5_rte_flow_action_set_tag *set_tag; 5373 struct rte_flow_action *tag_action; 5374 struct mlx5_rte_flow_item_tag *tag_item; 5375 struct rte_flow_item *item; 5376 char *addr; 5377 int push_vlan = 0; 5378 int encap = 0; 5379 5380 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5381 if (actions->type == RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN) 5382 push_vlan = 1; 5383 switch (actions->type) { 5384 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 5385 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 5386 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5387 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 5388 rte_memcpy(actions_tx, actions, 5389 sizeof(struct rte_flow_action)); 5390 actions_tx++; 5391 break; 5392 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5393 if (push_vlan) { 5394 rte_memcpy(actions_tx, actions, 5395 sizeof(struct rte_flow_action)); 5396 actions_tx++; 5397 } else { 5398 rte_memcpy(actions_rx, actions, 5399 sizeof(struct rte_flow_action)); 5400 actions_rx++; 5401 } 5402 break; 5403 case RTE_FLOW_ACTION_TYPE_COUNT: 5404 if (encap) { 5405 rte_memcpy(actions_tx, actions, 5406 sizeof(struct rte_flow_action)); 5407 actions_tx++; 5408 } else { 5409 rte_memcpy(actions_rx, actions, 5410 sizeof(struct rte_flow_action)); 5411 actions_rx++; 5412 } 5413 break; 5414 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5415 raw_encap = actions->conf; 5416 if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE) { 5417 memcpy(actions_tx, actions, 5418 sizeof(struct rte_flow_action)); 5419 actions_tx++; 5420 encap = 1; 5421 } else { 5422 rte_memcpy(actions_rx, actions, 5423 sizeof(struct rte_flow_action)); 5424 actions_rx++; 5425 } 5426 break; 5427 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5428 raw_decap = actions->conf; 5429 if (raw_decap->size < MLX5_ENCAPSULATION_DECISION_SIZE) { 5430 memcpy(actions_tx, actions, 5431 sizeof(struct rte_flow_action)); 5432 actions_tx++; 5433 } else { 5434 rte_memcpy(actions_rx, actions, 5435 sizeof(struct rte_flow_action)); 5436 actions_rx++; 5437 } 5438 break; 5439 default: 5440 rte_memcpy(actions_rx, actions, 5441 sizeof(struct rte_flow_action)); 5442 actions_rx++; 5443 break; 5444 } 5445 } 5446 /* Add set meta action and end action for the Rx flow. */ 5447 tag_action = actions_rx; 5448 tag_action->type = (enum rte_flow_action_type) 5449 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5450 actions_rx++; 5451 rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); 5452 actions_rx++; 5453 set_tag = (void *)actions_rx; 5454 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5455 .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), 5456 .data = flow_id, 5457 }; 5458 MLX5_ASSERT(set_tag->id > REG_NON); 5459 tag_action->conf = set_tag; 5460 /* Create Tx item list. */ 5461 rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); 5462 addr = (void *)&pattern_tx[2]; 5463 item = pattern_tx; 5464 item->type = (enum rte_flow_item_type) 5465 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5466 tag_item = (void *)addr; 5467 tag_item->data = flow_id; 5468 tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL); 5469 MLX5_ASSERT(set_tag->id > REG_NON); 5470 item->spec = tag_item; 5471 addr += sizeof(struct mlx5_rte_flow_item_tag); 5472 tag_item = (void *)addr; 5473 tag_item->data = UINT32_MAX; 5474 tag_item->id = UINT16_MAX; 5475 item->mask = tag_item; 5476 item->last = NULL; 5477 item++; 5478 item->type = RTE_FLOW_ITEM_TYPE_END; 5479 return 0; 5480 } 5481 5482 /** 5483 * The last stage of splitting chain, just creates the subflow 5484 * without any modification. 5485 * 5486 * @param[in] dev 5487 * Pointer to Ethernet device. 5488 * @param[in] flow 5489 * Parent flow structure pointer. 5490 * @param[in, out] sub_flow 5491 * Pointer to return the created subflow, may be NULL. 5492 * @param[in] attr 5493 * Flow rule attributes. 5494 * @param[in] items 5495 * Pattern specification (list terminated by the END pattern item). 5496 * @param[in] actions 5497 * Associated actions (list terminated by the END action). 5498 * @param[in] flow_split_info 5499 * Pointer to flow split info structure. 5500 * @param[out] error 5501 * Perform verbose error reporting if not NULL. 5502 * @return 5503 * 0 on success, negative value otherwise 5504 */ 5505 static int 5506 flow_create_split_inner(struct rte_eth_dev *dev, 5507 struct rte_flow *flow, 5508 struct mlx5_flow **sub_flow, 5509 const struct rte_flow_attr *attr, 5510 const struct rte_flow_item items[], 5511 const struct rte_flow_action actions[], 5512 struct mlx5_flow_split_info *flow_split_info, 5513 struct rte_flow_error *error) 5514 { 5515 struct mlx5_flow *dev_flow; 5516 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 5517 5518 dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, 5519 flow_split_info->flow_idx, error); 5520 if (!dev_flow) 5521 return -rte_errno; 5522 dev_flow->flow = flow; 5523 dev_flow->external = flow_split_info->external; 5524 dev_flow->skip_scale = flow_split_info->skip_scale; 5525 /* Subflow object was created, we must include one in the list. */ 5526 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 5527 dev_flow->handle, next); 5528 /* 5529 * If dev_flow is as one of the suffix flow, some actions in suffix 5530 * flow may need some user defined item layer flags, and pass the 5531 * Metadata rxq mark flag to suffix flow as well. 5532 */ 5533 if (flow_split_info->prefix_layers) 5534 dev_flow->handle->layers = flow_split_info->prefix_layers; 5535 if (flow_split_info->prefix_mark) { 5536 MLX5_ASSERT(wks); 5537 wks->mark = 1; 5538 } 5539 if (sub_flow) 5540 *sub_flow = dev_flow; 5541 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5542 dev_flow->dv.table_id = flow_split_info->table_id; 5543 #endif 5544 return flow_drv_translate(dev, dev_flow, attr, items, actions, error); 5545 } 5546 5547 /** 5548 * Get the sub policy of a meter. 5549 * 5550 * @param[in] dev 5551 * Pointer to Ethernet device. 5552 * @param[in] flow 5553 * Parent flow structure pointer. 5554 * @param wks 5555 * Pointer to thread flow work space. 5556 * @param[in] attr 5557 * Flow rule attributes. 5558 * @param[in] items 5559 * Pattern specification (list terminated by the END pattern item). 5560 * @param[out] error 5561 * Perform verbose error reporting if not NULL. 5562 * 5563 * @return 5564 * Pointer to the meter sub policy, NULL otherwise and rte_errno is set. 5565 */ 5566 static struct mlx5_flow_meter_sub_policy * 5567 get_meter_sub_policy(struct rte_eth_dev *dev, 5568 struct rte_flow *flow, 5569 struct mlx5_flow_workspace *wks, 5570 const struct rte_flow_attr *attr, 5571 const struct rte_flow_item items[], 5572 struct rte_flow_error *error) 5573 { 5574 struct mlx5_flow_meter_policy *policy; 5575 struct mlx5_flow_meter_policy *final_policy; 5576 struct mlx5_flow_meter_sub_policy *sub_policy = NULL; 5577 5578 policy = wks->policy; 5579 final_policy = policy->is_hierarchy ? wks->final_policy : policy; 5580 if (final_policy->is_rss || final_policy->is_queue) { 5581 struct mlx5_flow_rss_desc rss_desc_v[MLX5_MTR_RTE_COLORS]; 5582 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS] = {0}; 5583 uint32_t i; 5584 5585 /* 5586 * This is a tmp dev_flow, 5587 * no need to register any matcher for it in translate. 5588 */ 5589 wks->skip_matcher_reg = 1; 5590 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) { 5591 struct mlx5_flow dev_flow = {0}; 5592 struct mlx5_flow_handle dev_handle = { {0} }; 5593 uint8_t fate = final_policy->act_cnt[i].fate_action; 5594 5595 if (fate == MLX5_FLOW_FATE_SHARED_RSS) { 5596 const struct rte_flow_action_rss *rss_act = 5597 final_policy->act_cnt[i].rss->conf; 5598 struct rte_flow_action rss_actions[2] = { 5599 [0] = { 5600 .type = RTE_FLOW_ACTION_TYPE_RSS, 5601 .conf = rss_act, 5602 }, 5603 [1] = { 5604 .type = RTE_FLOW_ACTION_TYPE_END, 5605 .conf = NULL, 5606 } 5607 }; 5608 5609 dev_flow.handle = &dev_handle; 5610 dev_flow.ingress = attr->ingress; 5611 dev_flow.flow = flow; 5612 dev_flow.external = 0; 5613 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 5614 dev_flow.dv.transfer = attr->transfer; 5615 #endif 5616 /** 5617 * Translate RSS action to get rss hash fields. 5618 */ 5619 if (flow_drv_translate(dev, &dev_flow, attr, 5620 items, rss_actions, error)) 5621 goto exit; 5622 rss_desc_v[i] = wks->rss_desc; 5623 rss_desc_v[i].symmetric_hash_function = 5624 dev_flow.symmetric_hash_function; 5625 rss_desc_v[i].key_len = MLX5_RSS_HASH_KEY_LEN; 5626 rss_desc_v[i].hash_fields = 5627 dev_flow.hash_fields; 5628 rss_desc_v[i].queue_num = 5629 rss_desc_v[i].hash_fields ? 5630 rss_desc_v[i].queue_num : 1; 5631 rss_desc_v[i].tunnel = 5632 !!(dev_flow.handle->layers & 5633 MLX5_FLOW_LAYER_TUNNEL); 5634 /* Use the RSS queues in the containers. */ 5635 rss_desc_v[i].queue = 5636 (uint16_t *)(uintptr_t)rss_act->queue; 5637 rss_desc[i] = &rss_desc_v[i]; 5638 } else if (fate == MLX5_FLOW_FATE_QUEUE) { 5639 /* This is queue action. */ 5640 rss_desc_v[i] = wks->rss_desc; 5641 rss_desc_v[i].key_len = 0; 5642 rss_desc_v[i].hash_fields = 0; 5643 rss_desc_v[i].queue = 5644 &final_policy->act_cnt[i].queue; 5645 rss_desc_v[i].queue_num = 1; 5646 rss_desc[i] = &rss_desc_v[i]; 5647 } else { 5648 rss_desc[i] = NULL; 5649 } 5650 } 5651 sub_policy = flow_drv_meter_sub_policy_rss_prepare(dev, 5652 flow, policy, rss_desc); 5653 } else { 5654 enum mlx5_meter_domain mtr_domain = 5655 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5656 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5657 MLX5_MTR_DOMAIN_INGRESS); 5658 sub_policy = policy->sub_policys[mtr_domain][0]; 5659 } 5660 if (!sub_policy) 5661 rte_flow_error_set(error, EINVAL, 5662 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5663 "Failed to get meter sub-policy."); 5664 exit: 5665 return sub_policy; 5666 } 5667 5668 /** 5669 * Split the meter flow. 5670 * 5671 * As meter flow will split to three sub flow, other than meter 5672 * action, the other actions make sense to only meter accepts 5673 * the packet. If it need to be dropped, no other additional 5674 * actions should be take. 5675 * 5676 * One kind of special action which decapsulates the L3 tunnel 5677 * header will be in the prefix sub flow, as not to take the 5678 * L3 tunnel header into account. 5679 * 5680 * @param[in] dev 5681 * Pointer to Ethernet device. 5682 * @param[in] flow 5683 * Parent flow structure pointer. 5684 * @param wks 5685 * Pointer to thread flow work space. 5686 * @param[in] attr 5687 * Flow rule attributes. 5688 * @param[in] items 5689 * Pattern specification (list terminated by the END pattern item). 5690 * @param[out] sfx_items 5691 * Suffix flow match items (list terminated by the END pattern item). 5692 * @param[in] actions 5693 * Associated actions (list terminated by the END action). 5694 * @param[out] actions_sfx 5695 * Suffix flow actions. 5696 * @param[out] actions_pre 5697 * Prefix flow actions. 5698 * @param[out] mtr_flow_id 5699 * Pointer to meter flow id. 5700 * @param[out] error 5701 * Perform verbose error reporting if not NULL. 5702 * 5703 * @return 5704 * 0 on success, a negative errno value otherwise and rte_errno is set. 5705 */ 5706 static int 5707 flow_meter_split_prep(struct rte_eth_dev *dev, 5708 struct rte_flow *flow, 5709 struct mlx5_flow_workspace *wks, 5710 const struct rte_flow_attr *attr, 5711 const struct rte_flow_item items[], 5712 struct rte_flow_item sfx_items[], 5713 const struct rte_flow_action actions[], 5714 struct rte_flow_action actions_sfx[], 5715 struct rte_flow_action actions_pre[], 5716 uint32_t *mtr_flow_id, 5717 struct rte_flow_error *error) 5718 { 5719 struct mlx5_priv *priv = dev->data->dev_private; 5720 struct mlx5_flow_meter_info *fm = wks->fm; 5721 struct rte_flow_action *tag_action = NULL; 5722 struct rte_flow_item *tag_item; 5723 struct mlx5_rte_flow_action_set_tag *set_tag; 5724 const struct rte_flow_action_raw_encap *raw_encap; 5725 const struct rte_flow_action_raw_decap *raw_decap; 5726 struct mlx5_rte_flow_item_tag *tag_item_spec; 5727 struct mlx5_rte_flow_item_tag *tag_item_mask; 5728 uint32_t tag_id = 0; 5729 struct rte_flow_item *vlan_item_dst = NULL; 5730 const struct rte_flow_item *vlan_item_src = NULL; 5731 const struct rte_flow_item *orig_items = items; 5732 struct rte_flow_action *hw_mtr_action; 5733 struct rte_flow_action *action_pre_head = NULL; 5734 uint16_t flow_src_port = priv->representor_id; 5735 bool mtr_first; 5736 uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; 5737 uint8_t mtr_reg_bits = priv->mtr_reg_share ? 5738 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; 5739 uint32_t flow_id = 0; 5740 uint32_t flow_id_reversed = 0; 5741 uint8_t flow_id_bits = 0; 5742 bool after_meter = false; 5743 int shift; 5744 5745 /* Prepare the suffix subflow items. */ 5746 tag_item = sfx_items++; 5747 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 5748 int item_type = items->type; 5749 5750 switch (item_type) { 5751 case RTE_FLOW_ITEM_TYPE_PORT_ID: 5752 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 5753 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 5754 if (mlx5_flow_get_item_vport_id(dev, items, &flow_src_port, NULL, error)) 5755 return -rte_errno; 5756 if (!fm->def_policy && wks->policy->hierarchy_match_port && 5757 flow_src_port != priv->representor_id) { 5758 if (flow_drv_mtr_hierarchy_rule_create(dev, 5759 flow, fm, 5760 flow_src_port, 5761 items, 5762 error)) 5763 return -rte_errno; 5764 } 5765 memcpy(sfx_items, items, sizeof(*sfx_items)); 5766 sfx_items++; 5767 break; 5768 case RTE_FLOW_ITEM_TYPE_VLAN: 5769 /* Determine if copy vlan item below. */ 5770 vlan_item_src = items; 5771 vlan_item_dst = sfx_items++; 5772 vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID; 5773 break; 5774 default: 5775 break; 5776 } 5777 } 5778 sfx_items->type = RTE_FLOW_ITEM_TYPE_END; 5779 sfx_items++; 5780 mtr_first = priv->sh->meter_aso_en && 5781 (attr->egress || (attr->transfer && flow_src_port != UINT16_MAX)); 5782 /* For ASO meter, meter must be before tag in TX direction. */ 5783 if (mtr_first) { 5784 action_pre_head = actions_pre++; 5785 /* Leave space for tag action. */ 5786 tag_action = actions_pre++; 5787 } 5788 /* Prepare the actions for prefix and suffix flow. */ 5789 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 5790 struct rte_flow_action *action_cur = NULL; 5791 5792 switch (actions->type) { 5793 case RTE_FLOW_ACTION_TYPE_METER: 5794 if (mtr_first) { 5795 action_cur = action_pre_head; 5796 } else { 5797 /* Leave space for tag action. */ 5798 tag_action = actions_pre++; 5799 action_cur = actions_pre++; 5800 } 5801 after_meter = true; 5802 break; 5803 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 5804 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 5805 action_cur = actions_pre++; 5806 break; 5807 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 5808 raw_encap = actions->conf; 5809 if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) 5810 action_cur = actions_pre++; 5811 break; 5812 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 5813 raw_decap = actions->conf; 5814 if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) 5815 action_cur = actions_pre++; 5816 break; 5817 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 5818 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 5819 if (vlan_item_dst && vlan_item_src) { 5820 memcpy(vlan_item_dst, vlan_item_src, 5821 sizeof(*vlan_item_dst)); 5822 /* 5823 * Convert to internal match item, it is used 5824 * for vlan push and set vid. 5825 */ 5826 vlan_item_dst->type = (enum rte_flow_item_type) 5827 MLX5_RTE_FLOW_ITEM_TYPE_VLAN; 5828 } 5829 break; 5830 case RTE_FLOW_ACTION_TYPE_COUNT: 5831 if (fm->def_policy) 5832 action_cur = after_meter ? 5833 actions_sfx++ : actions_pre++; 5834 break; 5835 default: 5836 break; 5837 } 5838 if (!action_cur) 5839 action_cur = (fm->def_policy) ? 5840 actions_sfx++ : actions_pre++; 5841 memcpy(action_cur, actions, sizeof(struct rte_flow_action)); 5842 } 5843 /* Add end action to the actions. */ 5844 actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; 5845 if (priv->sh->meter_aso_en) { 5846 /** 5847 * For ASO meter, need to add an extra jump action explicitly, 5848 * to jump from meter to policer table. 5849 */ 5850 struct mlx5_flow_meter_sub_policy *sub_policy; 5851 struct mlx5_flow_tbl_data_entry *tbl_data; 5852 5853 if (!fm->def_policy) { 5854 sub_policy = get_meter_sub_policy(dev, flow, wks, 5855 attr, orig_items, 5856 error); 5857 if (!sub_policy) 5858 return -rte_errno; 5859 } else { 5860 enum mlx5_meter_domain mtr_domain = 5861 attr->transfer ? MLX5_MTR_DOMAIN_TRANSFER : 5862 (attr->egress ? MLX5_MTR_DOMAIN_EGRESS : 5863 MLX5_MTR_DOMAIN_INGRESS); 5864 5865 sub_policy = 5866 &priv->sh->mtrmng->def_policy[mtr_domain]->sub_policy; 5867 } 5868 tbl_data = container_of(sub_policy->tbl_rsc, 5869 struct mlx5_flow_tbl_data_entry, tbl); 5870 hw_mtr_action = actions_pre++; 5871 hw_mtr_action->type = (enum rte_flow_action_type) 5872 MLX5_RTE_FLOW_ACTION_TYPE_JUMP; 5873 hw_mtr_action->conf = tbl_data->jump.action; 5874 } 5875 actions_pre->type = RTE_FLOW_ACTION_TYPE_END; 5876 actions_pre++; 5877 if (!tag_action) 5878 return rte_flow_error_set(error, ENOMEM, 5879 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 5880 NULL, "No tag action space."); 5881 if (!mtr_flow_id) { 5882 tag_action->type = RTE_FLOW_ACTION_TYPE_VOID; 5883 goto exit; 5884 } 5885 /* Only default-policy Meter creates mtr flow id. */ 5886 if (fm->def_policy) { 5887 mlx5_ipool_malloc(fm->flow_ipool, &tag_id); 5888 if (!tag_id) 5889 return rte_flow_error_set(error, ENOMEM, 5890 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5891 "Failed to allocate meter flow id."); 5892 flow_id = tag_id - 1; 5893 flow_id_bits = (!flow_id) ? 1 : 5894 (MLX5_REG_BITS - rte_clz32(flow_id)); 5895 if ((flow_id_bits + priv->sh->mtrmng->max_mtr_bits) > 5896 mtr_reg_bits) { 5897 mlx5_ipool_free(fm->flow_ipool, tag_id); 5898 return rte_flow_error_set(error, EINVAL, 5899 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 5900 "Meter flow id exceeds max limit."); 5901 } 5902 if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits) 5903 priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits; 5904 } 5905 /* Build tag actions and items for meter_id/meter flow_id. */ 5906 set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; 5907 tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; 5908 tag_item_mask = tag_item_spec + 1; 5909 /* Both flow_id and meter_id share the same register. */ 5910 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 5911 .id = (enum modify_reg)mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 5912 0, error), 5913 .offset = mtr_id_offset, 5914 .length = mtr_reg_bits, 5915 .data = flow->meter, 5916 }; 5917 /* 5918 * The color Reg bits used by flow_id are growing from 5919 * msb to lsb, so must do bit reverse for flow_id val in RegC. 5920 */ 5921 for (shift = 0; shift < flow_id_bits; shift++) 5922 flow_id_reversed = (flow_id_reversed << 1) | 5923 ((flow_id >> shift) & 0x1); 5924 set_tag->data |= 5925 flow_id_reversed << (mtr_reg_bits - flow_id_bits); 5926 tag_item_spec->id = set_tag->id; 5927 tag_item_spec->data = set_tag->data << mtr_id_offset; 5928 tag_item_mask->data = UINT32_MAX << mtr_id_offset; 5929 tag_action->type = (enum rte_flow_action_type) 5930 MLX5_RTE_FLOW_ACTION_TYPE_TAG; 5931 tag_action->conf = set_tag; 5932 tag_item->type = (enum rte_flow_item_type) 5933 MLX5_RTE_FLOW_ITEM_TYPE_TAG; 5934 tag_item->spec = tag_item_spec; 5935 tag_item->last = NULL; 5936 tag_item->mask = tag_item_mask; 5937 exit: 5938 if (mtr_flow_id) 5939 *mtr_flow_id = tag_id; 5940 return 0; 5941 } 5942 5943 /** 5944 * Split action list having QUEUE/RSS for metadata register copy. 5945 * 5946 * Once Q/RSS action is detected in user's action list, the flow action 5947 * should be split in order to copy metadata registers, which will happen in 5948 * RX_CP_TBL like, 5949 * - CQE->flow_tag := reg_c[1] (MARK) 5950 * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META) 5951 * The Q/RSS action will be performed on RX_ACT_TBL after passing by RX_CP_TBL. 5952 * This is because the last action of each flow must be a terminal action 5953 * (QUEUE, RSS or DROP). 5954 * 5955 * Flow ID must be allocated to identify actions in the RX_ACT_TBL and it is 5956 * stored and kept in the mlx5_flow structure per each sub_flow. 5957 * 5958 * The Q/RSS action is replaced with, 5959 * - SET_TAG, setting the allocated flow ID to reg_c[2]. 5960 * And the following JUMP action is added at the end, 5961 * - JUMP, to RX_CP_TBL. 5962 * 5963 * A flow to perform remained Q/RSS action will be created in RX_ACT_TBL by 5964 * flow_create_split_metadata() routine. The flow will look like, 5965 * - If flow ID matches (reg_c[2]), perform Q/RSS. 5966 * 5967 * @param dev 5968 * Pointer to Ethernet device. 5969 * @param[out] split_actions 5970 * Pointer to store split actions to jump to CP_TBL. 5971 * @param[in] actions 5972 * Pointer to the list of original flow actions. 5973 * @param[in] qrss 5974 * Pointer to the Q/RSS action. 5975 * @param[in] actions_n 5976 * Number of original actions. 5977 * @param[in] mtr_sfx 5978 * Check if it is in meter suffix table. 5979 * @param[out] error 5980 * Perform verbose error reporting if not NULL. 5981 * 5982 * @return 5983 * non-zero unique flow_id on success, otherwise 0 and 5984 * error/rte_error are set. 5985 */ 5986 static uint32_t 5987 flow_mreg_split_qrss_prep(struct rte_eth_dev *dev, 5988 struct rte_flow_action *split_actions, 5989 const struct rte_flow_action *actions, 5990 const struct rte_flow_action *qrss, 5991 int actions_n, int mtr_sfx, 5992 struct rte_flow_error *error) 5993 { 5994 struct mlx5_priv *priv = dev->data->dev_private; 5995 struct mlx5_rte_flow_action_set_tag *set_tag; 5996 struct rte_flow_action_jump *jump; 5997 const int qrss_idx = qrss - actions; 5998 uint32_t flow_id = 0; 5999 int ret = 0; 6000 6001 /* 6002 * Given actions will be split 6003 * - Replace QUEUE/RSS action with SET_TAG to set flow ID. 6004 * - Add jump to mreg CP_TBL. 6005 * As a result, there will be one more action. 6006 */ 6007 memcpy(split_actions, actions, sizeof(*split_actions) * actions_n); 6008 /* Count MLX5_RTE_FLOW_ACTION_TYPE_TAG. */ 6009 ++actions_n; 6010 set_tag = (void *)(split_actions + actions_n); 6011 /* 6012 * If we are not the meter suffix flow, add the tag action. 6013 * Since meter suffix flow already has the tag added. 6014 */ 6015 if (!mtr_sfx) { 6016 /* 6017 * Allocate the new subflow ID. This one is unique within 6018 * device and not shared with representors. Otherwise, 6019 * we would have to resolve multi-thread access synch 6020 * issue. Each flow on the shared device is appended 6021 * with source vport identifier, so the resulting 6022 * flows will be unique in the shared (by master and 6023 * representors) domain even if they have coinciding 6024 * IDs. 6025 */ 6026 mlx5_ipool_malloc(priv->sh->ipool 6027 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &flow_id); 6028 if (!flow_id) 6029 return rte_flow_error_set(error, ENOMEM, 6030 RTE_FLOW_ERROR_TYPE_ACTION, 6031 NULL, "can't allocate id " 6032 "for split Q/RSS subflow"); 6033 /* Internal SET_TAG action to set flow ID. */ 6034 *set_tag = (struct mlx5_rte_flow_action_set_tag){ 6035 .data = flow_id, 6036 }; 6037 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, error); 6038 if (ret < 0) 6039 return ret; 6040 set_tag->id = ret; 6041 /* Construct new actions array. */ 6042 /* Replace QUEUE/RSS action. */ 6043 split_actions[qrss_idx] = (struct rte_flow_action){ 6044 .type = (enum rte_flow_action_type) 6045 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6046 .conf = set_tag, 6047 }; 6048 } else { 6049 /* 6050 * If we are the suffix flow of meter, tag already exist. 6051 * Set the QUEUE/RSS action to void. 6052 */ 6053 split_actions[qrss_idx].type = RTE_FLOW_ACTION_TYPE_VOID; 6054 } 6055 /* JUMP action to jump to mreg copy table (CP_TBL). */ 6056 jump = (void *)(set_tag + 1); 6057 *jump = (struct rte_flow_action_jump){ 6058 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 6059 }; 6060 split_actions[actions_n - 2] = (struct rte_flow_action){ 6061 .type = RTE_FLOW_ACTION_TYPE_JUMP, 6062 .conf = jump, 6063 }; 6064 split_actions[actions_n - 1] = (struct rte_flow_action){ 6065 .type = RTE_FLOW_ACTION_TYPE_END, 6066 }; 6067 return flow_id; 6068 } 6069 6070 /** 6071 * Extend the given action list for Tx metadata copy. 6072 * 6073 * Copy the given action list to the ext_actions and add flow metadata register 6074 * copy action in order to copy reg_a set by WQE to reg_c[0]. 6075 * 6076 * @param[out] ext_actions 6077 * Pointer to the extended action list. 6078 * @param[in] actions 6079 * Pointer to the list of actions. 6080 * @param[in] actions_n 6081 * Number of actions in the list. 6082 * @param[out] error 6083 * Perform verbose error reporting if not NULL. 6084 * @param[in] encap_idx 6085 * The encap action index. 6086 * 6087 * @return 6088 * 0 on success, negative value otherwise 6089 */ 6090 static int 6091 flow_mreg_tx_copy_prep(struct rte_eth_dev *dev, 6092 struct rte_flow_action *ext_actions, 6093 const struct rte_flow_action *actions, 6094 int actions_n, struct rte_flow_error *error, 6095 int encap_idx) 6096 { 6097 struct mlx5_flow_action_copy_mreg *cp_mreg = 6098 (struct mlx5_flow_action_copy_mreg *) 6099 (ext_actions + actions_n + 1); 6100 int ret; 6101 6102 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error); 6103 if (ret < 0) 6104 return ret; 6105 cp_mreg->dst = ret; 6106 ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_TX, 0, error); 6107 if (ret < 0) 6108 return ret; 6109 cp_mreg->src = ret; 6110 if (encap_idx != 0) 6111 memcpy(ext_actions, actions, sizeof(*ext_actions) * encap_idx); 6112 if (encap_idx == actions_n - 1) { 6113 ext_actions[actions_n - 1] = (struct rte_flow_action){ 6114 .type = (enum rte_flow_action_type) 6115 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6116 .conf = cp_mreg, 6117 }; 6118 ext_actions[actions_n] = (struct rte_flow_action){ 6119 .type = RTE_FLOW_ACTION_TYPE_END, 6120 }; 6121 } else { 6122 ext_actions[encap_idx] = (struct rte_flow_action){ 6123 .type = (enum rte_flow_action_type) 6124 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 6125 .conf = cp_mreg, 6126 }; 6127 memcpy(ext_actions + encap_idx + 1, actions + encap_idx, 6128 sizeof(*ext_actions) * (actions_n - encap_idx)); 6129 } 6130 return 0; 6131 } 6132 6133 /** 6134 * Check the match action from the action list. 6135 * 6136 * @param[in] actions 6137 * Pointer to the list of actions. 6138 * @param[in] attr 6139 * Flow rule attributes. 6140 * @param[in] action 6141 * The action to be check if exist. 6142 * @param[out] match_action_pos 6143 * Pointer to the position of the matched action if exists, otherwise is -1. 6144 * @param[out] qrss_action_pos 6145 * Pointer to the position of the Queue/RSS action if exists, otherwise is -1. 6146 * @param[out] modify_after_mirror 6147 * Pointer to the flag of modify action after FDB mirroring. 6148 * 6149 * @return 6150 * > 0 the total number of actions. 6151 * 0 if not found match action in action list. 6152 */ 6153 static int 6154 flow_check_match_action(const struct rte_flow_action actions[], 6155 const struct rte_flow_attr *attr, 6156 enum rte_flow_action_type action, 6157 int *match_action_pos, int *qrss_action_pos, 6158 int *modify_after_mirror) 6159 { 6160 const struct rte_flow_action_sample *sample; 6161 const struct rte_flow_action_raw_decap *decap; 6162 int actions_n = 0; 6163 uint32_t ratio = 0; 6164 int sub_type = 0; 6165 int flag = 0; 6166 int fdb_mirror = 0; 6167 6168 *match_action_pos = -1; 6169 *qrss_action_pos = -1; 6170 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { 6171 if (actions->type == action) { 6172 flag = 1; 6173 *match_action_pos = actions_n; 6174 } 6175 switch (actions->type) { 6176 case RTE_FLOW_ACTION_TYPE_QUEUE: 6177 case RTE_FLOW_ACTION_TYPE_RSS: 6178 *qrss_action_pos = actions_n; 6179 break; 6180 case RTE_FLOW_ACTION_TYPE_SAMPLE: 6181 sample = actions->conf; 6182 ratio = sample->ratio; 6183 sub_type = ((const struct rte_flow_action *) 6184 (sample->actions))->type; 6185 if (ratio == 1 && attr->transfer && 6186 sub_type != RTE_FLOW_ACTION_TYPE_END) 6187 fdb_mirror = 1; 6188 break; 6189 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC: 6190 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST: 6191 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC: 6192 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST: 6193 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC: 6194 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST: 6195 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC: 6196 case RTE_FLOW_ACTION_TYPE_SET_TP_DST: 6197 case RTE_FLOW_ACTION_TYPE_DEC_TTL: 6198 case RTE_FLOW_ACTION_TYPE_SET_TTL: 6199 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ: 6200 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ: 6201 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK: 6202 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK: 6203 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: 6204 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP: 6205 case RTE_FLOW_ACTION_TYPE_FLAG: 6206 case RTE_FLOW_ACTION_TYPE_MARK: 6207 case RTE_FLOW_ACTION_TYPE_SET_META: 6208 case RTE_FLOW_ACTION_TYPE_SET_TAG: 6209 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: 6210 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6211 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6212 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: 6213 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: 6214 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: 6215 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD: 6216 case RTE_FLOW_ACTION_TYPE_METER: 6217 if (fdb_mirror) 6218 *modify_after_mirror = 1; 6219 break; 6220 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6221 decap = actions->conf; 6222 while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID) 6223 ; 6224 actions_n++; 6225 if (actions->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) { 6226 const struct rte_flow_action_raw_encap *encap = 6227 actions->conf; 6228 if (decap->size <= 6229 MLX5_ENCAPSULATION_DECISION_SIZE && 6230 encap->size > 6231 MLX5_ENCAPSULATION_DECISION_SIZE) 6232 /* L3 encap. */ 6233 break; 6234 } 6235 if (fdb_mirror) 6236 *modify_after_mirror = 1; 6237 break; 6238 default: 6239 break; 6240 } 6241 actions_n++; 6242 } 6243 if (flag && fdb_mirror && !*modify_after_mirror) { 6244 /* FDB mirroring uses the destination array to implement 6245 * instead of FLOW_SAMPLER object. 6246 */ 6247 if (sub_type != RTE_FLOW_ACTION_TYPE_END) 6248 flag = 0; 6249 } 6250 /* Count RTE_FLOW_ACTION_TYPE_END. */ 6251 return flag ? actions_n + 1 : 0; 6252 } 6253 6254 #define SAMPLE_SUFFIX_ITEM 3 6255 6256 /** 6257 * Split the sample flow. 6258 * 6259 * As sample flow will split to two sub flow, sample flow with 6260 * sample action, the other actions will move to new suffix flow. 6261 * 6262 * Also add unique tag id with tag action in the sample flow, 6263 * the same tag id will be as match in the suffix flow. 6264 * 6265 * @param dev 6266 * Pointer to Ethernet device. 6267 * @param[in] add_tag 6268 * Add extra tag action flag. 6269 * @param[out] sfx_items 6270 * Suffix flow match items (list terminated by the END pattern item). 6271 * @param[in] actions 6272 * Associated actions (list terminated by the END action). 6273 * @param[out] actions_sfx 6274 * Suffix flow actions. 6275 * @param[out] actions_pre 6276 * Prefix flow actions. 6277 * @param[in] actions_n 6278 * The total number of actions. 6279 * @param[in] sample_action_pos 6280 * The sample action position. 6281 * @param[in] qrss_action_pos 6282 * The Queue/RSS action position. 6283 * @param[in] jump_table 6284 * Add extra jump action flag. 6285 * @param[out] error 6286 * Perform verbose error reporting if not NULL. 6287 * 6288 * @return 6289 * 0 on success, or unique flow_id, a negative errno value 6290 * otherwise and rte_errno is set. 6291 */ 6292 static int 6293 flow_sample_split_prep(struct rte_eth_dev *dev, 6294 int add_tag, 6295 const struct rte_flow_item items[], 6296 struct rte_flow_item sfx_items[], 6297 const struct rte_flow_action actions[], 6298 struct rte_flow_action actions_sfx[], 6299 struct rte_flow_action actions_pre[], 6300 int actions_n, 6301 int sample_action_pos, 6302 int qrss_action_pos, 6303 int jump_table, 6304 struct rte_flow_error *error) 6305 { 6306 struct mlx5_priv *priv = dev->data->dev_private; 6307 struct mlx5_rte_flow_action_set_tag *set_tag; 6308 struct mlx5_rte_flow_item_tag *tag_spec; 6309 struct mlx5_rte_flow_item_tag *tag_mask; 6310 struct rte_flow_action_jump *jump_action; 6311 uint32_t tag_id = 0; 6312 int append_index = 0; 6313 int set_tag_idx = -1; 6314 int index; 6315 int ret; 6316 6317 if (sample_action_pos < 0) 6318 return rte_flow_error_set(error, EINVAL, 6319 RTE_FLOW_ERROR_TYPE_ACTION, 6320 NULL, "invalid position of sample " 6321 "action in list"); 6322 /* Prepare the actions for prefix and suffix flow. */ 6323 if (add_tag) { 6324 /* Update the new added tag action index preceding 6325 * the PUSH_VLAN or ENCAP action. 6326 */ 6327 const struct rte_flow_action_raw_encap *raw_encap; 6328 const struct rte_flow_action *action = actions; 6329 int encap_idx; 6330 int action_idx = 0; 6331 int raw_decap_idx = -1; 6332 int push_vlan_idx = -1; 6333 for (; action->type != RTE_FLOW_ACTION_TYPE_END; action++) { 6334 switch (action->type) { 6335 case RTE_FLOW_ACTION_TYPE_RAW_DECAP: 6336 raw_decap_idx = action_idx; 6337 break; 6338 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: 6339 raw_encap = action->conf; 6340 if (raw_encap->size > 6341 MLX5_ENCAPSULATION_DECISION_SIZE) { 6342 encap_idx = raw_decap_idx != -1 ? 6343 raw_decap_idx : action_idx; 6344 if (encap_idx < sample_action_pos && 6345 push_vlan_idx == -1) 6346 set_tag_idx = encap_idx; 6347 } 6348 break; 6349 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: 6350 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP: 6351 encap_idx = action_idx; 6352 if (encap_idx < sample_action_pos && 6353 push_vlan_idx == -1) 6354 set_tag_idx = encap_idx; 6355 break; 6356 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: 6357 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: 6358 if (action_idx < sample_action_pos && 6359 push_vlan_idx == -1) { 6360 set_tag_idx = action_idx; 6361 push_vlan_idx = action_idx; 6362 } 6363 break; 6364 default: 6365 break; 6366 } 6367 action_idx++; 6368 } 6369 } 6370 /* Prepare the actions for prefix and suffix flow. */ 6371 if (qrss_action_pos >= 0 && qrss_action_pos < sample_action_pos) { 6372 index = qrss_action_pos; 6373 /* Put the preceding the Queue/RSS action into prefix flow. */ 6374 if (index != 0) 6375 memcpy(actions_pre, actions, 6376 sizeof(struct rte_flow_action) * index); 6377 /* Put others preceding the sample action into prefix flow. */ 6378 if (sample_action_pos > index + 1) 6379 memcpy(actions_pre + index, actions + index + 1, 6380 sizeof(struct rte_flow_action) * 6381 (sample_action_pos - index - 1)); 6382 index = sample_action_pos - 1; 6383 /* Put Queue/RSS action into Suffix flow. */ 6384 memcpy(actions_sfx, actions + qrss_action_pos, 6385 sizeof(struct rte_flow_action)); 6386 actions_sfx++; 6387 } else if (add_tag && set_tag_idx >= 0) { 6388 if (set_tag_idx > 0) 6389 memcpy(actions_pre, actions, 6390 sizeof(struct rte_flow_action) * set_tag_idx); 6391 memcpy(actions_pre + set_tag_idx + 1, actions + set_tag_idx, 6392 sizeof(struct rte_flow_action) * 6393 (sample_action_pos - set_tag_idx)); 6394 index = sample_action_pos; 6395 } else { 6396 index = sample_action_pos; 6397 if (index != 0) 6398 memcpy(actions_pre, actions, 6399 sizeof(struct rte_flow_action) * index); 6400 } 6401 /* For CX5, add an extra tag action for NIC-RX and E-Switch ingress. 6402 * For CX6DX and above, metadata registers Cx preserve their value, 6403 * add an extra tag action for NIC-RX and E-Switch Domain. 6404 */ 6405 if (add_tag) { 6406 /* Prepare the prefix tag action. */ 6407 append_index++; 6408 set_tag = (void *)(actions_pre + actions_n + append_index); 6409 /* Trust VF/SF on CX5 not supported meter so that the reserved 6410 * metadata regC is REG_NON, back to use application tag 6411 * index 0. 6412 */ 6413 if (unlikely(priv->mtr_color_reg == REG_NON)) 6414 ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); 6415 else 6416 ret = mlx5_flow_get_reg_id(dev, MLX5_SAMPLE_ID, 0, error); 6417 if (ret < 0) 6418 return ret; 6419 mlx5_ipool_malloc(priv->sh->ipool 6420 [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); 6421 *set_tag = (struct mlx5_rte_flow_action_set_tag) { 6422 .id = ret, 6423 .data = tag_id, 6424 }; 6425 /* Prepare the suffix subflow items. */ 6426 tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); 6427 tag_spec->data = tag_id; 6428 tag_spec->id = set_tag->id; 6429 tag_mask = tag_spec + 1; 6430 tag_mask->data = UINT32_MAX; 6431 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { 6432 if (items->type == RTE_FLOW_ITEM_TYPE_PORT_ID || 6433 items->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR || 6434 items->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 6435 memcpy(sfx_items, items, sizeof(*sfx_items)); 6436 sfx_items++; 6437 break; 6438 } 6439 } 6440 sfx_items[0] = (struct rte_flow_item){ 6441 .type = (enum rte_flow_item_type) 6442 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6443 .spec = tag_spec, 6444 .last = NULL, 6445 .mask = tag_mask, 6446 }; 6447 sfx_items[1] = (struct rte_flow_item){ 6448 .type = (enum rte_flow_item_type) 6449 RTE_FLOW_ITEM_TYPE_END, 6450 }; 6451 /* Prepare the tag action in prefix subflow. */ 6452 set_tag_idx = (set_tag_idx == -1) ? index : set_tag_idx; 6453 actions_pre[set_tag_idx] = 6454 (struct rte_flow_action){ 6455 .type = (enum rte_flow_action_type) 6456 MLX5_RTE_FLOW_ACTION_TYPE_TAG, 6457 .conf = set_tag, 6458 }; 6459 /* Update next sample position due to add one tag action */ 6460 index += 1; 6461 } 6462 /* Copy the sample action into prefix flow. */ 6463 memcpy(actions_pre + index, actions + sample_action_pos, 6464 sizeof(struct rte_flow_action)); 6465 index += 1; 6466 /* For the modify action after the sample action in E-Switch mirroring, 6467 * Add the extra jump action in prefix subflow and jump into the next 6468 * table, then do the modify action in the new table. 6469 */ 6470 if (jump_table) { 6471 /* Prepare the prefix jump action. */ 6472 append_index++; 6473 jump_action = (void *)(actions_pre + actions_n + append_index); 6474 jump_action->group = jump_table; 6475 actions_pre[index++] = 6476 (struct rte_flow_action){ 6477 .type = (enum rte_flow_action_type) 6478 RTE_FLOW_ACTION_TYPE_JUMP, 6479 .conf = jump_action, 6480 }; 6481 } 6482 actions_pre[index] = (struct rte_flow_action){ 6483 .type = (enum rte_flow_action_type) 6484 RTE_FLOW_ACTION_TYPE_END, 6485 }; 6486 /* Put the actions after sample into Suffix flow. */ 6487 memcpy(actions_sfx, actions + sample_action_pos + 1, 6488 sizeof(struct rte_flow_action) * 6489 (actions_n - sample_action_pos - 1)); 6490 return tag_id; 6491 } 6492 6493 /** 6494 * The splitting for metadata feature. 6495 * 6496 * - Q/RSS action on NIC Rx should be split in order to pass by 6497 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6498 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6499 * 6500 * - All the actions on NIC Tx should have a mreg copy action to 6501 * copy reg_a from WQE to reg_c[0]. 6502 * 6503 * @param dev 6504 * Pointer to Ethernet device. 6505 * @param[in] flow 6506 * Parent flow structure pointer. 6507 * @param[in] attr 6508 * Flow rule attributes. 6509 * @param[in] items 6510 * Pattern specification (list terminated by the END pattern item). 6511 * @param[in] actions 6512 * Associated actions (list terminated by the END action). 6513 * @param[in] flow_split_info 6514 * Pointer to flow split info structure. 6515 * @param[out] error 6516 * Perform verbose error reporting if not NULL. 6517 * @return 6518 * 0 on success, negative value otherwise 6519 */ 6520 static int 6521 flow_create_split_metadata(struct rte_eth_dev *dev, 6522 struct rte_flow *flow, 6523 const struct rte_flow_attr *attr, 6524 const struct rte_flow_item items[], 6525 const struct rte_flow_action actions[], 6526 struct mlx5_flow_split_info *flow_split_info, 6527 struct rte_flow_error *error) 6528 { 6529 struct mlx5_priv *priv = dev->data->dev_private; 6530 struct mlx5_sh_config *config = &priv->sh->config; 6531 const struct rte_flow_action *qrss = NULL; 6532 struct rte_flow_action *ext_actions = NULL; 6533 struct mlx5_flow *dev_flow = NULL; 6534 uint32_t qrss_id = 0; 6535 int mtr_sfx = 0; 6536 size_t act_size; 6537 int actions_n; 6538 int encap_idx; 6539 int ret; 6540 6541 /* Check whether extensive metadata feature is engaged. */ 6542 if (!config->dv_flow_en || 6543 config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY || 6544 !mlx5_flow_ext_mreg_supported(dev)) 6545 return flow_create_split_inner(dev, flow, NULL, attr, items, 6546 actions, flow_split_info, error); 6547 actions_n = flow_parse_metadata_split_actions_info(actions, &qrss, 6548 &encap_idx); 6549 if (qrss) { 6550 /* Exclude hairpin flows from splitting. */ 6551 if (qrss->type == RTE_FLOW_ACTION_TYPE_QUEUE) { 6552 const struct rte_flow_action_queue *queue; 6553 6554 queue = qrss->conf; 6555 if (mlx5_rxq_is_hairpin(dev, queue->index)) 6556 qrss = NULL; 6557 } else if (qrss->type == RTE_FLOW_ACTION_TYPE_RSS) { 6558 const struct rte_flow_action_rss *rss; 6559 6560 rss = qrss->conf; 6561 if (mlx5_rxq_is_hairpin(dev, rss->queue[0])) 6562 qrss = NULL; 6563 } 6564 } 6565 if (qrss) { 6566 /* Check if it is in meter suffix table. */ 6567 mtr_sfx = attr->group == 6568 ((attr->transfer && priv->fdb_def_rule) ? 6569 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6570 MLX5_FLOW_TABLE_LEVEL_METER); 6571 /* 6572 * Q/RSS action on NIC Rx should be split in order to pass by 6573 * the mreg copy table (RX_CP_TBL) and then it jumps to the 6574 * action table (RX_ACT_TBL) which has the split Q/RSS action. 6575 */ 6576 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6577 sizeof(struct rte_flow_action_set_tag) + 6578 sizeof(struct rte_flow_action_jump); 6579 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6580 SOCKET_ID_ANY); 6581 if (!ext_actions) 6582 return rte_flow_error_set(error, ENOMEM, 6583 RTE_FLOW_ERROR_TYPE_ACTION, 6584 NULL, "no memory to split " 6585 "metadata flow"); 6586 /* 6587 * Create the new actions list with removed Q/RSS action 6588 * and appended set tag and jump to register copy table 6589 * (RX_CP_TBL). We should preallocate unique tag ID here 6590 * in advance, because it is needed for set tag action. 6591 */ 6592 qrss_id = flow_mreg_split_qrss_prep(dev, ext_actions, actions, 6593 qrss, actions_n, 6594 mtr_sfx, error); 6595 if (!mtr_sfx && !qrss_id) { 6596 ret = -rte_errno; 6597 goto exit; 6598 } 6599 } else if (attr->egress) { 6600 /* 6601 * All the actions on NIC Tx should have a metadata register 6602 * copy action to copy reg_a from WQE to reg_c[meta] 6603 */ 6604 act_size = sizeof(struct rte_flow_action) * (actions_n + 1) + 6605 sizeof(struct mlx5_flow_action_copy_mreg); 6606 ext_actions = mlx5_malloc(MLX5_MEM_ZERO, act_size, 0, 6607 SOCKET_ID_ANY); 6608 if (!ext_actions) 6609 return rte_flow_error_set(error, ENOMEM, 6610 RTE_FLOW_ERROR_TYPE_ACTION, 6611 NULL, "no memory to split " 6612 "metadata flow"); 6613 /* Create the action list appended with copy register. */ 6614 ret = flow_mreg_tx_copy_prep(dev, ext_actions, actions, 6615 actions_n, error, encap_idx); 6616 if (ret < 0) 6617 goto exit; 6618 } 6619 /* Add the unmodified original or prefix subflow. */ 6620 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 6621 items, ext_actions ? ext_actions : 6622 actions, flow_split_info, error); 6623 if (ret < 0) 6624 goto exit; 6625 MLX5_ASSERT(dev_flow); 6626 if (qrss) { 6627 const struct rte_flow_attr q_attr = { 6628 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 6629 .ingress = 1, 6630 }; 6631 /* Internal PMD action to set register. */ 6632 struct mlx5_rte_flow_item_tag q_tag_spec = { 6633 .data = qrss_id, 6634 .id = REG_NON, 6635 }; 6636 struct rte_flow_item q_items[] = { 6637 { 6638 .type = (enum rte_flow_item_type) 6639 MLX5_RTE_FLOW_ITEM_TYPE_TAG, 6640 .spec = &q_tag_spec, 6641 .last = NULL, 6642 .mask = NULL, 6643 }, 6644 { 6645 .type = RTE_FLOW_ITEM_TYPE_END, 6646 }, 6647 }; 6648 struct rte_flow_action q_actions[] = { 6649 { 6650 .type = qrss->type, 6651 .conf = qrss->conf, 6652 }, 6653 { 6654 .type = RTE_FLOW_ACTION_TYPE_END, 6655 }, 6656 }; 6657 uint64_t layers = flow_get_prefix_layer_flags(dev_flow); 6658 6659 /* 6660 * Configure the tag item only if there is no meter subflow. 6661 * Since tag is already marked in the meter suffix subflow 6662 * we can just use the meter suffix items as is. 6663 */ 6664 if (qrss_id) { 6665 /* Not meter subflow. */ 6666 MLX5_ASSERT(!mtr_sfx); 6667 /* 6668 * Put unique id in prefix flow due to it is destroyed 6669 * after suffix flow and id will be freed after there 6670 * is no actual flows with this id and identifier 6671 * reallocation becomes possible (for example, for 6672 * other flows in other threads). 6673 */ 6674 dev_flow->handle->split_flow_id = qrss_id; 6675 ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0, 6676 error); 6677 if (ret < 0) 6678 goto exit; 6679 q_tag_spec.id = ret; 6680 } 6681 dev_flow = NULL; 6682 /* Add suffix subflow to execute Q/RSS. */ 6683 flow_split_info->prefix_layers = layers; 6684 flow_split_info->prefix_mark = 0; 6685 flow_split_info->table_id = 0; 6686 ret = flow_create_split_inner(dev, flow, &dev_flow, 6687 &q_attr, mtr_sfx ? items : 6688 q_items, q_actions, 6689 flow_split_info, error); 6690 if (ret < 0) 6691 goto exit; 6692 /* qrss ID should be freed if failed. */ 6693 qrss_id = 0; 6694 MLX5_ASSERT(dev_flow); 6695 } 6696 6697 exit: 6698 /* 6699 * We do not destroy the partially created sub_flows in case of error. 6700 * These ones are included into parent flow list and will be destroyed 6701 * by flow_drv_destroy. 6702 */ 6703 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], 6704 qrss_id); 6705 mlx5_free(ext_actions); 6706 return ret; 6707 } 6708 6709 /** 6710 * Create meter internal drop flow with the original pattern. 6711 * 6712 * @param dev 6713 * Pointer to Ethernet device. 6714 * @param[in] flow 6715 * Parent flow structure pointer. 6716 * @param[in] attr 6717 * Flow rule attributes. 6718 * @param[in] items 6719 * Pattern specification (list terminated by the END pattern item). 6720 * @param[in] flow_split_info 6721 * Pointer to flow split info structure. 6722 * @param[in] fm 6723 * Pointer to flow meter structure. 6724 * @param[out] error 6725 * Perform verbose error reporting if not NULL. 6726 * @return 6727 * 0 on success, negative value otherwise 6728 */ 6729 static uint32_t 6730 flow_meter_create_drop_flow_with_org_pattern(struct rte_eth_dev *dev, 6731 struct rte_flow *flow, 6732 const struct rte_flow_attr *attr, 6733 const struct rte_flow_item items[], 6734 struct mlx5_flow_split_info *flow_split_info, 6735 struct mlx5_flow_meter_info *fm, 6736 struct rte_flow_error *error) 6737 { 6738 struct mlx5_flow *dev_flow = NULL; 6739 struct rte_flow_attr drop_attr = *attr; 6740 struct rte_flow_action drop_actions[3]; 6741 struct mlx5_flow_split_info drop_split_info = *flow_split_info; 6742 6743 MLX5_ASSERT(fm->drop_cnt); 6744 drop_actions[0].type = 6745 (enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_COUNT; 6746 drop_actions[0].conf = (void *)(uintptr_t)fm->drop_cnt; 6747 drop_actions[1].type = RTE_FLOW_ACTION_TYPE_DROP; 6748 drop_actions[1].conf = NULL; 6749 drop_actions[2].type = RTE_FLOW_ACTION_TYPE_END; 6750 drop_actions[2].conf = NULL; 6751 drop_split_info.external = false; 6752 drop_split_info.skip_scale |= 1 << MLX5_SCALE_FLOW_GROUP_BIT; 6753 drop_split_info.table_id = MLX5_MTR_TABLE_ID_DROP; 6754 drop_attr.group = MLX5_FLOW_TABLE_LEVEL_METER; 6755 return flow_create_split_inner(dev, flow, &dev_flow, 6756 &drop_attr, items, drop_actions, 6757 &drop_split_info, error); 6758 } 6759 6760 /** 6761 * The splitting for meter feature. 6762 * 6763 * - The meter flow will be split to two flows as prefix and 6764 * suffix flow. The packets make sense only it pass the prefix 6765 * meter action. 6766 * 6767 * - Reg_C_5 is used for the packet to match betweend prefix and 6768 * suffix flow. 6769 * 6770 * @param dev 6771 * Pointer to Ethernet device. 6772 * @param[in] flow 6773 * Parent flow structure pointer. 6774 * @param[in] attr 6775 * Flow rule attributes. 6776 * @param[in] items 6777 * Pattern specification (list terminated by the END pattern item). 6778 * @param[in] actions 6779 * Associated actions (list terminated by the END action). 6780 * @param[in] flow_split_info 6781 * Pointer to flow split info structure. 6782 * @param[out] error 6783 * Perform verbose error reporting if not NULL. 6784 * @return 6785 * 0 on success, negative value otherwise 6786 */ 6787 static int 6788 flow_create_split_meter(struct rte_eth_dev *dev, 6789 struct rte_flow *flow, 6790 const struct rte_flow_attr *attr, 6791 const struct rte_flow_item items[], 6792 const struct rte_flow_action actions[], 6793 struct mlx5_flow_split_info *flow_split_info, 6794 struct rte_flow_error *error) 6795 { 6796 struct mlx5_priv *priv = dev->data->dev_private; 6797 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 6798 struct rte_flow_action *sfx_actions = NULL; 6799 struct rte_flow_action *pre_actions = NULL; 6800 struct rte_flow_item *sfx_items = NULL; 6801 struct mlx5_flow *dev_flow = NULL; 6802 struct rte_flow_attr sfx_attr = *attr; 6803 struct mlx5_flow_meter_info *fm = NULL; 6804 uint8_t skip_scale_restore; 6805 bool has_mtr = false; 6806 bool has_modify = false; 6807 bool set_mtr_reg = true; 6808 bool is_mtr_hierarchy = false; 6809 uint32_t meter_id = 0; 6810 uint32_t mtr_idx = 0; 6811 uint32_t mtr_flow_id = 0; 6812 size_t act_size; 6813 size_t item_size; 6814 int actions_n = 0; 6815 int ret = 0; 6816 6817 if (priv->mtr_en) 6818 actions_n = flow_check_meter_action(dev, actions, &has_mtr, 6819 &has_modify, &meter_id); 6820 if (has_mtr) { 6821 if (flow->meter) { 6822 fm = flow_dv_meter_find_by_idx(priv, flow->meter); 6823 if (!fm) 6824 return rte_flow_error_set(error, EINVAL, 6825 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6826 NULL, "Meter not found."); 6827 } else { 6828 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); 6829 if (!fm) 6830 return rte_flow_error_set(error, EINVAL, 6831 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 6832 NULL, "Meter not found."); 6833 ret = mlx5_flow_meter_attach(priv, fm, 6834 &sfx_attr, error); 6835 if (ret) 6836 return -rte_errno; 6837 flow->meter = mtr_idx; 6838 } 6839 MLX5_ASSERT(wks); 6840 wks->fm = fm; 6841 if (!fm->def_policy) { 6842 wks->policy = mlx5_flow_meter_policy_find(dev, 6843 fm->policy_id, 6844 NULL); 6845 MLX5_ASSERT(wks->policy); 6846 if (wks->policy->mark) 6847 wks->mark = 1; 6848 if (wks->policy->is_hierarchy) { 6849 wks->final_policy = 6850 mlx5_flow_meter_hierarchy_get_final_policy(dev, 6851 wks->policy); 6852 if (!wks->final_policy) 6853 return rte_flow_error_set(error, 6854 EINVAL, 6855 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 6856 "Failed to find terminal policy of hierarchy."); 6857 is_mtr_hierarchy = true; 6858 } 6859 } 6860 /* 6861 * If it isn't default-policy Meter, and 6862 * 1. Not meter hierarchy and there's no action in flow to change 6863 * packet (modify/encap/decap etc.), OR 6864 * 2. No drop count needed for this meter. 6865 * Then no need to use regC to save meter id anymore. 6866 */ 6867 if (!fm->def_policy && ((!has_modify && !is_mtr_hierarchy) || !fm->drop_cnt)) 6868 set_mtr_reg = false; 6869 /* Prefix actions: meter, decap, encap, tag, jump, end, cnt. */ 6870 #define METER_PREFIX_ACTION 7 6871 act_size = (sizeof(struct rte_flow_action) * 6872 (actions_n + METER_PREFIX_ACTION)) + 6873 sizeof(struct mlx5_rte_flow_action_set_tag); 6874 /* Suffix items: tag, vlan, port id, end. */ 6875 #define METER_SUFFIX_ITEM 4 6876 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + 6877 sizeof(struct mlx5_rte_flow_item_tag) * 2; 6878 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + item_size), 6879 0, SOCKET_ID_ANY); 6880 if (!sfx_actions) 6881 return rte_flow_error_set(error, ENOMEM, 6882 RTE_FLOW_ERROR_TYPE_ACTION, 6883 NULL, "no memory to split " 6884 "meter flow"); 6885 sfx_items = (struct rte_flow_item *)((char *)sfx_actions + 6886 act_size); 6887 /* There's no suffix flow for meter of non-default policy. */ 6888 if (!fm->def_policy) 6889 pre_actions = sfx_actions + 1; 6890 else 6891 pre_actions = sfx_actions + actions_n; 6892 ret = flow_meter_split_prep(dev, flow, wks, &sfx_attr, 6893 items, sfx_items, actions, 6894 sfx_actions, pre_actions, 6895 (set_mtr_reg ? &mtr_flow_id : NULL), 6896 error); 6897 if (ret) { 6898 ret = -rte_errno; 6899 goto exit; 6900 } 6901 /* Add the prefix subflow. */ 6902 skip_scale_restore = flow_split_info->skip_scale; 6903 flow_split_info->skip_scale |= 6904 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 6905 ret = flow_create_split_inner(dev, flow, &dev_flow, 6906 attr, items, pre_actions, 6907 flow_split_info, error); 6908 flow_split_info->skip_scale = skip_scale_restore; 6909 if (ret) { 6910 if (mtr_flow_id) 6911 mlx5_ipool_free(fm->flow_ipool, mtr_flow_id); 6912 ret = -rte_errno; 6913 goto exit; 6914 } 6915 if (mtr_flow_id) { 6916 dev_flow->handle->split_flow_id = mtr_flow_id; 6917 dev_flow->handle->is_meter_flow_id = 1; 6918 } 6919 if (!fm->def_policy) { 6920 if (!set_mtr_reg && fm->drop_cnt) 6921 ret = 6922 flow_meter_create_drop_flow_with_org_pattern(dev, flow, 6923 &sfx_attr, items, 6924 flow_split_info, 6925 fm, error); 6926 goto exit; 6927 } 6928 /* Setting the sfx group atrr. */ 6929 sfx_attr.group = sfx_attr.transfer ? 6930 (MLX5_FLOW_TABLE_LEVEL_METER - 1) : 6931 MLX5_FLOW_TABLE_LEVEL_METER; 6932 flow_split_info->prefix_layers = 6933 flow_get_prefix_layer_flags(dev_flow); 6934 flow_split_info->prefix_mark |= wks->mark; 6935 flow_split_info->table_id = MLX5_MTR_TABLE_ID_SUFFIX; 6936 } 6937 /* Add the prefix subflow. */ 6938 ret = flow_create_split_metadata(dev, flow, 6939 &sfx_attr, sfx_items ? 6940 sfx_items : items, 6941 sfx_actions ? sfx_actions : actions, 6942 flow_split_info, error); 6943 exit: 6944 if (sfx_actions) 6945 mlx5_free(sfx_actions); 6946 return ret; 6947 } 6948 6949 /** 6950 * The splitting for sample feature. 6951 * 6952 * Once Sample action is detected in the action list, the flow actions should 6953 * be split into prefix sub flow and suffix sub flow. 6954 * 6955 * The original items remain in the prefix sub flow, all actions preceding the 6956 * sample action and the sample action itself will be copied to the prefix 6957 * sub flow, the actions following the sample action will be copied to the 6958 * suffix sub flow, Queue action always be located in the suffix sub flow. 6959 * 6960 * In order to make the packet from prefix sub flow matches with suffix sub 6961 * flow, an extra tag action be added into prefix sub flow, and the suffix sub 6962 * flow uses tag item with the unique flow id. 6963 * 6964 * @param dev 6965 * Pointer to Ethernet device. 6966 * @param[in] flow 6967 * Parent flow structure pointer. 6968 * @param[in] attr 6969 * Flow rule attributes. 6970 * @param[in] items 6971 * Pattern specification (list terminated by the END pattern item). 6972 * @param[in] actions 6973 * Associated actions (list terminated by the END action). 6974 * @param[in] flow_split_info 6975 * Pointer to flow split info structure. 6976 * @param[out] error 6977 * Perform verbose error reporting if not NULL. 6978 * @return 6979 * 0 on success, negative value otherwise 6980 */ 6981 static int 6982 flow_create_split_sample(struct rte_eth_dev *dev, 6983 struct rte_flow *flow, 6984 const struct rte_flow_attr *attr, 6985 const struct rte_flow_item items[], 6986 const struct rte_flow_action actions[], 6987 struct mlx5_flow_split_info *flow_split_info, 6988 struct rte_flow_error *error) 6989 { 6990 struct mlx5_priv *priv = dev->data->dev_private; 6991 struct rte_flow_action *sfx_actions = NULL; 6992 struct rte_flow_action *pre_actions = NULL; 6993 struct rte_flow_item *sfx_items = NULL; 6994 struct mlx5_flow *dev_flow = NULL; 6995 struct rte_flow_attr sfx_attr = *attr; 6996 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 6997 struct mlx5_flow_dv_sample_resource *sample_res; 6998 struct mlx5_flow_tbl_data_entry *sfx_tbl_data; 6999 struct mlx5_flow_tbl_resource *sfx_tbl; 7000 struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); 7001 #endif 7002 size_t act_size; 7003 size_t item_size; 7004 uint32_t fdb_tx = 0; 7005 int32_t tag_id = 0; 7006 int actions_n = 0; 7007 int sample_action_pos; 7008 int qrss_action_pos; 7009 int add_tag = 0; 7010 int modify_after_mirror = 0; 7011 uint16_t jump_table = 0; 7012 const uint32_t next_ft_step = 1; 7013 int ret = 0; 7014 struct mlx5_priv *item_port_priv = NULL; 7015 const struct rte_flow_item *item; 7016 7017 if (priv->sampler_en) 7018 actions_n = flow_check_match_action(actions, attr, 7019 RTE_FLOW_ACTION_TYPE_SAMPLE, 7020 &sample_action_pos, &qrss_action_pos, 7021 &modify_after_mirror); 7022 if (actions_n) { 7023 /* The prefix actions must includes sample, tag, end. */ 7024 act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1) 7025 + sizeof(struct mlx5_rte_flow_action_set_tag); 7026 item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM + 7027 sizeof(struct mlx5_rte_flow_item_tag) * 2; 7028 sfx_actions = mlx5_malloc(MLX5_MEM_ZERO, (act_size + 7029 item_size), 0, SOCKET_ID_ANY); 7030 if (!sfx_actions) 7031 return rte_flow_error_set(error, ENOMEM, 7032 RTE_FLOW_ERROR_TYPE_ACTION, 7033 NULL, "no memory to split " 7034 "sample flow"); 7035 for (item = items; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 7036 if (item->type == RTE_FLOW_ITEM_TYPE_PORT_ID) { 7037 const struct rte_flow_item_port_id *spec; 7038 7039 spec = (const struct rte_flow_item_port_id *)item->spec; 7040 if (spec) 7041 item_port_priv = 7042 mlx5_port_to_eswitch_info(spec->id, true); 7043 break; 7044 } else if (item->type == RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT) { 7045 const struct rte_flow_item_ethdev *spec; 7046 7047 spec = (const struct rte_flow_item_ethdev *)item->spec; 7048 if (spec) 7049 item_port_priv = 7050 mlx5_port_to_eswitch_info(spec->port_id, true); 7051 break; 7052 } else if (item->type == RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR) { 7053 const struct rte_flow_item_ethdev *spec; 7054 7055 spec = (const struct rte_flow_item_ethdev *)item->spec; 7056 if (spec) 7057 item_port_priv = 7058 mlx5_port_to_eswitch_info(spec->port_id, true); 7059 break; 7060 } 7061 } 7062 /* The representor_id is UINT16_MAX for uplink. */ 7063 fdb_tx = (attr->transfer && 7064 flow_source_vport_representor(priv, item_port_priv)); 7065 /* 7066 * When reg_c_preserve is set, metadata registers Cx preserve 7067 * their value even through packet duplication. 7068 */ 7069 add_tag = (!fdb_tx || 7070 priv->sh->cdev->config.hca_attr.reg_c_preserve); 7071 if (add_tag) 7072 sfx_items = (struct rte_flow_item *)((char *)sfx_actions 7073 + act_size); 7074 if (modify_after_mirror) 7075 jump_table = attr->group * MLX5_FLOW_TABLE_FACTOR + 7076 next_ft_step; 7077 pre_actions = sfx_actions + actions_n; 7078 tag_id = flow_sample_split_prep(dev, add_tag, items, sfx_items, 7079 actions, sfx_actions, 7080 pre_actions, actions_n, 7081 sample_action_pos, 7082 qrss_action_pos, jump_table, 7083 error); 7084 if (tag_id < 0 || (add_tag && !tag_id)) { 7085 ret = -rte_errno; 7086 goto exit; 7087 } 7088 if (modify_after_mirror) 7089 flow_split_info->skip_scale = 7090 1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT; 7091 /* Add the prefix subflow. */ 7092 ret = flow_create_split_inner(dev, flow, &dev_flow, attr, 7093 items, pre_actions, 7094 flow_split_info, error); 7095 if (ret) { 7096 ret = -rte_errno; 7097 goto exit; 7098 } 7099 dev_flow->handle->split_flow_id = tag_id; 7100 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7101 if (!modify_after_mirror) { 7102 /* Set the sfx group attr. */ 7103 sample_res = (struct mlx5_flow_dv_sample_resource *) 7104 dev_flow->dv.sample_res; 7105 sfx_tbl = (struct mlx5_flow_tbl_resource *) 7106 sample_res->normal_path_tbl; 7107 sfx_tbl_data = container_of(sfx_tbl, 7108 struct mlx5_flow_tbl_data_entry, 7109 tbl); 7110 sfx_attr.group = sfx_attr.transfer ? 7111 (sfx_tbl_data->level - 1) : sfx_tbl_data->level; 7112 } else { 7113 MLX5_ASSERT(attr->transfer); 7114 sfx_attr.group = jump_table; 7115 } 7116 flow_split_info->prefix_layers = 7117 flow_get_prefix_layer_flags(dev_flow); 7118 MLX5_ASSERT(wks); 7119 flow_split_info->prefix_mark |= wks->mark; 7120 /* Suffix group level already be scaled with factor, set 7121 * MLX5_SCALE_FLOW_GROUP_BIT of skip_scale to 1 to avoid scale 7122 * again in translation. 7123 */ 7124 flow_split_info->skip_scale = 1 << MLX5_SCALE_FLOW_GROUP_BIT; 7125 #endif 7126 } 7127 /* Add the suffix subflow. */ 7128 ret = flow_create_split_meter(dev, flow, &sfx_attr, 7129 sfx_items ? sfx_items : items, 7130 sfx_actions ? sfx_actions : actions, 7131 flow_split_info, error); 7132 exit: 7133 if (sfx_actions) 7134 mlx5_free(sfx_actions); 7135 return ret; 7136 } 7137 7138 /** 7139 * Split the flow to subflow set. The splitters might be linked 7140 * in the chain, like this: 7141 * flow_create_split_outer() calls: 7142 * flow_create_split_meter() calls: 7143 * flow_create_split_metadata(meter_subflow_0) calls: 7144 * flow_create_split_inner(metadata_subflow_0) 7145 * flow_create_split_inner(metadata_subflow_1) 7146 * flow_create_split_inner(metadata_subflow_2) 7147 * flow_create_split_metadata(meter_subflow_1) calls: 7148 * flow_create_split_inner(metadata_subflow_0) 7149 * flow_create_split_inner(metadata_subflow_1) 7150 * flow_create_split_inner(metadata_subflow_2) 7151 * 7152 * This provide flexible way to add new levels of flow splitting. 7153 * The all of successfully created subflows are included to the 7154 * parent flow dev_flow list. 7155 * 7156 * @param dev 7157 * Pointer to Ethernet device. 7158 * @param[in] flow 7159 * Parent flow structure pointer. 7160 * @param[in] attr 7161 * Flow rule attributes. 7162 * @param[in] items 7163 * Pattern specification (list terminated by the END pattern item). 7164 * @param[in] actions 7165 * Associated actions (list terminated by the END action). 7166 * @param[in] flow_split_info 7167 * Pointer to flow split info structure. 7168 * @param[out] error 7169 * Perform verbose error reporting if not NULL. 7170 * @return 7171 * 0 on success, negative value otherwise 7172 */ 7173 static int 7174 flow_create_split_outer(struct rte_eth_dev *dev, 7175 struct rte_flow *flow, 7176 const struct rte_flow_attr *attr, 7177 const struct rte_flow_item items[], 7178 const struct rte_flow_action actions[], 7179 struct mlx5_flow_split_info *flow_split_info, 7180 struct rte_flow_error *error) 7181 { 7182 int ret; 7183 7184 ret = flow_create_split_sample(dev, flow, attr, items, 7185 actions, flow_split_info, error); 7186 MLX5_ASSERT(ret <= 0); 7187 return ret; 7188 } 7189 7190 static inline struct mlx5_flow_tunnel * 7191 flow_tunnel_from_rule(const struct mlx5_flow *flow) 7192 { 7193 struct mlx5_flow_tunnel *tunnel; 7194 7195 #pragma GCC diagnostic push 7196 #pragma GCC diagnostic ignored "-Wcast-qual" 7197 tunnel = (typeof(tunnel))flow->tunnel; 7198 #pragma GCC diagnostic pop 7199 7200 return tunnel; 7201 } 7202 7203 /** 7204 * Create a flow and add it to @p list. 7205 * 7206 * @param dev 7207 * Pointer to Ethernet device. 7208 * @param list 7209 * Pointer to a TAILQ flow list. If this parameter NULL, 7210 * no list insertion occurred, flow is just created, 7211 * this is caller's responsibility to track the 7212 * created flow. 7213 * @param[in] attr 7214 * Flow rule attributes. 7215 * @param[in] items 7216 * Pattern specification (list terminated by the END pattern item). 7217 * @param[in] actions 7218 * Associated actions (list terminated by the END action). 7219 * @param[in] external 7220 * This flow rule is created by request external to PMD. 7221 * @param[out] error 7222 * Perform verbose error reporting if not NULL. 7223 * 7224 * @return 7225 * A flow index on success, 0 otherwise and rte_errno is set. 7226 */ 7227 static uint32_t 7228 flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7229 const struct rte_flow_attr *attr, 7230 const struct rte_flow_item items[], 7231 const struct rte_flow_action original_actions[], 7232 bool external, struct rte_flow_error *error) 7233 { 7234 struct mlx5_priv *priv = dev->data->dev_private; 7235 struct rte_flow *flow = NULL; 7236 struct mlx5_flow *dev_flow; 7237 const struct rte_flow_action_rss *rss = NULL; 7238 struct mlx5_translated_action_handle 7239 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7240 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7241 union { 7242 struct mlx5_flow_expand_rss buf; 7243 uint8_t buffer[8192]; 7244 } expand_buffer; 7245 union { 7246 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7247 uint8_t buffer[2048]; 7248 } actions_rx; 7249 union { 7250 struct rte_flow_action actions[MLX5_MAX_SPLIT_ACTIONS]; 7251 uint8_t buffer[2048]; 7252 } actions_hairpin_tx; 7253 union { 7254 struct rte_flow_item items[MLX5_MAX_SPLIT_ITEMS]; 7255 uint8_t buffer[2048]; 7256 } items_tx; 7257 struct mlx5_rte_flow_item_sq sq_specs[RTE_MAX_QUEUES_PER_PORT]; 7258 struct mlx5_flow_expand_rss *buf = &expand_buffer.buf; 7259 struct mlx5_flow_rss_desc *rss_desc; 7260 const struct rte_flow_action *p_actions_rx; 7261 uint32_t i; 7262 uint32_t idx = 0; 7263 int hairpin_flow; 7264 struct rte_flow_attr attr_tx = { .priority = 0 }; 7265 const struct rte_flow_action *actions; 7266 struct rte_flow_action *translated_actions = NULL; 7267 struct mlx5_flow_tunnel *tunnel; 7268 struct tunnel_default_miss_ctx default_miss_ctx = { 0, }; 7269 struct mlx5_flow_workspace *wks = mlx5_flow_push_thread_workspace(); 7270 struct mlx5_flow_split_info flow_split_info = { 7271 .external = !!external, 7272 .skip_scale = 0, 7273 .flow_idx = 0, 7274 .prefix_mark = 0, 7275 .prefix_layers = 0, 7276 .table_id = 0 7277 }; 7278 int ret; 7279 7280 MLX5_ASSERT(wks); 7281 rss_desc = &wks->rss_desc; 7282 ret = flow_action_handles_translate(dev, original_actions, 7283 indir_actions, 7284 &indir_actions_n, 7285 &translated_actions, error); 7286 if (ret < 0) { 7287 MLX5_ASSERT(translated_actions == NULL); 7288 return 0; 7289 } 7290 actions = translated_actions ? translated_actions : original_actions; 7291 p_actions_rx = actions; 7292 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7293 ret = flow_drv_validate(dev, attr, items, p_actions_rx, 7294 external, hairpin_flow, error); 7295 if (ret < 0) 7296 goto error_before_hairpin_split; 7297 flow = mlx5_ipool_zmalloc(priv->flows[type], &idx); 7298 if (!flow) { 7299 rte_errno = ENOMEM; 7300 goto error_before_hairpin_split; 7301 } 7302 if (hairpin_flow > 0) { 7303 if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) { 7304 rte_errno = EINVAL; 7305 goto error_before_hairpin_split; 7306 } 7307 flow_hairpin_split(dev, actions, actions_rx.actions, 7308 actions_hairpin_tx.actions, items_tx.items, 7309 idx); 7310 p_actions_rx = actions_rx.actions; 7311 } 7312 flow_split_info.flow_idx = idx; 7313 flow->drv_type = flow_get_drv_type(dev, attr); 7314 MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN && 7315 flow->drv_type < MLX5_FLOW_TYPE_MAX); 7316 memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue)); 7317 /* RSS Action only works on NIC RX domain */ 7318 if (attr->ingress) 7319 rss = flow_get_rss_action(dev, p_actions_rx); 7320 if (rss) { 7321 MLX5_ASSERT(rss->queue_num <= RTE_ETH_RSS_RETA_SIZE_512); 7322 rss_desc->symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func); 7323 /* 7324 * The following information is required by 7325 * mlx5_flow_hashfields_adjust() in advance. 7326 */ 7327 rss_desc->level = rss->level; 7328 /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */ 7329 rss_desc->types = !rss->types ? RTE_ETH_RSS_IP : rss->types; 7330 } 7331 flow->dev_handles = 0; 7332 if (rss && rss->types) { 7333 unsigned int graph_root; 7334 7335 graph_root = find_graph_root(rss->level); 7336 ret = mlx5_flow_expand_rss(buf, sizeof(expand_buffer.buffer), 7337 items, rss->types, 7338 mlx5_support_expansion, graph_root); 7339 MLX5_ASSERT(ret > 0 && 7340 (unsigned int)ret < sizeof(expand_buffer.buffer)); 7341 if (rte_log_can_log(mlx5_logtype, RTE_LOG_DEBUG)) { 7342 for (i = 0; i < buf->entries; ++i) 7343 mlx5_dbg__print_pattern(buf->entry[i].pattern); 7344 } 7345 } else { 7346 ret = mlx5_flow_expand_sqn((struct mlx5_flow_expand_sqn *)buf, 7347 sizeof(expand_buffer.buffer), 7348 items, sq_specs); 7349 if (ret) { 7350 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, 7351 NULL, "not enough memory for rte_flow"); 7352 goto error; 7353 } 7354 if (buf->entries == 0) { 7355 buf->entries = 1; 7356 buf->entry[0].pattern = (void *)(uintptr_t)items; 7357 } 7358 } 7359 rss_desc->shared_rss = flow_get_shared_rss_action(dev, indir_actions, 7360 indir_actions_n); 7361 for (i = 0; i < buf->entries; ++i) { 7362 /* Initialize flow split data. */ 7363 flow_split_info.prefix_layers = 0; 7364 flow_split_info.prefix_mark = 0; 7365 flow_split_info.skip_scale = 0; 7366 /* 7367 * The splitter may create multiple dev_flows, 7368 * depending on configuration. In the simplest 7369 * case it just creates unmodified original flow. 7370 */ 7371 ret = flow_create_split_outer(dev, flow, attr, 7372 buf->entry[i].pattern, 7373 p_actions_rx, &flow_split_info, 7374 error); 7375 if (ret < 0) 7376 goto error; 7377 if (is_flow_tunnel_steer_rule(wks->flows[0].tof_type)) { 7378 ret = flow_tunnel_add_default_miss(dev, flow, attr, 7379 p_actions_rx, 7380 idx, 7381 wks->flows[0].tunnel, 7382 &default_miss_ctx, 7383 error); 7384 if (ret < 0) { 7385 mlx5_free(default_miss_ctx.queue); 7386 goto error; 7387 } 7388 } 7389 } 7390 /* Create the tx flow. */ 7391 if (hairpin_flow) { 7392 attr_tx.group = MLX5_HAIRPIN_TX_TABLE; 7393 attr_tx.ingress = 0; 7394 attr_tx.egress = 1; 7395 dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items, 7396 actions_hairpin_tx.actions, 7397 idx, error); 7398 if (!dev_flow) 7399 goto error; 7400 dev_flow->flow = flow; 7401 dev_flow->external = 0; 7402 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 7403 dev_flow->handle, next); 7404 ret = flow_drv_translate(dev, dev_flow, &attr_tx, 7405 items_tx.items, 7406 actions_hairpin_tx.actions, error); 7407 if (ret < 0) 7408 goto error; 7409 } 7410 /* 7411 * Update the metadata register copy table. If extensive 7412 * metadata feature is enabled and registers are supported 7413 * we might create the extra rte_flow for each unique 7414 * MARK/FLAG action ID. 7415 * 7416 * The table is updated for ingress and transfer flows only, because 7417 * the egress Flows belong to the different device and 7418 * copy table should be updated in peer NIC Rx domain. 7419 */ 7420 if ((attr->ingress || attr->transfer) && 7421 (external || attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP)) { 7422 ret = flow_mreg_update_copy_table(dev, flow, actions, error); 7423 if (ret) 7424 goto error; 7425 } 7426 /* 7427 * If the flow is external (from application) OR device is started, 7428 * OR mreg discover, then apply immediately. 7429 */ 7430 if (external || dev->data->dev_started || 7431 (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP && 7432 attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) { 7433 ret = flow_drv_apply(dev, flow, error); 7434 if (ret < 0) 7435 goto error; 7436 } 7437 flow->type = type; 7438 flow_rxq_flags_set(dev, flow); 7439 rte_free(translated_actions); 7440 tunnel = flow_tunnel_from_rule(wks->flows); 7441 if (tunnel) { 7442 flow->tunnel = 1; 7443 flow->tunnel_id = tunnel->tunnel_id; 7444 __atomic_fetch_add(&tunnel->refctn, 1, __ATOMIC_RELAXED); 7445 mlx5_free(default_miss_ctx.queue); 7446 } 7447 mlx5_flow_pop_thread_workspace(); 7448 return idx; 7449 error: 7450 MLX5_ASSERT(flow); 7451 ret = rte_errno; /* Save rte_errno before cleanup. */ 7452 flow_mreg_del_copy_action(dev, flow); 7453 flow_drv_destroy(dev, flow); 7454 if (rss_desc->shared_rss) 7455 __atomic_fetch_sub(&((struct mlx5_shared_action_rss *) 7456 mlx5_ipool_get 7457 (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 7458 rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED); 7459 mlx5_ipool_free(priv->flows[type], idx); 7460 rte_errno = ret; /* Restore rte_errno. */ 7461 ret = rte_errno; 7462 rte_errno = ret; 7463 error_before_hairpin_split: 7464 mlx5_flow_pop_thread_workspace(); 7465 rte_free(translated_actions); 7466 return 0; 7467 } 7468 7469 /** 7470 * Create a dedicated flow rule on e-switch table 0 (root table), to direct all 7471 * incoming packets to table 1. 7472 * 7473 * Other flow rules, requested for group n, will be created in 7474 * e-switch table n+1. 7475 * Jump action to e-switch group n will be created to group n+1. 7476 * 7477 * Used when working in switchdev mode, to utilise advantages of table 1 7478 * and above. 7479 * 7480 * @param dev 7481 * Pointer to Ethernet device. 7482 * 7483 * @return 7484 * Pointer to flow on success, NULL otherwise and rte_errno is set. 7485 */ 7486 struct rte_flow * 7487 mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev) 7488 { 7489 const struct rte_flow_attr attr = { 7490 .group = 0, 7491 .priority = 0, 7492 .ingress = 0, 7493 .egress = 0, 7494 .transfer = 1, 7495 }; 7496 const struct rte_flow_item pattern = { 7497 .type = RTE_FLOW_ITEM_TYPE_END, 7498 }; 7499 struct rte_flow_action_jump jump = { 7500 .group = 1, 7501 }; 7502 const struct rte_flow_action actions[] = { 7503 { 7504 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7505 .conf = &jump, 7506 }, 7507 { 7508 .type = RTE_FLOW_ACTION_TYPE_END, 7509 }, 7510 }; 7511 struct rte_flow_error error; 7512 7513 return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 7514 &attr, &pattern, 7515 actions, false, &error); 7516 } 7517 7518 /** 7519 * Create a dedicated flow rule on e-switch table 1, matches ESW manager 7520 * and sq number, directs all packets to peer vport. 7521 * 7522 * @param dev 7523 * Pointer to Ethernet device. 7524 * @param sq_num 7525 * SQ number. 7526 * 7527 * @return 7528 * Flow ID on success, 0 otherwise and rte_errno is set. 7529 */ 7530 uint32_t 7531 mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num) 7532 { 7533 struct rte_flow_attr attr = { 7534 .group = 0, 7535 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 7536 .ingress = 0, 7537 .egress = 0, 7538 .transfer = 1, 7539 }; 7540 struct rte_flow_item_port_id port_spec = { 7541 .id = MLX5_PORT_ESW_MGR, 7542 }; 7543 struct mlx5_rte_flow_item_sq sq_spec = { 7544 .queue = sq_num, 7545 }; 7546 struct rte_flow_item pattern[] = { 7547 { 7548 .type = RTE_FLOW_ITEM_TYPE_PORT_ID, 7549 .spec = &port_spec, 7550 }, 7551 { 7552 .type = (enum rte_flow_item_type) 7553 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 7554 .spec = &sq_spec, 7555 }, 7556 { 7557 .type = RTE_FLOW_ITEM_TYPE_END, 7558 }, 7559 }; 7560 struct rte_flow_action_jump jump = { 7561 .group = 1, 7562 }; 7563 struct rte_flow_action_port_id port = { 7564 .id = dev->data->port_id, 7565 }; 7566 struct rte_flow_action actions[] = { 7567 { 7568 .type = RTE_FLOW_ACTION_TYPE_JUMP, 7569 .conf = &jump, 7570 }, 7571 { 7572 .type = RTE_FLOW_ACTION_TYPE_END, 7573 }, 7574 }; 7575 struct rte_flow_error error; 7576 7577 /* 7578 * Creates group 0, highest priority jump flow. 7579 * Matches txq to bypass kernel packets. 7580 */ 7581 if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions, 7582 false, &error) == 0) 7583 return 0; 7584 /* Create group 1, lowest priority redirect flow for txq. */ 7585 attr.group = 1; 7586 actions[0].conf = &port; 7587 actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID; 7588 return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, 7589 actions, false, &error); 7590 } 7591 7592 /** 7593 * Validate a flow supported by the NIC. 7594 * 7595 * @see rte_flow_validate() 7596 * @see rte_flow_ops 7597 */ 7598 int 7599 mlx5_flow_validate(struct rte_eth_dev *dev, 7600 const struct rte_flow_attr *attr, 7601 const struct rte_flow_item items[], 7602 const struct rte_flow_action original_actions[], 7603 struct rte_flow_error *error) 7604 { 7605 int hairpin_flow; 7606 struct mlx5_translated_action_handle 7607 indir_actions[MLX5_MAX_INDIRECT_ACTIONS]; 7608 int indir_actions_n = MLX5_MAX_INDIRECT_ACTIONS; 7609 const struct rte_flow_action *actions; 7610 struct rte_flow_action *translated_actions = NULL; 7611 int ret = flow_action_handles_translate(dev, original_actions, 7612 indir_actions, 7613 &indir_actions_n, 7614 &translated_actions, error); 7615 7616 if (ret) 7617 return ret; 7618 actions = translated_actions ? translated_actions : original_actions; 7619 hairpin_flow = flow_check_hairpin_split(dev, attr, actions); 7620 ret = flow_drv_validate(dev, attr, items, actions, 7621 true, hairpin_flow, error); 7622 rte_free(translated_actions); 7623 return ret; 7624 } 7625 7626 static int 7627 mlx5_flow_cache_flow_info(struct rte_eth_dev *dev, 7628 const struct rte_flow_attr *attr, 7629 const uint32_t orig_prio, 7630 const struct rte_flow_item *items, 7631 const struct rte_flow_action *actions, 7632 uint32_t flow_idx) 7633 { 7634 struct mlx5_priv *priv = dev->data->dev_private; 7635 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7636 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7637 struct rte_flow_error error; 7638 int len, ret; 7639 7640 flow_info = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_info), 0, SOCKET_ID_ANY); 7641 if (!flow_info) { 7642 DRV_LOG(ERR, "No enough memory for flow_info caching."); 7643 return -1; 7644 } 7645 flow_info->orig_prio = orig_prio; 7646 flow_info->attr = *attr; 7647 /* Standby mode rule awlays saves it in low priority entry. */ 7648 flow_info->flow_idx_low_prio = flow_idx; 7649 7650 /* Store matching items. */ 7651 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, NULL, 0, items, &error); 7652 if (ret <= 0) { 7653 DRV_LOG(ERR, "Can't get items length."); 7654 goto end; 7655 } 7656 len = RTE_ALIGN(ret, 16); 7657 flow_info->items = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7658 if (!flow_info->items) { 7659 DRV_LOG(ERR, "No enough memory for items caching."); 7660 goto end; 7661 } 7662 ret = rte_flow_conv(RTE_FLOW_CONV_OP_PATTERN, flow_info->items, ret, items, &error); 7663 if (ret <= 0) { 7664 DRV_LOG(ERR, "Can't duplicate items."); 7665 goto end; 7666 } 7667 7668 /* Store flow actions. */ 7669 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, actions, &error); 7670 if (ret <= 0) { 7671 DRV_LOG(ERR, "Can't get actions length."); 7672 goto end; 7673 } 7674 len = RTE_ALIGN(ret, 16); 7675 flow_info->actions = mlx5_malloc(MLX5_MEM_ZERO, len, 0, SOCKET_ID_ANY); 7676 if (!flow_info->actions) { 7677 DRV_LOG(ERR, "No enough memory for actions caching."); 7678 goto end; 7679 } 7680 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, flow_info->actions, ret, actions, &error); 7681 if (ret <= 0) { 7682 DRV_LOG(ERR, "Can't duplicate actions."); 7683 goto end; 7684 } 7685 7686 /* Insert to the list end. */ 7687 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7688 LIST_INSERT_HEAD(&mode_info->hot_upgrade, flow_info, next); 7689 } else { 7690 tmp_info = LIST_FIRST(&mode_info->hot_upgrade); 7691 while (LIST_NEXT(tmp_info, next)) 7692 tmp_info = LIST_NEXT(tmp_info, next); 7693 LIST_INSERT_AFTER(tmp_info, flow_info, next); 7694 } 7695 return 0; 7696 end: 7697 if (flow_info->items) 7698 mlx5_free(flow_info->items); 7699 if (flow_info->actions) 7700 mlx5_free(flow_info->actions); 7701 mlx5_free(flow_info); 7702 return -1; 7703 } 7704 7705 static int 7706 mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio) 7707 { 7708 struct mlx5_priv *priv = dev->data->dev_private; 7709 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7710 struct mlx5_dv_flow_info *flow_info; 7711 struct rte_flow_attr attr; 7712 struct rte_flow_error error; 7713 struct rte_flow *high, *low; 7714 7715 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7716 while (flow_info) { 7717 /* DUP flow may have the same priority. */ 7718 if (flow_info->orig_prio != flow_info->attr.priority) { 7719 attr = flow_info->attr; 7720 if (orig_prio) 7721 attr.priority = flow_info->orig_prio; 7722 flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, 7723 &attr, flow_info->items, flow_info->actions, 7724 true, &error); 7725 if (!flow_info->flow_idx_high_prio) { 7726 DRV_LOG(ERR, "Priority toggle failed internally."); 7727 goto err; 7728 } 7729 } 7730 flow_info = LIST_NEXT(flow_info, next); 7731 } 7732 /* Delete the low priority rules and swap the flow handle. */ 7733 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7734 while (flow_info) { 7735 MLX5_ASSERT(flow_info->flow_idx_low_prio); 7736 if (flow_info->orig_prio != flow_info->attr.priority) { 7737 high = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7738 flow_info->flow_idx_high_prio); 7739 low = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 7740 flow_info->flow_idx_low_prio); 7741 if (high && low) { 7742 RTE_SWAP(*low, *high); 7743 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7744 flow_info->flow_idx_low_prio); 7745 flow_info->flow_idx_high_prio = 0; 7746 } 7747 } 7748 flow_info = LIST_NEXT(flow_info, next); 7749 } 7750 return 0; 7751 err: 7752 /* Destroy preceding successful high priority rules. */ 7753 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7754 while (flow_info) { 7755 if (flow_info->orig_prio != flow_info->attr.priority) { 7756 if (flow_info->flow_idx_high_prio) 7757 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 7758 flow_info->flow_idx_high_prio); 7759 else 7760 break; 7761 flow_info->flow_idx_high_prio = 0; 7762 } 7763 flow_info = LIST_NEXT(flow_info, next); 7764 } 7765 return -1; 7766 } 7767 7768 /** 7769 * Set the mode of the flow engine of a process to active or standby during live migration. 7770 * 7771 * @param[in] mode 7772 * MLX5 flow engine mode, @see `enum mlx5_flow_engine_mode`. 7773 * @param[in] flags 7774 * Flow engine mode specific flags. 7775 * 7776 * @return 7777 * Negative value on error, positive on success. 7778 */ 7779 int 7780 rte_pmd_mlx5_flow_engine_set_mode(enum mlx5_flow_engine_mode mode, uint32_t flags) 7781 { 7782 struct mlx5_priv *priv; 7783 struct mlx5_flow_engine_mode_info *mode_info; 7784 struct mlx5_dv_flow_info *flow_info, *tmp_info; 7785 uint16_t port, port_id; 7786 uint16_t toggle_num = 0; 7787 struct rte_eth_dev *dev; 7788 enum mlx5_flow_engine_mode orig_mode; 7789 uint32_t orig_flags; 7790 bool need_toggle = false; 7791 7792 /* Check if flags combinations are supported. */ 7793 if (flags && flags != MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS) { 7794 DRV_LOG(ERR, "Doesn't support such flags %u", flags); 7795 return -1; 7796 } 7797 MLX5_ETH_FOREACH_DEV(port, NULL) { 7798 dev = &rte_eth_devices[port]; 7799 priv = dev->data->dev_private; 7800 mode_info = &priv->mode_info; 7801 /* No mode change. Assume all devices hold the same mode. */ 7802 if (mode_info->mode == mode) { 7803 DRV_LOG(INFO, "Process flow engine has been in mode %u", mode); 7804 if (mode_info->mode_flag != flags && !LIST_EMPTY(&mode_info->hot_upgrade)) { 7805 DRV_LOG(ERR, "Port %u has rule cache with different flag %u\n", 7806 port, mode_info->mode_flag); 7807 orig_mode = mode_info->mode; 7808 orig_flags = mode_info->mode_flag; 7809 goto err; 7810 } 7811 mode_info->mode_flag = flags; 7812 toggle_num++; 7813 continue; 7814 } 7815 /* Active -> standby. */ 7816 if (mode == MLX5_FLOW_ENGINE_MODE_STANDBY) { 7817 if (!LIST_EMPTY(&mode_info->hot_upgrade)) { 7818 DRV_LOG(ERR, "Cached rule existed"); 7819 orig_mode = mode_info->mode; 7820 orig_flags = mode_info->mode_flag; 7821 goto err; 7822 } 7823 mode_info->mode_flag = flags; 7824 mode_info->mode = mode; 7825 toggle_num++; 7826 /* Standby -> active. */ 7827 } else if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7828 if (LIST_EMPTY(&mode_info->hot_upgrade)) { 7829 DRV_LOG(INFO, "No cached rule existed"); 7830 } else { 7831 if (mlx5_flow_cache_flow_toggle(dev, true)) { 7832 orig_mode = mode_info->mode; 7833 orig_flags = mode_info->mode_flag; 7834 need_toggle = true; 7835 goto err; 7836 } 7837 } 7838 toggle_num++; 7839 } 7840 } 7841 if (mode == MLX5_FLOW_ENGINE_MODE_ACTIVE) { 7842 /* Clear cache flow rules. */ 7843 MLX5_ETH_FOREACH_DEV(port, NULL) { 7844 priv = rte_eth_devices[port].data->dev_private; 7845 mode_info = &priv->mode_info; 7846 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 7847 while (flow_info) { 7848 tmp_info = LIST_NEXT(flow_info, next); 7849 LIST_REMOVE(flow_info, next); 7850 mlx5_free(flow_info->actions); 7851 mlx5_free(flow_info->items); 7852 mlx5_free(flow_info); 7853 flow_info = tmp_info; 7854 } 7855 MLX5_ASSERT(LIST_EMPTY(&mode_info->hot_upgrade)); 7856 } 7857 } 7858 return toggle_num; 7859 err: 7860 /* Rollback all preceding successful ports. */ 7861 MLX5_ETH_FOREACH_DEV(port_id, NULL) { 7862 if (port_id == port) 7863 break; 7864 priv = rte_eth_devices[port_id].data->dev_private; 7865 mode_info = &priv->mode_info; 7866 if (need_toggle && !LIST_EMPTY(&mode_info->hot_upgrade) && 7867 mlx5_flow_cache_flow_toggle(dev, false)) 7868 return -EPERM; 7869 mode_info->mode = orig_mode; 7870 mode_info->mode_flag = orig_flags; 7871 } 7872 return -EINVAL; 7873 } 7874 /** 7875 * Create a flow. 7876 * 7877 * @see rte_flow_create() 7878 * @see rte_flow_ops 7879 */ 7880 struct rte_flow * 7881 mlx5_flow_create(struct rte_eth_dev *dev, 7882 const struct rte_flow_attr *attr, 7883 const struct rte_flow_item items[], 7884 const struct rte_flow_action actions[], 7885 struct rte_flow_error *error) 7886 { 7887 struct mlx5_priv *priv = dev->data->dev_private; 7888 struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr; 7889 uint32_t prio = attr->priority; 7890 uint32_t flow_idx; 7891 7892 if (priv->sh->config.dv_flow_en == 2) { 7893 rte_flow_error_set(error, ENOTSUP, 7894 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7895 NULL, 7896 "Flow non-Q creation not supported"); 7897 return NULL; 7898 } 7899 /* 7900 * If the device is not started yet, it is not allowed to created a 7901 * flow from application. PMD default flows and traffic control flows 7902 * are not affected. 7903 */ 7904 if (unlikely(!dev->data->dev_started)) { 7905 DRV_LOG(DEBUG, "port %u is not started when " 7906 "inserting a flow", dev->data->port_id); 7907 rte_flow_error_set(error, ENODEV, 7908 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 7909 NULL, 7910 "port not started"); 7911 return NULL; 7912 } 7913 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7914 if (attr->transfer || 7915 (attr->ingress && 7916 !(priv->mode_info.mode_flag & MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS))) 7917 new_attr->priority += 1; 7918 } 7919 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error); 7920 if (!flow_idx) 7921 return NULL; 7922 if (unlikely(mlx5_need_cache_flow(priv, attr))) { 7923 if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) { 7924 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 7925 flow_idx = 0; 7926 } 7927 } 7928 return (void *)(uintptr_t)flow_idx; 7929 } 7930 7931 /** 7932 * Destroy a flow in a list. 7933 * 7934 * @param dev 7935 * Pointer to Ethernet device. 7936 * @param[in] flow_idx 7937 * Index of flow to destroy. 7938 */ 7939 static void 7940 flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7941 uint32_t flow_idx) 7942 { 7943 struct mlx5_priv *priv = dev->data->dev_private; 7944 struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx); 7945 7946 if (!flow) 7947 return; 7948 MLX5_ASSERT(flow->type == type); 7949 /* 7950 * Update RX queue flags only if port is started, otherwise it is 7951 * already clean. 7952 */ 7953 if (dev->data->dev_started) 7954 flow_rxq_flags_trim(dev, flow); 7955 flow_drv_destroy(dev, flow); 7956 if (flow->tunnel) { 7957 struct mlx5_flow_tunnel *tunnel; 7958 7959 tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id); 7960 RTE_VERIFY(tunnel); 7961 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 7962 mlx5_flow_tunnel_free(dev, tunnel); 7963 } 7964 flow_mreg_del_copy_action(dev, flow); 7965 mlx5_ipool_free(priv->flows[type], flow_idx); 7966 } 7967 7968 /** 7969 * Destroy all flows. 7970 * 7971 * @param dev 7972 * Pointer to Ethernet device. 7973 * @param type 7974 * Flow type to be flushed. 7975 * @param active 7976 * If flushing is called actively. 7977 */ 7978 void 7979 mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type, 7980 bool active) 7981 { 7982 struct mlx5_priv *priv = dev->data->dev_private; 7983 uint32_t num_flushed = 0, fidx = 1; 7984 struct rte_flow *flow; 7985 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 7986 struct mlx5_dv_flow_info *flow_info; 7987 7988 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 7989 if (priv->sh->config.dv_flow_en == 2 && 7990 type == MLX5_FLOW_TYPE_GEN) { 7991 flow_hw_q_flow_flush(dev, NULL); 7992 return; 7993 } 7994 #endif 7995 7996 MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) { 7997 flow_list_destroy(dev, type, fidx); 7998 if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) { 7999 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8000 while (flow_info) { 8001 /* Romove the cache flow info. */ 8002 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)fidx) { 8003 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8004 LIST_REMOVE(flow_info, next); 8005 mlx5_free(flow_info->items); 8006 mlx5_free(flow_info->actions); 8007 mlx5_free(flow_info); 8008 break; 8009 } 8010 flow_info = LIST_NEXT(flow_info, next); 8011 } 8012 } 8013 num_flushed++; 8014 } 8015 if (active) { 8016 DRV_LOG(INFO, "port %u: %u flows flushed before stopping", 8017 dev->data->port_id, num_flushed); 8018 } 8019 } 8020 8021 /** 8022 * Stop all default actions for flows. 8023 * 8024 * @param dev 8025 * Pointer to Ethernet device. 8026 */ 8027 void 8028 mlx5_flow_stop_default(struct rte_eth_dev *dev) 8029 { 8030 flow_mreg_del_default_copy_action(dev); 8031 flow_rxq_flags_clear(dev); 8032 } 8033 8034 /** 8035 * Set rxq flag. 8036 * 8037 * @param[in] dev 8038 * Pointer to the rte_eth_dev structure. 8039 * @param[in] enable 8040 * Flag to enable or not. 8041 */ 8042 void 8043 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable) 8044 { 8045 struct mlx5_priv *priv = dev->data->dev_private; 8046 unsigned int i; 8047 8048 if ((!priv->mark_enabled && !enable) || 8049 (priv->mark_enabled && enable)) 8050 return; 8051 for (i = 0; i < priv->rxqs_n; ++i) { 8052 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 8053 8054 /* With RXQ start/stop feature, RXQ might be stopped. */ 8055 if (!rxq_ctrl) 8056 continue; 8057 rxq_ctrl->rxq.mark = enable; 8058 } 8059 priv->mark_enabled = enable; 8060 } 8061 8062 /** 8063 * Start all default actions for flows. 8064 * 8065 * @param dev 8066 * Pointer to Ethernet device. 8067 * @return 8068 * 0 on success, a negative errno value otherwise and rte_errno is set. 8069 */ 8070 int 8071 mlx5_flow_start_default(struct rte_eth_dev *dev) 8072 { 8073 struct rte_flow_error error; 8074 8075 /* Make sure default copy action (reg_c[0] -> reg_b) is created. */ 8076 return flow_mreg_add_default_copy_action(dev, &error); 8077 } 8078 8079 /** 8080 * Release key of thread specific flow workspace data. 8081 */ 8082 void 8083 flow_release_workspace(void *data) 8084 { 8085 struct mlx5_flow_workspace *wks = data; 8086 struct mlx5_flow_workspace *next; 8087 8088 while (wks) { 8089 next = wks->next; 8090 free(wks); 8091 wks = next; 8092 } 8093 } 8094 8095 static struct mlx5_flow_workspace *gc_head; 8096 static rte_spinlock_t mlx5_flow_workspace_lock = RTE_SPINLOCK_INITIALIZER; 8097 8098 static void 8099 mlx5_flow_workspace_gc_add(struct mlx5_flow_workspace *ws) 8100 { 8101 rte_spinlock_lock(&mlx5_flow_workspace_lock); 8102 ws->gc = gc_head; 8103 gc_head = ws; 8104 rte_spinlock_unlock(&mlx5_flow_workspace_lock); 8105 } 8106 8107 void 8108 mlx5_flow_workspace_gc_release(void) 8109 { 8110 while (gc_head) { 8111 struct mlx5_flow_workspace *wks = gc_head; 8112 8113 gc_head = wks->gc; 8114 flow_release_workspace(wks); 8115 } 8116 } 8117 8118 /** 8119 * Get thread specific current flow workspace. 8120 * 8121 * @return pointer to thread specific flow workspace data, NULL on error. 8122 */ 8123 struct mlx5_flow_workspace* 8124 mlx5_flow_get_thread_workspace(void) 8125 { 8126 struct mlx5_flow_workspace *data; 8127 8128 data = mlx5_flow_os_get_specific_workspace(); 8129 MLX5_ASSERT(data && data->inuse); 8130 if (!data || !data->inuse) 8131 DRV_LOG(ERR, "flow workspace not initialized."); 8132 return data; 8133 } 8134 8135 /** 8136 * Allocate and init new flow workspace. 8137 * 8138 * @return pointer to flow workspace data, NULL on error. 8139 */ 8140 static struct mlx5_flow_workspace* 8141 flow_alloc_thread_workspace(void) 8142 { 8143 size_t data_size = RTE_ALIGN(sizeof(struct mlx5_flow_workspace), sizeof(long)); 8144 size_t rss_queue_array_size = sizeof(uint16_t) * RTE_ETH_RSS_RETA_SIZE_512; 8145 struct mlx5_flow_workspace *data = calloc(1, data_size + 8146 rss_queue_array_size); 8147 8148 if (!data) { 8149 DRV_LOG(ERR, "Failed to allocate flow workspace memory."); 8150 return NULL; 8151 } 8152 data->rss_desc.queue = RTE_PTR_ADD(data, data_size); 8153 return data; 8154 } 8155 8156 /** 8157 * Get new thread specific flow workspace. 8158 * 8159 * If current workspace inuse, create new one and set as current. 8160 * 8161 * @return pointer to thread specific flow workspace data, NULL on error. 8162 */ 8163 struct mlx5_flow_workspace* 8164 mlx5_flow_push_thread_workspace(void) 8165 { 8166 struct mlx5_flow_workspace *curr; 8167 struct mlx5_flow_workspace *data; 8168 8169 curr = mlx5_flow_os_get_specific_workspace(); 8170 if (!curr) { 8171 data = flow_alloc_thread_workspace(); 8172 if (!data) 8173 return NULL; 8174 mlx5_flow_workspace_gc_add(data); 8175 } else if (!curr->inuse) { 8176 data = curr; 8177 } else if (curr->next) { 8178 data = curr->next; 8179 } else { 8180 data = flow_alloc_thread_workspace(); 8181 if (!data) 8182 return NULL; 8183 curr->next = data; 8184 data->prev = curr; 8185 } 8186 data->inuse = 1; 8187 data->flow_idx = 0; 8188 /* Set as current workspace */ 8189 if (mlx5_flow_os_set_specific_workspace(data)) 8190 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8191 return data; 8192 } 8193 8194 /** 8195 * Close current thread specific flow workspace. 8196 * 8197 * If previous workspace available, set it as current. 8198 * 8199 * @return pointer to thread specific flow workspace data, NULL on error. 8200 */ 8201 void 8202 mlx5_flow_pop_thread_workspace(void) 8203 { 8204 struct mlx5_flow_workspace *data = mlx5_flow_get_thread_workspace(); 8205 8206 if (!data) 8207 return; 8208 if (!data->inuse) { 8209 DRV_LOG(ERR, "Failed to close unused flow workspace."); 8210 return; 8211 } 8212 data->inuse = 0; 8213 if (!data->prev) 8214 return; 8215 if (mlx5_flow_os_set_specific_workspace(data->prev)) 8216 DRV_LOG(ERR, "Failed to set flow workspace to thread."); 8217 } 8218 8219 /** 8220 * Verify the flow list is empty 8221 * 8222 * @param dev 8223 * Pointer to Ethernet device. 8224 * 8225 * @return the number of flows not released. 8226 */ 8227 int 8228 mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused) 8229 { 8230 struct mlx5_priv *priv = dev->data->dev_private; 8231 struct rte_flow *flow; 8232 uint32_t idx = 0; 8233 int ret = 0, i; 8234 8235 for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) { 8236 MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) { 8237 DRV_LOG(DEBUG, "port %u flow %p still referenced", 8238 dev->data->port_id, (void *)flow); 8239 ret++; 8240 } 8241 } 8242 return ret; 8243 } 8244 8245 /** 8246 * Enable default hairpin egress flow. 8247 * 8248 * @param dev 8249 * Pointer to Ethernet device. 8250 * @param sq_num 8251 * The SQ hw number. 8252 * 8253 * @return 8254 * 0 on success, a negative errno value otherwise and rte_errno is set. 8255 */ 8256 int 8257 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, 8258 uint32_t sq_num) 8259 { 8260 const struct rte_flow_attr attr = { 8261 .egress = 1, 8262 .priority = 0, 8263 }; 8264 struct mlx5_rte_flow_item_sq queue_spec = { 8265 .queue = sq_num, 8266 }; 8267 struct mlx5_rte_flow_item_sq queue_mask = { 8268 .queue = UINT32_MAX, 8269 }; 8270 struct rte_flow_item items[] = { 8271 { 8272 .type = (enum rte_flow_item_type) 8273 MLX5_RTE_FLOW_ITEM_TYPE_SQ, 8274 .spec = &queue_spec, 8275 .last = NULL, 8276 .mask = &queue_mask, 8277 }, 8278 { 8279 .type = RTE_FLOW_ITEM_TYPE_END, 8280 }, 8281 }; 8282 struct rte_flow_action_jump jump = { 8283 .group = MLX5_HAIRPIN_TX_TABLE, 8284 }; 8285 struct rte_flow_action actions[2]; 8286 uint32_t flow_idx; 8287 struct rte_flow_error error; 8288 8289 actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP; 8290 actions[0].conf = &jump; 8291 actions[1].type = RTE_FLOW_ACTION_TYPE_END; 8292 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8293 &attr, items, actions, false, &error); 8294 if (!flow_idx) { 8295 DRV_LOG(DEBUG, 8296 "Failed to create ctrl flow: rte_errno(%d)," 8297 " type(%d), message(%s)", 8298 rte_errno, error.type, 8299 error.message ? error.message : " (no stated reason)"); 8300 return -rte_errno; 8301 } 8302 return 0; 8303 } 8304 8305 /** 8306 * Enable a control flow configured from the control plane. 8307 * 8308 * @param dev 8309 * Pointer to Ethernet device. 8310 * @param eth_spec 8311 * An Ethernet flow spec to apply. 8312 * @param eth_mask 8313 * An Ethernet flow mask to apply. 8314 * @param vlan_spec 8315 * A VLAN flow spec to apply. 8316 * @param vlan_mask 8317 * A VLAN flow mask to apply. 8318 * 8319 * @return 8320 * 0 on success, a negative errno value otherwise and rte_errno is set. 8321 */ 8322 int 8323 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, 8324 struct rte_flow_item_eth *eth_spec, 8325 struct rte_flow_item_eth *eth_mask, 8326 struct rte_flow_item_vlan *vlan_spec, 8327 struct rte_flow_item_vlan *vlan_mask) 8328 { 8329 struct mlx5_priv *priv = dev->data->dev_private; 8330 const struct rte_flow_attr attr = { 8331 .ingress = 1, 8332 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 8333 }; 8334 struct rte_flow_item items[] = { 8335 { 8336 .type = RTE_FLOW_ITEM_TYPE_ETH, 8337 .spec = eth_spec, 8338 .last = NULL, 8339 .mask = eth_mask, 8340 }, 8341 { 8342 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN : 8343 RTE_FLOW_ITEM_TYPE_END, 8344 .spec = vlan_spec, 8345 .last = NULL, 8346 .mask = vlan_mask, 8347 }, 8348 { 8349 .type = RTE_FLOW_ITEM_TYPE_END, 8350 }, 8351 }; 8352 uint16_t queue[priv->reta_idx_n]; 8353 struct rte_flow_action_rss action_rss = { 8354 .func = RTE_ETH_HASH_FUNCTION_DEFAULT, 8355 .level = 0, 8356 .types = priv->rss_conf.rss_hf, 8357 .key_len = priv->rss_conf.rss_key_len, 8358 .queue_num = priv->reta_idx_n, 8359 .key = priv->rss_conf.rss_key, 8360 .queue = queue, 8361 }; 8362 struct rte_flow_action actions[] = { 8363 { 8364 .type = RTE_FLOW_ACTION_TYPE_RSS, 8365 .conf = &action_rss, 8366 }, 8367 { 8368 .type = RTE_FLOW_ACTION_TYPE_END, 8369 }, 8370 }; 8371 uint32_t flow_idx; 8372 struct rte_flow_error error; 8373 unsigned int i; 8374 8375 if (!priv->reta_idx_n || !priv->rxqs_n) { 8376 return 0; 8377 } 8378 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 8379 action_rss.types = 0; 8380 for (i = 0; i != priv->reta_idx_n; ++i) 8381 queue[i] = (*priv->reta_idx)[i]; 8382 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8383 &attr, items, actions, false, &error); 8384 if (!flow_idx) 8385 return -rte_errno; 8386 return 0; 8387 } 8388 8389 /** 8390 * Enable a flow control configured from the control plane. 8391 * 8392 * @param dev 8393 * Pointer to Ethernet device. 8394 * @param eth_spec 8395 * An Ethernet flow spec to apply. 8396 * @param eth_mask 8397 * An Ethernet flow mask to apply. 8398 * 8399 * @return 8400 * 0 on success, a negative errno value otherwise and rte_errno is set. 8401 */ 8402 int 8403 mlx5_ctrl_flow(struct rte_eth_dev *dev, 8404 struct rte_flow_item_eth *eth_spec, 8405 struct rte_flow_item_eth *eth_mask) 8406 { 8407 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL); 8408 } 8409 8410 /** 8411 * Create default miss flow rule matching lacp traffic 8412 * 8413 * @param dev 8414 * Pointer to Ethernet device. 8415 * @param eth_spec 8416 * An Ethernet flow spec to apply. 8417 * 8418 * @return 8419 * 0 on success, a negative errno value otherwise and rte_errno is set. 8420 */ 8421 int 8422 mlx5_flow_lacp_miss(struct rte_eth_dev *dev) 8423 { 8424 /* 8425 * The LACP matching is done by only using ether type since using 8426 * a multicast dst mac causes kernel to give low priority to this flow. 8427 */ 8428 static const struct rte_flow_item_eth lacp_spec = { 8429 .hdr.ether_type = RTE_BE16(0x8809), 8430 }; 8431 static const struct rte_flow_item_eth lacp_mask = { 8432 .hdr.ether_type = 0xffff, 8433 }; 8434 const struct rte_flow_attr attr = { 8435 .ingress = 1, 8436 }; 8437 struct rte_flow_item items[] = { 8438 { 8439 .type = RTE_FLOW_ITEM_TYPE_ETH, 8440 .spec = &lacp_spec, 8441 .mask = &lacp_mask, 8442 }, 8443 { 8444 .type = RTE_FLOW_ITEM_TYPE_END, 8445 }, 8446 }; 8447 struct rte_flow_action actions[] = { 8448 { 8449 .type = (enum rte_flow_action_type) 8450 MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS, 8451 }, 8452 { 8453 .type = RTE_FLOW_ACTION_TYPE_END, 8454 }, 8455 }; 8456 struct rte_flow_error error; 8457 uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL, 8458 &attr, items, actions, 8459 false, &error); 8460 8461 if (!flow_idx) 8462 return -rte_errno; 8463 return 0; 8464 } 8465 8466 /** 8467 * Destroy a flow. 8468 * 8469 * @see rte_flow_destroy() 8470 * @see rte_flow_ops 8471 */ 8472 int 8473 mlx5_flow_destroy(struct rte_eth_dev *dev, 8474 struct rte_flow *flow, 8475 struct rte_flow_error *error __rte_unused) 8476 { 8477 struct mlx5_priv *priv = dev->data->dev_private; 8478 struct mlx5_flow_engine_mode_info *mode_info = &priv->mode_info; 8479 struct mlx5_dv_flow_info *flow_info; 8480 8481 if (priv->sh->config.dv_flow_en == 2) 8482 return rte_flow_error_set(error, ENOTSUP, 8483 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8484 NULL, 8485 "Flow non-Q destruction not supported"); 8486 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, 8487 (uintptr_t)(void *)flow); 8488 if (unlikely(mlx5_need_cache_flow(priv, NULL))) { 8489 flow_info = LIST_FIRST(&mode_info->hot_upgrade); 8490 while (flow_info) { 8491 /* Romove the cache flow info. */ 8492 if (flow_info->flow_idx_low_prio == (uint32_t)(uintptr_t)flow) { 8493 MLX5_ASSERT(!flow_info->flow_idx_high_prio); 8494 LIST_REMOVE(flow_info, next); 8495 mlx5_free(flow_info->items); 8496 mlx5_free(flow_info->actions); 8497 mlx5_free(flow_info); 8498 break; 8499 } 8500 flow_info = LIST_NEXT(flow_info, next); 8501 } 8502 } 8503 return 0; 8504 } 8505 8506 /** 8507 * Destroy all flows. 8508 * 8509 * @see rte_flow_flush() 8510 * @see rte_flow_ops 8511 */ 8512 int 8513 mlx5_flow_flush(struct rte_eth_dev *dev, 8514 struct rte_flow_error *error __rte_unused) 8515 { 8516 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false); 8517 return 0; 8518 } 8519 8520 /** 8521 * Isolated mode. 8522 * 8523 * @see rte_flow_isolate() 8524 * @see rte_flow_ops 8525 */ 8526 int 8527 mlx5_flow_isolate(struct rte_eth_dev *dev, 8528 int enable, 8529 struct rte_flow_error *error) 8530 { 8531 struct mlx5_priv *priv = dev->data->dev_private; 8532 8533 if (dev->data->dev_started) { 8534 rte_flow_error_set(error, EBUSY, 8535 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8536 NULL, 8537 "port must be stopped first"); 8538 return -rte_errno; 8539 } 8540 if (!enable && !priv->sh->config.repr_matching) 8541 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 8542 "isolated mode cannot be disabled when " 8543 "representor matching is disabled"); 8544 priv->isolated = !!enable; 8545 if (enable) 8546 dev->dev_ops = &mlx5_dev_ops_isolate; 8547 else 8548 dev->dev_ops = &mlx5_dev_ops; 8549 8550 dev->rx_descriptor_status = mlx5_rx_descriptor_status; 8551 dev->tx_descriptor_status = mlx5_tx_descriptor_status; 8552 8553 return 0; 8554 } 8555 8556 /** 8557 * Query a flow. 8558 * 8559 * @see rte_flow_query() 8560 * @see rte_flow_ops 8561 */ 8562 static int 8563 flow_drv_query(struct rte_eth_dev *dev, 8564 struct rte_flow *eflow, 8565 const struct rte_flow_action *actions, 8566 void *data, 8567 struct rte_flow_error *error) 8568 { 8569 struct mlx5_priv *priv = dev->data->dev_private; 8570 const struct mlx5_flow_driver_ops *fops; 8571 struct rte_flow *flow = NULL; 8572 enum mlx5_flow_drv_type ftype = MLX5_FLOW_TYPE_MIN; 8573 8574 if (priv->sh->config.dv_flow_en == 2) { 8575 #ifdef HAVE_MLX5_HWS_SUPPORT 8576 flow = eflow; 8577 ftype = MLX5_FLOW_TYPE_HW; 8578 #endif 8579 } else { 8580 flow = (struct rte_flow *)mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 8581 (uintptr_t)(void *)eflow); 8582 } 8583 if (!flow) { 8584 return rte_flow_error_set(error, ENOENT, 8585 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 8586 NULL, 8587 "invalid flow handle"); 8588 } 8589 if (ftype == MLX5_FLOW_TYPE_MIN) 8590 ftype = flow->drv_type; 8591 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && ftype < MLX5_FLOW_TYPE_MAX); 8592 fops = flow_get_drv_ops(ftype); 8593 8594 return fops->query(dev, flow, actions, data, error); 8595 } 8596 8597 /** 8598 * Query a flow. 8599 * 8600 * @see rte_flow_query() 8601 * @see rte_flow_ops 8602 */ 8603 int 8604 mlx5_flow_query(struct rte_eth_dev *dev, 8605 struct rte_flow *flow, 8606 const struct rte_flow_action *actions, 8607 void *data, 8608 struct rte_flow_error *error) 8609 { 8610 int ret; 8611 8612 ret = flow_drv_query(dev, flow, actions, data, 8613 error); 8614 if (ret < 0) 8615 return ret; 8616 return 0; 8617 } 8618 8619 /** 8620 * Get rte_flow callbacks. 8621 * 8622 * @param dev 8623 * Pointer to Ethernet device structure. 8624 * @param ops 8625 * Pointer to operation-specific structure. 8626 * 8627 * @return 0 8628 */ 8629 int 8630 mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, 8631 const struct rte_flow_ops **ops) 8632 { 8633 *ops = &mlx5_flow_ops; 8634 return 0; 8635 } 8636 8637 /** 8638 * Validate meter policy actions. 8639 * Dispatcher for action type specific validation. 8640 * 8641 * @param[in] dev 8642 * Pointer to the Ethernet device structure. 8643 * @param[in] action 8644 * The meter policy action object to validate. 8645 * @param[in] attr 8646 * Attributes of flow to determine steering domain. 8647 * @param[out] is_rss 8648 * Is RSS or not. 8649 * @param[out] domain_bitmap 8650 * Domain bitmap. 8651 * @param[out] is_def_policy 8652 * Is default policy or not. 8653 * @param[out] error 8654 * Perform verbose error reporting if not NULL. Initialized in case of 8655 * error only. 8656 * 8657 * @return 8658 * 0 on success, otherwise negative errno value. 8659 */ 8660 int 8661 mlx5_flow_validate_mtr_acts(struct rte_eth_dev *dev, 8662 const struct rte_flow_action *actions[RTE_COLORS], 8663 struct rte_flow_attr *attr, 8664 bool *is_rss, 8665 uint8_t *domain_bitmap, 8666 uint8_t *policy_mode, 8667 struct rte_mtr_error *error) 8668 { 8669 const struct mlx5_flow_driver_ops *fops; 8670 8671 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8672 return fops->validate_mtr_acts(dev, actions, attr, is_rss, 8673 domain_bitmap, policy_mode, error); 8674 } 8675 8676 /** 8677 * Destroy the meter table set. 8678 * 8679 * @param[in] dev 8680 * Pointer to Ethernet device. 8681 * @param[in] mtr_policy 8682 * Meter policy struct. 8683 */ 8684 void 8685 mlx5_flow_destroy_mtr_acts(struct rte_eth_dev *dev, 8686 struct mlx5_flow_meter_policy *mtr_policy) 8687 { 8688 const struct mlx5_flow_driver_ops *fops; 8689 8690 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8691 fops->destroy_mtr_acts(dev, mtr_policy); 8692 } 8693 8694 /** 8695 * Create policy action, lock free, 8696 * (mutex should be acquired by caller). 8697 * Dispatcher for action type specific call. 8698 * 8699 * @param[in] dev 8700 * Pointer to the Ethernet device structure. 8701 * @param[in] mtr_policy 8702 * Meter policy struct. 8703 * @param[in] action 8704 * Action specification used to create meter actions. 8705 * @param[in] attr 8706 * Flow rule attributes. 8707 * @param[out] error 8708 * Perform verbose error reporting if not NULL. Initialized in case of 8709 * error only. 8710 * 8711 * @return 8712 * 0 on success, otherwise negative errno value. 8713 */ 8714 int 8715 mlx5_flow_create_mtr_acts(struct rte_eth_dev *dev, 8716 struct mlx5_flow_meter_policy *mtr_policy, 8717 const struct rte_flow_action *actions[RTE_COLORS], 8718 struct rte_flow_attr *attr, 8719 struct rte_mtr_error *error) 8720 { 8721 const struct mlx5_flow_driver_ops *fops; 8722 8723 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8724 return fops->create_mtr_acts(dev, mtr_policy, actions, attr, error); 8725 } 8726 8727 /** 8728 * Create policy rules, lock free, 8729 * (mutex should be acquired by caller). 8730 * Dispatcher for action type specific call. 8731 * 8732 * @param[in] dev 8733 * Pointer to the Ethernet device structure. 8734 * @param[in] mtr_policy 8735 * Meter policy struct. 8736 * 8737 * @return 8738 * 0 on success, -1 otherwise. 8739 */ 8740 int 8741 mlx5_flow_create_policy_rules(struct rte_eth_dev *dev, 8742 struct mlx5_flow_meter_policy *mtr_policy) 8743 { 8744 const struct mlx5_flow_driver_ops *fops; 8745 8746 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8747 return fops->create_policy_rules(dev, mtr_policy); 8748 } 8749 8750 /** 8751 * Destroy policy rules, lock free, 8752 * (mutex should be acquired by caller). 8753 * Dispatcher for action type specific call. 8754 * 8755 * @param[in] dev 8756 * Pointer to the Ethernet device structure. 8757 * @param[in] mtr_policy 8758 * Meter policy struct. 8759 */ 8760 void 8761 mlx5_flow_destroy_policy_rules(struct rte_eth_dev *dev, 8762 struct mlx5_flow_meter_policy *mtr_policy) 8763 { 8764 const struct mlx5_flow_driver_ops *fops; 8765 8766 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8767 fops->destroy_policy_rules(dev, mtr_policy); 8768 } 8769 8770 /** 8771 * Destroy the default policy table set. 8772 * 8773 * @param[in] dev 8774 * Pointer to Ethernet device. 8775 */ 8776 void 8777 mlx5_flow_destroy_def_policy(struct rte_eth_dev *dev) 8778 { 8779 const struct mlx5_flow_driver_ops *fops; 8780 8781 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8782 fops->destroy_def_policy(dev); 8783 } 8784 8785 /** 8786 * Destroy the default policy table set. 8787 * 8788 * @param[in] dev 8789 * Pointer to Ethernet device. 8790 * 8791 * @return 8792 * 0 on success, -1 otherwise. 8793 */ 8794 int 8795 mlx5_flow_create_def_policy(struct rte_eth_dev *dev) 8796 { 8797 const struct mlx5_flow_driver_ops *fops; 8798 8799 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8800 return fops->create_def_policy(dev); 8801 } 8802 8803 /** 8804 * Create the needed meter and suffix tables. 8805 * 8806 * @param[in] dev 8807 * Pointer to Ethernet device. 8808 * 8809 * @return 8810 * 0 on success, -1 otherwise. 8811 */ 8812 int 8813 mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, 8814 struct mlx5_flow_meter_info *fm, 8815 uint32_t mtr_idx, 8816 uint8_t domain_bitmap) 8817 { 8818 const struct mlx5_flow_driver_ops *fops; 8819 8820 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8821 return fops->create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap); 8822 } 8823 8824 /** 8825 * Destroy the meter table set. 8826 * 8827 * @param[in] dev 8828 * Pointer to Ethernet device. 8829 * @param[in] tbl 8830 * Pointer to the meter table set. 8831 */ 8832 void 8833 mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, 8834 struct mlx5_flow_meter_info *fm) 8835 { 8836 const struct mlx5_flow_driver_ops *fops; 8837 8838 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8839 fops->destroy_mtr_tbls(dev, fm); 8840 } 8841 8842 /** 8843 * Destroy the global meter drop table. 8844 * 8845 * @param[in] dev 8846 * Pointer to Ethernet device. 8847 */ 8848 void 8849 mlx5_flow_destroy_mtr_drop_tbls(struct rte_eth_dev *dev) 8850 { 8851 const struct mlx5_flow_driver_ops *fops; 8852 8853 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8854 fops->destroy_mtr_drop_tbls(dev); 8855 } 8856 8857 /** 8858 * Destroy the sub policy table with RX queue. 8859 * 8860 * @param[in] dev 8861 * Pointer to Ethernet device. 8862 * @param[in] mtr_policy 8863 * Pointer to meter policy table. 8864 */ 8865 void 8866 mlx5_flow_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev, 8867 struct mlx5_flow_meter_policy *mtr_policy) 8868 { 8869 const struct mlx5_flow_driver_ops *fops; 8870 8871 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8872 fops->destroy_sub_policy_with_rxq(dev, mtr_policy); 8873 } 8874 8875 /** 8876 * Allocate the needed aso flow meter id. 8877 * 8878 * @param[in] dev 8879 * Pointer to Ethernet device. 8880 * 8881 * @return 8882 * Index to aso flow meter on success, NULL otherwise. 8883 */ 8884 uint32_t 8885 mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) 8886 { 8887 const struct mlx5_flow_driver_ops *fops; 8888 8889 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8890 return fops->create_meter(dev); 8891 } 8892 8893 /** 8894 * Free the aso flow meter id. 8895 * 8896 * @param[in] dev 8897 * Pointer to Ethernet device. 8898 * @param[in] mtr_idx 8899 * Index to aso flow meter to be free. 8900 * 8901 * @return 8902 * 0 on success. 8903 */ 8904 void 8905 mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) 8906 { 8907 const struct mlx5_flow_driver_ops *fops; 8908 8909 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); 8910 fops->free_meter(dev, mtr_idx); 8911 } 8912 8913 /** 8914 * Allocate a counter. 8915 * 8916 * @param[in] dev 8917 * Pointer to Ethernet device structure. 8918 * 8919 * @return 8920 * Index to allocated counter on success, 0 otherwise. 8921 */ 8922 uint32_t 8923 mlx5_counter_alloc(struct rte_eth_dev *dev) 8924 { 8925 struct rte_flow_attr attr = { .transfer = 0 }; 8926 8927 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_alloc 8928 (dev); 8929 } 8930 8931 /** 8932 * Free a counter. 8933 * 8934 * @param[in] dev 8935 * Pointer to Ethernet device structure. 8936 * @param[in] cnt 8937 * Index to counter to be free. 8938 */ 8939 void 8940 mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt) 8941 { 8942 struct rte_flow_attr attr = { .transfer = 0 }; 8943 8944 flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_free(dev, cnt); 8945 } 8946 8947 /** 8948 * Query counter statistics. 8949 * 8950 * @param[in] dev 8951 * Pointer to Ethernet device structure. 8952 * @param[in] cnt 8953 * Index to counter to query. 8954 * @param[in] clear 8955 * Set to clear counter statistics. 8956 * @param[out] pkts 8957 * The counter hits packets number to save. 8958 * @param[out] bytes 8959 * The counter hits bytes number to save. 8960 * 8961 * @return 8962 * 0 on success, a negative errno value otherwise. 8963 */ 8964 int 8965 mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, 8966 bool clear, uint64_t *pkts, uint64_t *bytes, void **action) 8967 { 8968 struct rte_flow_attr attr = { .transfer = 0 }; 8969 8970 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->counter_query 8971 (dev, cnt, clear, pkts, bytes, action); 8972 } 8973 8974 /** 8975 * Get information about HWS pre-configurable resources. 8976 * 8977 * @param[in] dev 8978 * Pointer to the rte_eth_dev structure. 8979 * @param[out] port_info 8980 * Pointer to port information. 8981 * @param[out] queue_info 8982 * Pointer to queue information. 8983 * @param[out] error 8984 * Pointer to error structure. 8985 * 8986 * @return 8987 * 0 on success, a negative errno value otherwise and rte_errno is set. 8988 */ 8989 static int 8990 mlx5_flow_info_get(struct rte_eth_dev *dev, 8991 struct rte_flow_port_info *port_info, 8992 struct rte_flow_queue_info *queue_info, 8993 struct rte_flow_error *error) 8994 { 8995 const struct mlx5_flow_driver_ops *fops; 8996 struct rte_flow_attr attr = {0}; 8997 8998 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 8999 return rte_flow_error_set(error, ENOTSUP, 9000 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9001 NULL, 9002 "info get with incorrect steering mode"); 9003 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9004 return fops->info_get(dev, port_info, queue_info, error); 9005 } 9006 9007 /** 9008 * Configure port HWS resources. 9009 * 9010 * @param[in] dev 9011 * Pointer to the rte_eth_dev structure. 9012 * @param[in] port_attr 9013 * Port configuration attributes. 9014 * @param[in] nb_queue 9015 * Number of queue. 9016 * @param[in] queue_attr 9017 * Array that holds attributes for each flow queue. 9018 * @param[out] error 9019 * Pointer to error structure. 9020 * 9021 * @return 9022 * 0 on success, a negative errno value otherwise and rte_errno is set. 9023 */ 9024 static int 9025 mlx5_flow_port_configure(struct rte_eth_dev *dev, 9026 const struct rte_flow_port_attr *port_attr, 9027 uint16_t nb_queue, 9028 const struct rte_flow_queue_attr *queue_attr[], 9029 struct rte_flow_error *error) 9030 { 9031 const struct mlx5_flow_driver_ops *fops; 9032 struct rte_flow_attr attr = {0}; 9033 9034 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9035 return rte_flow_error_set(error, ENOTSUP, 9036 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9037 NULL, 9038 "port configure with incorrect steering mode"); 9039 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9040 return fops->configure(dev, port_attr, nb_queue, queue_attr, error); 9041 } 9042 9043 /** 9044 * Validate item template. 9045 * 9046 * @param[in] dev 9047 * Pointer to the rte_eth_dev structure. 9048 * @param[in] attr 9049 * Pointer to the item template attributes. 9050 * @param[in] items 9051 * The template item pattern. 9052 * @param[out] error 9053 * Pointer to error structure. 9054 * 9055 * @return 9056 * 0 on success, a negative errno value otherwise and rte_errno is set. 9057 */ 9058 int 9059 mlx5_flow_pattern_validate(struct rte_eth_dev *dev, 9060 const struct rte_flow_pattern_template_attr *attr, 9061 const struct rte_flow_item items[], 9062 struct rte_flow_error *error) 9063 { 9064 const struct mlx5_flow_driver_ops *fops; 9065 struct rte_flow_attr fattr = {0}; 9066 9067 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9068 rte_flow_error_set(error, ENOTSUP, 9069 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9070 "pattern validate with incorrect steering mode"); 9071 return -ENOTSUP; 9072 } 9073 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9074 return fops->pattern_validate(dev, attr, items, error); 9075 } 9076 9077 /** 9078 * Create flow item template. 9079 * 9080 * @param[in] dev 9081 * Pointer to the rte_eth_dev structure. 9082 * @param[in] attr 9083 * Pointer to the item template attributes. 9084 * @param[in] items 9085 * The template item pattern. 9086 * @param[out] error 9087 * Pointer to error structure. 9088 * 9089 * @return 9090 * 0 on success, a negative errno value otherwise and rte_errno is set. 9091 */ 9092 static struct rte_flow_pattern_template * 9093 mlx5_flow_pattern_template_create(struct rte_eth_dev *dev, 9094 const struct rte_flow_pattern_template_attr *attr, 9095 const struct rte_flow_item items[], 9096 struct rte_flow_error *error) 9097 { 9098 const struct mlx5_flow_driver_ops *fops; 9099 struct rte_flow_attr fattr = {0}; 9100 9101 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9102 rte_flow_error_set(error, ENOTSUP, 9103 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9104 NULL, 9105 "pattern create with incorrect steering mode"); 9106 return NULL; 9107 } 9108 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9109 return fops->pattern_template_create(dev, attr, items, error); 9110 } 9111 9112 /** 9113 * Destroy flow item template. 9114 * 9115 * @param[in] dev 9116 * Pointer to the rte_eth_dev structure. 9117 * @param[in] template 9118 * Pointer to the item template to be destroyed. 9119 * @param[out] error 9120 * Pointer to error structure. 9121 * 9122 * @return 9123 * 0 on success, a negative errno value otherwise and rte_errno is set. 9124 */ 9125 static int 9126 mlx5_flow_pattern_template_destroy(struct rte_eth_dev *dev, 9127 struct rte_flow_pattern_template *template, 9128 struct rte_flow_error *error) 9129 { 9130 const struct mlx5_flow_driver_ops *fops; 9131 struct rte_flow_attr attr = {0}; 9132 9133 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9134 return rte_flow_error_set(error, ENOTSUP, 9135 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9136 NULL, 9137 "pattern destroy with incorrect steering mode"); 9138 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9139 return fops->pattern_template_destroy(dev, template, error); 9140 } 9141 9142 /** 9143 * Validate flow actions template. 9144 * 9145 * @param[in] dev 9146 * Pointer to the rte_eth_dev structure. 9147 * @param[in] attr 9148 * Pointer to the action template attributes. 9149 * @param[in] actions 9150 * Associated actions (list terminated by the END action). 9151 * @param[in] masks 9152 * List of actions that marks which of the action's member is constant. 9153 * @param[out] error 9154 * Pointer to error structure. 9155 * 9156 * @return 9157 * 0 on success, a negative errno value otherwise and rte_errno is set. 9158 */ 9159 int 9160 mlx5_flow_actions_validate(struct rte_eth_dev *dev, 9161 const struct rte_flow_actions_template_attr *attr, 9162 const struct rte_flow_action actions[], 9163 const struct rte_flow_action masks[], 9164 struct rte_flow_error *error) 9165 { 9166 const struct mlx5_flow_driver_ops *fops; 9167 struct rte_flow_attr fattr = {0}; 9168 9169 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9170 rte_flow_error_set(error, ENOTSUP, 9171 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 9172 "actions validate with incorrect steering mode"); 9173 return -ENOTSUP; 9174 } 9175 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9176 return fops->actions_validate(dev, attr, actions, masks, error); 9177 } 9178 9179 /** 9180 * Create flow item template. 9181 * 9182 * @param[in] dev 9183 * Pointer to the rte_eth_dev structure. 9184 * @param[in] attr 9185 * Pointer to the action template attributes. 9186 * @param[in] actions 9187 * Associated actions (list terminated by the END action). 9188 * @param[in] masks 9189 * List of actions that marks which of the action's member is constant. 9190 * @param[out] error 9191 * Pointer to error structure. 9192 * 9193 * @return 9194 * 0 on success, a negative errno value otherwise and rte_errno is set. 9195 */ 9196 static struct rte_flow_actions_template * 9197 mlx5_flow_actions_template_create(struct rte_eth_dev *dev, 9198 const struct rte_flow_actions_template_attr *attr, 9199 const struct rte_flow_action actions[], 9200 const struct rte_flow_action masks[], 9201 struct rte_flow_error *error) 9202 { 9203 const struct mlx5_flow_driver_ops *fops; 9204 struct rte_flow_attr fattr = {0}; 9205 9206 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9207 rte_flow_error_set(error, ENOTSUP, 9208 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9209 NULL, 9210 "action create with incorrect steering mode"); 9211 return NULL; 9212 } 9213 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9214 return fops->actions_template_create(dev, attr, actions, masks, error); 9215 } 9216 9217 /** 9218 * Destroy flow action template. 9219 * 9220 * @param[in] dev 9221 * Pointer to the rte_eth_dev structure. 9222 * @param[in] template 9223 * Pointer to the action template to be destroyed. 9224 * @param[out] error 9225 * Pointer to error structure. 9226 * 9227 * @return 9228 * 0 on success, a negative errno value otherwise and rte_errno is set. 9229 */ 9230 static int 9231 mlx5_flow_actions_template_destroy(struct rte_eth_dev *dev, 9232 struct rte_flow_actions_template *template, 9233 struct rte_flow_error *error) 9234 { 9235 const struct mlx5_flow_driver_ops *fops; 9236 struct rte_flow_attr attr = {0}; 9237 9238 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9239 return rte_flow_error_set(error, ENOTSUP, 9240 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9241 NULL, 9242 "action destroy with incorrect steering mode"); 9243 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9244 return fops->actions_template_destroy(dev, template, error); 9245 } 9246 9247 /** 9248 * Create flow table. 9249 * 9250 * @param[in] dev 9251 * Pointer to the rte_eth_dev structure. 9252 * @param[in] attr 9253 * Pointer to the table attributes. 9254 * @param[in] item_templates 9255 * Item template array to be binded to the table. 9256 * @param[in] nb_item_templates 9257 * Number of item template. 9258 * @param[in] action_templates 9259 * Action template array to be binded to the table. 9260 * @param[in] nb_action_templates 9261 * Number of action template. 9262 * @param[out] error 9263 * Pointer to error structure. 9264 * 9265 * @return 9266 * Table on success, NULL otherwise and rte_errno is set. 9267 */ 9268 static struct rte_flow_template_table * 9269 mlx5_flow_table_create(struct rte_eth_dev *dev, 9270 const struct rte_flow_template_table_attr *attr, 9271 struct rte_flow_pattern_template *item_templates[], 9272 uint8_t nb_item_templates, 9273 struct rte_flow_actions_template *action_templates[], 9274 uint8_t nb_action_templates, 9275 struct rte_flow_error *error) 9276 { 9277 const struct mlx5_flow_driver_ops *fops; 9278 struct rte_flow_attr fattr = {0}; 9279 9280 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9281 rte_flow_error_set(error, ENOTSUP, 9282 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9283 NULL, 9284 "table create with incorrect steering mode"); 9285 return NULL; 9286 } 9287 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9288 return fops->template_table_create(dev, 9289 attr, 9290 item_templates, 9291 nb_item_templates, 9292 action_templates, 9293 nb_action_templates, 9294 error); 9295 } 9296 9297 /** 9298 * PMD destroy flow table. 9299 * 9300 * @param[in] dev 9301 * Pointer to the rte_eth_dev structure. 9302 * @param[in] table 9303 * Pointer to the table to be destroyed. 9304 * @param[out] error 9305 * Pointer to error structure. 9306 * 9307 * @return 9308 * 0 on success, a negative errno value otherwise and rte_errno is set. 9309 */ 9310 static int 9311 mlx5_flow_table_destroy(struct rte_eth_dev *dev, 9312 struct rte_flow_template_table *table, 9313 struct rte_flow_error *error) 9314 { 9315 const struct mlx5_flow_driver_ops *fops; 9316 struct rte_flow_attr attr = {0}; 9317 9318 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9319 return rte_flow_error_set(error, ENOTSUP, 9320 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9321 NULL, 9322 "table destroy with incorrect steering mode"); 9323 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9324 return fops->template_table_destroy(dev, table, error); 9325 } 9326 9327 /** 9328 * Enqueue flow creation. 9329 * 9330 * @param[in] dev 9331 * Pointer to the rte_eth_dev structure. 9332 * @param[in] queue_id 9333 * The queue to create the flow. 9334 * @param[in] attr 9335 * Pointer to the flow operation attributes. 9336 * @param[in] items 9337 * Items with flow spec value. 9338 * @param[in] pattern_template_index 9339 * The item pattern flow follows from the table. 9340 * @param[in] actions 9341 * Action with flow spec value. 9342 * @param[in] action_template_index 9343 * The action pattern flow follows from the table. 9344 * @param[in] user_data 9345 * Pointer to the user_data. 9346 * @param[out] error 9347 * Pointer to error structure. 9348 * 9349 * @return 9350 * Flow pointer on success, NULL otherwise and rte_errno is set. 9351 */ 9352 static struct rte_flow * 9353 mlx5_flow_async_flow_create(struct rte_eth_dev *dev, 9354 uint32_t queue_id, 9355 const struct rte_flow_op_attr *attr, 9356 struct rte_flow_template_table *table, 9357 const struct rte_flow_item items[], 9358 uint8_t pattern_template_index, 9359 const struct rte_flow_action actions[], 9360 uint8_t action_template_index, 9361 void *user_data, 9362 struct rte_flow_error *error) 9363 { 9364 const struct mlx5_flow_driver_ops *fops; 9365 struct rte_flow_attr fattr = {0}; 9366 9367 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9368 rte_flow_error_set(error, ENOTSUP, 9369 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9370 NULL, 9371 "flow_q create with incorrect steering mode"); 9372 return NULL; 9373 } 9374 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9375 return fops->async_flow_create(dev, queue_id, attr, table, 9376 items, pattern_template_index, 9377 actions, action_template_index, 9378 user_data, error); 9379 } 9380 9381 /** 9382 * Enqueue flow creation by index. 9383 * 9384 * @param[in] dev 9385 * Pointer to the rte_eth_dev structure. 9386 * @param[in] queue_id 9387 * The queue to create the flow. 9388 * @param[in] attr 9389 * Pointer to the flow operation attributes. 9390 * @param[in] rule_index 9391 * The item pattern flow follows from the table. 9392 * @param[in] actions 9393 * Action with flow spec value. 9394 * @param[in] action_template_index 9395 * The action pattern flow follows from the table. 9396 * @param[in] user_data 9397 * Pointer to the user_data. 9398 * @param[out] error 9399 * Pointer to error structure. 9400 * 9401 * @return 9402 * Flow pointer on success, NULL otherwise and rte_errno is set. 9403 */ 9404 static struct rte_flow * 9405 mlx5_flow_async_flow_create_by_index(struct rte_eth_dev *dev, 9406 uint32_t queue_id, 9407 const struct rte_flow_op_attr *attr, 9408 struct rte_flow_template_table *table, 9409 uint32_t rule_index, 9410 const struct rte_flow_action actions[], 9411 uint8_t action_template_index, 9412 void *user_data, 9413 struct rte_flow_error *error) 9414 { 9415 const struct mlx5_flow_driver_ops *fops; 9416 struct rte_flow_attr fattr = {0}; 9417 9418 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) { 9419 rte_flow_error_set(error, ENOTSUP, 9420 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9421 NULL, 9422 "flow_q create with incorrect steering mode"); 9423 return NULL; 9424 } 9425 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9426 return fops->async_flow_create_by_index(dev, queue_id, attr, table, 9427 rule_index, actions, action_template_index, 9428 user_data, error); 9429 } 9430 9431 /** 9432 * Enqueue flow update. 9433 * 9434 * @param[in] dev 9435 * Pointer to the rte_eth_dev structure. 9436 * @param[in] queue 9437 * The queue to destroy the flow. 9438 * @param[in] attr 9439 * Pointer to the flow operation attributes. 9440 * @param[in] flow 9441 * Pointer to the flow to be destroyed. 9442 * @param[in] actions 9443 * Action with flow spec value. 9444 * @param[in] action_template_index 9445 * The action pattern flow follows from the table. 9446 * @param[in] user_data 9447 * Pointer to the user_data. 9448 * @param[out] error 9449 * Pointer to error structure. 9450 * 9451 * @return 9452 * 0 on success, negative value otherwise and rte_errno is set. 9453 */ 9454 static int 9455 mlx5_flow_async_flow_update(struct rte_eth_dev *dev, 9456 uint32_t queue, 9457 const struct rte_flow_op_attr *attr, 9458 struct rte_flow *flow, 9459 const struct rte_flow_action actions[], 9460 uint8_t action_template_index, 9461 void *user_data, 9462 struct rte_flow_error *error) 9463 { 9464 const struct mlx5_flow_driver_ops *fops; 9465 struct rte_flow_attr fattr = {0}; 9466 9467 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9468 return rte_flow_error_set(error, ENOTSUP, 9469 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9470 NULL, 9471 "flow_q update with incorrect steering mode"); 9472 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9473 return fops->async_flow_update(dev, queue, attr, flow, 9474 actions, action_template_index, user_data, error); 9475 } 9476 9477 /** 9478 * Enqueue flow destruction. 9479 * 9480 * @param[in] dev 9481 * Pointer to the rte_eth_dev structure. 9482 * @param[in] queue 9483 * The queue to destroy the flow. 9484 * @param[in] attr 9485 * Pointer to the flow operation attributes. 9486 * @param[in] flow 9487 * Pointer to the flow to be destroyed. 9488 * @param[in] user_data 9489 * Pointer to the user_data. 9490 * @param[out] error 9491 * Pointer to error structure. 9492 * 9493 * @return 9494 * 0 on success, negative value otherwise and rte_errno is set. 9495 */ 9496 static int 9497 mlx5_flow_async_flow_destroy(struct rte_eth_dev *dev, 9498 uint32_t queue, 9499 const struct rte_flow_op_attr *attr, 9500 struct rte_flow *flow, 9501 void *user_data, 9502 struct rte_flow_error *error) 9503 { 9504 const struct mlx5_flow_driver_ops *fops; 9505 struct rte_flow_attr fattr = {0}; 9506 9507 if (flow_get_drv_type(dev, &fattr) != MLX5_FLOW_TYPE_HW) 9508 return rte_flow_error_set(error, ENOTSUP, 9509 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9510 NULL, 9511 "flow_q destroy with incorrect steering mode"); 9512 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9513 return fops->async_flow_destroy(dev, queue, attr, flow, 9514 user_data, error); 9515 } 9516 9517 /** 9518 * Pull the enqueued flows. 9519 * 9520 * @param[in] dev 9521 * Pointer to the rte_eth_dev structure. 9522 * @param[in] queue 9523 * The queue to pull the result. 9524 * @param[in/out] res 9525 * Array to save the results. 9526 * @param[in] n_res 9527 * Available result with the array. 9528 * @param[out] error 9529 * Pointer to error structure. 9530 * 9531 * @return 9532 * Result number on success, negative value otherwise and rte_errno is set. 9533 */ 9534 static int 9535 mlx5_flow_pull(struct rte_eth_dev *dev, 9536 uint32_t queue, 9537 struct rte_flow_op_result res[], 9538 uint16_t n_res, 9539 struct rte_flow_error *error) 9540 { 9541 const struct mlx5_flow_driver_ops *fops; 9542 struct rte_flow_attr attr = {0}; 9543 9544 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9545 return rte_flow_error_set(error, ENOTSUP, 9546 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9547 NULL, 9548 "flow_q pull with incorrect steering mode"); 9549 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9550 return fops->pull(dev, queue, res, n_res, error); 9551 } 9552 9553 /** 9554 * Push the enqueued flows. 9555 * 9556 * @param[in] dev 9557 * Pointer to the rte_eth_dev structure. 9558 * @param[in] queue 9559 * The queue to push the flows. 9560 * @param[out] error 9561 * Pointer to error structure. 9562 * 9563 * @return 9564 * 0 on success, negative value otherwise and rte_errno is set. 9565 */ 9566 static int 9567 mlx5_flow_push(struct rte_eth_dev *dev, 9568 uint32_t queue, 9569 struct rte_flow_error *error) 9570 { 9571 const struct mlx5_flow_driver_ops *fops; 9572 struct rte_flow_attr attr = {0}; 9573 9574 if (flow_get_drv_type(dev, &attr) != MLX5_FLOW_TYPE_HW) 9575 return rte_flow_error_set(error, ENOTSUP, 9576 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 9577 NULL, 9578 "flow_q push with incorrect steering mode"); 9579 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9580 return fops->push(dev, queue, error); 9581 } 9582 9583 /** 9584 * Create shared action. 9585 * 9586 * @param[in] dev 9587 * Pointer to the rte_eth_dev structure. 9588 * @param[in] queue 9589 * Which queue to be used.. 9590 * @param[in] attr 9591 * Operation attribute. 9592 * @param[in] conf 9593 * Indirect action configuration. 9594 * @param[in] action 9595 * rte_flow action detail. 9596 * @param[in] user_data 9597 * Pointer to the user_data. 9598 * @param[out] error 9599 * Pointer to error structure. 9600 * 9601 * @return 9602 * Action handle on success, NULL otherwise and rte_errno is set. 9603 */ 9604 static struct rte_flow_action_handle * 9605 mlx5_flow_async_action_handle_create(struct rte_eth_dev *dev, uint32_t queue, 9606 const struct rte_flow_op_attr *attr, 9607 const struct rte_flow_indir_action_conf *conf, 9608 const struct rte_flow_action *action, 9609 void *user_data, 9610 struct rte_flow_error *error) 9611 { 9612 const struct mlx5_flow_driver_ops *fops = 9613 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9614 9615 return fops->async_action_create(dev, queue, attr, conf, action, 9616 user_data, error); 9617 } 9618 9619 /** 9620 * Update shared action. 9621 * 9622 * @param[in] dev 9623 * Pointer to the rte_eth_dev structure. 9624 * @param[in] queue 9625 * Which queue to be used.. 9626 * @param[in] attr 9627 * Operation attribute. 9628 * @param[in] handle 9629 * Action handle to be updated. 9630 * @param[in] update 9631 * Update value. 9632 * @param[in] user_data 9633 * Pointer to the user_data. 9634 * @param[out] error 9635 * Pointer to error structure. 9636 * 9637 * @return 9638 * 0 on success, negative value otherwise and rte_errno is set. 9639 */ 9640 static int 9641 mlx5_flow_async_action_handle_update(struct rte_eth_dev *dev, uint32_t queue, 9642 const struct rte_flow_op_attr *attr, 9643 struct rte_flow_action_handle *handle, 9644 const void *update, 9645 void *user_data, 9646 struct rte_flow_error *error) 9647 { 9648 const struct mlx5_flow_driver_ops *fops = 9649 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9650 9651 return fops->async_action_update(dev, queue, attr, handle, 9652 update, user_data, error); 9653 } 9654 9655 static int 9656 mlx5_flow_async_action_handle_query_update 9657 (struct rte_eth_dev *dev, uint32_t queue_id, 9658 const struct rte_flow_op_attr *op_attr, 9659 struct rte_flow_action_handle *action_handle, 9660 const void *update, void *query, 9661 enum rte_flow_query_update_mode qu_mode, 9662 void *user_data, struct rte_flow_error *error) 9663 { 9664 const struct mlx5_flow_driver_ops *fops = 9665 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9666 9667 if (!fops || !fops->async_action_query_update) 9668 return rte_flow_error_set(error, ENOTSUP, 9669 RTE_FLOW_ERROR_TYPE_ACTION, NULL, 9670 "async query_update not supported"); 9671 return fops->async_action_query_update 9672 (dev, queue_id, op_attr, action_handle, 9673 update, query, qu_mode, user_data, error); 9674 } 9675 9676 /** 9677 * Query shared action. 9678 * 9679 * @param[in] dev 9680 * Pointer to the rte_eth_dev structure. 9681 * @param[in] queue 9682 * Which queue to be used.. 9683 * @param[in] attr 9684 * Operation attribute. 9685 * @param[in] handle 9686 * Action handle to be updated. 9687 * @param[in] data 9688 * Pointer query result data. 9689 * @param[in] user_data 9690 * Pointer to the user_data. 9691 * @param[out] error 9692 * Pointer to error structure. 9693 * 9694 * @return 9695 * 0 on success, negative value otherwise and rte_errno is set. 9696 */ 9697 static int 9698 mlx5_flow_async_action_handle_query(struct rte_eth_dev *dev, uint32_t queue, 9699 const struct rte_flow_op_attr *attr, 9700 const struct rte_flow_action_handle *handle, 9701 void *data, 9702 void *user_data, 9703 struct rte_flow_error *error) 9704 { 9705 const struct mlx5_flow_driver_ops *fops = 9706 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9707 9708 return fops->async_action_query(dev, queue, attr, handle, 9709 data, user_data, error); 9710 } 9711 9712 /** 9713 * Destroy shared action. 9714 * 9715 * @param[in] dev 9716 * Pointer to the rte_eth_dev structure. 9717 * @param[in] queue 9718 * Which queue to be used.. 9719 * @param[in] attr 9720 * Operation attribute. 9721 * @param[in] handle 9722 * Action handle to be destroyed. 9723 * @param[in] user_data 9724 * Pointer to the user_data. 9725 * @param[out] error 9726 * Pointer to error structure. 9727 * 9728 * @return 9729 * 0 on success, negative value otherwise and rte_errno is set. 9730 */ 9731 static int 9732 mlx5_flow_async_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue, 9733 const struct rte_flow_op_attr *attr, 9734 struct rte_flow_action_handle *handle, 9735 void *user_data, 9736 struct rte_flow_error *error) 9737 { 9738 const struct mlx5_flow_driver_ops *fops = 9739 flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 9740 9741 return fops->async_action_destroy(dev, queue, attr, handle, 9742 user_data, error); 9743 } 9744 9745 /** 9746 * Allocate a new memory for the counter values wrapped by all the needed 9747 * management. 9748 * 9749 * @param[in] sh 9750 * Pointer to mlx5_dev_ctx_shared object. 9751 * 9752 * @return 9753 * 0 on success, a negative errno value otherwise. 9754 */ 9755 static int 9756 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh) 9757 { 9758 struct mlx5_counter_stats_mem_mng *mem_mng; 9759 volatile struct flow_counter_stats *raw_data; 9760 int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES; 9761 int size = (sizeof(struct flow_counter_stats) * 9762 MLX5_COUNTERS_PER_POOL + 9763 sizeof(struct mlx5_counter_stats_raw)) * raws_n + 9764 sizeof(struct mlx5_counter_stats_mem_mng); 9765 size_t pgsize = rte_mem_page_size(); 9766 uint8_t *mem; 9767 int ret; 9768 int i; 9769 9770 if (pgsize == (size_t)-1) { 9771 DRV_LOG(ERR, "Failed to get mem page size"); 9772 rte_errno = ENOMEM; 9773 return -ENOMEM; 9774 } 9775 mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize, SOCKET_ID_ANY); 9776 if (!mem) { 9777 rte_errno = ENOMEM; 9778 return -ENOMEM; 9779 } 9780 mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1; 9781 size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n; 9782 ret = mlx5_os_wrapped_mkey_create(sh->cdev->ctx, sh->cdev->pd, 9783 sh->cdev->pdn, mem, size, 9784 &mem_mng->wm); 9785 if (ret) { 9786 rte_errno = errno; 9787 mlx5_free(mem); 9788 return -rte_errno; 9789 } 9790 mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size); 9791 raw_data = (volatile struct flow_counter_stats *)mem; 9792 for (i = 0; i < raws_n; ++i) { 9793 mem_mng->raws[i].mem_mng = mem_mng; 9794 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL; 9795 } 9796 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i) 9797 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, 9798 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i, 9799 next); 9800 LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next); 9801 sh->sws_cmng.mem_mng = mem_mng; 9802 return 0; 9803 } 9804 9805 /** 9806 * Set the statistic memory to the new counter pool. 9807 * 9808 * @param[in] sh 9809 * Pointer to mlx5_dev_ctx_shared object. 9810 * @param[in] pool 9811 * Pointer to the pool to set the statistic memory. 9812 * 9813 * @return 9814 * 0 on success, a negative errno value otherwise. 9815 */ 9816 static int 9817 mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh, 9818 struct mlx5_flow_counter_pool *pool) 9819 { 9820 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9821 /* Resize statistic memory once used out. */ 9822 if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) && 9823 mlx5_flow_create_counter_stat_mem_mng(sh)) { 9824 DRV_LOG(ERR, "Cannot resize counter stat mem."); 9825 return -1; 9826 } 9827 rte_spinlock_lock(&pool->sl); 9828 pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK; 9829 rte_spinlock_unlock(&pool->sl); 9830 pool->raw_hw = NULL; 9831 return 0; 9832 } 9833 9834 #define MLX5_POOL_QUERY_FREQ_US 1000000 9835 9836 /** 9837 * Set the periodic procedure for triggering asynchronous batch queries for all 9838 * the counter pools. 9839 * 9840 * @param[in] sh 9841 * Pointer to mlx5_dev_ctx_shared object. 9842 */ 9843 void 9844 mlx5_set_query_alarm(struct mlx5_dev_ctx_shared *sh) 9845 { 9846 uint32_t pools_n, us; 9847 9848 pools_n = __atomic_load_n(&sh->sws_cmng.n_valid, __ATOMIC_RELAXED); 9849 us = MLX5_POOL_QUERY_FREQ_US / pools_n; 9850 DRV_LOG(DEBUG, "Set alarm for %u pools each %u us", pools_n, us); 9851 if (rte_eal_alarm_set(us, mlx5_flow_query_alarm, sh)) { 9852 sh->sws_cmng.query_thread_on = 0; 9853 DRV_LOG(ERR, "Cannot reinitialize query alarm"); 9854 } else { 9855 sh->sws_cmng.query_thread_on = 1; 9856 } 9857 } 9858 9859 /** 9860 * The periodic procedure for triggering asynchronous batch queries for all the 9861 * counter pools. This function is probably called by the host thread. 9862 * 9863 * @param[in] arg 9864 * The parameter for the alarm process. 9865 */ 9866 void 9867 mlx5_flow_query_alarm(void *arg) 9868 { 9869 struct mlx5_dev_ctx_shared *sh = arg; 9870 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 9871 uint16_t pool_index = cmng->pool_index; 9872 struct mlx5_flow_counter_pool *pool; 9873 uint16_t n_valid; 9874 int ret; 9875 9876 if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES) 9877 goto set_alarm; 9878 rte_spinlock_lock(&cmng->pool_update_sl); 9879 pool = cmng->pools[pool_index]; 9880 n_valid = cmng->n_valid; 9881 rte_spinlock_unlock(&cmng->pool_update_sl); 9882 /* Set the statistic memory to the new created pool. */ 9883 if ((!pool->raw && mlx5_flow_set_counter_stat_mem(sh, pool))) 9884 goto set_alarm; 9885 if (pool->raw_hw) 9886 /* There is a pool query in progress. */ 9887 goto set_alarm; 9888 pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws); 9889 if (!pool->raw_hw) 9890 /* No free counter statistics raw memory. */ 9891 goto set_alarm; 9892 /* 9893 * Identify the counters released between query trigger and query 9894 * handle more efficiently. The counter released in this gap period 9895 * should wait for a new round of query as the new arrived packets 9896 * will not be taken into account. 9897 */ 9898 pool->query_gen++; 9899 ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0, 9900 MLX5_COUNTERS_PER_POOL, 9901 NULL, NULL, 9902 pool->raw_hw->mem_mng->wm.lkey, 9903 (void *)(uintptr_t) 9904 pool->raw_hw->data, 9905 sh->devx_comp, 9906 (uint64_t)(uintptr_t)pool); 9907 if (ret) { 9908 DRV_LOG(ERR, "Failed to trigger asynchronous query for dcs ID" 9909 " %d", pool->min_dcs->id); 9910 pool->raw_hw = NULL; 9911 goto set_alarm; 9912 } 9913 LIST_REMOVE(pool->raw_hw, next); 9914 cmng->pending_queries++; 9915 pool_index++; 9916 if (pool_index >= n_valid) 9917 pool_index = 0; 9918 set_alarm: 9919 cmng->pool_index = pool_index; 9920 mlx5_set_query_alarm(sh); 9921 } 9922 9923 /** 9924 * Check and callback event for new aged flow in the counter pool 9925 * 9926 * @param[in] sh 9927 * Pointer to mlx5_dev_ctx_shared object. 9928 * @param[in] pool 9929 * Pointer to Current counter pool. 9930 */ 9931 static void 9932 mlx5_flow_aging_check(struct mlx5_dev_ctx_shared *sh, 9933 struct mlx5_flow_counter_pool *pool) 9934 { 9935 struct mlx5_priv *priv; 9936 struct mlx5_flow_counter *cnt; 9937 struct mlx5_age_info *age_info; 9938 struct mlx5_age_param *age_param; 9939 struct mlx5_counter_stats_raw *cur = pool->raw_hw; 9940 struct mlx5_counter_stats_raw *prev = pool->raw; 9941 const uint64_t curr_time = MLX5_CURR_TIME_SEC; 9942 const uint32_t time_delta = curr_time - pool->time_of_last_age_check; 9943 uint16_t expected = AGE_CANDIDATE; 9944 uint32_t i; 9945 9946 pool->time_of_last_age_check = curr_time; 9947 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { 9948 cnt = MLX5_POOL_GET_CNT(pool, i); 9949 age_param = MLX5_CNT_TO_AGE(cnt); 9950 if (__atomic_load_n(&age_param->state, 9951 __ATOMIC_RELAXED) != AGE_CANDIDATE) 9952 continue; 9953 if (cur->data[i].hits != prev->data[i].hits) { 9954 __atomic_store_n(&age_param->sec_since_last_hit, 0, 9955 __ATOMIC_RELAXED); 9956 continue; 9957 } 9958 if (__atomic_fetch_add(&age_param->sec_since_last_hit, 9959 time_delta, 9960 __ATOMIC_RELAXED) + time_delta <= age_param->timeout) 9961 continue; 9962 /** 9963 * Hold the lock first, or if between the 9964 * state AGE_TMOUT and tailq operation the 9965 * release happened, the release procedure 9966 * may delete a non-existent tailq node. 9967 */ 9968 priv = rte_eth_devices[age_param->port_id].data->dev_private; 9969 age_info = GET_PORT_AGE_INFO(priv); 9970 rte_spinlock_lock(&age_info->aged_sl); 9971 if (__atomic_compare_exchange_n(&age_param->state, &expected, 9972 AGE_TMOUT, false, 9973 __ATOMIC_RELAXED, 9974 __ATOMIC_RELAXED)) { 9975 TAILQ_INSERT_TAIL(&age_info->aged_counters, cnt, next); 9976 MLX5_AGE_SET(age_info, MLX5_AGE_EVENT_NEW); 9977 } 9978 rte_spinlock_unlock(&age_info->aged_sl); 9979 } 9980 mlx5_age_event_prepare(sh); 9981 } 9982 9983 /** 9984 * Handler for the HW respond about ready values from an asynchronous batch 9985 * query. This function is probably called by the host thread. 9986 * 9987 * @param[in] sh 9988 * The pointer to the shared device context. 9989 * @param[in] async_id 9990 * The Devx async ID. 9991 * @param[in] status 9992 * The status of the completion. 9993 */ 9994 void 9995 mlx5_flow_async_pool_query_handle(struct mlx5_dev_ctx_shared *sh, 9996 uint64_t async_id, int status) 9997 { 9998 struct mlx5_flow_counter_pool *pool = 9999 (struct mlx5_flow_counter_pool *)(uintptr_t)async_id; 10000 struct mlx5_counter_stats_raw *raw_to_free; 10001 uint8_t query_gen = pool->query_gen ^ 1; 10002 struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng; 10003 enum mlx5_counter_type cnt_type = 10004 pool->is_aged ? MLX5_COUNTER_TYPE_AGE : 10005 MLX5_COUNTER_TYPE_ORIGIN; 10006 10007 if (unlikely(status)) { 10008 raw_to_free = pool->raw_hw; 10009 } else { 10010 raw_to_free = pool->raw; 10011 if (pool->is_aged) 10012 mlx5_flow_aging_check(sh, pool); 10013 rte_spinlock_lock(&pool->sl); 10014 pool->raw = pool->raw_hw; 10015 rte_spinlock_unlock(&pool->sl); 10016 /* Be sure the new raw counters data is updated in memory. */ 10017 rte_io_wmb(); 10018 if (!TAILQ_EMPTY(&pool->counters[query_gen])) { 10019 rte_spinlock_lock(&cmng->csl[cnt_type]); 10020 TAILQ_CONCAT(&cmng->counters[cnt_type], 10021 &pool->counters[query_gen], next); 10022 rte_spinlock_unlock(&cmng->csl[cnt_type]); 10023 } 10024 } 10025 LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws, raw_to_free, next); 10026 pool->raw_hw = NULL; 10027 sh->sws_cmng.pending_queries--; 10028 } 10029 10030 static int 10031 flow_group_to_table(uint32_t port_id, uint32_t group, uint32_t *table, 10032 const struct flow_grp_info *grp_info, 10033 struct rte_flow_error *error) 10034 { 10035 if (grp_info->transfer && grp_info->external && 10036 grp_info->fdb_def_rule) { 10037 if (group == UINT32_MAX) 10038 return rte_flow_error_set 10039 (error, EINVAL, 10040 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 10041 NULL, 10042 "group index not supported"); 10043 *table = group + 1; 10044 } else { 10045 *table = group; 10046 } 10047 DRV_LOG(DEBUG, "port %u group=%#x table=%#x", port_id, group, *table); 10048 return 0; 10049 } 10050 10051 /** 10052 * Translate the rte_flow group index to HW table value. 10053 * 10054 * If tunnel offload is disabled, all group ids converted to flow table 10055 * id using the standard method. 10056 * If tunnel offload is enabled, group id can be converted using the 10057 * standard or tunnel conversion method. Group conversion method 10058 * selection depends on flags in `grp_info` parameter: 10059 * - Internal (grp_info.external == 0) groups conversion uses the 10060 * standard method. 10061 * - Group ids in JUMP action converted with the tunnel conversion. 10062 * - Group id in rule attribute conversion depends on a rule type and 10063 * group id value: 10064 * ** non zero group attributes converted with the tunnel method 10065 * ** zero group attribute in non-tunnel rule is converted using the 10066 * standard method - there's only one root table 10067 * ** zero group attribute in steer tunnel rule is converted with the 10068 * standard method - single root table 10069 * ** zero group attribute in match tunnel rule is a special OvS 10070 * case: that value is used for portability reasons. That group 10071 * id is converted with the tunnel conversion method. 10072 * 10073 * @param[in] dev 10074 * Port device 10075 * @param[in] tunnel 10076 * PMD tunnel offload object 10077 * @param[in] group 10078 * rte_flow group index value. 10079 * @param[out] table 10080 * HW table value. 10081 * @param[in] grp_info 10082 * flags used for conversion 10083 * @param[out] error 10084 * Pointer to error structure. 10085 * 10086 * @return 10087 * 0 on success, a negative errno value otherwise and rte_errno is set. 10088 */ 10089 int 10090 mlx5_flow_group_to_table(struct rte_eth_dev *dev, 10091 const struct mlx5_flow_tunnel *tunnel, 10092 uint32_t group, uint32_t *table, 10093 const struct flow_grp_info *grp_info, 10094 struct rte_flow_error *error) 10095 { 10096 int ret; 10097 bool standard_translation; 10098 10099 if (!grp_info->skip_scale && grp_info->external && 10100 group < MLX5_MAX_TABLES_EXTERNAL) 10101 group *= MLX5_FLOW_TABLE_FACTOR; 10102 if (is_tunnel_offload_active(dev)) { 10103 standard_translation = !grp_info->external || 10104 grp_info->std_tbl_fix; 10105 } else { 10106 standard_translation = true; 10107 } 10108 DRV_LOG(DEBUG, 10109 "port %u group=%u transfer=%d external=%d fdb_def_rule=%d translate=%s", 10110 dev->data->port_id, group, grp_info->transfer, 10111 grp_info->external, grp_info->fdb_def_rule, 10112 standard_translation ? "STANDARD" : "TUNNEL"); 10113 if (standard_translation) 10114 ret = flow_group_to_table(dev->data->port_id, group, table, 10115 grp_info, error); 10116 else 10117 ret = tunnel_flow_group_to_flow_table(dev, tunnel, group, 10118 table, error); 10119 10120 return ret; 10121 } 10122 10123 /** 10124 * Discover availability of metadata reg_c's. 10125 * 10126 * Iteratively use test flows to check availability. 10127 * 10128 * @param[in] dev 10129 * Pointer to the Ethernet device structure. 10130 * 10131 * @return 10132 * 0 on success, a negative errno value otherwise and rte_errno is set. 10133 */ 10134 int 10135 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev) 10136 { 10137 struct mlx5_priv *priv = dev->data->dev_private; 10138 enum modify_reg idx; 10139 int n = 0; 10140 10141 /* reg_c[0] and reg_c[1] are reserved. */ 10142 priv->sh->flow_mreg_c[n++] = REG_C_0; 10143 priv->sh->flow_mreg_c[n++] = REG_C_1; 10144 /* Discover availability of other reg_c's. */ 10145 for (idx = REG_C_2; idx <= REG_C_7; ++idx) { 10146 struct rte_flow_attr attr = { 10147 .group = MLX5_FLOW_MREG_CP_TABLE_GROUP, 10148 .priority = MLX5_FLOW_LOWEST_PRIO_INDICATOR, 10149 .ingress = 1, 10150 }; 10151 struct rte_flow_item items[] = { 10152 [0] = { 10153 .type = RTE_FLOW_ITEM_TYPE_END, 10154 }, 10155 }; 10156 struct rte_flow_action actions[] = { 10157 [0] = { 10158 .type = (enum rte_flow_action_type) 10159 MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG, 10160 .conf = &(struct mlx5_flow_action_copy_mreg){ 10161 .src = REG_C_1, 10162 .dst = idx, 10163 }, 10164 }, 10165 [1] = { 10166 .type = RTE_FLOW_ACTION_TYPE_JUMP, 10167 .conf = &(struct rte_flow_action_jump){ 10168 .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP, 10169 }, 10170 }, 10171 [2] = { 10172 .type = RTE_FLOW_ACTION_TYPE_END, 10173 }, 10174 }; 10175 uint32_t flow_idx; 10176 struct rte_flow *flow; 10177 struct rte_flow_error error; 10178 10179 if (!priv->sh->config.dv_flow_en) 10180 break; 10181 /* Create internal flow, validation skips copy action. */ 10182 flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, 10183 items, actions, false, &error); 10184 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10185 flow_idx); 10186 if (!flow) 10187 continue; 10188 priv->sh->flow_mreg_c[n++] = idx; 10189 flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx); 10190 } 10191 for (; n < MLX5_MREG_C_NUM; ++n) 10192 priv->sh->flow_mreg_c[n] = REG_NON; 10193 priv->sh->metadata_regc_check_flag = 1; 10194 return 0; 10195 } 10196 10197 int 10198 save_dump_file(const uint8_t *data, uint32_t size, 10199 uint32_t type, uint64_t id, void *arg, FILE *file) 10200 { 10201 char line[BUF_SIZE]; 10202 uint32_t out = 0; 10203 uint32_t k; 10204 uint32_t actions_num; 10205 struct rte_flow_query_count *count; 10206 10207 memset(line, 0, BUF_SIZE); 10208 switch (type) { 10209 case DR_DUMP_REC_TYPE_PMD_MODIFY_HDR: 10210 actions_num = *(uint32_t *)(arg); 10211 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",%d,", 10212 type, id, actions_num); 10213 break; 10214 case DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT: 10215 out += snprintf(line + out, BUF_SIZE - out, "%d,0x%" PRIx64 ",", 10216 type, id); 10217 break; 10218 case DR_DUMP_REC_TYPE_PMD_COUNTER: 10219 count = (struct rte_flow_query_count *)arg; 10220 fprintf(file, 10221 "%d,0x%" PRIx64 ",%" PRIu64 ",%" PRIu64 "\n", 10222 type, id, count->hits, count->bytes); 10223 return 0; 10224 default: 10225 return -1; 10226 } 10227 10228 for (k = 0; k < size; k++) { 10229 /* Make sure we do not overrun the line buffer length. */ 10230 if (out >= BUF_SIZE - 4) { 10231 line[out] = '\0'; 10232 break; 10233 } 10234 out += snprintf(line + out, BUF_SIZE - out, "%02x", 10235 (data[k]) & 0xff); 10236 } 10237 fprintf(file, "%s\n", line); 10238 return 0; 10239 } 10240 10241 int 10242 mlx5_flow_query_counter(struct rte_eth_dev *dev, struct rte_flow *flow, 10243 struct rte_flow_query_count *count, struct rte_flow_error *error) 10244 { 10245 struct rte_flow_action action[2]; 10246 enum mlx5_flow_drv_type ftype; 10247 const struct mlx5_flow_driver_ops *fops; 10248 10249 if (!flow) { 10250 return rte_flow_error_set(error, ENOENT, 10251 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10252 NULL, 10253 "invalid flow handle"); 10254 } 10255 action[0].type = RTE_FLOW_ACTION_TYPE_COUNT; 10256 action[1].type = RTE_FLOW_ACTION_TYPE_END; 10257 if (flow->counter) { 10258 memset(count, 0, sizeof(struct rte_flow_query_count)); 10259 ftype = (enum mlx5_flow_drv_type)(flow->drv_type); 10260 MLX5_ASSERT(ftype > MLX5_FLOW_TYPE_MIN && 10261 ftype < MLX5_FLOW_TYPE_MAX); 10262 fops = flow_get_drv_ops(ftype); 10263 return fops->query(dev, flow, action, count, error); 10264 } 10265 return -1; 10266 } 10267 10268 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10269 /** 10270 * Dump flow ipool data to file 10271 * 10272 * @param[in] dev 10273 * The pointer to Ethernet device. 10274 * @param[in] file 10275 * A pointer to a file for output. 10276 * @param[out] error 10277 * Perform verbose error reporting if not NULL. PMDs initialize this 10278 * structure in case of error only. 10279 * @return 10280 * 0 on success, a negative value otherwise. 10281 */ 10282 int 10283 mlx5_flow_dev_dump_ipool(struct rte_eth_dev *dev, 10284 struct rte_flow *flow, FILE *file, 10285 struct rte_flow_error *error) 10286 { 10287 struct mlx5_priv *priv = dev->data->dev_private; 10288 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10289 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10290 uint32_t handle_idx; 10291 struct mlx5_flow_handle *dh; 10292 struct rte_flow_query_count count; 10293 uint32_t actions_num; 10294 const uint8_t *data; 10295 size_t size; 10296 uint64_t id; 10297 uint32_t type; 10298 void *action = NULL; 10299 10300 if (!flow) { 10301 return rte_flow_error_set(error, ENOENT, 10302 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 10303 NULL, 10304 "invalid flow handle"); 10305 } 10306 handle_idx = flow->dev_handles; 10307 /* query counter */ 10308 if (flow->counter && 10309 (!mlx5_counter_query(dev, flow->counter, false, 10310 &count.hits, &count.bytes, &action)) && action) { 10311 id = (uint64_t)(uintptr_t)action; 10312 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10313 save_dump_file(NULL, 0, type, 10314 id, (void *)&count, file); 10315 } 10316 10317 while (handle_idx) { 10318 dh = mlx5_ipool_get(priv->sh->ipool 10319 [MLX5_IPOOL_MLX5_FLOW], handle_idx); 10320 if (!dh) 10321 continue; 10322 handle_idx = dh->next.next; 10323 10324 /* Get modify_hdr and encap_decap buf from ipools. */ 10325 encap_decap = NULL; 10326 modify_hdr = dh->dvh.modify_hdr; 10327 10328 if (dh->dvh.rix_encap_decap) { 10329 encap_decap = mlx5_ipool_get(priv->sh->ipool 10330 [MLX5_IPOOL_DECAP_ENCAP], 10331 dh->dvh.rix_encap_decap); 10332 } 10333 if (modify_hdr) { 10334 data = (const uint8_t *)modify_hdr->actions; 10335 size = (size_t)(modify_hdr->actions_num) * 8; 10336 id = (uint64_t)(uintptr_t)modify_hdr->action; 10337 actions_num = modify_hdr->actions_num; 10338 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10339 save_dump_file(data, size, type, id, 10340 (void *)(&actions_num), file); 10341 } 10342 if (encap_decap) { 10343 data = encap_decap->buf; 10344 size = encap_decap->size; 10345 id = (uint64_t)(uintptr_t)encap_decap->action; 10346 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10347 save_dump_file(data, size, type, 10348 id, NULL, file); 10349 } 10350 } 10351 return 0; 10352 } 10353 10354 /** 10355 * Dump all flow's encap_decap/modify_hdr/counter data to file 10356 * 10357 * @param[in] dev 10358 * The pointer to Ethernet device. 10359 * @param[in] file 10360 * A pointer to a file for output. 10361 * @param[out] error 10362 * Perform verbose error reporting if not NULL. PMDs initialize this 10363 * structure in case of error only. 10364 * @return 10365 * 0 on success, a negative value otherwise. 10366 */ 10367 static int 10368 mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev, 10369 FILE *file, struct rte_flow_error *error __rte_unused) 10370 { 10371 struct mlx5_priv *priv = dev->data->dev_private; 10372 struct mlx5_dev_ctx_shared *sh = priv->sh; 10373 struct mlx5_hlist *h; 10374 struct mlx5_flow_dv_modify_hdr_resource *modify_hdr; 10375 struct mlx5_flow_dv_encap_decap_resource *encap_decap; 10376 struct rte_flow_query_count count; 10377 uint32_t actions_num; 10378 const uint8_t *data; 10379 size_t size; 10380 uint64_t id; 10381 uint32_t type; 10382 uint32_t i; 10383 uint32_t j; 10384 struct mlx5_list_inconst *l_inconst; 10385 struct mlx5_list_entry *e; 10386 int lcore_index; 10387 struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng; 10388 uint32_t max; 10389 void *action; 10390 10391 /* encap_decap hlist is lcore_share, get global core cache. */ 10392 i = MLX5_LIST_GLOBAL; 10393 h = sh->encaps_decaps; 10394 if (h) { 10395 for (j = 0; j <= h->mask; j++) { 10396 l_inconst = &h->buckets[j].l; 10397 if (!l_inconst || !l_inconst->cache[i]) 10398 continue; 10399 10400 e = LIST_FIRST(&l_inconst->cache[i]->h); 10401 while (e) { 10402 encap_decap = 10403 (struct mlx5_flow_dv_encap_decap_resource *)e; 10404 data = encap_decap->buf; 10405 size = encap_decap->size; 10406 id = (uint64_t)(uintptr_t)encap_decap->action; 10407 type = DR_DUMP_REC_TYPE_PMD_PKT_REFORMAT; 10408 save_dump_file(data, size, type, 10409 id, NULL, file); 10410 e = LIST_NEXT(e, next); 10411 } 10412 } 10413 } 10414 10415 /* get modify_hdr */ 10416 h = sh->modify_cmds; 10417 if (h) { 10418 lcore_index = rte_lcore_index(rte_lcore_id()); 10419 if (unlikely(lcore_index == -1)) { 10420 lcore_index = MLX5_LIST_NLCORE; 10421 rte_spinlock_lock(&h->l_const.lcore_lock); 10422 } 10423 i = lcore_index; 10424 10425 if (lcore_index == MLX5_LIST_NLCORE) { 10426 for (i = 0; i <= (uint32_t)lcore_index; i++) { 10427 for (j = 0; j <= h->mask; j++) { 10428 l_inconst = &h->buckets[j].l; 10429 if (!l_inconst || !l_inconst->cache[i]) 10430 continue; 10431 10432 e = LIST_FIRST(&l_inconst->cache[i]->h); 10433 while (e) { 10434 modify_hdr = 10435 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10436 data = (const uint8_t *)modify_hdr->actions; 10437 size = (size_t)(modify_hdr->actions_num) * 8; 10438 actions_num = modify_hdr->actions_num; 10439 id = (uint64_t)(uintptr_t)modify_hdr->action; 10440 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10441 save_dump_file(data, size, type, id, 10442 (void *)(&actions_num), file); 10443 e = LIST_NEXT(e, next); 10444 } 10445 } 10446 } 10447 } else { 10448 for (j = 0; j <= h->mask; j++) { 10449 l_inconst = &h->buckets[j].l; 10450 if (!l_inconst || !l_inconst->cache[i]) 10451 continue; 10452 10453 e = LIST_FIRST(&l_inconst->cache[i]->h); 10454 while (e) { 10455 modify_hdr = 10456 (struct mlx5_flow_dv_modify_hdr_resource *)e; 10457 data = (const uint8_t *)modify_hdr->actions; 10458 size = (size_t)(modify_hdr->actions_num) * 8; 10459 actions_num = modify_hdr->actions_num; 10460 id = (uint64_t)(uintptr_t)modify_hdr->action; 10461 type = DR_DUMP_REC_TYPE_PMD_MODIFY_HDR; 10462 save_dump_file(data, size, type, id, 10463 (void *)(&actions_num), file); 10464 e = LIST_NEXT(e, next); 10465 } 10466 } 10467 } 10468 10469 if (unlikely(lcore_index == MLX5_LIST_NLCORE)) 10470 rte_spinlock_unlock(&h->l_const.lcore_lock); 10471 } 10472 10473 /* get counter */ 10474 MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM); 10475 max = MLX5_COUNTERS_PER_POOL * cmng->n_valid; 10476 for (j = 1; j <= max; j++) { 10477 action = NULL; 10478 if ((!mlx5_counter_query(dev, j, false, &count.hits, 10479 &count.bytes, &action)) && action) { 10480 id = (uint64_t)(uintptr_t)action; 10481 type = DR_DUMP_REC_TYPE_PMD_COUNTER; 10482 save_dump_file(NULL, 0, type, 10483 id, (void *)&count, file); 10484 } 10485 } 10486 return 0; 10487 } 10488 #endif 10489 10490 /** 10491 * Dump flow raw hw data to file 10492 * 10493 * @param[in] dev 10494 * The pointer to Ethernet device. 10495 * @param[in] file 10496 * A pointer to a file for output. 10497 * @param[out] error 10498 * Perform verbose error reporting if not NULL. PMDs initialize this 10499 * structure in case of error only. 10500 * @return 10501 * 0 on success, a negative value otherwise. 10502 */ 10503 int 10504 mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx, 10505 FILE *file, 10506 struct rte_flow_error *error __rte_unused) 10507 { 10508 struct mlx5_priv *priv = dev->data->dev_private; 10509 struct mlx5_dev_ctx_shared *sh = priv->sh; 10510 uint32_t handle_idx; 10511 int ret; 10512 struct mlx5_flow_handle *dh; 10513 struct rte_flow *flow; 10514 10515 if (!sh->config.dv_flow_en) { 10516 if (fputs("device dv flow disabled\n", file) <= 0) 10517 return -errno; 10518 return -ENOTSUP; 10519 } 10520 10521 /* dump all */ 10522 if (!flow_idx) { 10523 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10524 if (mlx5_flow_dev_dump_sh_all(dev, file, error)) 10525 return -EINVAL; 10526 10527 if (sh->config.dv_flow_en == 2) 10528 return mlx5dr_debug_dump(priv->dr_ctx, file); 10529 #endif 10530 return mlx5_devx_cmd_flow_dump(sh->fdb_domain, 10531 sh->rx_domain, 10532 sh->tx_domain, file); 10533 } 10534 /* dump one */ 10535 flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN], 10536 (uintptr_t)(void *)flow_idx); 10537 if (!flow) 10538 return -EINVAL; 10539 10540 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 10541 mlx5_flow_dev_dump_ipool(dev, flow, file, error); 10542 #endif 10543 handle_idx = flow->dev_handles; 10544 while (handle_idx) { 10545 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], 10546 handle_idx); 10547 if (!dh) 10548 return -ENOENT; 10549 if (dh->drv_flow) { 10550 if (sh->config.dv_flow_en == 2) 10551 return -ENOTSUP; 10552 10553 ret = mlx5_devx_cmd_flow_single_dump(dh->drv_flow, 10554 file); 10555 if (ret) 10556 return -ENOENT; 10557 } 10558 handle_idx = dh->next.next; 10559 } 10560 return 0; 10561 } 10562 10563 /** 10564 * Get aged-out flows. 10565 * 10566 * @param[in] dev 10567 * Pointer to the Ethernet device structure. 10568 * @param[in] context 10569 * The address of an array of pointers to the aged-out flows contexts. 10570 * @param[in] nb_countexts 10571 * The length of context array pointers. 10572 * @param[out] error 10573 * Perform verbose error reporting if not NULL. Initialized in case of 10574 * error only. 10575 * 10576 * @return 10577 * how many contexts get in success, otherwise negative errno value. 10578 * if nb_contexts is 0, return the amount of all aged contexts. 10579 * if nb_contexts is not 0 , return the amount of aged flows reported 10580 * in the context array. 10581 */ 10582 int 10583 mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts, 10584 uint32_t nb_contexts, struct rte_flow_error *error) 10585 { 10586 struct rte_flow_attr attr = { .transfer = 0 }; 10587 10588 return flow_get_drv_ops(flow_get_drv_type(dev, &attr))->get_aged_flows 10589 (dev, contexts, nb_contexts, error); 10590 } 10591 10592 /** 10593 * Get aged-out flows per HWS queue. 10594 * 10595 * @param[in] dev 10596 * Pointer to the Ethernet device structure. 10597 * @param[in] queue_id 10598 * Flow queue to query. 10599 * @param[in] context 10600 * The address of an array of pointers to the aged-out flows contexts. 10601 * @param[in] nb_countexts 10602 * The length of context array pointers. 10603 * @param[out] error 10604 * Perform verbose error reporting if not NULL. Initialized in case of 10605 * error only. 10606 * 10607 * @return 10608 * how many contexts get in success, otherwise negative errno value. 10609 * if nb_contexts is 0, return the amount of all aged contexts. 10610 * if nb_contexts is not 0 , return the amount of aged flows reported 10611 * in the context array. 10612 */ 10613 int 10614 mlx5_flow_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id, 10615 void **contexts, uint32_t nb_contexts, 10616 struct rte_flow_error *error) 10617 { 10618 const struct mlx5_flow_driver_ops *fops; 10619 struct rte_flow_attr attr = { 0 }; 10620 10621 if (flow_get_drv_type(dev, &attr) == MLX5_FLOW_TYPE_HW) { 10622 fops = flow_get_drv_ops(MLX5_FLOW_TYPE_HW); 10623 return fops->get_q_aged_flows(dev, queue_id, contexts, 10624 nb_contexts, error); 10625 } 10626 DRV_LOG(ERR, "port %u queue %u get aged flows is not supported.", 10627 dev->data->port_id, queue_id); 10628 return rte_flow_error_set(error, ENOTSUP, 10629 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 10630 "get Q aged flows with incorrect steering mode"); 10631 } 10632 10633 /* Wrapper for driver action_validate op callback */ 10634 static int 10635 flow_drv_action_validate(struct rte_eth_dev *dev, 10636 const struct rte_flow_indir_action_conf *conf, 10637 const struct rte_flow_action *action, 10638 const struct mlx5_flow_driver_ops *fops, 10639 struct rte_flow_error *error) 10640 { 10641 static const char err_msg[] = "indirect action validation unsupported"; 10642 10643 if (!fops->action_validate) { 10644 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10645 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10646 NULL, err_msg); 10647 return -rte_errno; 10648 } 10649 return fops->action_validate(dev, conf, action, error); 10650 } 10651 10652 /** 10653 * Destroys the shared action by handle. 10654 * 10655 * @param dev 10656 * Pointer to Ethernet device structure. 10657 * @param[in] handle 10658 * Handle for the indirect action object to be destroyed. 10659 * @param[out] error 10660 * Perform verbose error reporting if not NULL. PMDs initialize this 10661 * structure in case of error only. 10662 * 10663 * @return 10664 * 0 on success, a negative errno value otherwise and rte_errno is set. 10665 * 10666 * @note: wrapper for driver action_create op callback. 10667 */ 10668 static int 10669 mlx5_action_handle_destroy(struct rte_eth_dev *dev, 10670 struct rte_flow_action_handle *handle, 10671 struct rte_flow_error *error) 10672 { 10673 static const char err_msg[] = "indirect action destruction unsupported"; 10674 struct rte_flow_attr attr = { .transfer = 0 }; 10675 const struct mlx5_flow_driver_ops *fops = 10676 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10677 10678 if (!fops->action_destroy) { 10679 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10680 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10681 NULL, err_msg); 10682 return -rte_errno; 10683 } 10684 return fops->action_destroy(dev, handle, error); 10685 } 10686 10687 /* Wrapper for driver action_destroy op callback */ 10688 static int 10689 flow_drv_action_update(struct rte_eth_dev *dev, 10690 struct rte_flow_action_handle *handle, 10691 const void *update, 10692 const struct mlx5_flow_driver_ops *fops, 10693 struct rte_flow_error *error) 10694 { 10695 static const char err_msg[] = "indirect action update unsupported"; 10696 10697 if (!fops->action_update) { 10698 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10699 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10700 NULL, err_msg); 10701 return -rte_errno; 10702 } 10703 return fops->action_update(dev, handle, update, error); 10704 } 10705 10706 /* Wrapper for driver action_destroy op callback */ 10707 static int 10708 flow_drv_action_query(struct rte_eth_dev *dev, 10709 const struct rte_flow_action_handle *handle, 10710 void *data, 10711 const struct mlx5_flow_driver_ops *fops, 10712 struct rte_flow_error *error) 10713 { 10714 static const char err_msg[] = "indirect action query unsupported"; 10715 10716 if (!fops->action_query) { 10717 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10718 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10719 NULL, err_msg); 10720 return -rte_errno; 10721 } 10722 return fops->action_query(dev, handle, data, error); 10723 } 10724 10725 /** 10726 * Create indirect action for reuse in multiple flow rules. 10727 * 10728 * @param dev 10729 * Pointer to Ethernet device structure. 10730 * @param conf 10731 * Pointer to indirect action object configuration. 10732 * @param[in] action 10733 * Action configuration for indirect action object creation. 10734 * @param[out] error 10735 * Perform verbose error reporting if not NULL. PMDs initialize this 10736 * structure in case of error only. 10737 * @return 10738 * A valid handle in case of success, NULL otherwise and rte_errno is set. 10739 */ 10740 static struct rte_flow_action_handle * 10741 mlx5_action_handle_create(struct rte_eth_dev *dev, 10742 const struct rte_flow_indir_action_conf *conf, 10743 const struct rte_flow_action *action, 10744 struct rte_flow_error *error) 10745 { 10746 static const char err_msg[] = "indirect action creation unsupported"; 10747 struct rte_flow_attr attr = { .transfer = 0 }; 10748 const struct mlx5_flow_driver_ops *fops = 10749 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10750 10751 if (flow_drv_action_validate(dev, conf, action, fops, error)) 10752 return NULL; 10753 if (!fops->action_create) { 10754 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 10755 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 10756 NULL, err_msg); 10757 return NULL; 10758 } 10759 return fops->action_create(dev, conf, action, error); 10760 } 10761 10762 /** 10763 * Updates inplace the indirect action configuration pointed by *handle* 10764 * with the configuration provided as *update* argument. 10765 * The update of the indirect action configuration effects all flow rules 10766 * reusing the action via handle. 10767 * 10768 * @param dev 10769 * Pointer to Ethernet device structure. 10770 * @param[in] handle 10771 * Handle for the indirect action to be updated. 10772 * @param[in] update 10773 * Action specification used to modify the action pointed by handle. 10774 * *update* could be of same type with the action pointed by the *handle* 10775 * handle argument, or some other structures like a wrapper, depending on 10776 * the indirect action type. 10777 * @param[out] error 10778 * Perform verbose error reporting if not NULL. PMDs initialize this 10779 * structure in case of error only. 10780 * 10781 * @return 10782 * 0 on success, a negative errno value otherwise and rte_errno is set. 10783 */ 10784 static int 10785 mlx5_action_handle_update(struct rte_eth_dev *dev, 10786 struct rte_flow_action_handle *handle, 10787 const void *update, 10788 struct rte_flow_error *error) 10789 { 10790 struct rte_flow_attr attr = { .transfer = 0 }; 10791 const struct mlx5_flow_driver_ops *fops = 10792 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10793 int ret; 10794 uint32_t act_idx = (uint32_t)(uintptr_t)handle; 10795 uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET; 10796 10797 switch (type) { 10798 case MLX5_INDIRECT_ACTION_TYPE_CT: 10799 case MLX5_INDIRECT_ACTION_TYPE_METER_MARK: 10800 ret = 0; 10801 break; 10802 default: 10803 ret = flow_drv_action_validate(dev, NULL, 10804 (const struct rte_flow_action *)update, 10805 fops, error); 10806 } 10807 if (ret) 10808 return ret; 10809 return flow_drv_action_update(dev, handle, update, fops, 10810 error); 10811 } 10812 10813 /** 10814 * Query the indirect action by handle. 10815 * 10816 * This function allows retrieving action-specific data such as counters. 10817 * Data is gathered by special action which may be present/referenced in 10818 * more than one flow rule definition. 10819 * 10820 * see @RTE_FLOW_ACTION_TYPE_COUNT 10821 * 10822 * @param dev 10823 * Pointer to Ethernet device structure. 10824 * @param[in] handle 10825 * Handle for the indirect action to query. 10826 * @param[in, out] data 10827 * Pointer to storage for the associated query data type. 10828 * @param[out] error 10829 * Perform verbose error reporting if not NULL. PMDs initialize this 10830 * structure in case of error only. 10831 * 10832 * @return 10833 * 0 on success, a negative errno value otherwise and rte_errno is set. 10834 */ 10835 static int 10836 mlx5_action_handle_query(struct rte_eth_dev *dev, 10837 const struct rte_flow_action_handle *handle, 10838 void *data, 10839 struct rte_flow_error *error) 10840 { 10841 struct rte_flow_attr attr = { .transfer = 0 }; 10842 const struct mlx5_flow_driver_ops *fops = 10843 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 10844 10845 return flow_drv_action_query(dev, handle, data, fops, error); 10846 } 10847 10848 static int 10849 mlx5_action_handle_query_update(struct rte_eth_dev *dev, 10850 struct rte_flow_action_handle *handle, 10851 const void *update, void *query, 10852 enum rte_flow_query_update_mode qu_mode, 10853 struct rte_flow_error *error) 10854 { 10855 struct rte_flow_attr attr = { .transfer = 0 }; 10856 enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr); 10857 const struct mlx5_flow_driver_ops *fops; 10858 10859 if (drv_type == MLX5_FLOW_TYPE_MIN || drv_type == MLX5_FLOW_TYPE_MAX) 10860 return rte_flow_error_set(error, ENOTSUP, 10861 RTE_FLOW_ERROR_TYPE_ACTION, 10862 NULL, "invalid driver type"); 10863 fops = flow_get_drv_ops(drv_type); 10864 if (!fops || !fops->action_query_update) 10865 return rte_flow_error_set(error, ENOTSUP, 10866 RTE_FLOW_ERROR_TYPE_ACTION, 10867 NULL, "no query_update handler"); 10868 return fops->action_query_update(dev, handle, update, 10869 query, qu_mode, error); 10870 } 10871 10872 /** 10873 * Destroy all indirect actions (shared RSS). 10874 * 10875 * @param dev 10876 * Pointer to Ethernet device. 10877 * 10878 * @return 10879 * 0 on success, a negative errno value otherwise and rte_errno is set. 10880 */ 10881 int 10882 mlx5_action_handle_flush(struct rte_eth_dev *dev) 10883 { 10884 struct rte_flow_error error; 10885 struct mlx5_priv *priv = dev->data->dev_private; 10886 struct mlx5_shared_action_rss *shared_rss; 10887 int ret = 0; 10888 uint32_t idx; 10889 10890 ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], 10891 priv->rss_shared_actions, idx, shared_rss, next) { 10892 ret |= mlx5_action_handle_destroy(dev, 10893 (struct rte_flow_action_handle *)(uintptr_t)idx, &error); 10894 } 10895 return ret; 10896 } 10897 10898 /** 10899 * Validate existing indirect actions against current device configuration 10900 * and attach them to device resources. 10901 * 10902 * @param dev 10903 * Pointer to Ethernet device. 10904 * 10905 * @return 10906 * 0 on success, a negative errno value otherwise and rte_errno is set. 10907 */ 10908 int 10909 mlx5_action_handle_attach(struct rte_eth_dev *dev) 10910 { 10911 struct mlx5_priv *priv = dev->data->dev_private; 10912 int ret = 0; 10913 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10914 10915 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10916 const char *message; 10917 uint32_t queue_idx; 10918 10919 ret = mlx5_validate_rss_queues(dev, ind_tbl->queues, 10920 ind_tbl->queues_n, 10921 &message, &queue_idx); 10922 if (ret != 0) { 10923 DRV_LOG(ERR, "Port %u cannot use queue %u in RSS: %s", 10924 dev->data->port_id, ind_tbl->queues[queue_idx], 10925 message); 10926 break; 10927 } 10928 } 10929 if (ret != 0) 10930 return ret; 10931 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10932 ret = mlx5_ind_table_obj_attach(dev, ind_tbl); 10933 if (ret != 0) { 10934 DRV_LOG(ERR, "Port %u could not attach " 10935 "indirection table obj %p", 10936 dev->data->port_id, (void *)ind_tbl); 10937 goto error; 10938 } 10939 } 10940 10941 return 0; 10942 error: 10943 ind_tbl_last = ind_tbl; 10944 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10945 if (ind_tbl == ind_tbl_last) 10946 break; 10947 if (mlx5_ind_table_obj_detach(dev, ind_tbl) != 0) 10948 DRV_LOG(CRIT, "Port %u could not detach " 10949 "indirection table obj %p on rollback", 10950 dev->data->port_id, (void *)ind_tbl); 10951 } 10952 return ret; 10953 } 10954 10955 /** 10956 * Detach indirect actions of the device from its resources. 10957 * 10958 * @param dev 10959 * Pointer to Ethernet device. 10960 * 10961 * @return 10962 * 0 on success, a negative errno value otherwise and rte_errno is set. 10963 */ 10964 int 10965 mlx5_action_handle_detach(struct rte_eth_dev *dev) 10966 { 10967 struct mlx5_priv *priv = dev->data->dev_private; 10968 int ret = 0; 10969 struct mlx5_ind_table_obj *ind_tbl, *ind_tbl_last; 10970 10971 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10972 ret = mlx5_ind_table_obj_detach(dev, ind_tbl); 10973 if (ret != 0) { 10974 DRV_LOG(ERR, "Port %u could not detach " 10975 "indirection table obj %p", 10976 dev->data->port_id, (void *)ind_tbl); 10977 goto error; 10978 } 10979 } 10980 return 0; 10981 error: 10982 ind_tbl_last = ind_tbl; 10983 LIST_FOREACH(ind_tbl, &priv->standalone_ind_tbls, next) { 10984 if (ind_tbl == ind_tbl_last) 10985 break; 10986 if (mlx5_ind_table_obj_attach(dev, ind_tbl) != 0) 10987 DRV_LOG(CRIT, "Port %u could not attach " 10988 "indirection table obj %p on rollback", 10989 dev->data->port_id, (void *)ind_tbl); 10990 } 10991 return ret; 10992 } 10993 10994 #ifndef HAVE_MLX5DV_DR 10995 #define MLX5_DOMAIN_SYNC_FLOW ((1 << 0) | (1 << 1)) 10996 #else 10997 #define MLX5_DOMAIN_SYNC_FLOW \ 10998 (MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW | MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW) 10999 #endif 11000 11001 int rte_pmd_mlx5_sync_flow(uint16_t port_id, uint32_t domains) 11002 { 11003 struct rte_eth_dev *dev = &rte_eth_devices[port_id]; 11004 const struct mlx5_flow_driver_ops *fops; 11005 int ret; 11006 struct rte_flow_attr attr = { .transfer = 0 }; 11007 11008 fops = flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11009 ret = fops->sync_domain(dev, domains, MLX5_DOMAIN_SYNC_FLOW); 11010 if (ret > 0) 11011 ret = -ret; 11012 return ret; 11013 } 11014 11015 const struct mlx5_flow_tunnel * 11016 mlx5_get_tof(const struct rte_flow_item *item, 11017 const struct rte_flow_action *action, 11018 enum mlx5_tof_rule_type *rule_type) 11019 { 11020 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11021 if (item->type == (typeof(item->type)) 11022 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL) { 11023 *rule_type = MLX5_TUNNEL_OFFLOAD_MATCH_RULE; 11024 return flow_items_to_tunnel(item); 11025 } 11026 } 11027 for (; action->conf != RTE_FLOW_ACTION_TYPE_END; action++) { 11028 if (action->type == (typeof(action->type)) 11029 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET) { 11030 *rule_type = MLX5_TUNNEL_OFFLOAD_SET_RULE; 11031 return flow_actions_to_tunnel(action); 11032 } 11033 } 11034 return NULL; 11035 } 11036 11037 /** 11038 * tunnel offload functionality is defined for DV environment only 11039 */ 11040 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 11041 __extension__ 11042 union tunnel_offload_mark { 11043 uint32_t val; 11044 struct { 11045 uint32_t app_reserve:8; 11046 uint32_t table_id:15; 11047 uint32_t transfer:1; 11048 uint32_t _unused_:8; 11049 }; 11050 }; 11051 11052 static bool 11053 mlx5_access_tunnel_offload_db 11054 (struct rte_eth_dev *dev, 11055 bool (*match)(struct rte_eth_dev *, 11056 struct mlx5_flow_tunnel *, const void *), 11057 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11058 void (*miss)(struct rte_eth_dev *, void *), 11059 void *ctx, bool lock_op); 11060 11061 static int 11062 flow_tunnel_add_default_miss(struct rte_eth_dev *dev, 11063 struct rte_flow *flow, 11064 const struct rte_flow_attr *attr, 11065 const struct rte_flow_action *app_actions, 11066 uint32_t flow_idx, 11067 const struct mlx5_flow_tunnel *tunnel, 11068 struct tunnel_default_miss_ctx *ctx, 11069 struct rte_flow_error *error) 11070 { 11071 struct mlx5_priv *priv = dev->data->dev_private; 11072 struct mlx5_flow *dev_flow; 11073 struct rte_flow_attr miss_attr = *attr; 11074 const struct rte_flow_item miss_items[2] = { 11075 { 11076 .type = RTE_FLOW_ITEM_TYPE_ETH, 11077 .spec = NULL, 11078 .last = NULL, 11079 .mask = NULL 11080 }, 11081 { 11082 .type = RTE_FLOW_ITEM_TYPE_END, 11083 .spec = NULL, 11084 .last = NULL, 11085 .mask = NULL 11086 } 11087 }; 11088 union tunnel_offload_mark mark_id; 11089 struct rte_flow_action_mark miss_mark; 11090 struct rte_flow_action miss_actions[3] = { 11091 [0] = { .type = RTE_FLOW_ACTION_TYPE_MARK, .conf = &miss_mark }, 11092 [2] = { .type = RTE_FLOW_ACTION_TYPE_END, .conf = NULL } 11093 }; 11094 const struct rte_flow_action_jump *jump_data; 11095 uint32_t i, flow_table = 0; /* prevent compilation warning */ 11096 struct flow_grp_info grp_info = { 11097 .external = 1, 11098 .transfer = attr->transfer, 11099 .fdb_def_rule = !!priv->fdb_def_rule, 11100 .std_tbl_fix = 0, 11101 }; 11102 int ret; 11103 11104 if (!attr->transfer) { 11105 uint32_t q_size; 11106 11107 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_RSS; 11108 q_size = priv->reta_idx_n * sizeof(ctx->queue[0]); 11109 ctx->queue = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, q_size, 11110 0, SOCKET_ID_ANY); 11111 if (!ctx->queue) 11112 return rte_flow_error_set 11113 (error, ENOMEM, 11114 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11115 NULL, "invalid default miss RSS"); 11116 ctx->action_rss.func = RTE_ETH_HASH_FUNCTION_DEFAULT, 11117 ctx->action_rss.level = 0, 11118 ctx->action_rss.types = priv->rss_conf.rss_hf, 11119 ctx->action_rss.key_len = priv->rss_conf.rss_key_len, 11120 ctx->action_rss.queue_num = priv->reta_idx_n, 11121 ctx->action_rss.key = priv->rss_conf.rss_key, 11122 ctx->action_rss.queue = ctx->queue; 11123 if (!priv->reta_idx_n || !priv->rxqs_n) 11124 return rte_flow_error_set 11125 (error, EINVAL, 11126 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11127 NULL, "invalid port configuration"); 11128 if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)) 11129 ctx->action_rss.types = 0; 11130 for (i = 0; i != priv->reta_idx_n; ++i) 11131 ctx->queue[i] = (*priv->reta_idx)[i]; 11132 } else { 11133 miss_actions[1].type = RTE_FLOW_ACTION_TYPE_JUMP; 11134 ctx->miss_jump.group = MLX5_TNL_MISS_FDB_JUMP_GRP; 11135 } 11136 miss_actions[1].conf = (typeof(miss_actions[1].conf))ctx->raw; 11137 for (; app_actions->type != RTE_FLOW_ACTION_TYPE_JUMP; app_actions++); 11138 jump_data = app_actions->conf; 11139 miss_attr.priority = MLX5_TNL_MISS_RULE_PRIORITY; 11140 miss_attr.group = jump_data->group; 11141 ret = mlx5_flow_group_to_table(dev, tunnel, jump_data->group, 11142 &flow_table, &grp_info, error); 11143 if (ret) 11144 return rte_flow_error_set(error, EINVAL, 11145 RTE_FLOW_ERROR_TYPE_ACTION_CONF, 11146 NULL, "invalid tunnel id"); 11147 mark_id.app_reserve = 0; 11148 mark_id.table_id = tunnel_flow_tbl_to_id(flow_table); 11149 mark_id.transfer = !!attr->transfer; 11150 mark_id._unused_ = 0; 11151 miss_mark.id = mark_id.val; 11152 dev_flow = flow_drv_prepare(dev, flow, &miss_attr, 11153 miss_items, miss_actions, flow_idx, error); 11154 if (!dev_flow) 11155 return -rte_errno; 11156 dev_flow->flow = flow; 11157 dev_flow->external = true; 11158 dev_flow->tunnel = tunnel; 11159 dev_flow->tof_type = MLX5_TUNNEL_OFFLOAD_MISS_RULE; 11160 /* Subflow object was created, we must include one in the list. */ 11161 SILIST_INSERT(&flow->dev_handles, dev_flow->handle_idx, 11162 dev_flow->handle, next); 11163 DRV_LOG(DEBUG, 11164 "port %u tunnel type=%d id=%u miss rule priority=%u group=%u", 11165 dev->data->port_id, tunnel->app_tunnel.type, 11166 tunnel->tunnel_id, miss_attr.priority, miss_attr.group); 11167 ret = flow_drv_translate(dev, dev_flow, &miss_attr, miss_items, 11168 miss_actions, error); 11169 if (!ret) 11170 ret = flow_mreg_update_copy_table(dev, flow, miss_actions, 11171 error); 11172 11173 return ret; 11174 } 11175 11176 static const struct mlx5_flow_tbl_data_entry * 11177 tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark) 11178 { 11179 struct mlx5_priv *priv = dev->data->dev_private; 11180 struct mlx5_dev_ctx_shared *sh = priv->sh; 11181 struct mlx5_list_entry *he; 11182 union tunnel_offload_mark mbits = { .val = mark }; 11183 union mlx5_flow_tbl_key table_key = { 11184 { 11185 .level = tunnel_id_to_flow_tbl(mbits.table_id), 11186 .id = 0, 11187 .reserved = 0, 11188 .dummy = 0, 11189 .is_fdb = !!mbits.transfer, 11190 .is_egress = 0, 11191 } 11192 }; 11193 struct mlx5_flow_cb_ctx ctx = { 11194 .data = &table_key.v64, 11195 }; 11196 11197 he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx); 11198 return he ? 11199 container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL; 11200 } 11201 11202 static void 11203 mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx, 11204 struct mlx5_list_entry *entry) 11205 { 11206 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11207 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11208 11209 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11210 tunnel_flow_tbl_to_id(tte->flow_table)); 11211 mlx5_free(tte); 11212 } 11213 11214 static int 11215 mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused, 11216 struct mlx5_list_entry *entry, void *cb_ctx) 11217 { 11218 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11219 union tunnel_tbl_key tbl = { 11220 .val = *(uint64_t *)(ctx->data), 11221 }; 11222 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11223 11224 return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group; 11225 } 11226 11227 static struct mlx5_list_entry * 11228 mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx) 11229 { 11230 struct mlx5_dev_ctx_shared *sh = tool_ctx; 11231 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 11232 struct tunnel_tbl_entry *tte; 11233 union tunnel_tbl_key tbl = { 11234 .val = *(uint64_t *)(ctx->data), 11235 }; 11236 11237 tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, 11238 sizeof(*tte), 0, 11239 SOCKET_ID_ANY); 11240 if (!tte) 11241 goto err; 11242 mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11243 &tte->flow_table); 11244 if (tte->flow_table >= MLX5_MAX_TABLES) { 11245 DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.", 11246 tte->flow_table); 11247 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID], 11248 tte->flow_table); 11249 goto err; 11250 } else if (!tte->flow_table) { 11251 goto err; 11252 } 11253 tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table); 11254 tte->tunnel_id = tbl.tunnel_id; 11255 tte->group = tbl.group; 11256 return &tte->hash; 11257 err: 11258 if (tte) 11259 mlx5_free(tte); 11260 return NULL; 11261 } 11262 11263 static struct mlx5_list_entry * 11264 mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused, 11265 struct mlx5_list_entry *oentry, 11266 void *cb_ctx __rte_unused) 11267 { 11268 struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte), 11269 0, SOCKET_ID_ANY); 11270 11271 if (!tte) 11272 return NULL; 11273 memcpy(tte, oentry, sizeof(*tte)); 11274 return &tte->hash; 11275 } 11276 11277 static void 11278 mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused, 11279 struct mlx5_list_entry *entry) 11280 { 11281 struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash); 11282 11283 mlx5_free(tte); 11284 } 11285 11286 static uint32_t 11287 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev, 11288 const struct mlx5_flow_tunnel *tunnel, 11289 uint32_t group, uint32_t *table, 11290 struct rte_flow_error *error) 11291 { 11292 struct mlx5_list_entry *he; 11293 struct tunnel_tbl_entry *tte; 11294 union tunnel_tbl_key key = { 11295 .tunnel_id = tunnel ? tunnel->tunnel_id : 0, 11296 .group = group 11297 }; 11298 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11299 struct mlx5_hlist *group_hash; 11300 struct mlx5_flow_cb_ctx ctx = { 11301 .data = &key.val, 11302 }; 11303 11304 group_hash = tunnel ? tunnel->groups : thub->groups; 11305 he = mlx5_hlist_register(group_hash, key.val, &ctx); 11306 if (!he) 11307 return rte_flow_error_set(error, EINVAL, 11308 RTE_FLOW_ERROR_TYPE_ATTR_GROUP, 11309 NULL, 11310 "tunnel group index not supported"); 11311 tte = container_of(he, typeof(*tte), hash); 11312 *table = tte->flow_table; 11313 DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x", 11314 dev->data->port_id, key.tunnel_id, group, *table); 11315 return 0; 11316 } 11317 11318 static void 11319 mlx5_flow_tunnel_free(struct rte_eth_dev *dev, 11320 struct mlx5_flow_tunnel *tunnel) 11321 { 11322 struct mlx5_priv *priv = dev->data->dev_private; 11323 struct mlx5_indexed_pool *ipool; 11324 11325 DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x", 11326 dev->data->port_id, tunnel->tunnel_id); 11327 LIST_REMOVE(tunnel, chain); 11328 mlx5_hlist_destroy(tunnel->groups); 11329 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11330 mlx5_ipool_free(ipool, tunnel->tunnel_id); 11331 } 11332 11333 static bool 11334 mlx5_access_tunnel_offload_db 11335 (struct rte_eth_dev *dev, 11336 bool (*match)(struct rte_eth_dev *, 11337 struct mlx5_flow_tunnel *, const void *), 11338 void (*hit)(struct rte_eth_dev *, struct mlx5_flow_tunnel *, void *), 11339 void (*miss)(struct rte_eth_dev *, void *), 11340 void *ctx, bool lock_op) 11341 { 11342 bool verdict = false; 11343 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11344 struct mlx5_flow_tunnel *tunnel; 11345 11346 rte_spinlock_lock(&thub->sl); 11347 LIST_FOREACH(tunnel, &thub->tunnels, chain) { 11348 verdict = match(dev, tunnel, (const void *)ctx); 11349 if (verdict) 11350 break; 11351 } 11352 if (!lock_op) 11353 rte_spinlock_unlock(&thub->sl); 11354 if (verdict && hit) 11355 hit(dev, tunnel, ctx); 11356 if (!verdict && miss) 11357 miss(dev, ctx); 11358 if (lock_op) 11359 rte_spinlock_unlock(&thub->sl); 11360 11361 return verdict; 11362 } 11363 11364 struct tunnel_db_find_tunnel_id_ctx { 11365 uint32_t tunnel_id; 11366 struct mlx5_flow_tunnel *tunnel; 11367 }; 11368 11369 static bool 11370 find_tunnel_id_match(struct rte_eth_dev *dev, 11371 struct mlx5_flow_tunnel *tunnel, const void *x) 11372 { 11373 const struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11374 11375 RTE_SET_USED(dev); 11376 return tunnel->tunnel_id == ctx->tunnel_id; 11377 } 11378 11379 static void 11380 find_tunnel_id_hit(struct rte_eth_dev *dev, 11381 struct mlx5_flow_tunnel *tunnel, void *x) 11382 { 11383 struct tunnel_db_find_tunnel_id_ctx *ctx = x; 11384 RTE_SET_USED(dev); 11385 ctx->tunnel = tunnel; 11386 } 11387 11388 static struct mlx5_flow_tunnel * 11389 mlx5_find_tunnel_id(struct rte_eth_dev *dev, uint32_t id) 11390 { 11391 struct tunnel_db_find_tunnel_id_ctx ctx = { 11392 .tunnel_id = id, 11393 }; 11394 11395 mlx5_access_tunnel_offload_db(dev, find_tunnel_id_match, 11396 find_tunnel_id_hit, NULL, &ctx, true); 11397 11398 return ctx.tunnel; 11399 } 11400 11401 static struct mlx5_flow_tunnel * 11402 mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev, 11403 const struct rte_flow_tunnel *app_tunnel) 11404 { 11405 struct mlx5_priv *priv = dev->data->dev_private; 11406 struct mlx5_indexed_pool *ipool; 11407 struct mlx5_flow_tunnel *tunnel; 11408 uint32_t id; 11409 11410 ipool = priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID]; 11411 tunnel = mlx5_ipool_zmalloc(ipool, &id); 11412 if (!tunnel) 11413 return NULL; 11414 if (id >= MLX5_MAX_TUNNELS) { 11415 mlx5_ipool_free(ipool, id); 11416 DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id); 11417 return NULL; 11418 } 11419 tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true, 11420 priv->sh, 11421 mlx5_flow_tunnel_grp2tbl_create_cb, 11422 mlx5_flow_tunnel_grp2tbl_match_cb, 11423 mlx5_flow_tunnel_grp2tbl_remove_cb, 11424 mlx5_flow_tunnel_grp2tbl_clone_cb, 11425 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11426 if (!tunnel->groups) { 11427 mlx5_ipool_free(ipool, id); 11428 return NULL; 11429 } 11430 /* initiate new PMD tunnel */ 11431 memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel)); 11432 tunnel->tunnel_id = id; 11433 tunnel->action.type = (typeof(tunnel->action.type)) 11434 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET; 11435 tunnel->action.conf = tunnel; 11436 tunnel->item.type = (typeof(tunnel->item.type)) 11437 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL; 11438 tunnel->item.spec = tunnel; 11439 tunnel->item.last = NULL; 11440 tunnel->item.mask = NULL; 11441 11442 DRV_LOG(DEBUG, "port %u new pmd tunnel id=0x%x", 11443 dev->data->port_id, tunnel->tunnel_id); 11444 11445 return tunnel; 11446 } 11447 11448 struct tunnel_db_get_tunnel_ctx { 11449 const struct rte_flow_tunnel *app_tunnel; 11450 struct mlx5_flow_tunnel *tunnel; 11451 }; 11452 11453 static bool get_tunnel_match(struct rte_eth_dev *dev, 11454 struct mlx5_flow_tunnel *tunnel, const void *x) 11455 { 11456 const struct tunnel_db_get_tunnel_ctx *ctx = x; 11457 11458 RTE_SET_USED(dev); 11459 return !memcmp(ctx->app_tunnel, &tunnel->app_tunnel, 11460 sizeof(*ctx->app_tunnel)); 11461 } 11462 11463 static void get_tunnel_hit(struct rte_eth_dev *dev, 11464 struct mlx5_flow_tunnel *tunnel, void *x) 11465 { 11466 /* called under tunnel spinlock protection */ 11467 struct tunnel_db_get_tunnel_ctx *ctx = x; 11468 11469 RTE_SET_USED(dev); 11470 tunnel->refctn++; 11471 ctx->tunnel = tunnel; 11472 } 11473 11474 static void get_tunnel_miss(struct rte_eth_dev *dev, void *x) 11475 { 11476 /* called under tunnel spinlock protection */ 11477 struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev); 11478 struct tunnel_db_get_tunnel_ctx *ctx = x; 11479 11480 rte_spinlock_unlock(&thub->sl); 11481 ctx->tunnel = mlx5_flow_tunnel_allocate(dev, ctx->app_tunnel); 11482 rte_spinlock_lock(&thub->sl); 11483 if (ctx->tunnel) { 11484 ctx->tunnel->refctn = 1; 11485 LIST_INSERT_HEAD(&thub->tunnels, ctx->tunnel, chain); 11486 } 11487 } 11488 11489 11490 static int 11491 mlx5_get_flow_tunnel(struct rte_eth_dev *dev, 11492 const struct rte_flow_tunnel *app_tunnel, 11493 struct mlx5_flow_tunnel **tunnel) 11494 { 11495 struct tunnel_db_get_tunnel_ctx ctx = { 11496 .app_tunnel = app_tunnel, 11497 }; 11498 11499 mlx5_access_tunnel_offload_db(dev, get_tunnel_match, get_tunnel_hit, 11500 get_tunnel_miss, &ctx, true); 11501 *tunnel = ctx.tunnel; 11502 return ctx.tunnel ? 0 : -ENOMEM; 11503 } 11504 11505 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id) 11506 { 11507 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub; 11508 11509 if (!thub) 11510 return; 11511 if (!LIST_EMPTY(&thub->tunnels)) 11512 DRV_LOG(WARNING, "port %u tunnels present", port_id); 11513 mlx5_hlist_destroy(thub->groups); 11514 mlx5_free(thub); 11515 } 11516 11517 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh) 11518 { 11519 int err; 11520 struct mlx5_flow_tunnel_hub *thub; 11521 11522 thub = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO, sizeof(*thub), 11523 0, SOCKET_ID_ANY); 11524 if (!thub) 11525 return -ENOMEM; 11526 LIST_INIT(&thub->tunnels); 11527 rte_spinlock_init(&thub->sl); 11528 thub->groups = mlx5_hlist_create("flow groups", 64, 11529 false, true, sh, 11530 mlx5_flow_tunnel_grp2tbl_create_cb, 11531 mlx5_flow_tunnel_grp2tbl_match_cb, 11532 mlx5_flow_tunnel_grp2tbl_remove_cb, 11533 mlx5_flow_tunnel_grp2tbl_clone_cb, 11534 mlx5_flow_tunnel_grp2tbl_clone_free_cb); 11535 if (!thub->groups) { 11536 err = -rte_errno; 11537 goto err; 11538 } 11539 sh->tunnel_hub = thub; 11540 11541 return 0; 11542 11543 err: 11544 if (thub->groups) 11545 mlx5_hlist_destroy(thub->groups); 11546 if (thub) 11547 mlx5_free(thub); 11548 return err; 11549 } 11550 11551 static inline int 11552 mlx5_flow_tunnel_validate(struct rte_eth_dev *dev, 11553 struct rte_flow_tunnel *tunnel, 11554 struct rte_flow_error *error) 11555 { 11556 struct mlx5_priv *priv = dev->data->dev_private; 11557 11558 if (!priv->sh->config.dv_flow_en) 11559 return rte_flow_error_set(error, ENOTSUP, 11560 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11561 "flow DV interface is off"); 11562 if (!is_tunnel_offload_active(dev)) 11563 return rte_flow_error_set(error, ENOTSUP, 11564 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11565 "tunnel offload was not activated, consider setting dv_xmeta_en=3"); 11566 if (!tunnel) 11567 return rte_flow_error_set(error, EINVAL, 11568 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11569 "no application tunnel"); 11570 switch (tunnel->type) { 11571 default: 11572 return rte_flow_error_set(error, EINVAL, 11573 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11574 "unsupported tunnel type"); 11575 case RTE_FLOW_ITEM_TYPE_VXLAN: 11576 case RTE_FLOW_ITEM_TYPE_GRE: 11577 case RTE_FLOW_ITEM_TYPE_NVGRE: 11578 case RTE_FLOW_ITEM_TYPE_GENEVE: 11579 break; 11580 } 11581 return 0; 11582 } 11583 11584 static int 11585 mlx5_flow_tunnel_decap_set(struct rte_eth_dev *dev, 11586 struct rte_flow_tunnel *app_tunnel, 11587 struct rte_flow_action **actions, 11588 uint32_t *num_of_actions, 11589 struct rte_flow_error *error) 11590 { 11591 struct mlx5_flow_tunnel *tunnel; 11592 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11593 11594 if (ret) 11595 return ret; 11596 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11597 if (ret < 0) { 11598 return rte_flow_error_set(error, ret, 11599 RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, 11600 "failed to initialize pmd tunnel"); 11601 } 11602 *actions = &tunnel->action; 11603 *num_of_actions = 1; 11604 return 0; 11605 } 11606 11607 static int 11608 mlx5_flow_tunnel_match(struct rte_eth_dev *dev, 11609 struct rte_flow_tunnel *app_tunnel, 11610 struct rte_flow_item **items, 11611 uint32_t *num_of_items, 11612 struct rte_flow_error *error) 11613 { 11614 struct mlx5_flow_tunnel *tunnel; 11615 int ret = mlx5_flow_tunnel_validate(dev, app_tunnel, error); 11616 11617 if (ret) 11618 return ret; 11619 ret = mlx5_get_flow_tunnel(dev, app_tunnel, &tunnel); 11620 if (ret < 0) { 11621 return rte_flow_error_set(error, ret, 11622 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11623 "failed to initialize pmd tunnel"); 11624 } 11625 *items = &tunnel->item; 11626 *num_of_items = 1; 11627 return 0; 11628 } 11629 11630 struct tunnel_db_element_release_ctx { 11631 struct rte_flow_item *items; 11632 struct rte_flow_action *actions; 11633 uint32_t num_elements; 11634 struct rte_flow_error *error; 11635 int ret; 11636 }; 11637 11638 static bool 11639 tunnel_element_release_match(struct rte_eth_dev *dev, 11640 struct mlx5_flow_tunnel *tunnel, const void *x) 11641 { 11642 const struct tunnel_db_element_release_ctx *ctx = x; 11643 11644 RTE_SET_USED(dev); 11645 if (ctx->num_elements != 1) 11646 return false; 11647 else if (ctx->items) 11648 return ctx->items == &tunnel->item; 11649 else if (ctx->actions) 11650 return ctx->actions == &tunnel->action; 11651 11652 return false; 11653 } 11654 11655 static void 11656 tunnel_element_release_hit(struct rte_eth_dev *dev, 11657 struct mlx5_flow_tunnel *tunnel, void *x) 11658 { 11659 struct tunnel_db_element_release_ctx *ctx = x; 11660 ctx->ret = 0; 11661 if (!(__atomic_fetch_sub(&tunnel->refctn, 1, __ATOMIC_RELAXED) - 1)) 11662 mlx5_flow_tunnel_free(dev, tunnel); 11663 } 11664 11665 static void 11666 tunnel_element_release_miss(struct rte_eth_dev *dev, void *x) 11667 { 11668 struct tunnel_db_element_release_ctx *ctx = x; 11669 RTE_SET_USED(dev); 11670 ctx->ret = rte_flow_error_set(ctx->error, EINVAL, 11671 RTE_FLOW_ERROR_TYPE_HANDLE, NULL, 11672 "invalid argument"); 11673 } 11674 11675 static int 11676 mlx5_flow_tunnel_item_release(struct rte_eth_dev *dev, 11677 struct rte_flow_item *pmd_items, 11678 uint32_t num_items, struct rte_flow_error *err) 11679 { 11680 struct tunnel_db_element_release_ctx ctx = { 11681 .items = pmd_items, 11682 .actions = NULL, 11683 .num_elements = num_items, 11684 .error = err, 11685 }; 11686 11687 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11688 tunnel_element_release_hit, 11689 tunnel_element_release_miss, &ctx, false); 11690 11691 return ctx.ret; 11692 } 11693 11694 static int 11695 mlx5_flow_tunnel_action_release(struct rte_eth_dev *dev, 11696 struct rte_flow_action *pmd_actions, 11697 uint32_t num_actions, struct rte_flow_error *err) 11698 { 11699 struct tunnel_db_element_release_ctx ctx = { 11700 .items = NULL, 11701 .actions = pmd_actions, 11702 .num_elements = num_actions, 11703 .error = err, 11704 }; 11705 11706 mlx5_access_tunnel_offload_db(dev, tunnel_element_release_match, 11707 tunnel_element_release_hit, 11708 tunnel_element_release_miss, &ctx, false); 11709 11710 return ctx.ret; 11711 } 11712 11713 static int 11714 mlx5_flow_tunnel_get_restore_info(struct rte_eth_dev *dev, 11715 struct rte_mbuf *m, 11716 struct rte_flow_restore_info *info, 11717 struct rte_flow_error *err) 11718 { 11719 uint64_t ol_flags = m->ol_flags; 11720 const struct mlx5_flow_tbl_data_entry *tble; 11721 const uint64_t mask = RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 11722 struct mlx5_priv *priv = dev->data->dev_private; 11723 11724 if (priv->tunnel_enabled == 0) 11725 goto err; 11726 if ((ol_flags & mask) != mask) 11727 goto err; 11728 tble = tunnel_mark_decode(dev, m->hash.fdir.hi); 11729 if (!tble) { 11730 DRV_LOG(DEBUG, "port %u invalid miss tunnel mark %#x", 11731 dev->data->port_id, m->hash.fdir.hi); 11732 goto err; 11733 } 11734 MLX5_ASSERT(tble->tunnel); 11735 memcpy(&info->tunnel, &tble->tunnel->app_tunnel, sizeof(info->tunnel)); 11736 info->group_id = tble->group_id; 11737 info->flags = RTE_FLOW_RESTORE_INFO_TUNNEL | 11738 RTE_FLOW_RESTORE_INFO_GROUP_ID | 11739 RTE_FLOW_RESTORE_INFO_ENCAPSULATED; 11740 11741 return 0; 11742 11743 err: 11744 return rte_flow_error_set(err, EINVAL, 11745 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11746 "failed to get restore info"); 11747 } 11748 11749 #else /* HAVE_IBV_FLOW_DV_SUPPORT */ 11750 static int 11751 mlx5_flow_tunnel_decap_set(__rte_unused struct rte_eth_dev *dev, 11752 __rte_unused struct rte_flow_tunnel *app_tunnel, 11753 __rte_unused struct rte_flow_action **actions, 11754 __rte_unused uint32_t *num_of_actions, 11755 __rte_unused struct rte_flow_error *error) 11756 { 11757 return -ENOTSUP; 11758 } 11759 11760 static int 11761 mlx5_flow_tunnel_match(__rte_unused struct rte_eth_dev *dev, 11762 __rte_unused struct rte_flow_tunnel *app_tunnel, 11763 __rte_unused struct rte_flow_item **items, 11764 __rte_unused uint32_t *num_of_items, 11765 __rte_unused struct rte_flow_error *error) 11766 { 11767 return -ENOTSUP; 11768 } 11769 11770 static int 11771 mlx5_flow_tunnel_item_release(__rte_unused struct rte_eth_dev *dev, 11772 __rte_unused struct rte_flow_item *pmd_items, 11773 __rte_unused uint32_t num_items, 11774 __rte_unused struct rte_flow_error *err) 11775 { 11776 return -ENOTSUP; 11777 } 11778 11779 static int 11780 mlx5_flow_tunnel_action_release(__rte_unused struct rte_eth_dev *dev, 11781 __rte_unused struct rte_flow_action *pmd_action, 11782 __rte_unused uint32_t num_actions, 11783 __rte_unused struct rte_flow_error *err) 11784 { 11785 return -ENOTSUP; 11786 } 11787 11788 static int 11789 mlx5_flow_tunnel_get_restore_info(__rte_unused struct rte_eth_dev *dev, 11790 __rte_unused struct rte_mbuf *m, 11791 __rte_unused struct rte_flow_restore_info *i, 11792 __rte_unused struct rte_flow_error *err) 11793 { 11794 return -ENOTSUP; 11795 } 11796 11797 static int 11798 flow_tunnel_add_default_miss(__rte_unused struct rte_eth_dev *dev, 11799 __rte_unused struct rte_flow *flow, 11800 __rte_unused const struct rte_flow_attr *attr, 11801 __rte_unused const struct rte_flow_action *actions, 11802 __rte_unused uint32_t flow_idx, 11803 __rte_unused const struct mlx5_flow_tunnel *tunnel, 11804 __rte_unused struct tunnel_default_miss_ctx *ctx, 11805 __rte_unused struct rte_flow_error *error) 11806 { 11807 return -ENOTSUP; 11808 } 11809 11810 static struct mlx5_flow_tunnel * 11811 mlx5_find_tunnel_id(__rte_unused struct rte_eth_dev *dev, 11812 __rte_unused uint32_t id) 11813 { 11814 return NULL; 11815 } 11816 11817 static void 11818 mlx5_flow_tunnel_free(__rte_unused struct rte_eth_dev *dev, 11819 __rte_unused struct mlx5_flow_tunnel *tunnel) 11820 { 11821 } 11822 11823 static uint32_t 11824 tunnel_flow_group_to_flow_table(__rte_unused struct rte_eth_dev *dev, 11825 __rte_unused const struct mlx5_flow_tunnel *t, 11826 __rte_unused uint32_t group, 11827 __rte_unused uint32_t *table, 11828 struct rte_flow_error *error) 11829 { 11830 return rte_flow_error_set(error, ENOTSUP, 11831 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11832 "tunnel offload requires DV support"); 11833 } 11834 11835 void 11836 mlx5_release_tunnel_hub(__rte_unused struct mlx5_dev_ctx_shared *sh, 11837 __rte_unused uint16_t port_id) 11838 { 11839 } 11840 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */ 11841 11842 /* Flex flow item API */ 11843 static struct rte_flow_item_flex_handle * 11844 mlx5_flow_flex_item_create(struct rte_eth_dev *dev, 11845 const struct rte_flow_item_flex_conf *conf, 11846 struct rte_flow_error *error) 11847 { 11848 static const char err_msg[] = "flex item creation unsupported"; 11849 struct mlx5_priv *priv = dev->data->dev_private; 11850 struct rte_flow_attr attr = { .transfer = 0 }; 11851 const struct mlx5_flow_driver_ops *fops = 11852 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11853 11854 if (!priv->pci_dev) { 11855 rte_flow_error_set(error, ENOTSUP, 11856 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11857 "create flex item on PF only"); 11858 return NULL; 11859 } 11860 switch (priv->pci_dev->id.device_id) { 11861 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 11862 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 11863 break; 11864 default: 11865 rte_flow_error_set(error, ENOTSUP, 11866 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, 11867 "flex item available on BlueField ports only"); 11868 return NULL; 11869 } 11870 if (!fops->item_create) { 11871 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11872 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11873 NULL, err_msg); 11874 return NULL; 11875 } 11876 return fops->item_create(dev, conf, error); 11877 } 11878 11879 static int 11880 mlx5_flow_flex_item_release(struct rte_eth_dev *dev, 11881 const struct rte_flow_item_flex_handle *handle, 11882 struct rte_flow_error *error) 11883 { 11884 static const char err_msg[] = "flex item release unsupported"; 11885 struct rte_flow_attr attr = { .transfer = 0 }; 11886 const struct mlx5_flow_driver_ops *fops = 11887 flow_get_drv_ops(flow_get_drv_type(dev, &attr)); 11888 11889 if (!fops->item_release) { 11890 DRV_LOG(ERR, "port %u %s.", dev->data->port_id, err_msg); 11891 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, 11892 NULL, err_msg); 11893 return -rte_errno; 11894 } 11895 return fops->item_release(dev, handle, error); 11896 } 11897 11898 static void 11899 mlx5_dbg__print_pattern(const struct rte_flow_item *item) 11900 { 11901 int ret; 11902 struct rte_flow_error error; 11903 11904 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { 11905 char *item_name; 11906 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ITEM_NAME_PTR, &item_name, 11907 sizeof(item_name), 11908 (void *)(uintptr_t)item->type, &error); 11909 if (ret > 0) 11910 printf("%s ", item_name); 11911 else 11912 printf("%d\n", (int)item->type); 11913 } 11914 printf("END\n"); 11915 } 11916 11917 static int 11918 mlx5_flow_is_std_vxlan_port(const struct rte_flow_item *udp_item) 11919 { 11920 const struct rte_flow_item_udp *spec = udp_item->spec; 11921 const struct rte_flow_item_udp *mask = udp_item->mask; 11922 uint16_t udp_dport = 0; 11923 11924 if (spec != NULL) { 11925 if (!mask) 11926 mask = &rte_flow_item_udp_mask; 11927 udp_dport = rte_be_to_cpu_16(spec->hdr.dst_port & 11928 mask->hdr.dst_port); 11929 } 11930 return (!udp_dport || udp_dport == MLX5_UDP_PORT_VXLAN); 11931 } 11932 11933 static const struct mlx5_flow_expand_node * 11934 mlx5_flow_expand_rss_adjust_node(const struct rte_flow_item *pattern, 11935 unsigned int item_idx, 11936 const struct mlx5_flow_expand_node graph[], 11937 const struct mlx5_flow_expand_node *node) 11938 { 11939 const struct rte_flow_item *item = pattern + item_idx, *prev_item; 11940 11941 if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN && 11942 node != NULL && 11943 node->type == RTE_FLOW_ITEM_TYPE_VXLAN) { 11944 /* 11945 * The expansion node is VXLAN and it is also the last 11946 * expandable item in the pattern, so need to continue 11947 * expansion of the inner tunnel. 11948 */ 11949 MLX5_ASSERT(item_idx > 0); 11950 prev_item = pattern + item_idx - 1; 11951 MLX5_ASSERT(prev_item->type == RTE_FLOW_ITEM_TYPE_UDP); 11952 if (mlx5_flow_is_std_vxlan_port(prev_item)) 11953 return &graph[MLX5_EXPANSION_STD_VXLAN]; 11954 return &graph[MLX5_EXPANSION_L3_VXLAN]; 11955 } 11956 return node; 11957 } 11958 11959 /* Map of Verbs to Flow priority with 8 Verbs priorities. */ 11960 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { 11961 { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, 11962 }; 11963 11964 /* Map of Verbs to Flow priority with 16 Verbs priorities. */ 11965 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { 11966 { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, 11967 { 9, 10, 11 }, { 12, 13, 14 }, 11968 }; 11969 11970 /** 11971 * Discover the number of available flow priorities. 11972 * 11973 * @param dev 11974 * Ethernet device. 11975 * 11976 * @return 11977 * On success, number of available flow priorities. 11978 * On failure, a negative errno-style code and rte_errno is set. 11979 */ 11980 int 11981 mlx5_flow_discover_priorities(struct rte_eth_dev *dev) 11982 { 11983 static const uint16_t vprio[] = {8, 16}; 11984 const struct mlx5_priv *priv = dev->data->dev_private; 11985 const struct mlx5_flow_driver_ops *fops; 11986 enum mlx5_flow_drv_type type; 11987 int ret; 11988 11989 type = mlx5_flow_os_get_type(); 11990 if (type == MLX5_FLOW_TYPE_MAX) { 11991 type = MLX5_FLOW_TYPE_VERBS; 11992 if (priv->sh->cdev->config.devx && priv->sh->config.dv_flow_en) 11993 type = MLX5_FLOW_TYPE_DV; 11994 } 11995 fops = flow_get_drv_ops(type); 11996 if (fops->discover_priorities == NULL) { 11997 DRV_LOG(ERR, "Priority discovery not supported"); 11998 rte_errno = ENOTSUP; 11999 return -rte_errno; 12000 } 12001 ret = fops->discover_priorities(dev, vprio, RTE_DIM(vprio)); 12002 if (ret < 0) 12003 return ret; 12004 switch (ret) { 12005 case 8: 12006 ret = RTE_DIM(priority_map_3); 12007 break; 12008 case 16: 12009 ret = RTE_DIM(priority_map_5); 12010 break; 12011 default: 12012 rte_errno = ENOTSUP; 12013 DRV_LOG(ERR, 12014 "port %u maximum priority: %d expected 8/16", 12015 dev->data->port_id, ret); 12016 return -rte_errno; 12017 } 12018 DRV_LOG(INFO, "port %u supported flow priorities:" 12019 " 0-%d for ingress or egress root table," 12020 " 0-%d for non-root table or transfer root table.", 12021 dev->data->port_id, ret - 2, 12022 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); 12023 return ret; 12024 } 12025 12026 /** 12027 * Adjust flow priority based on the highest layer and the request priority. 12028 * 12029 * @param[in] dev 12030 * Pointer to the Ethernet device structure. 12031 * @param[in] priority 12032 * The rule base priority. 12033 * @param[in] subpriority 12034 * The priority based on the items. 12035 * 12036 * @return 12037 * The new priority. 12038 */ 12039 uint32_t 12040 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, 12041 uint32_t subpriority) 12042 { 12043 uint32_t res = 0; 12044 struct mlx5_priv *priv = dev->data->dev_private; 12045 12046 switch (priv->sh->flow_max_priority) { 12047 case RTE_DIM(priority_map_3): 12048 res = priority_map_3[priority][subpriority]; 12049 break; 12050 case RTE_DIM(priority_map_5): 12051 res = priority_map_5[priority][subpriority]; 12052 break; 12053 } 12054 return res; 12055 } 12056 12057 /** 12058 * Get the priority for sending traffic to kernel table. 12059 * 12060 * @param[in] dev 12061 * Pointer to the Ethernet device structure. 12062 * 12063 * @return 12064 * On success: the value of priority for sending traffic to kernel table 12065 * On failure: -1 12066 */ 12067 uint32_t 12068 mlx5_get_send_to_kernel_priority(struct rte_eth_dev *dev) 12069 { 12070 struct mlx5_priv *priv = dev->data->dev_private; 12071 uint32_t res; 12072 12073 switch (priv->sh->flow_max_priority) { 12074 case RTE_DIM(priority_map_5): 12075 res = 15; 12076 break; 12077 case RTE_DIM(priority_map_3): 12078 res = 7; 12079 break; 12080 default: 12081 DRV_LOG(ERR, 12082 "port %u maximum priority: %d expected 8/16", 12083 dev->data->port_id, priv->sh->flow_max_priority); 12084 res = (uint32_t)-1; 12085 } 12086 return res; 12087 } 12088 12089 /** 12090 * Get the E-Switch Manager vport id. 12091 * 12092 * @param[in] dev 12093 * Pointer to the Ethernet device structure. 12094 * 12095 * @return 12096 * The vport id. 12097 */ 12098 int16_t mlx5_flow_get_esw_manager_vport_id(struct rte_eth_dev *dev) 12099 { 12100 struct mlx5_priv *priv = dev->data->dev_private; 12101 struct mlx5_common_device *cdev = priv->sh->cdev; 12102 12103 /* New FW exposes E-Switch Manager vport ID, can use it directly. */ 12104 if (cdev->config.hca_attr.esw_mgr_vport_id_valid) 12105 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id; 12106 12107 if (priv->pci_dev == NULL) 12108 return 0; 12109 switch (priv->pci_dev->id.device_id) { 12110 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD: 12111 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD2: 12112 case PCI_DEVICE_ID_MELLANOX_BLUEFIELD3: 12113 /* 12114 * In old FW which doesn't expose the E-Switch Manager vport ID in the capability, 12115 * only the BF embedded CPUs control the E-Switch Manager port. Hence, 12116 * ECPF vport ID is selected and not the host port (0) in any BF case. 12117 */ 12118 return (int16_t)MLX5_ECPF_VPORT_ID; 12119 default: 12120 return MLX5_PF_VPORT_ID; 12121 } 12122 } 12123 12124 /** 12125 * Parse item to get the vport id. 12126 * 12127 * @param[in] dev 12128 * Pointer to the Ethernet device structure. 12129 * @param[in] item 12130 * The src port id match item. 12131 * @param[out] vport_id 12132 * Pointer to put the vport id. 12133 * @param[out] all_ports 12134 * Indicate if the item matches all ports. 12135 * @param[out] error 12136 * Pointer to error structure. 12137 * 12138 * @return 12139 * 0 on success, a negative errno value otherwise and rte_errno is set. 12140 */ 12141 int mlx5_flow_get_item_vport_id(struct rte_eth_dev *dev, 12142 const struct rte_flow_item *item, 12143 uint16_t *vport_id, 12144 bool *all_ports, 12145 struct rte_flow_error *error) 12146 { 12147 struct mlx5_priv *port_priv; 12148 const struct rte_flow_item_port_id *pid_v = NULL; 12149 const struct rte_flow_item_ethdev *dev_v = NULL; 12150 uint32_t esw_mgr_port; 12151 uint32_t src_port; 12152 12153 if (all_ports) 12154 *all_ports = false; 12155 switch (item->type) { 12156 case RTE_FLOW_ITEM_TYPE_PORT_ID: 12157 pid_v = item->spec; 12158 if (!pid_v) 12159 return 0; 12160 src_port = pid_v->id; 12161 esw_mgr_port = MLX5_PORT_ESW_MGR; 12162 break; 12163 case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT: 12164 dev_v = item->spec; 12165 if (!dev_v) { 12166 if (all_ports) 12167 *all_ports = true; 12168 return 0; 12169 } 12170 src_port = dev_v->port_id; 12171 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12172 break; 12173 case RTE_FLOW_ITEM_TYPE_PORT_REPRESENTOR: 12174 src_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12175 esw_mgr_port = MLX5_REPRESENTED_PORT_ESW_MGR; 12176 break; 12177 default: 12178 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12179 NULL, "Incorrect item type."); 12180 } 12181 if (src_port == esw_mgr_port) { 12182 *vport_id = mlx5_flow_get_esw_manager_vport_id(dev); 12183 } else { 12184 port_priv = mlx5_port_to_eswitch_info(src_port, false); 12185 if (!port_priv) 12186 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_SPEC, 12187 NULL, "Failed to get port info."); 12188 *vport_id = port_priv->representor_id; 12189 } 12190 12191 return 0; 12192 } 12193 12194 int 12195 mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev, 12196 uint16_t *proxy_port_id, 12197 struct rte_flow_error *error) 12198 { 12199 const struct mlx5_priv *priv = dev->data->dev_private; 12200 uint16_t port_id; 12201 12202 if (!priv->sh->config.dv_esw_en) 12203 return rte_flow_error_set(error, EINVAL, 12204 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12205 NULL, 12206 "unable to provide a proxy port" 12207 " without E-Switch configured"); 12208 if (!priv->master && !priv->representor) 12209 return rte_flow_error_set(error, EINVAL, 12210 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12211 NULL, 12212 "unable to provide a proxy port" 12213 " for port which is not a master" 12214 " or a representor port"); 12215 if (priv->master) { 12216 *proxy_port_id = dev->data->port_id; 12217 return 0; 12218 } 12219 MLX5_ETH_FOREACH_DEV(port_id, dev->device) { 12220 const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id]; 12221 const struct mlx5_priv *port_priv = port_dev->data->dev_private; 12222 12223 if (port_priv->master && 12224 port_priv->domain_id == priv->domain_id) { 12225 *proxy_port_id = port_id; 12226 return 0; 12227 } 12228 } 12229 return rte_flow_error_set(error, ENODEV, 12230 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, 12231 NULL, "unable to find a proxy port"); 12232 } 12233